diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ChainingGlTextureProcessorListener.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ChainingGlTextureProcessorListener.java new file mode 100644 index 0000000000..3017107b2f --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ChainingGlTextureProcessorListener.java @@ -0,0 +1,106 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import android.util.Pair; +import androidx.annotation.Nullable; +import java.util.ArrayDeque; +import java.util.Queue; + +/** + * A {@link GlTextureProcessor.Listener} that connects the {@link GlTextureProcessor} it is + * {@linkplain GlTextureProcessor#setListener(GlTextureProcessor.Listener) set} on to a previous and + * next {@link GlTextureProcessor}. + */ +/* package */ final class ChainingGlTextureProcessorListener + implements GlTextureProcessor.Listener { + + @Nullable private final GlTextureProcessor previousGlTextureProcessor; + @Nullable private final GlTextureProcessor nextGlTextureProcessor; + private final FrameProcessingTaskExecutor frameProcessingTaskExecutor; + private final FrameProcessorChain.Listener frameProcessorChainListener; + private final Queue> pendingFrames; + + /** + * Creates a new instance. + * + * @param previousGlTextureProcessor The {@link GlTextureProcessor} that comes before the {@link + * GlTextureProcessor} this listener is set on or {@code null} if not applicable. + * @param nextGlTextureProcessor The {@link GlTextureProcessor} that comes after the {@link + * GlTextureProcessor} this listener is set on or {@code null} if not applicable. + * @param frameProcessingTaskExecutor The {@link FrameProcessingTaskExecutor} that is used for + * OpenGL calls. All calls to the previous/next {@link GlTextureProcessor} will be executed by + * the {@link FrameProcessingTaskExecutor}. The caller is responsible for releasing the {@link + * FrameProcessingTaskExecutor}. + * @param frameProcessorChainListener The {@link FrameProcessorChain.Listener} to forward + * exceptions to. + */ + public ChainingGlTextureProcessorListener( + @Nullable GlTextureProcessor previousGlTextureProcessor, + @Nullable GlTextureProcessor nextGlTextureProcessor, + FrameProcessingTaskExecutor frameProcessingTaskExecutor, + FrameProcessorChain.Listener frameProcessorChainListener) { + this.previousGlTextureProcessor = previousGlTextureProcessor; + this.nextGlTextureProcessor = nextGlTextureProcessor; + this.frameProcessingTaskExecutor = frameProcessingTaskExecutor; + this.frameProcessorChainListener = frameProcessorChainListener; + pendingFrames = new ArrayDeque<>(); + } + + @Override + public void onInputFrameProcessed(TextureInfo inputTexture) { + if (previousGlTextureProcessor != null) { + GlTextureProcessor nonNullPreviousGlTextureProcessor = previousGlTextureProcessor; + frameProcessingTaskExecutor.submit( + () -> nonNullPreviousGlTextureProcessor.releaseOutputFrame(inputTexture)); + } + } + + @Override + public void onOutputFrameAvailable(TextureInfo outputTexture, long presentationTimeUs) { + if (nextGlTextureProcessor != null) { + GlTextureProcessor nonNullNextGlTextureProcessor = nextGlTextureProcessor; + frameProcessingTaskExecutor.submit( + () -> { + pendingFrames.add(new Pair<>(outputTexture, presentationTimeUs)); + processFrameNowOrLater(nonNullNextGlTextureProcessor); + }); + } + } + + private void processFrameNowOrLater(GlTextureProcessor nextGlTextureProcessor) { + Pair pendingFrame = pendingFrames.element(); + TextureInfo outputTexture = pendingFrame.first; + long presentationTimeUs = pendingFrame.second; + if (nextGlTextureProcessor.maybeQueueInputFrame(outputTexture, presentationTimeUs)) { + pendingFrames.remove(); + } else { + frameProcessingTaskExecutor.submit(() -> processFrameNowOrLater(nextGlTextureProcessor)); + } + } + + @Override + public void onOutputStreamEnded() { + if (nextGlTextureProcessor != null) { + frameProcessingTaskExecutor.submit(nextGlTextureProcessor::signalEndOfInputStream); + } + } + + @Override + public void onFrameProcessingError(FrameProcessingException e) { + frameProcessorChainListener.onFrameProcessingError(e); + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java index 510160e3f1..f86eda288c 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java @@ -146,11 +146,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } /** - * Creates the OpenGL textures and framebuffers, initializes the {@link + * Creates the OpenGL context, surfaces, textures, and framebuffers, initializes the {@link * SingleFrameGlTextureProcessor SingleFrameGlTextureProcessors} corresponding to the {@link * GlEffect GlEffects}, and returns a new {@code FrameProcessorChain}. * - *

This method must be executed using the {@code singleThreadExecutorService}. + *

This method must be executed using the {@code singleThreadExecutorService}, as all later + * OpenGL commands will be called on that thread. */ @WorkerThread @Nullable diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/ChainingGlTextureProcessorListenerTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/ChainingGlTextureProcessorListenerTest.java new file mode 100644 index 0000000000..9c74cb9306 --- /dev/null +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/ChainingGlTextureProcessorListenerTest.java @@ -0,0 +1,163 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import androidx.media3.common.util.Util; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import org.junit.After; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Tests for {@link ChainingGlTextureProcessorListener}. */ +@RunWith(AndroidJUnit4.class) +public final class ChainingGlTextureProcessorListenerTest { + private static final long EXECUTOR_WAIT_TIME_MS = 100; + + private final FrameProcessorChain.Listener mockFrameProcessorChainListener = + mock(FrameProcessorChain.Listener.class); + private final FrameProcessingTaskExecutor frameProcessingTaskExecutor = + new FrameProcessingTaskExecutor( + Util.newSingleThreadExecutor("Test"), mockFrameProcessorChainListener); + private final GlTextureProcessor mockPreviousGlTextureProcessor = mock(GlTextureProcessor.class); + private final FakeGlTextureProcessor fakeNextGlTextureProcessor = + spy(new FakeGlTextureProcessor()); + private final ChainingGlTextureProcessorListener chainingGlTextureProcessorListener = + new ChainingGlTextureProcessorListener( + mockPreviousGlTextureProcessor, + fakeNextGlTextureProcessor, + frameProcessingTaskExecutor, + mockFrameProcessorChainListener); + + @After + public void release() throws InterruptedException { + frameProcessingTaskExecutor.release(/* releaseTask= */ () -> {}, EXECUTOR_WAIT_TIME_MS); + } + + @Test + public void onFrameProcessingError_callsListener() { + FrameProcessingException exception = new FrameProcessingException("message"); + + chainingGlTextureProcessorListener.onFrameProcessingError(exception); + + verify(mockFrameProcessorChainListener, times(1)).onFrameProcessingError(exception); + } + + @Test + public void onInputFrameProcessed_surrendersFrameToPreviousGlTextureProcessor() + throws InterruptedException { + TextureInfo texture = + new TextureInfo(/* texId= */ 1, /* fboId= */ 1, /* width= */ 100, /* height= */ 100); + + chainingGlTextureProcessorListener.onInputFrameProcessed(texture); + Thread.sleep(EXECUTOR_WAIT_TIME_MS); + + verify(mockPreviousGlTextureProcessor, times(1)).releaseOutputFrame(texture); + } + + @Test + public void onOutputFrameAvailable_passesFrameToNextGlTextureProcessor() + throws InterruptedException { + TextureInfo texture = + new TextureInfo(/* texId= */ 1, /* fboId= */ 1, /* width= */ 100, /* height= */ 100); + long presentationTimeUs = 123; + + chainingGlTextureProcessorListener.onOutputFrameAvailable(texture, presentationTimeUs); + Thread.sleep(EXECUTOR_WAIT_TIME_MS); + + verify(fakeNextGlTextureProcessor, times(1)).maybeQueueInputFrame(texture, presentationTimeUs); + } + + @Test + public void onOutputFrameAvailable_nextGlTextureProcessorRejectsFrame_triesAgain() + throws InterruptedException { + TextureInfo texture = + new TextureInfo(/* texId= */ 1, /* fboId= */ 1, /* width= */ 100, /* height= */ 100); + long presentationTimeUs = 123; + fakeNextGlTextureProcessor.rejectNextFrame(); + + chainingGlTextureProcessorListener.onOutputFrameAvailable(texture, presentationTimeUs); + Thread.sleep(EXECUTOR_WAIT_TIME_MS); + + verify(fakeNextGlTextureProcessor, times(2)).maybeQueueInputFrame(texture, presentationTimeUs); + } + + @Test + public void onOutputFrameAvailable_twoFramesWithFirstRejected_retriesFirstBeforeSecond() + throws InterruptedException { + TextureInfo firstTexture = + new TextureInfo(/* texId= */ 1, /* fboId= */ 1, /* width= */ 100, /* height= */ 100); + long firstPresentationTimeUs = 123; + TextureInfo secondTexture = + new TextureInfo(/* texId= */ 2, /* fboId= */ 2, /* width= */ 100, /* height= */ 100); + long secondPresentationTimeUs = 567; + fakeNextGlTextureProcessor.rejectNextFrame(); + + chainingGlTextureProcessorListener.onOutputFrameAvailable( + firstTexture, firstPresentationTimeUs); + chainingGlTextureProcessorListener.onOutputFrameAvailable( + secondTexture, secondPresentationTimeUs); + Thread.sleep(EXECUTOR_WAIT_TIME_MS); + + verify(fakeNextGlTextureProcessor, times(2)) + .maybeQueueInputFrame(firstTexture, firstPresentationTimeUs); + verify(fakeNextGlTextureProcessor, times(1)) + .maybeQueueInputFrame(secondTexture, secondPresentationTimeUs); + } + + @Test + public void onOutputStreamEnded_signalsInputStreamEndedToNextGlTextureProcessor() + throws InterruptedException { + chainingGlTextureProcessorListener.onOutputStreamEnded(); + Thread.sleep(EXECUTOR_WAIT_TIME_MS); + + verify(fakeNextGlTextureProcessor, times(1)).signalEndOfInputStream(); + } + + private static class FakeGlTextureProcessor implements GlTextureProcessor { + + private volatile boolean rejectNextFrame; + + public void rejectNextFrame() { + rejectNextFrame = true; + } + + @Override + public void setListener(Listener listener) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) { + boolean acceptFrame = !rejectNextFrame; + rejectNextFrame = false; + return acceptFrame; + } + + @Override + public void releaseOutputFrame(TextureInfo outputTexture) {} + + @Override + public void signalEndOfInputStream() {} + + @Override + public void release() {} + } +}