loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-04-30 06:46:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Abstract the interface of DefaultVideoFrameProcessor's input.

Browse Source 
PiperOrigin-RevId: 526642898
This commit is contained in:
claincly 2023-04-24 15:59:20 +01:00 committed by Ian Baker
parent 2d968a5d97
commit f3c0256bb4
5 changed files with 136 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java
View File

@ -208,11 +208,8 @@ public interface VideoFrameProcessor {
void registerInputFrame();
/**
* Returns the number of input frames that have been {@linkplain #registerInputFrame() registered}
* but not processed off the {@linkplain #getInputSurface() input surface} yet.
*
* <p>This method must only be called when the {@link VideoFrameProcessor} is {@linkplain
* Factory#create created} with {@link #INPUT_TYPE_SURFACE}.
* returns the number of input frames that have been made available to the {@code
* VideoFrameProcessor} but have not been processed yet.
*
* <p>Can be called on any thread.
*/

50
libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java
View File

@ -21,10 +21,12 @@ import static java.lang.Math.round;
import android.graphics.Bitmap;
import android.opengl.GLES20;
import android.opengl.GLUtils;
import android.view.Surface;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.FrameInfo;
import androidx.media3.common.GlTextureInfo;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.UnstableApi;
import java.util.Queue;
@ -38,7 +40,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* <p>Public methods in this class can be called from any thread.
*/
@UnstableApi
/* package */ final class BitmapTextureManager implements GlShaderProgram.InputListener {
/* package */ final class BitmapTextureManager implements InputHandler {
private final GlShaderProgram shaderProgram;
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
// The queue holds all bitmaps with one or more frames pending to be sent downstream.
@ -77,22 +79,35 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
});
}
/**
* Provides an input {@link Bitmap} to put into the video frames.
*
* @see VideoFrameProcessor#queueInputBitmap
*/
@Override
public void setDefaultBufferSize(int width, int height) {
throw new UnsupportedOperationException();
}
@Override
public void queueInputBitmap(
Bitmap inputBitmap, long durationUs, float frameRate, boolean useHdr) {
videoFrameProcessingTaskExecutor.submit(
() -> setupBitmap(inputBitmap, durationUs, frameRate, useHdr));
}
/**
* Signals the end of the input.
*
* @see VideoFrameProcessor#signalEndOfInput()
*/
@Override
public Surface getInputSurface() {
throw new UnsupportedOperationException();
}
@Override
public void registerInputFrame(FrameInfo frameInfo) {
// Do nothing.
}
@Override
public int getPendingFrameCount() {
// Always treat all queued bitmaps as immediately processed.
return 0;
}
@Override
public void signalEndOfInput() {
videoFrameProcessingTaskExecutor.submit(
() -> {
@ -101,11 +116,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
});
}
/**
* Releases all resources.
*
* @see VideoFrameProcessor#release()
*/
@Override
public void setOnFlushCompleteListener(@Nullable VideoFrameProcessingTask task) {
// Do nothing.
}
@Override
public void release() {
videoFrameProcessingTaskExecutor.submit(
() -> {

40
libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java
View File

@ -17,7 +17,6 @@ package androidx.media3.effect;
import static androidx.annotation.VisibleForTesting.PACKAGE_PRIVATE;
import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static com.google.common.collect.Iterables.getLast;
@ -247,8 +246,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
private final EGLDisplay eglDisplay;
private final EGLContext eglContext;
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
private @MonotonicNonNull BitmapTextureManager inputBitmapTextureManager;
private @MonotonicNonNull ExternalTextureManager inputExternalTextureManager;
private final InputHandler inputHandler;
private final boolean releaseFramesAutomatically;
private final FinalShaderProgramWrapper finalShaderProgramWrapper;
private final ImmutableList<GlShaderProgram> allShaderPrograms;
@ -278,20 +276,19 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
switch (inputType) {
case VideoFrameProcessor.INPUT_TYPE_SURFACE:
checkState(inputShaderProgram instanceof ExternalShaderProgram);
inputExternalTextureManager =
inputHandler =
new ExternalTextureManager(
(ExternalShaderProgram) inputShaderProgram, videoFrameProcessingTaskExecutor);
inputShaderProgram.setInputListener(inputExternalTextureManager);
break;
case VideoFrameProcessor.INPUT_TYPE_BITMAP:
inputBitmapTextureManager =
inputHandler =
new BitmapTextureManager(inputShaderProgram, videoFrameProcessingTaskExecutor);
inputShaderProgram.setInputListener(inputBitmapTextureManager);
break;
case VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID: // fall through
default:
throw new VideoFrameProcessingException("Input type not supported yet");
}
inputShaderProgram.setInputListener(inputHandler);
finalShaderProgramWrapper = (FinalShaderProgramWrapper) getLast(shaderPrograms);
allShaderPrograms = shaderPrograms;
@ -319,18 +316,17 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
* @param height The default height for input buffers, in pixels.
*/
public void setInputDefaultBufferSize(int width, int height) {
checkNotNull(inputExternalTextureManager).setDefaultBufferSize(width, height);
inputHandler.setDefaultBufferSize(width, height);
}
@Override
public void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate) {
checkNotNull(inputBitmapTextureManager)
.queueInputBitmap(inputBitmap, durationUs, frameRate, /* useHdr= */ false);
inputHandler.queueInputBitmap(inputBitmap, durationUs, frameRate, /* useHdr= */ false);
}
@Override
public Surface getInputSurface() {
return checkNotNull(inputExternalTextureManager).getInputSurface();
return inputHandler.getInputSurface();
}
@Override
@ -344,12 +340,12 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
checkStateNotNull(
nextInputFrameInfo, "setInputFrameInfo must be called before registering input frames");
checkNotNull(inputExternalTextureManager).registerInputFrame(nextInputFrameInfo);
inputHandler.registerInputFrame(nextInputFrameInfo);
}
@Override
public int getPendingInputFrameCount() {
return checkNotNull(inputExternalTextureManager).getPendingFrameCount();
return inputHandler.getPendingFrameCount();
}
@Override
@ -370,12 +366,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
public void signalEndOfInput() {
checkState(!inputStreamEnded);
inputStreamEnded = true;
if (inputBitmapTextureManager != null) {
videoFrameProcessingTaskExecutor.submit(inputBitmapTextureManager::signalEndOfInput);
}
if (inputExternalTextureManager != null) {
videoFrameProcessingTaskExecutor.submit(inputExternalTextureManager::signalEndOfInput);
}
videoFrameProcessingTaskExecutor.submit(inputHandler::signalEndOfInput);
}
@Override
@ -383,10 +374,10 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
try {
videoFrameProcessingTaskExecutor.flush();
CountDownLatch latch = new CountDownLatch(1);
checkNotNull(inputExternalTextureManager).setOnFlushCompleteListener(latch::countDown);
inputHandler.setOnFlushCompleteListener(latch::countDown);
videoFrameProcessingTaskExecutor.submit(finalShaderProgramWrapper::flush);
latch.await();
inputExternalTextureManager.setOnFlushCompleteListener(null);
inputHandler.setOnFlushCompleteListener(null);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
@ -401,12 +392,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
Thread.currentThread().interrupt();
throw new IllegalStateException(unexpected);
}
if (inputExternalTextureManager != null) {
inputExternalTextureManager.release();
}
if (inputBitmapTextureManager != null) {
inputBitmapTextureManager.release();
}
inputHandler.release();
}
/**

30
libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java
View File

@ -17,6 +17,7 @@ package androidx.media3.effect;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import android.graphics.Bitmap;
import android.graphics.SurfaceTexture;
import android.view.Surface;
import androidx.annotation.Nullable;
@ -24,7 +25,6 @@ import androidx.media3.common.C;
import androidx.media3.common.FrameInfo;
import androidx.media3.common.GlTextureInfo;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.util.GlUtil;
import androidx.media3.effect.GlShaderProgram.InputListener;
import java.util.Queue;
@ -35,7 +35,7 @@ import java.util.concurrent.atomic.AtomicInteger;
* Forwards externally produced frames that become available via a {@link SurfaceTexture} to an
* {@link ExternalShaderProgram} for consumption.
*/
/* package */ final class ExternalTextureManager implements InputListener {
/* package */ final class ExternalTextureManager implements InputHandler {
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
private final ExternalShaderProgram externalShaderProgram;
@ -105,12 +105,18 @@ import java.util.concurrent.atomic.AtomicInteger;
surface = new Surface(surfaceTexture);
}
/** See {@link DefaultVideoFrameProcessor#setInputDefaultBufferSize}. */
@Override
public void setDefaultBufferSize(int width, int height) {
surfaceTexture.setDefaultBufferSize(width, height);
}
/** Returns the {@linkplain Surface input surface} that wraps the external texture. */
@Override
public void queueInputBitmap(
Bitmap inputBitmap, long durationUs, float frameRate, boolean useHdr) {
throw new UnsupportedOperationException();
}
@Override
public Surface getInputSurface() {
return surface;
}
@ -137,7 +143,7 @@ import java.util.concurrent.atomic.AtomicInteger;
});
}
/** Sets the task to run on completing flushing, or {@code null} to clear any task. */
@Override
public void setOnFlushCompleteListener(@Nullable VideoFrameProcessingTask task) {
onFlushCompleteTask = task;
}
@ -154,6 +160,7 @@ import java.util.concurrent.atomic.AtomicInteger;
* <p>Can be called on any thread. The caller must ensure that frames are registered in the
* correct order.
*/
@Override
public void registerInputFrame(FrameInfo frame) {
pendingFrames.add(frame);
}
@ -164,15 +171,12 @@ import java.util.concurrent.atomic.AtomicInteger;
*
* <p>Can be called on any thread.
*/
@Override
public int getPendingFrameCount() {
return pendingFrames.size();
}
/**
* Signals the end of the input.
*
* @see VideoFrameProcessor#signalEndOfInput()
*/
@Override
public void signalEndOfInput() {
videoFrameProcessingTaskExecutor.submit(
() -> {
@ -183,11 +187,7 @@ import java.util.concurrent.atomic.AtomicInteger;
});
}
/**
* Releases all resources.
*
* @see VideoFrameProcessor#release()
*/
@Override
public void release() {
surfaceTexture.release();
surface.release();

73
libraries/effect/src/main/java/androidx/media3/effect/InputHandler.java Normal file
View File

@ -0,0 +1,73 @@
/*
* Copyright 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package androidx.media3.effect;
import android.graphics.Bitmap;
import android.graphics.SurfaceTexture;
import android.view.Surface;
import androidx.annotation.Nullable;
import androidx.media3.common.FrameInfo;
import androidx.media3.common.VideoFrameProcessor;
/** A component that handles {@code DefaultVideoFrameProcessor}'s input. */
/* package */ interface InputHandler extends GlShaderProgram.InputListener {
/**
* See {@link DefaultVideoFrameProcessor#setInputDefaultBufferSize}.
*
* <p>Only works when the input is received on a {@link SurfaceTexture}.
*/
void setDefaultBufferSize(int width, int height);
/**
* Provides an input {@link Bitmap} to put into the video frames.
*
* @see VideoFrameProcessor#queueInputBitmap
*/
void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate, boolean useHdr);
/**
* See {@link VideoFrameProcessor#getInputSurface}.
*
* <p>Only works when the input is received on a {@link SurfaceTexture}.
*/
Surface getInputSurface();
/** Informs the {@code InputHandler} that a frame will be queued. */
void registerInputFrame(FrameInfo frameInfo);
/** See {@link VideoFrameProcessor#getPendingInputFrameCount}. */
int getPendingFrameCount();
/**
* Signals the end of the input.
*
* @see VideoFrameProcessor#signalEndOfInput()
*/
void signalEndOfInput();
/** Sets the task to run on completing flushing, or {@code null} to clear any task. */
void setOnFlushCompleteListener(@Nullable VideoFrameProcessingTask task);
/**
* Releases all resources.
*
* @see VideoFrameProcessor#release()
*/
void release();
}