Create GlObjectsProvider
To create this file TextureInfo has been moved to common and renamed to GLTextureInfo. We'll look to expand the interface in future to cover more of the methods around GL object maintenance in future as required. PiperOrigin-RevId: 514445397
This commit is contained in:
tofunmi
Rohit Singh
parent
5115df1147
commit
7303caffb5
@ -24,11 +24,11 @@ import android.content.Context;
|
||||
import android.opengl.EGL14;
|
||||
import androidx.annotation.Nullable;
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.GlTextureInfo;
|
||||
import androidx.media3.common.VideoFrameProcessingException;
|
||||
import androidx.media3.common.util.LibraryLoader;
|
||||
import androidx.media3.common.util.Util;
|
||||
import androidx.media3.effect.GlShaderProgram;
|
||||
import androidx.media3.effect.TextureInfo;
|
||||
import com.google.common.util.concurrent.MoreExecutors;
|
||||
import com.google.mediapipe.components.FrameProcessor;
|
||||
import com.google.mediapipe.framework.AppTextureFrame;
|
||||
@ -68,7 +68,7 @@ import java.util.concurrent.Future;
|
||||
}
|
||||
|
||||
private final FrameProcessor frameProcessor;
|
||||
private final ConcurrentHashMap<TextureInfo, TextureFrame> outputFrames;
|
||||
private final ConcurrentHashMap<GlTextureInfo, TextureFrame> outputFrames;
|
||||
private final boolean isSingleFrameGraph;
|
||||
@Nullable private final ExecutorService singleThreadExecutorService;
|
||||
private final Queue<Future<?>> futures;
|
||||
@ -137,10 +137,11 @@ import java.util.concurrent.Future;
|
||||
this.outputListener = outputListener;
|
||||
frameProcessor.setConsumer(
|
||||
frame -> {
|
||||
TextureInfo texture =
|
||||
new TextureInfo(
|
||||
GlTextureInfo texture =
|
||||
new GlTextureInfo(
|
||||
frame.getTextureName(),
|
||||
/* fboId= */ C.INDEX_UNSET,
|
||||
/* rboId= */ C.INDEX_UNSET,
|
||||
frame.getWidth(),
|
||||
frame.getHeight());
|
||||
outputFrames.put(texture, frame);
|
||||
@ -159,7 +160,7 @@ import java.util.concurrent.Future;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
|
||||
public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) {
|
||||
AppTextureFrame appTextureFrame =
|
||||
new AppTextureFrame(inputTexture.texId, inputTexture.width, inputTexture.height);
|
||||
// TODO(b/238302213): Handle timestamps restarting from 0 when applying effects to a playlist.
|
||||
@ -183,7 +184,7 @@ import java.util.concurrent.Future;
|
||||
}
|
||||
|
||||
private boolean maybeQueueInputFrameSynchronous(
|
||||
AppTextureFrame appTextureFrame, TextureInfo inputTexture) {
|
||||
AppTextureFrame appTextureFrame, GlTextureInfo inputTexture) {
|
||||
acceptedFrame = false;
|
||||
frameProcessor.onNewFrame(appTextureFrame);
|
||||
try {
|
||||
@ -200,7 +201,7 @@ import java.util.concurrent.Future;
|
||||
}
|
||||
|
||||
private void queueInputFrameAsynchronous(
|
||||
AppTextureFrame appTextureFrame, TextureInfo inputTexture) {
|
||||
AppTextureFrame appTextureFrame, GlTextureInfo inputTexture) {
|
||||
removeFinishedFutures();
|
||||
futures.add(
|
||||
checkStateNotNull(singleThreadExecutorService)
|
||||
@ -222,7 +223,7 @@ import java.util.concurrent.Future;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void releaseOutputFrame(TextureInfo outputTexture) {
|
||||
public void releaseOutputFrame(GlTextureInfo outputTexture) {
|
||||
checkStateNotNull(outputFrames.get(outputTexture)).release();
|
||||
if (isSingleFrameGraph) {
|
||||
inputListener.onReadyToAcceptInputFrame();
|
||||
|
||||
@ -0,0 +1,87 @@
|
||||
/*
|
||||
* Copyright (C) 2023 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package androidx.media3.common;
|
||||
|
||||
import android.opengl.EGLContext;
|
||||
import android.opengl.EGLDisplay;
|
||||
import androidx.annotation.IntRange;
|
||||
import androidx.annotation.RequiresApi;
|
||||
import androidx.media3.common.util.GlUtil;
|
||||
import androidx.media3.common.util.GlUtil.GlException;
|
||||
import androidx.media3.common.util.UnstableApi;
|
||||
|
||||
// TODO(271433904): Expand this class to cover more methods in GlUtil.
|
||||
/** Provider to customize the creation and maintenance of GL objects. */
|
||||
@UnstableApi
|
||||
public interface GlObjectsProvider {
|
||||
/**
|
||||
* Provider for GL objects that configures a GL context with 8-bit RGB or 10-bit RGB attributes,
|
||||
* and no depth buffer or render buffers.
|
||||
*/
|
||||
GlObjectsProvider DEFAULT =
|
||||
new GlObjectsProvider() {
|
||||
@Override
|
||||
@RequiresApi(17)
|
||||
public EGLContext createEglContext(
|
||||
EGLDisplay eglDisplay, int openGlVersion, int[] configAttributes) throws GlException {
|
||||
return GlUtil.createEglContext(eglDisplay, openGlVersion, configAttributes);
|
||||
}
|
||||
|
||||
@Override
|
||||
public GlTextureInfo createBuffersForTexture(int texId, int width, int height)
|
||||
throws GlException {
|
||||
int fboId = GlUtil.createFboForTexture(texId);
|
||||
return new GlTextureInfo(texId, fboId, /* rboId= */ C.INDEX_UNSET, width, height);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clearOutputFrame() throws GlException {
|
||||
GlUtil.clearOutputFrame();
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Creates a new {@link EGLContext} for the specified {@link EGLDisplay}.
|
||||
*
|
||||
* @param eglDisplay The {@link EGLDisplay} to create an {@link EGLContext} for.
|
||||
* @param openGlVersion The version of OpenGL ES to configure. Accepts either {@code 2}, for
|
||||
* OpenGL ES 2.0, or {@code 3}, for OpenGL ES 3.0.
|
||||
* @param configAttributes The attributes to configure EGL with.
|
||||
* @throws GlException If an error occurs during creation.
|
||||
*/
|
||||
@RequiresApi(17)
|
||||
EGLContext createEglContext(
|
||||
EGLDisplay eglDisplay, @IntRange(from = 2, to = 3) int openGlVersion, int[] configAttributes)
|
||||
throws GlException;
|
||||
|
||||
/**
|
||||
* Returns a {@link GlTextureInfo} containing the identifiers of the newly created buffers.
|
||||
*
|
||||
* @param texId The identifier of the texture to attach to the buffers.
|
||||
* @param width The width of the texture in pixels.
|
||||
* @param height The height of the texture in pixels.
|
||||
* @throws GlException If an error occurs during creation.
|
||||
*/
|
||||
GlTextureInfo createBuffersForTexture(int texId, int width, int height) throws GlException;
|
||||
|
||||
/**
|
||||
* Clears the current render target.
|
||||
*
|
||||
* @throws GlException If an error occurs during clearing.
|
||||
*/
|
||||
void clearOutputFrame() throws GlException;
|
||||
}
|
||||
@ -0,0 +1,68 @@
|
||||
/*
|
||||
* Copyright 2022 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package androidx.media3.common;
|
||||
|
||||
import androidx.media3.common.util.UnstableApi;
|
||||
|
||||
/** Contains information describing an OpenGL texture. */
|
||||
@UnstableApi
|
||||
public final class GlTextureInfo {
|
||||
|
||||
/** A {@link GlTextureInfo} instance with all fields unset. */
|
||||
public static final GlTextureInfo UNSET =
|
||||
new GlTextureInfo(
|
||||
/* texId= */ C.INDEX_UNSET,
|
||||
/* fboId= */ C.INDEX_UNSET,
|
||||
/* rboId= */ C.INDEX_UNSET,
|
||||
/* width= */ C.LENGTH_UNSET,
|
||||
/* height= */ C.LENGTH_UNSET);
|
||||
|
||||
/** The OpenGL texture identifier, or {@link C#INDEX_UNSET} if not specified. */
|
||||
public final int texId;
|
||||
/**
|
||||
* Identifier of a framebuffer object associated with the texture, or {@link C#INDEX_UNSET} if not
|
||||
* specified.
|
||||
*/
|
||||
public final int fboId;
|
||||
/**
|
||||
* Identifier of a renderbuffer object attached with the framebuffer, or {@link C#INDEX_UNSET} if
|
||||
* not specified.
|
||||
*/
|
||||
public final int rboId;
|
||||
/** The width of the texture, in pixels, or {@link C#LENGTH_UNSET} if not specified. */
|
||||
public final int width;
|
||||
/** The height of the texture, in pixels, or {@link C#LENGTH_UNSET} if not specified. */
|
||||
public final int height;
|
||||
|
||||
/**
|
||||
* Creates a new instance.
|
||||
*
|
||||
* @param texId The OpenGL texture identifier, or {@link C#INDEX_UNSET} if not specified.
|
||||
* @param fboId Identifier of a framebuffer object associated with the texture, or {@link
|
||||
* C#INDEX_UNSET} if not specified.
|
||||
* @param rboId Identifier of a renderbuffer object associated with the texture, or {@link
|
||||
* C#INDEX_UNSET} if not specified.
|
||||
* @param width The width of the texture, in pixels, or {@link C#LENGTH_UNSET} if not specified.
|
||||
* @param height The height of the texture, in pixels, or {@link C#LENGTH_UNSET} if not specified.
|
||||
*/
|
||||
public GlTextureInfo(int texId, int fboId, int rboId, int width, int height) {
|
||||
this.texId = texId;
|
||||
this.fboId = fboId;
|
||||
this.rboId = rboId;
|
||||
this.width = width;
|
||||
this.height = height;
|
||||
}
|
||||
}
|
||||
@ -24,9 +24,11 @@ import android.graphics.PixelFormat;
|
||||
import android.media.Image;
|
||||
import android.media.ImageReader;
|
||||
import androidx.annotation.Nullable;
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.ColorInfo;
|
||||
import androidx.media3.common.DebugViewProvider;
|
||||
import androidx.media3.common.FrameInfo;
|
||||
import androidx.media3.common.GlTextureInfo;
|
||||
import androidx.media3.common.SurfaceInfo;
|
||||
import androidx.media3.common.VideoFrameProcessingException;
|
||||
import androidx.media3.common.VideoFrameProcessor;
|
||||
@ -373,7 +375,7 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest {
|
||||
/** Produces blank frames with the given timestamps. */
|
||||
private static final class BlankFrameProducer implements GlShaderProgram {
|
||||
|
||||
private @MonotonicNonNull TextureInfo blankTexture;
|
||||
private @MonotonicNonNull GlTextureInfo blankTexture;
|
||||
private @MonotonicNonNull OutputListener outputListener;
|
||||
|
||||
public void configureGlObjects() throws VideoFrameProcessingException {
|
||||
@ -381,7 +383,7 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest {
|
||||
int texId =
|
||||
GlUtil.createTexture(WIDTH, HEIGHT, /* useHighPrecisionColorComponents= */ false);
|
||||
int fboId = GlUtil.createFboForTexture(texId);
|
||||
blankTexture = new TextureInfo(texId, fboId, WIDTH, HEIGHT);
|
||||
blankTexture = new GlTextureInfo(texId, fboId, /* rboId= */ C.INDEX_UNSET, WIDTH, HEIGHT);
|
||||
GlUtil.focusFramebufferUsingCurrentContext(fboId, WIDTH, HEIGHT);
|
||||
GlUtil.clearOutputFrame();
|
||||
} catch (GlUtil.GlException e) {
|
||||
@ -409,13 +411,13 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest {
|
||||
public void setErrorListener(Executor executor, ErrorListener errorListener) {}
|
||||
|
||||
@Override
|
||||
public void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
|
||||
public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) {
|
||||
// No input is queued in these tests. The BlankFrameProducer is used to produce frames.
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void releaseOutputFrame(TextureInfo outputTexture) {}
|
||||
public void releaseOutputFrame(GlTextureInfo outputTexture) {}
|
||||
|
||||
@Override
|
||||
public void signalEndOfCurrentInputStream() {
|
||||
|
||||
@ -19,6 +19,7 @@ import android.util.Pair;
|
||||
import androidx.annotation.GuardedBy;
|
||||
import androidx.annotation.Nullable;
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.GlTextureInfo;
|
||||
import androidx.media3.effect.GlShaderProgram.InputListener;
|
||||
import androidx.media3.effect.GlShaderProgram.OutputListener;
|
||||
import java.util.ArrayDeque;
|
||||
@ -38,7 +39,7 @@ import java.util.Queue;
|
||||
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
|
||||
|
||||
@GuardedBy("this")
|
||||
private final Queue<Pair<TextureInfo, Long>> availableFrames;
|
||||
private final Queue<Pair<GlTextureInfo, Long>> availableFrames;
|
||||
|
||||
@GuardedBy("this")
|
||||
private int consumingGlShaderProgramInputCapacity;
|
||||
@ -67,7 +68,7 @@ import java.util.Queue;
|
||||
|
||||
@Override
|
||||
public synchronized void onReadyToAcceptInputFrame() {
|
||||
@Nullable Pair<TextureInfo, Long> pendingFrame = availableFrames.poll();
|
||||
@Nullable Pair<GlTextureInfo, Long> pendingFrame = availableFrames.poll();
|
||||
if (pendingFrame == null) {
|
||||
consumingGlShaderProgramInputCapacity++;
|
||||
return;
|
||||
@ -86,7 +87,7 @@ import java.util.Queue;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onInputFrameProcessed(TextureInfo inputTexture) {
|
||||
public void onInputFrameProcessed(GlTextureInfo inputTexture) {
|
||||
videoFrameProcessingTaskExecutor.submit(
|
||||
() -> producingGlShaderProgram.releaseOutputFrame(inputTexture));
|
||||
}
|
||||
@ -100,7 +101,7 @@ import java.util.Queue;
|
||||
|
||||
@Override
|
||||
public synchronized void onOutputFrameAvailable(
|
||||
TextureInfo outputTexture, long presentationTimeUs) {
|
||||
GlTextureInfo outputTexture, long presentationTimeUs) {
|
||||
if (consumingGlShaderProgramInputCapacity > 0) {
|
||||
videoFrameProcessingTaskExecutor.submit(
|
||||
() ->
|
||||
@ -115,7 +116,7 @@ import java.util.Queue;
|
||||
@Override
|
||||
public synchronized void onCurrentOutputStreamEnded() {
|
||||
if (!availableFrames.isEmpty()) {
|
||||
availableFrames.add(new Pair<>(TextureInfo.UNSET, C.TIME_END_OF_SOURCE));
|
||||
availableFrames.add(new Pair<>(GlTextureInfo.UNSET, C.TIME_END_OF_SOURCE));
|
||||
} else {
|
||||
videoFrameProcessingTaskExecutor.submit(
|
||||
consumingGlShaderProgram::signalEndOfCurrentInputStream);
|
||||
|
||||
@ -22,6 +22,7 @@ import android.view.Surface;
|
||||
import androidx.annotation.Nullable;
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.FrameInfo;
|
||||
import androidx.media3.common.GlTextureInfo;
|
||||
import androidx.media3.common.VideoFrameProcessingException;
|
||||
import androidx.media3.common.VideoFrameProcessor;
|
||||
import androidx.media3.common.util.GlUtil;
|
||||
@ -126,7 +127,7 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onInputFrameProcessed(TextureInfo inputTexture) {
|
||||
public void onInputFrameProcessed(GlTextureInfo inputTexture) {
|
||||
videoFrameProcessingTaskExecutor.submit(
|
||||
() -> {
|
||||
currentFrame = null;
|
||||
@ -234,8 +235,12 @@ import java.util.concurrent.atomic.AtomicInteger;
|
||||
// Correct the presentation time so that GlShaderPrograms don't see the stream offset.
|
||||
long presentationTimeUs = (frameTimeNs / 1000) + offsetToAddUs - streamOffsetUs;
|
||||
externalShaderProgram.queueInputFrame(
|
||||
new TextureInfo(
|
||||
externalTexId, /* fboId= */ C.INDEX_UNSET, currentFrame.width, currentFrame.height),
|
||||
new GlTextureInfo(
|
||||
externalTexId,
|
||||
/* fboId= */ C.INDEX_UNSET,
|
||||
/* rboId= */ C.INDEX_UNSET,
|
||||
currentFrame.width,
|
||||
currentFrame.height),
|
||||
presentationTimeUs);
|
||||
checkStateNotNull(pendingFrames.remove());
|
||||
|
||||
|
||||
@ -35,6 +35,7 @@ import androidx.annotation.Nullable;
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.ColorInfo;
|
||||
import androidx.media3.common.DebugViewProvider;
|
||||
import androidx.media3.common.GlTextureInfo;
|
||||
import androidx.media3.common.SurfaceInfo;
|
||||
import androidx.media3.common.VideoFrameProcessingException;
|
||||
import androidx.media3.common.VideoFrameProcessor;
|
||||
@ -79,7 +80,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
private final VideoFrameProcessor.Listener videoFrameProcessorListener;
|
||||
private final float[] textureTransformMatrix;
|
||||
private final Queue<Long> streamOffsetUsQueue;
|
||||
private final Queue<Pair<TextureInfo, Long>> availableFrames;
|
||||
private final Queue<Pair<GlTextureInfo, Long>> availableFrames;
|
||||
|
||||
private int inputWidth;
|
||||
private int inputHeight;
|
||||
@ -176,7 +177,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
// Methods that must be called on the GL thread.
|
||||
|
||||
@Override
|
||||
public void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
|
||||
public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) {
|
||||
long streamOffsetUs =
|
||||
checkStateNotNull(streamOffsetUsQueue.peek(), "No input stream specified.");
|
||||
long offsetPresentationTimeUs = presentationTimeUs + streamOffsetUs;
|
||||
@ -192,14 +193,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void releaseOutputFrame(TextureInfo outputTexture) {
|
||||
public void releaseOutputFrame(GlTextureInfo outputTexture) {
|
||||
// The final shader program writes to a surface so there is no texture to release.
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
public void releaseOutputFrame(long releaseTimeNs) {
|
||||
checkState(!releaseFramesAutomatically);
|
||||
Pair<TextureInfo, Long> oldestAvailableFrame = availableFrames.remove();
|
||||
Pair<GlTextureInfo, Long> oldestAvailableFrame = availableFrames.remove();
|
||||
renderFrameToSurfaces(
|
||||
/* inputTexture= */ oldestAvailableFrame.first,
|
||||
/* presentationTimeUs= */ oldestAvailableFrame.second,
|
||||
@ -272,7 +273,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
}
|
||||
|
||||
private void renderFrameToSurfaces(
|
||||
TextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs) {
|
||||
GlTextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs) {
|
||||
try {
|
||||
maybeRenderFrameToOutputSurface(inputTexture, presentationTimeUs, releaseTimeNs);
|
||||
} catch (VideoFrameProcessingException | GlUtil.GlException e) {
|
||||
@ -286,7 +287,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
}
|
||||
|
||||
private synchronized void maybeRenderFrameToOutputSurface(
|
||||
TextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs)
|
||||
GlTextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs)
|
||||
throws VideoFrameProcessingException, GlUtil.GlException {
|
||||
if (releaseTimeNs == VideoFrameProcessor.DROP_OUTPUT_FRAME
|
||||
|| !ensureConfigured(inputTexture.width, inputTexture.height)) {
|
||||
@ -436,7 +437,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
return defaultShaderProgram;
|
||||
}
|
||||
|
||||
private void maybeRenderFrameToDebugSurface(TextureInfo inputTexture, long presentationTimeUs) {
|
||||
private void maybeRenderFrameToDebugSurface(GlTextureInfo inputTexture, long presentationTimeUs) {
|
||||
if (debugSurfaceViewWrapper == null || this.defaultShaderProgram == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
@ -19,6 +19,8 @@ import static androidx.media3.common.util.Assertions.checkState;
|
||||
|
||||
import android.content.Context;
|
||||
import android.opengl.GLES20;
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.GlTextureInfo;
|
||||
import androidx.media3.common.VideoFrameProcessingException;
|
||||
import androidx.media3.common.util.GlProgram;
|
||||
import androidx.media3.common.util.GlUtil;
|
||||
@ -31,7 +33,7 @@ import java.util.NoSuchElementException;
|
||||
import java.util.concurrent.Executor;
|
||||
|
||||
/**
|
||||
* Manages a pool of {@linkplain TextureInfo textures}, and caches the input frame.
|
||||
* Manages a pool of {@linkplain GlTextureInfo textures}, and caches the input frame.
|
||||
*
|
||||
* <p>Implements {@link FrameCache}.
|
||||
*/
|
||||
@ -41,8 +43,8 @@ import java.util.concurrent.Executor;
|
||||
private static final String FRAGMENT_SHADER_TRANSFORMATION_ES2_PATH =
|
||||
"shaders/fragment_shader_transformation_es2.glsl";
|
||||
|
||||
private final ArrayDeque<TextureInfo> freeOutputTextures;
|
||||
private final ArrayDeque<TextureInfo> inUseOutputTextures;
|
||||
private final ArrayDeque<GlTextureInfo> freeOutputTextures;
|
||||
private final ArrayDeque<GlTextureInfo> inUseOutputTextures;
|
||||
private final GlProgram copyProgram;
|
||||
private final int capacity;
|
||||
private final boolean useHdr;
|
||||
@ -112,12 +114,12 @@ import java.util.concurrent.Executor;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
|
||||
public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) {
|
||||
try {
|
||||
configureAllOutputTextures(inputTexture.width, inputTexture.height);
|
||||
|
||||
// Focus on the next free buffer.
|
||||
TextureInfo outputTexture = freeOutputTextures.remove();
|
||||
GlTextureInfo outputTexture = freeOutputTextures.remove();
|
||||
inUseOutputTextures.add(outputTexture);
|
||||
|
||||
// Copy frame to fbo.
|
||||
@ -144,7 +146,7 @@ import java.util.concurrent.Executor;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void releaseOutputFrame(TextureInfo outputTexture) {
|
||||
public void releaseOutputFrame(GlTextureInfo outputTexture) {
|
||||
checkState(inUseOutputTextures.contains(outputTexture));
|
||||
inUseOutputTextures.remove(outputTexture);
|
||||
freeOutputTextures.add(outputTexture);
|
||||
@ -177,13 +179,13 @@ import java.util.concurrent.Executor;
|
||||
|
||||
private void configureAllOutputTextures(int inputWidth, int inputHeight)
|
||||
throws GlUtil.GlException {
|
||||
Iterator<TextureInfo> allTextures = getIteratorToAllTextures();
|
||||
Iterator<GlTextureInfo> allTextures = getIteratorToAllTextures();
|
||||
if (!allTextures.hasNext()) {
|
||||
createAllOutputTextures(inputWidth, inputHeight);
|
||||
return;
|
||||
}
|
||||
TextureInfo outputTextureInfo = allTextures.next();
|
||||
if (outputTextureInfo.width != inputWidth || outputTextureInfo.height != inputHeight) {
|
||||
GlTextureInfo outputGlTextureInfo = allTextures.next();
|
||||
if (outputGlTextureInfo.width != inputWidth || outputGlTextureInfo.height != inputHeight) {
|
||||
deleteAllOutputTextures();
|
||||
createAllOutputTextures(inputWidth, inputHeight);
|
||||
}
|
||||
@ -195,15 +197,16 @@ import java.util.concurrent.Executor;
|
||||
for (int i = 0; i < capacity; i++) {
|
||||
int outputTexId = GlUtil.createTexture(width, height, useHdr);
|
||||
int outputFboId = GlUtil.createFboForTexture(outputTexId);
|
||||
TextureInfo outputTexture = new TextureInfo(outputTexId, outputFboId, width, height);
|
||||
GlTextureInfo outputTexture =
|
||||
new GlTextureInfo(outputTexId, outputFboId, /* rboId= */ C.INDEX_UNSET, width, height);
|
||||
freeOutputTextures.add(outputTexture);
|
||||
}
|
||||
}
|
||||
|
||||
private void deleteAllOutputTextures() throws GlUtil.GlException {
|
||||
Iterator<TextureInfo> allTextures = getIteratorToAllTextures();
|
||||
Iterator<GlTextureInfo> allTextures = getIteratorToAllTextures();
|
||||
while (allTextures.hasNext()) {
|
||||
TextureInfo textureInfo = allTextures.next();
|
||||
GlTextureInfo textureInfo = allTextures.next();
|
||||
GlUtil.deleteTexture(textureInfo.texId);
|
||||
GlUtil.deleteFbo(textureInfo.fboId);
|
||||
}
|
||||
@ -211,7 +214,7 @@ import java.util.concurrent.Executor;
|
||||
inUseOutputTextures.clear();
|
||||
}
|
||||
|
||||
private Iterator<TextureInfo> getIteratorToAllTextures() {
|
||||
private Iterator<GlTextureInfo> getIteratorToAllTextures() {
|
||||
return Iterables.concat(freeOutputTextures, inUseOutputTextures).iterator();
|
||||
}
|
||||
}
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
*/
|
||||
package androidx.media3.effect;
|
||||
|
||||
import androidx.media3.common.GlTextureInfo;
|
||||
import androidx.media3.common.VideoFrameProcessingException;
|
||||
import androidx.media3.common.util.UnstableApi;
|
||||
import java.util.concurrent.Executor;
|
||||
@ -23,14 +24,14 @@ import java.util.concurrent.Executor;
|
||||
* Processes frames from one OpenGL 2D texture to another.
|
||||
*
|
||||
* <p>The {@code GlShaderProgram} consumes input frames it accepts via {@link
|
||||
* #queueInputFrame(TextureInfo, long)} and surrenders each texture back to the caller via its
|
||||
* {@linkplain InputListener#onInputFrameProcessed(TextureInfo) listener} once the texture's
|
||||
* #queueInputFrame(GlTextureInfo, long)} and surrenders each texture back to the caller via its
|
||||
* {@linkplain InputListener#onInputFrameProcessed(GlTextureInfo) listener} once the texture's
|
||||
* contents have been processed.
|
||||
*
|
||||
* <p>The {@code GlShaderProgram} produces output frames asynchronously and notifies its owner when
|
||||
* they are available via its {@linkplain OutputListener#onOutputFrameAvailable(TextureInfo, long)
|
||||
* they are available via its {@linkplain OutputListener#onOutputFrameAvailable(GlTextureInfo, long)
|
||||
* listener}. The {@code GlShaderProgram} instance's owner must surrender the texture back to the
|
||||
* {@code GlShaderProgram} via {@link #releaseOutputFrame(TextureInfo)} when it has finished
|
||||
* {@code GlShaderProgram} via {@link #releaseOutputFrame(GlTextureInfo)} when it has finished
|
||||
* processing it.
|
||||
*
|
||||
* <p>{@code GlShaderProgram} implementations can choose to produce output frames before receiving
|
||||
@ -55,7 +56,7 @@ public interface GlShaderProgram {
|
||||
/**
|
||||
* Called when the {@link GlShaderProgram} is ready to accept another input frame.
|
||||
*
|
||||
* <p>For each time this method is called, {@link #queueInputFrame(TextureInfo, long)} can be
|
||||
* <p>For each time this method is called, {@link #queueInputFrame(GlTextureInfo, long)} can be
|
||||
* called once.
|
||||
*/
|
||||
default void onReadyToAcceptInputFrame() {}
|
||||
@ -66,10 +67,10 @@ public interface GlShaderProgram {
|
||||
* <p>The implementation shall not assume the {@link GlShaderProgram} is {@linkplain
|
||||
* #onReadyToAcceptInputFrame ready to accept another input frame} when this method is called.
|
||||
*
|
||||
* @param inputTexture The {@link TextureInfo} that was used to {@linkplain
|
||||
* #queueInputFrame(TextureInfo, long) queue} the input frame.
|
||||
* @param inputTexture The {@link GlTextureInfo} that was used to {@linkplain
|
||||
* #queueInputFrame(GlTextureInfo, long) queue} the input frame.
|
||||
*/
|
||||
default void onInputFrameProcessed(TextureInfo inputTexture) {}
|
||||
default void onInputFrameProcessed(GlTextureInfo inputTexture) {}
|
||||
|
||||
/**
|
||||
* Called when the {@link GlShaderProgram} has been flushed.
|
||||
@ -90,15 +91,15 @@ public interface GlShaderProgram {
|
||||
* Called when the {@link GlShaderProgram} has produced an output frame.
|
||||
*
|
||||
* <p>After the listener's owner has processed the output frame, it must call {@link
|
||||
* #releaseOutputFrame(TextureInfo)}. The output frame should be released as soon as possible,
|
||||
* #releaseOutputFrame(GlTextureInfo)}. The output frame should be released as soon as possible,
|
||||
* as there is no guarantee that the {@link GlShaderProgram} will produce further output frames
|
||||
* before this output frame is released.
|
||||
*
|
||||
* @param outputTexture A {@link TextureInfo} describing the texture containing the output
|
||||
* @param outputTexture A {@link GlTextureInfo} describing the texture containing the output
|
||||
* frame.
|
||||
* @param presentationTimeUs The presentation timestamp of the output frame, in microseconds.
|
||||
*/
|
||||
default void onOutputFrameAvailable(TextureInfo outputTexture, long presentationTimeUs) {}
|
||||
default void onOutputFrameAvailable(GlTextureInfo outputTexture, long presentationTimeUs) {}
|
||||
|
||||
/**
|
||||
* Called when the {@link GlShaderProgram} will not produce further output frames belonging to
|
||||
@ -151,22 +152,22 @@ public interface GlShaderProgram {
|
||||
* Processes an input frame if possible.
|
||||
*
|
||||
* <p>The {@code GlShaderProgram} owns the accepted frame until it calls {@link
|
||||
* InputListener#onInputFrameProcessed(TextureInfo)}. The caller should not overwrite or release
|
||||
* InputListener#onInputFrameProcessed(GlTextureInfo)}. The caller should not overwrite or release
|
||||
* the texture before the {@code GlShaderProgram} has finished processing it.
|
||||
*
|
||||
* <p>This method must only be called when the {@code GlShaderProgram} can {@linkplain
|
||||
* InputListener#onReadyToAcceptInputFrame() accept an input frame}.
|
||||
*
|
||||
* @param inputTexture A {@link TextureInfo} describing the texture containing the input frame.
|
||||
* @param inputTexture A {@link GlTextureInfo} describing the texture containing the input frame.
|
||||
* @param presentationTimeUs The presentation timestamp of the input frame, in microseconds.
|
||||
*/
|
||||
void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs);
|
||||
void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs);
|
||||
|
||||
/**
|
||||
* Notifies the {@code GlShaderProgram} that the frame on the given output texture is no longer
|
||||
* used and can be overwritten.
|
||||
*/
|
||||
void releaseOutputFrame(TextureInfo outputTexture);
|
||||
void releaseOutputFrame(GlTextureInfo outputTexture);
|
||||
|
||||
/**
|
||||
* Notifies the {@code GlShaderProgram} that no further input frames belonging to the current
|
||||
|
||||
@ -22,6 +22,7 @@ import android.graphics.Bitmap;
|
||||
import android.opengl.GLES20;
|
||||
import android.opengl.GLUtils;
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.GlTextureInfo;
|
||||
import androidx.media3.common.VideoFrameProcessingException;
|
||||
import androidx.media3.common.VideoFrameProcessor;
|
||||
import androidx.media3.common.util.GlUtil;
|
||||
@ -43,7 +44,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
// The queue holds all bitmaps with one or more frames pending to be sent downstream.
|
||||
private final Queue<BitmapFrameSequenceInfo> pendingBitmaps;
|
||||
|
||||
private @MonotonicNonNull TextureInfo currentTextureInfo;
|
||||
private @MonotonicNonNull GlTextureInfo currentGlTextureInfo;
|
||||
private int downstreamShaderProgramCapacity;
|
||||
private int framesToQueueForCurrentBitmap;
|
||||
private long currentPresentationTimeUs;
|
||||
@ -126,8 +127,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
framesToQueueForCurrentBitmap = currentBitmapInfo.numberOfFrames;
|
||||
int currentTexId;
|
||||
try {
|
||||
if (currentTextureInfo != null) {
|
||||
GlUtil.deleteTexture(currentTextureInfo.texId);
|
||||
if (currentGlTextureInfo != null) {
|
||||
GlUtil.deleteTexture(currentGlTextureInfo.texId);
|
||||
}
|
||||
currentTexId =
|
||||
GlUtil.createTexture(
|
||||
@ -140,15 +141,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
} catch (GlUtil.GlException e) {
|
||||
throw VideoFrameProcessingException.from(e);
|
||||
}
|
||||
currentTextureInfo =
|
||||
new TextureInfo(
|
||||
currentTexId, /* fboId= */ C.INDEX_UNSET, bitmap.getWidth(), bitmap.getHeight());
|
||||
currentGlTextureInfo =
|
||||
new GlTextureInfo(
|
||||
currentTexId,
|
||||
/* fboId= */ C.INDEX_UNSET,
|
||||
/* rboId= */ C.INDEX_UNSET,
|
||||
bitmap.getWidth(),
|
||||
bitmap.getHeight());
|
||||
}
|
||||
|
||||
framesToQueueForCurrentBitmap--;
|
||||
downstreamShaderProgramCapacity--;
|
||||
|
||||
shaderProgram.queueInputFrame(checkNotNull(currentTextureInfo), currentPresentationTimeUs);
|
||||
shaderProgram.queueInputFrame(checkNotNull(currentGlTextureInfo), currentPresentationTimeUs);
|
||||
|
||||
currentPresentationTimeUs += currentBitmapInfo.frameDurationUs;
|
||||
if (framesToQueueForCurrentBitmap == 0) {
|
||||
|
||||
@ -18,6 +18,8 @@ package androidx.media3.effect;
|
||||
import static androidx.media3.common.util.Assertions.checkState;
|
||||
|
||||
import androidx.annotation.CallSuper;
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.GlTextureInfo;
|
||||
import androidx.media3.common.VideoFrameProcessingException;
|
||||
import androidx.media3.common.util.GlUtil;
|
||||
import androidx.media3.common.util.Size;
|
||||
@ -48,7 +50,7 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram {
|
||||
private Executor errorListenerExecutor;
|
||||
private int inputWidth;
|
||||
private int inputHeight;
|
||||
private @MonotonicNonNull TextureInfo outputTexture;
|
||||
private @MonotonicNonNull GlTextureInfo outputTexture;
|
||||
private boolean outputTextureInUse;
|
||||
|
||||
/**
|
||||
@ -116,7 +118,7 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram {
|
||||
}
|
||||
|
||||
@Override
|
||||
public final void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
|
||||
public final void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) {
|
||||
checkState(
|
||||
!outputTextureInUse,
|
||||
"The shader program does not currently accept input frames. Release prior output frames"
|
||||
@ -161,12 +163,17 @@ public abstract class SingleFrameGlShaderProgram implements GlShaderProgram {
|
||||
int outputTexId = GlUtil.createTexture(outputSize.getWidth(), outputSize.getHeight(), useHdr);
|
||||
int outputFboId = GlUtil.createFboForTexture(outputTexId);
|
||||
outputTexture =
|
||||
new TextureInfo(outputTexId, outputFboId, outputSize.getWidth(), outputSize.getHeight());
|
||||
new GlTextureInfo(
|
||||
outputTexId,
|
||||
outputFboId,
|
||||
/* rboId= */ C.INDEX_UNSET,
|
||||
outputSize.getWidth(),
|
||||
outputSize.getHeight());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public final void releaseOutputFrame(TextureInfo outputTexture) {
|
||||
public final void releaseOutputFrame(GlTextureInfo outputTexture) {
|
||||
outputTextureInUse = false;
|
||||
inputListener.onReadyToAcceptInputFrame();
|
||||
}
|
||||
|
||||
@ -1,52 +0,0 @@
|
||||
/*
|
||||
* Copyright 2022 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package androidx.media3.effect;
|
||||
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.util.UnstableApi;
|
||||
|
||||
/** Contains information describing an OpenGL texture. */
|
||||
@UnstableApi
|
||||
public final class TextureInfo {
|
||||
|
||||
/** A {@link TextureInfo} instance with all fields unset. */
|
||||
public static final TextureInfo UNSET =
|
||||
new TextureInfo(C.INDEX_UNSET, C.INDEX_UNSET, C.LENGTH_UNSET, C.LENGTH_UNSET);
|
||||
|
||||
/** The OpenGL texture identifier. */
|
||||
public final int texId;
|
||||
/** Identifier of a framebuffer object associated with the texture. */
|
||||
public final int fboId;
|
||||
/** The width of the texture, in pixels. */
|
||||
public final int width;
|
||||
/** The height of the texture, in pixels. */
|
||||
public final int height;
|
||||
|
||||
/**
|
||||
* Creates a new instance.
|
||||
*
|
||||
* @param texId The OpenGL texture identifier.
|
||||
* @param fboId Identifier of a framebuffer object associated with the texture.
|
||||
* @param width The width of the texture, in pixels.
|
||||
* @param height The height of the texture, in pixels.
|
||||
*/
|
||||
public TextureInfo(int texId, int fboId, int width, int height) {
|
||||
this.texId = texId;
|
||||
this.fboId = fboId;
|
||||
this.width = width;
|
||||
this.height = height;
|
||||
}
|
||||
}
|
||||
@ -18,6 +18,8 @@ package androidx.media3.effect;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.GlTextureInfo;
|
||||
import androidx.media3.common.VideoFrameProcessor;
|
||||
import androidx.media3.common.util.Util;
|
||||
import androidx.test.ext.junit.runners.AndroidJUnit4;
|
||||
@ -51,8 +53,13 @@ public final class ChainingGlShaderProgramListenerTest {
|
||||
@Test
|
||||
public void onInputFrameProcessed_surrendersFrameToPreviousGlShaderProgram()
|
||||
throws InterruptedException {
|
||||
TextureInfo texture =
|
||||
new TextureInfo(/* texId= */ 1, /* fboId= */ 1, /* width= */ 100, /* height= */ 100);
|
||||
GlTextureInfo texture =
|
||||
new GlTextureInfo(
|
||||
/* texId= */ 1,
|
||||
/* fboId= */ 1,
|
||||
/* rboId= */ C.INDEX_UNSET,
|
||||
/* width= */ 100,
|
||||
/* height= */ 100);
|
||||
|
||||
chainingGlShaderProgramListener.onInputFrameProcessed(texture);
|
||||
Thread.sleep(EXECUTOR_WAIT_TIME_MS);
|
||||
@ -63,8 +70,13 @@ public final class ChainingGlShaderProgramListenerTest {
|
||||
@Test
|
||||
public void onOutputFrameAvailable_afterAcceptsInputFrame_passesFrameToNextGlShaderProgram()
|
||||
throws InterruptedException {
|
||||
TextureInfo texture =
|
||||
new TextureInfo(/* texId= */ 1, /* fboId= */ 1, /* width= */ 100, /* height= */ 100);
|
||||
GlTextureInfo texture =
|
||||
new GlTextureInfo(
|
||||
/* texId= */ 1,
|
||||
/* fboId= */ 1,
|
||||
/* rboId= */ C.INDEX_UNSET,
|
||||
/* width= */ 100,
|
||||
/* height= */ 100);
|
||||
long presentationTimeUs = 123;
|
||||
|
||||
chainingGlShaderProgramListener.onReadyToAcceptInputFrame();
|
||||
@ -77,8 +89,13 @@ public final class ChainingGlShaderProgramListenerTest {
|
||||
@Test
|
||||
public void onOutputFrameAvailable_beforeAcceptsInputFrame_passesFrameToNextGlShaderProgram()
|
||||
throws InterruptedException {
|
||||
TextureInfo texture =
|
||||
new TextureInfo(/* texId= */ 1, /* fboId= */ 1, /* width= */ 100, /* height= */ 100);
|
||||
GlTextureInfo texture =
|
||||
new GlTextureInfo(
|
||||
/* texId= */ 1,
|
||||
/* fboId= */ 1,
|
||||
/* rboId= */ C.INDEX_UNSET,
|
||||
/* width= */ 100,
|
||||
/* height= */ 100);
|
||||
long presentationTimeUs = 123;
|
||||
|
||||
chainingGlShaderProgramListener.onOutputFrameAvailable(texture, presentationTimeUs);
|
||||
@ -91,11 +108,21 @@ public final class ChainingGlShaderProgramListenerTest {
|
||||
@Test
|
||||
public void onOutputFrameAvailable_twoFrames_passesFirstBeforeSecondToNextGlShaderProgram()
|
||||
throws InterruptedException {
|
||||
TextureInfo firstTexture =
|
||||
new TextureInfo(/* texId= */ 1, /* fboId= */ 1, /* width= */ 100, /* height= */ 100);
|
||||
GlTextureInfo firstTexture =
|
||||
new GlTextureInfo(
|
||||
/* texId= */ 1,
|
||||
/* fboId= */ 1,
|
||||
/* rboId= */ C.INDEX_UNSET,
|
||||
/* width= */ 100,
|
||||
/* height= */ 100);
|
||||
long firstPresentationTimeUs = 123;
|
||||
TextureInfo secondTexture =
|
||||
new TextureInfo(/* texId= */ 2, /* fboId= */ 2, /* width= */ 100, /* height= */ 100);
|
||||
GlTextureInfo secondTexture =
|
||||
new GlTextureInfo(
|
||||
/* texId= */ 2,
|
||||
/* fboId= */ 2,
|
||||
/* rboId= */ C.INDEX_UNSET,
|
||||
/* width= */ 100,
|
||||
/* height= */ 100);
|
||||
long secondPresentationTimeUs = 567;
|
||||
|
||||
chainingGlShaderProgramListener.onOutputFrameAvailable(firstTexture, firstPresentationTimeUs);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user