mirror of
https://github.com/androidx/media.git
synced 2025-04-30 06:46:50 +08:00
parent
9f60eb3825
commit
fafd12bcfe
@ -0,0 +1,47 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2025 The Android Open Source Project
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package androidx.media3.effect;
|
||||||
|
|
||||||
|
import android.content.Context;
|
||||||
|
import android.view.SurfaceView;
|
||||||
|
import androidx.media3.common.ColorInfo;
|
||||||
|
import androidx.media3.common.DebugViewProvider;
|
||||||
|
import androidx.media3.common.util.UnstableApi;
|
||||||
|
|
||||||
|
/** {@link GlEffect} that renders to a {@link SurfaceView} provided by {@link DebugViewProvider}. */
|
||||||
|
@UnstableApi
|
||||||
|
public final class DebugViewEffect implements GlEffect {
|
||||||
|
|
||||||
|
private final DebugViewProvider debugViewProvider;
|
||||||
|
private final ColorInfo outputColorInfo;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a new instance.
|
||||||
|
*
|
||||||
|
* @param debugViewProvider The class that provides the {@link SurfaceView} that the debug preview
|
||||||
|
* will be rendered to.
|
||||||
|
* @param outputColorInfo The {@link ColorInfo} of the output preview.
|
||||||
|
*/
|
||||||
|
public DebugViewEffect(DebugViewProvider debugViewProvider, ColorInfo outputColorInfo) {
|
||||||
|
this.debugViewProvider = debugViewProvider;
|
||||||
|
this.outputColorInfo = outputColorInfo;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) {
|
||||||
|
return new DebugViewShaderProgram(context, debugViewProvider, outputColorInfo);
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,270 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2025 The Android Open Source Project
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package androidx.media3.effect;
|
||||||
|
|
||||||
|
import static androidx.media3.common.util.Assertions.checkNotNull;
|
||||||
|
import static androidx.media3.common.util.GlUtil.getDefaultEglDisplay;
|
||||||
|
import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_DEFAULT;
|
||||||
|
import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_LINEAR;
|
||||||
|
import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
|
||||||
|
|
||||||
|
import android.content.Context;
|
||||||
|
import android.opengl.EGL14;
|
||||||
|
import android.opengl.EGLContext;
|
||||||
|
import android.opengl.EGLDisplay;
|
||||||
|
import android.opengl.EGLSurface;
|
||||||
|
import android.opengl.GLES20;
|
||||||
|
import android.view.Surface;
|
||||||
|
import android.view.SurfaceHolder;
|
||||||
|
import android.view.SurfaceView;
|
||||||
|
import androidx.annotation.GuardedBy;
|
||||||
|
import androidx.annotation.Nullable;
|
||||||
|
import androidx.media3.common.C;
|
||||||
|
import androidx.media3.common.ColorInfo;
|
||||||
|
import androidx.media3.common.DebugViewProvider;
|
||||||
|
import androidx.media3.common.GlObjectsProvider;
|
||||||
|
import androidx.media3.common.GlTextureInfo;
|
||||||
|
import androidx.media3.common.VideoFrameProcessingException;
|
||||||
|
import androidx.media3.common.util.GlUtil;
|
||||||
|
import androidx.media3.common.util.Log;
|
||||||
|
import androidx.media3.common.util.UnstableApi;
|
||||||
|
import androidx.media3.common.util.Util;
|
||||||
|
import com.google.common.collect.ImmutableList;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.concurrent.Executor;
|
||||||
|
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@link GlShaderProgram} that renders to a {@link SurfaceView} provided by {@link
|
||||||
|
* DebugViewProvider}.
|
||||||
|
*/
|
||||||
|
@UnstableApi
|
||||||
|
public final class DebugViewShaderProgram implements GlShaderProgram {
|
||||||
|
private static final String TAG = "DebugViewShaderProgram";
|
||||||
|
|
||||||
|
private final Context context;
|
||||||
|
private final DebugViewProvider debugViewProvider;
|
||||||
|
@Nullable private SurfaceView debugSurfaceView;
|
||||||
|
@Nullable private DefaultShaderProgram defaultShaderProgram;
|
||||||
|
@Nullable private SurfaceViewWrapper debugSurfaceViewWrapper;
|
||||||
|
|
||||||
|
private final ColorInfo outputColorInfo;
|
||||||
|
private InputListener inputListener;
|
||||||
|
private OutputListener outputListener;
|
||||||
|
private ErrorListener errorListener;
|
||||||
|
private Executor errorListenerExecutor;
|
||||||
|
|
||||||
|
private @MonotonicNonNull EGLDisplay eglDisplay;
|
||||||
|
|
||||||
|
public DebugViewShaderProgram(
|
||||||
|
Context context, DebugViewProvider debugViewProvider, ColorInfo outputColorInfo) {
|
||||||
|
this.context = context;
|
||||||
|
this.debugViewProvider = debugViewProvider;
|
||||||
|
this.outputColorInfo = outputColorInfo;
|
||||||
|
inputListener = new InputListener() {};
|
||||||
|
outputListener = new OutputListener() {};
|
||||||
|
errorListener =
|
||||||
|
(frameProcessingException) ->
|
||||||
|
Log.e(TAG, "Exception caught by errorListener.", frameProcessingException);
|
||||||
|
errorListenerExecutor = directExecutor();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setInputListener(InputListener inputListener) {
|
||||||
|
this.inputListener = inputListener;
|
||||||
|
inputListener.onReadyToAcceptInputFrame();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setOutputListener(OutputListener outputListener) {
|
||||||
|
this.outputListener = outputListener;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setErrorListener(Executor executor, ErrorListener errorListener) {
|
||||||
|
this.errorListener = errorListener;
|
||||||
|
this.errorListenerExecutor = executor;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void queueInputFrame(
|
||||||
|
GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) {
|
||||||
|
try {
|
||||||
|
ensureConfigured(inputTexture.width, inputTexture.height);
|
||||||
|
DefaultShaderProgram defaultShaderProgram = checkNotNull(this.defaultShaderProgram);
|
||||||
|
checkNotNull(this.debugSurfaceViewWrapper)
|
||||||
|
.maybeRenderToSurfaceView(
|
||||||
|
() -> defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs),
|
||||||
|
glObjectsProvider);
|
||||||
|
outputListener.onOutputFrameAvailable(inputTexture, presentationTimeUs);
|
||||||
|
} catch (VideoFrameProcessingException | GlUtil.GlException e) {
|
||||||
|
errorListenerExecutor.execute(
|
||||||
|
() -> errorListener.onError(VideoFrameProcessingException.from(e, presentationTimeUs)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void releaseOutputFrame(GlTextureInfo outputTexture) {
|
||||||
|
inputListener.onInputFrameProcessed(outputTexture);
|
||||||
|
inputListener.onReadyToAcceptInputFrame();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void signalEndOfCurrentInputStream() {
|
||||||
|
outputListener.onCurrentOutputStreamEnded();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void flush() {
|
||||||
|
if (defaultShaderProgram != null) {
|
||||||
|
defaultShaderProgram.flush();
|
||||||
|
}
|
||||||
|
inputListener.onFlush();
|
||||||
|
inputListener.onReadyToAcceptInputFrame();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void release() throws VideoFrameProcessingException {
|
||||||
|
if (defaultShaderProgram != null) {
|
||||||
|
defaultShaderProgram.release();
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
GlUtil.checkGlError();
|
||||||
|
} catch (GlUtil.GlException e) {
|
||||||
|
throw new VideoFrameProcessingException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ensureConfigured(int inputWidth, int inputHeight)
|
||||||
|
throws VideoFrameProcessingException, GlUtil.GlException {
|
||||||
|
if (eglDisplay == null) {
|
||||||
|
eglDisplay = getDefaultEglDisplay();
|
||||||
|
}
|
||||||
|
EGLContext eglContext = GlUtil.getCurrentContext();
|
||||||
|
@Nullable
|
||||||
|
SurfaceView debugSurfaceView =
|
||||||
|
debugViewProvider.getDebugPreviewSurfaceView(inputWidth, inputHeight);
|
||||||
|
if (debugSurfaceView != null && !Objects.equals(this.debugSurfaceView, debugSurfaceView)) {
|
||||||
|
debugSurfaceViewWrapper =
|
||||||
|
new SurfaceViewWrapper(
|
||||||
|
eglDisplay, eglContext, debugSurfaceView, outputColorInfo.colorTransfer);
|
||||||
|
}
|
||||||
|
this.debugSurfaceView = debugSurfaceView;
|
||||||
|
if (defaultShaderProgram == null) {
|
||||||
|
defaultShaderProgram =
|
||||||
|
DefaultShaderProgram.createApplyingOetf(
|
||||||
|
context,
|
||||||
|
/* matrixTransformations= */ ImmutableList.of(),
|
||||||
|
/* rgbMatrices= */ ImmutableList.of(),
|
||||||
|
outputColorInfo,
|
||||||
|
outputColorInfo.colorTransfer == C.COLOR_TRANSFER_LINEAR
|
||||||
|
? WORKING_COLOR_SPACE_LINEAR
|
||||||
|
: WORKING_COLOR_SPACE_DEFAULT);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wrapper around a {@link SurfaceView} that keeps track of whether the output surface is valid,
|
||||||
|
* and makes rendering a no-op if not.
|
||||||
|
*/
|
||||||
|
private static final class SurfaceViewWrapper implements SurfaceHolder.Callback {
|
||||||
|
public final @C.ColorTransfer int outputColorTransfer;
|
||||||
|
private final EGLDisplay eglDisplay;
|
||||||
|
private final EGLContext eglContext;
|
||||||
|
|
||||||
|
@GuardedBy("this")
|
||||||
|
@Nullable
|
||||||
|
private Surface surface;
|
||||||
|
|
||||||
|
@GuardedBy("this")
|
||||||
|
@Nullable
|
||||||
|
private EGLSurface eglSurface;
|
||||||
|
|
||||||
|
private int width;
|
||||||
|
private int height;
|
||||||
|
|
||||||
|
public SurfaceViewWrapper(
|
||||||
|
EGLDisplay eglDisplay,
|
||||||
|
EGLContext eglContext,
|
||||||
|
SurfaceView surfaceView,
|
||||||
|
@C.ColorTransfer int outputColorTransfer) {
|
||||||
|
this.eglDisplay = eglDisplay;
|
||||||
|
this.eglContext = eglContext;
|
||||||
|
// PQ SurfaceView output is supported from API 33, but HLG output is supported from API 34.
|
||||||
|
// Therefore, convert HLG to PQ below API 34, so that HLG input can be displayed properly on
|
||||||
|
// API 33.
|
||||||
|
this.outputColorTransfer =
|
||||||
|
outputColorTransfer == C.COLOR_TRANSFER_HLG && Util.SDK_INT < 34
|
||||||
|
? C.COLOR_TRANSFER_ST2084
|
||||||
|
: outputColorTransfer;
|
||||||
|
surfaceView.getHolder().addCallback(this);
|
||||||
|
surface = surfaceView.getHolder().getSurface();
|
||||||
|
width = surfaceView.getWidth();
|
||||||
|
height = surfaceView.getHeight();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void surfaceCreated(SurfaceHolder holder) {}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public synchronized void surfaceChanged(
|
||||||
|
SurfaceHolder holder, int format, int width, int height) {
|
||||||
|
this.width = width;
|
||||||
|
this.height = height;
|
||||||
|
Surface newSurface = holder.getSurface();
|
||||||
|
if (!newSurface.equals(surface)) {
|
||||||
|
surface = newSurface;
|
||||||
|
eglSurface = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public synchronized void surfaceDestroyed(SurfaceHolder holder) {
|
||||||
|
surface = null;
|
||||||
|
eglSurface = null;
|
||||||
|
width = C.LENGTH_UNSET;
|
||||||
|
height = C.LENGTH_UNSET;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Focuses the wrapped surface view's surface as an {@link EGLSurface}, renders using {@code
|
||||||
|
* renderingTask} and swaps buffers, if the view's holder has a valid surface. Does nothing
|
||||||
|
* otherwise.
|
||||||
|
*
|
||||||
|
* <p>Must be called on the GL thread.
|
||||||
|
*/
|
||||||
|
public synchronized void maybeRenderToSurfaceView(
|
||||||
|
VideoFrameProcessingTaskExecutor.Task renderingTask, GlObjectsProvider glObjectsProvider)
|
||||||
|
throws GlUtil.GlException, VideoFrameProcessingException {
|
||||||
|
if (surface == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (eglSurface == null) {
|
||||||
|
eglSurface =
|
||||||
|
glObjectsProvider.createEglSurface(
|
||||||
|
eglDisplay, surface, outputColorTransfer, /* isEncoderInputSurface= */ false);
|
||||||
|
}
|
||||||
|
EGLSurface eglSurface = this.eglSurface;
|
||||||
|
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, width, height);
|
||||||
|
renderingTask.run();
|
||||||
|
EGL14.eglSwapBuffers(eglDisplay, eglSurface);
|
||||||
|
// Prevents white flashing on the debug SurfaceView when frames are rendered too fast.
|
||||||
|
// TODO: b/393316699 - Investigate removing this to speed up transcoding.
|
||||||
|
GLES20.glFinish();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -485,6 +485,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
private final List<Effect> activeEffects;
|
private final List<Effect> activeEffects;
|
||||||
private final Object lock;
|
private final Object lock;
|
||||||
private final ColorInfo outputColorInfo;
|
private final ColorInfo outputColorInfo;
|
||||||
|
private final DebugViewProvider debugViewProvider;
|
||||||
|
|
||||||
private volatile @MonotonicNonNull FrameInfo nextInputFrameInfo;
|
private volatile @MonotonicNonNull FrameInfo nextInputFrameInfo;
|
||||||
private volatile boolean inputStreamEnded;
|
private volatile boolean inputStreamEnded;
|
||||||
@ -499,7 +500,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
Executor listenerExecutor,
|
Executor listenerExecutor,
|
||||||
FinalShaderProgramWrapper finalShaderProgramWrapper,
|
FinalShaderProgramWrapper finalShaderProgramWrapper,
|
||||||
boolean renderFramesAutomatically,
|
boolean renderFramesAutomatically,
|
||||||
ColorInfo outputColorInfo) {
|
ColorInfo outputColorInfo,
|
||||||
|
DebugViewProvider debugViewProvider) {
|
||||||
this.context = context;
|
this.context = context;
|
||||||
this.glObjectsProvider = glObjectsProvider;
|
this.glObjectsProvider = glObjectsProvider;
|
||||||
this.eglDisplay = eglDisplay;
|
this.eglDisplay = eglDisplay;
|
||||||
@ -511,6 +513,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
this.activeEffects = new ArrayList<>();
|
this.activeEffects = new ArrayList<>();
|
||||||
this.lock = new Object();
|
this.lock = new Object();
|
||||||
this.outputColorInfo = outputColorInfo;
|
this.outputColorInfo = outputColorInfo;
|
||||||
|
this.debugViewProvider = debugViewProvider;
|
||||||
this.finalShaderProgramWrapper = finalShaderProgramWrapper;
|
this.finalShaderProgramWrapper = finalShaderProgramWrapper;
|
||||||
this.intermediateGlShaderPrograms = new ArrayList<>();
|
this.intermediateGlShaderPrograms = new ArrayList<>();
|
||||||
this.inputStreamRegisteredCondition = new ConditionVariable();
|
this.inputStreamRegisteredCondition = new ConditionVariable();
|
||||||
@ -875,7 +878,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
eglDisplay,
|
eglDisplay,
|
||||||
eglContextAndPlaceholderSurface.first,
|
eglContextAndPlaceholderSurface.first,
|
||||||
eglContextAndPlaceholderSurface.second,
|
eglContextAndPlaceholderSurface.second,
|
||||||
debugViewProvider,
|
|
||||||
outputColorInfo,
|
outputColorInfo,
|
||||||
videoFrameProcessingTaskExecutor,
|
videoFrameProcessingTaskExecutor,
|
||||||
videoFrameProcessorListenerExecutor,
|
videoFrameProcessorListenerExecutor,
|
||||||
@ -895,7 +897,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
videoFrameProcessorListenerExecutor,
|
videoFrameProcessorListenerExecutor,
|
||||||
finalShaderProgramWrapper,
|
finalShaderProgramWrapper,
|
||||||
renderFramesAutomatically,
|
renderFramesAutomatically,
|
||||||
outputColorInfo);
|
outputColorInfo,
|
||||||
|
debugViewProvider);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -940,10 +943,10 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
rgbMatrixListBuilder.add((RgbMatrix) glEffect);
|
rgbMatrixListBuilder.add((RgbMatrix) glEffect);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
boolean isOutputTransferHdr = ColorInfo.isTransferHdr(outputColorInfo);
|
||||||
ImmutableList<GlMatrixTransformation> matrixTransformations =
|
ImmutableList<GlMatrixTransformation> matrixTransformations =
|
||||||
matrixTransformationListBuilder.build();
|
matrixTransformationListBuilder.build();
|
||||||
ImmutableList<RgbMatrix> rgbMatrices = rgbMatrixListBuilder.build();
|
ImmutableList<RgbMatrix> rgbMatrices = rgbMatrixListBuilder.build();
|
||||||
boolean isOutputTransferHdr = ColorInfo.isTransferHdr(outputColorInfo);
|
|
||||||
if (!matrixTransformations.isEmpty() || !rgbMatrices.isEmpty()) {
|
if (!matrixTransformations.isEmpty() || !rgbMatrices.isEmpty()) {
|
||||||
DefaultShaderProgram defaultShaderProgram =
|
DefaultShaderProgram defaultShaderProgram =
|
||||||
DefaultShaderProgram.create(
|
DefaultShaderProgram.create(
|
||||||
@ -1024,11 +1027,16 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
|
|||||||
intermediateGlShaderPrograms.clear();
|
intermediateGlShaderPrograms.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ImmutableList.Builder<Effect> effectsListBuilder =
|
||||||
|
new ImmutableList.Builder<Effect>().addAll(inputStreamInfo.effects);
|
||||||
|
if (debugViewProvider != DebugViewProvider.NONE) {
|
||||||
|
effectsListBuilder.add(new DebugViewEffect(debugViewProvider, outputColorInfo));
|
||||||
|
}
|
||||||
// The GlShaderPrograms that should be inserted in between InputSwitcher and
|
// The GlShaderPrograms that should be inserted in between InputSwitcher and
|
||||||
// FinalShaderProgramWrapper.
|
// FinalShaderProgramWrapper.
|
||||||
intermediateGlShaderPrograms.addAll(
|
intermediateGlShaderPrograms.addAll(
|
||||||
createGlShaderPrograms(
|
createGlShaderPrograms(
|
||||||
context, inputStreamInfo.effects, outputColorInfo, finalShaderProgramWrapper));
|
context, effectsListBuilder.build(), outputColorInfo, finalShaderProgramWrapper));
|
||||||
inputSwitcher.setDownstreamShaderProgram(
|
inputSwitcher.setDownstreamShaderProgram(
|
||||||
getFirst(intermediateGlShaderPrograms, /* defaultValue= */ finalShaderProgramWrapper));
|
getFirst(intermediateGlShaderPrograms, /* defaultValue= */ finalShaderProgramWrapper));
|
||||||
chainShaderProgramsWithListeners(
|
chainShaderProgramsWithListeners(
|
||||||
|
@ -21,7 +21,6 @@ import static androidx.media3.common.util.Assertions.checkNotNull;
|
|||||||
import static androidx.media3.common.util.Assertions.checkState;
|
import static androidx.media3.common.util.Assertions.checkState;
|
||||||
import static androidx.media3.effect.DebugTraceUtil.COMPONENT_VFP;
|
import static androidx.media3.effect.DebugTraceUtil.COMPONENT_VFP;
|
||||||
import static androidx.media3.effect.DebugTraceUtil.EVENT_RENDERED_TO_OUTPUT_SURFACE;
|
import static androidx.media3.effect.DebugTraceUtil.EVENT_RENDERED_TO_OUTPUT_SURFACE;
|
||||||
import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_LINEAR;
|
|
||||||
|
|
||||||
import android.content.Context;
|
import android.content.Context;
|
||||||
import android.opengl.EGL14;
|
import android.opengl.EGL14;
|
||||||
@ -29,16 +28,11 @@ import android.opengl.EGLContext;
|
|||||||
import android.opengl.EGLDisplay;
|
import android.opengl.EGLDisplay;
|
||||||
import android.opengl.EGLExt;
|
import android.opengl.EGLExt;
|
||||||
import android.opengl.EGLSurface;
|
import android.opengl.EGLSurface;
|
||||||
import android.opengl.GLES20;
|
|
||||||
import android.util.Pair;
|
import android.util.Pair;
|
||||||
import android.view.Surface;
|
import android.view.Surface;
|
||||||
import android.view.SurfaceHolder;
|
|
||||||
import android.view.SurfaceView;
|
|
||||||
import androidx.annotation.GuardedBy;
|
|
||||||
import androidx.annotation.Nullable;
|
import androidx.annotation.Nullable;
|
||||||
import androidx.media3.common.C;
|
import androidx.media3.common.C;
|
||||||
import androidx.media3.common.ColorInfo;
|
import androidx.media3.common.ColorInfo;
|
||||||
import androidx.media3.common.DebugViewProvider;
|
|
||||||
import androidx.media3.common.GlObjectsProvider;
|
import androidx.media3.common.GlObjectsProvider;
|
||||||
import androidx.media3.common.GlTextureInfo;
|
import androidx.media3.common.GlTextureInfo;
|
||||||
import androidx.media3.common.SurfaceInfo;
|
import androidx.media3.common.SurfaceInfo;
|
||||||
@ -88,7 +82,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||||||
private final EGLDisplay eglDisplay;
|
private final EGLDisplay eglDisplay;
|
||||||
private final EGLContext eglContext;
|
private final EGLContext eglContext;
|
||||||
private final EGLSurface placeholderSurface;
|
private final EGLSurface placeholderSurface;
|
||||||
private final DebugViewProvider debugViewProvider;
|
|
||||||
private final ColorInfo outputColorInfo;
|
private final ColorInfo outputColorInfo;
|
||||||
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
|
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
|
||||||
private final Executor videoFrameProcessorListenerExecutor;
|
private final Executor videoFrameProcessorListenerExecutor;
|
||||||
@ -104,7 +97,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||||||
private int inputWidth;
|
private int inputWidth;
|
||||||
private int inputHeight;
|
private int inputHeight;
|
||||||
@Nullable private DefaultShaderProgram defaultShaderProgram;
|
@Nullable private DefaultShaderProgram defaultShaderProgram;
|
||||||
@Nullable private SurfaceViewWrapper debugSurfaceViewWrapper;
|
|
||||||
// Whether the input stream has ended, but not all input has been released. This is relevant only
|
// Whether the input stream has ended, but not all input has been released. This is relevant only
|
||||||
// when renderFramesAutomatically is false. Ensures all frames are rendered before reporting
|
// when renderFramesAutomatically is false. Ensures all frames are rendered before reporting
|
||||||
// onInputStreamProcessed.
|
// onInputStreamProcessed.
|
||||||
@ -112,7 +104,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||||||
private boolean isInputStreamEndedWithPendingAvailableFrames;
|
private boolean isInputStreamEndedWithPendingAvailableFrames;
|
||||||
private InputListener inputListener;
|
private InputListener inputListener;
|
||||||
private @MonotonicNonNull Size outputSizeBeforeSurfaceTransformation;
|
private @MonotonicNonNull Size outputSizeBeforeSurfaceTransformation;
|
||||||
@Nullable private SurfaceView debugSurfaceView;
|
|
||||||
@Nullable private OnInputStreamProcessedListener onInputStreamProcessedListener;
|
@Nullable private OnInputStreamProcessedListener onInputStreamProcessedListener;
|
||||||
private boolean matrixTransformationsChanged;
|
private boolean matrixTransformationsChanged;
|
||||||
private boolean outputSurfaceInfoChanged;
|
private boolean outputSurfaceInfoChanged;
|
||||||
@ -126,7 +117,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||||||
EGLDisplay eglDisplay,
|
EGLDisplay eglDisplay,
|
||||||
EGLContext eglContext,
|
EGLContext eglContext,
|
||||||
EGLSurface placeholderSurface,
|
EGLSurface placeholderSurface,
|
||||||
DebugViewProvider debugViewProvider,
|
|
||||||
ColorInfo outputColorInfo,
|
ColorInfo outputColorInfo,
|
||||||
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
|
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
|
||||||
Executor videoFrameProcessorListenerExecutor,
|
Executor videoFrameProcessorListenerExecutor,
|
||||||
@ -141,7 +131,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||||||
this.eglDisplay = eglDisplay;
|
this.eglDisplay = eglDisplay;
|
||||||
this.eglContext = eglContext;
|
this.eglContext = eglContext;
|
||||||
this.placeholderSurface = placeholderSurface;
|
this.placeholderSurface = placeholderSurface;
|
||||||
this.debugViewProvider = debugViewProvider;
|
|
||||||
this.outputColorInfo = outputColorInfo;
|
this.outputColorInfo = outputColorInfo;
|
||||||
this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor;
|
this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor;
|
||||||
this.videoFrameProcessorListenerExecutor = videoFrameProcessorListenerExecutor;
|
this.videoFrameProcessorListenerExecutor = videoFrameProcessorListenerExecutor;
|
||||||
@ -422,9 +411,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||||||
videoFrameProcessorListener.onError(
|
videoFrameProcessorListener.onError(
|
||||||
VideoFrameProcessingException.from(e, presentationTimeUs)));
|
VideoFrameProcessingException.from(e, presentationTimeUs)));
|
||||||
}
|
}
|
||||||
if (debugSurfaceViewWrapper != null && defaultShaderProgram != null) {
|
|
||||||
renderFrameToDebugSurface(glObjectsProvider, inputTexture, presentationTimeUs);
|
|
||||||
}
|
|
||||||
|
|
||||||
inputListener.onInputFrameProcessed(inputTexture);
|
inputListener.onInputFrameProcessed(inputTexture);
|
||||||
}
|
}
|
||||||
@ -537,16 +523,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||||||
outputTexturePool.ensureConfigured(glObjectsProvider, outputWidth, outputHeight);
|
outputTexturePool.ensureConfigured(glObjectsProvider, outputWidth, outputHeight);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Nullable
|
|
||||||
SurfaceView debugSurfaceView =
|
|
||||||
debugViewProvider.getDebugPreviewSurfaceView(outputWidth, outputHeight);
|
|
||||||
if (debugSurfaceView != null && !Util.areEqual(this.debugSurfaceView, debugSurfaceView)) {
|
|
||||||
debugSurfaceViewWrapper =
|
|
||||||
new SurfaceViewWrapper(
|
|
||||||
eglDisplay, eglContext, debugSurfaceView, outputColorInfo.colorTransfer);
|
|
||||||
}
|
|
||||||
this.debugSurfaceView = debugSurfaceView;
|
|
||||||
|
|
||||||
if (defaultShaderProgram != null
|
if (defaultShaderProgram != null
|
||||||
&& (outputSurfaceInfoChanged || inputSizeChanged || matrixTransformationsChanged)) {
|
&& (outputSurfaceInfoChanged || inputSizeChanged || matrixTransformationsChanged)) {
|
||||||
defaultShaderProgram.release();
|
defaultShaderProgram.release();
|
||||||
@ -600,123 +576,4 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
|||||||
}
|
}
|
||||||
return defaultShaderProgram;
|
return defaultShaderProgram;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void renderFrameToDebugSurface(
|
|
||||||
GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) {
|
|
||||||
DefaultShaderProgram defaultShaderProgram = checkNotNull(this.defaultShaderProgram);
|
|
||||||
SurfaceViewWrapper debugSurfaceViewWrapper = checkNotNull(this.debugSurfaceViewWrapper);
|
|
||||||
try {
|
|
||||||
checkNotNull(debugSurfaceViewWrapper)
|
|
||||||
.maybeRenderToSurfaceView(
|
|
||||||
() -> {
|
|
||||||
GlUtil.clearFocusedBuffers();
|
|
||||||
if (sdrWorkingColorSpace == WORKING_COLOR_SPACE_LINEAR) {
|
|
||||||
@C.ColorTransfer
|
|
||||||
int configuredColorTransfer = defaultShaderProgram.getOutputColorTransfer();
|
|
||||||
defaultShaderProgram.setOutputColorTransfer(
|
|
||||||
debugSurfaceViewWrapper.outputColorTransfer);
|
|
||||||
defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs);
|
|
||||||
defaultShaderProgram.setOutputColorTransfer(configuredColorTransfer);
|
|
||||||
} else {
|
|
||||||
defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs);
|
|
||||||
}
|
|
||||||
},
|
|
||||||
glObjectsProvider);
|
|
||||||
} catch (VideoFrameProcessingException | GlUtil.GlException e) {
|
|
||||||
Log.d(TAG, "Error rendering to debug preview", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Wrapper around a {@link SurfaceView} that keeps track of whether the output surface is valid,
|
|
||||||
* and makes rendering a no-op if not.
|
|
||||||
*
|
|
||||||
* <p>This class should only be used for displaying a debug preview.
|
|
||||||
*/
|
|
||||||
private static final class SurfaceViewWrapper implements SurfaceHolder.Callback {
|
|
||||||
public final @C.ColorTransfer int outputColorTransfer;
|
|
||||||
private final EGLDisplay eglDisplay;
|
|
||||||
private final EGLContext eglContext;
|
|
||||||
|
|
||||||
@GuardedBy("this")
|
|
||||||
@Nullable
|
|
||||||
private Surface surface;
|
|
||||||
|
|
||||||
@GuardedBy("this")
|
|
||||||
@Nullable
|
|
||||||
private EGLSurface eglSurface;
|
|
||||||
|
|
||||||
private int width;
|
|
||||||
private int height;
|
|
||||||
|
|
||||||
public SurfaceViewWrapper(
|
|
||||||
EGLDisplay eglDisplay,
|
|
||||||
EGLContext eglContext,
|
|
||||||
SurfaceView surfaceView,
|
|
||||||
@C.ColorTransfer int outputColorTransfer) {
|
|
||||||
this.eglDisplay = eglDisplay;
|
|
||||||
this.eglContext = eglContext;
|
|
||||||
// PQ SurfaceView output is supported from API 33, but HLG output is supported from API 34.
|
|
||||||
// Therefore, convert HLG to PQ below API 34, so that HLG input can be displayed properly on
|
|
||||||
// API 33.
|
|
||||||
this.outputColorTransfer =
|
|
||||||
outputColorTransfer == C.COLOR_TRANSFER_HLG && Util.SDK_INT < 34
|
|
||||||
? C.COLOR_TRANSFER_ST2084
|
|
||||||
: outputColorTransfer;
|
|
||||||
surfaceView.getHolder().addCallback(this);
|
|
||||||
surface = surfaceView.getHolder().getSurface();
|
|
||||||
width = surfaceView.getWidth();
|
|
||||||
height = surfaceView.getHeight();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void surfaceCreated(SurfaceHolder holder) {}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public synchronized void surfaceChanged(
|
|
||||||
SurfaceHolder holder, int format, int width, int height) {
|
|
||||||
this.width = width;
|
|
||||||
this.height = height;
|
|
||||||
Surface newSurface = holder.getSurface();
|
|
||||||
if (surface == null || !surface.equals(newSurface)) {
|
|
||||||
surface = newSurface;
|
|
||||||
eglSurface = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public synchronized void surfaceDestroyed(SurfaceHolder holder) {
|
|
||||||
surface = null;
|
|
||||||
eglSurface = null;
|
|
||||||
width = C.LENGTH_UNSET;
|
|
||||||
height = C.LENGTH_UNSET;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Focuses the wrapped surface view's surface as an {@link EGLSurface}, renders using {@code
|
|
||||||
* renderingTask} and swaps buffers, if the view's holder has a valid surface. Does nothing
|
|
||||||
* otherwise.
|
|
||||||
*
|
|
||||||
* <p>Must be called on the GL thread.
|
|
||||||
*/
|
|
||||||
public synchronized void maybeRenderToSurfaceView(
|
|
||||||
VideoFrameProcessingTaskExecutor.Task renderingTask, GlObjectsProvider glObjectsProvider)
|
|
||||||
throws GlUtil.GlException, VideoFrameProcessingException {
|
|
||||||
if (surface == null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (eglSurface == null) {
|
|
||||||
eglSurface =
|
|
||||||
glObjectsProvider.createEglSurface(
|
|
||||||
eglDisplay, surface, outputColorTransfer, /* isEncoderInputSurface= */ false);
|
|
||||||
}
|
|
||||||
EGLSurface eglSurface = this.eglSurface;
|
|
||||||
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, width, height);
|
|
||||||
renderingTask.run();
|
|
||||||
EGL14.eglSwapBuffers(eglDisplay, eglSurface);
|
|
||||||
// Prevents white flashing on the debug SurfaceView when frames are rendered too fast.
|
|
||||||
GLES20.glFinish();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user