loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-04-30 06:46:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Rollback of 0fb4e3ba11

Browse Source 
PiperOrigin-RevId: 722585306
This commit is contained in:
Googler 2025-02-03 03:15:21 -08:00 committed by Copybara-Service
parent 82cb1d8ac7
commit 4ed9abd05b
7 changed files with 162 additions and 379 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java
View File

@ -61,7 +61,6 @@ import androidx.media3.datasource.DataSourceBitmapLoader;
import androidx.media3.effect.BitmapOverlay; import androidx.media3.effect.BitmapOverlay;
import androidx.media3.effect.Contrast; import androidx.media3.effect.Contrast;
import androidx.media3.effect.DebugTraceUtil; import androidx.media3.effect.DebugTraceUtil;
import androidx.media3.effect.DebugViewEffect;
import androidx.media3.effect.DrawableOverlay; import androidx.media3.effect.DrawableOverlay;
import androidx.media3.effect.GlEffect; import androidx.media3.effect.GlEffect;
import androidx.media3.effect.GlShaderProgram; import androidx.media3.effect.GlShaderProgram;
@ -312,6 +311,10 @@ public final class TransformerActivity extends AppCompatActivity {
transformerBuilder.setMuxerFactory(new InAppFragmentedMp4Muxer.Factory()); transformerBuilder.setMuxerFactory(new InAppFragmentedMp4Muxer.Factory());
} }
if (bundle.getBoolean(ConfigurationActivity.ENABLE_DEBUG_PREVIEW)) {
transformerBuilder.setDebugViewProvider(new DemoDebugViewProvider());
}
if (bundle.getBoolean(ConfigurationActivity.ENABLE_ANALYZER_MODE)) { if (bundle.getBoolean(ConfigurationActivity.ENABLE_ANALYZER_MODE)) {
return ExperimentalAnalyzerModeFactory.buildAnalyzer( return ExperimentalAnalyzerModeFactory.buildAnalyzer(
this.getApplicationContext(), transformerBuilder.build()); this.getApplicationContext(), transformerBuilder.build());
@ -562,10 +565,6 @@ public final class TransformerActivity extends AppCompatActivity {
effects.add(Presentation.createForHeight(resolutionHeight)); effects.add(Presentation.createForHeight(resolutionHeight));
} }
if (bundle.getBoolean(ConfigurationActivity.ENABLE_DEBUG_PREVIEW)) {
effects.add(new DebugViewEffect(new DemoDebugViewProvider()));
}
return effects.build(); return effects.build();
} }

53
libraries/effect/src/main/java/androidx/media3/effect/DebugViewEffect.java
View File

@ -1,53 +0,0 @@
/*
* Copyright 2025 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.effect;
import android.content.Context;
import android.view.SurfaceView;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.DebugViewProvider;
import androidx.media3.common.util.UnstableApi;
/** {@link GlEffect} that renders to a {@link SurfaceView} provided by {@link DebugViewProvider}. */
@UnstableApi
public final class DebugViewEffect implements GlEffect {
private final DebugViewProvider debugViewProvider;
/**
* Creates a new instance.
*
* @param debugViewProvider The class that provides the {@link SurfaceView} that the debug preview
* will be rendered to.
*/
public DebugViewEffect(DebugViewProvider debugViewProvider) {
this.debugViewProvider = debugViewProvider;
}
/**
* Throws {@link UnsupportedOperationException} because {@link #toGlShaderProgram(Context,
* ColorInfo)} should be used instead.
*/
@Override
public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) {
throw new UnsupportedOperationException();
}
@Override
public GlShaderProgram toGlShaderProgram(Context context, ColorInfo colorInfo) {
return new DebugViewShaderProgram(context, debugViewProvider, colorInfo);
}
}

269
libraries/effect/src/main/java/androidx/media3/effect/DebugViewShaderProgram.java
View File

@ -1,269 +0,0 @@
/*
* Copyright 2025 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.effect;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.GlUtil.getDefaultEglDisplay;
import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_DEFAULT;
import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_LINEAR;
import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
import android.content.Context;
import android.opengl.EGL14;
import android.opengl.EGLContext;
import android.opengl.EGLDisplay;
import android.opengl.EGLSurface;
import android.opengl.GLES20;
import android.view.Surface;
import android.view.SurfaceHolder;
import android.view.SurfaceView;
import androidx.annotation.GuardedBy;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.DebugViewProvider;
import androidx.media3.common.GlObjectsProvider;
import androidx.media3.common.GlTextureInfo;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.Log;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import com.google.common.collect.ImmutableList;
import java.util.Objects;
import java.util.concurrent.Executor;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* {@link GlShaderProgram} that renders to a {@link SurfaceView} provided by {@link
* DebugViewProvider}.
*/
@UnstableApi
public final class DebugViewShaderProgram implements GlShaderProgram {
private static final String TAG = "DebugViewShaderProgram";
private final Context context;
private final DebugViewProvider debugViewProvider;
@Nullable private SurfaceView debugSurfaceView;
@Nullable private DefaultShaderProgram defaultShaderProgram;
@Nullable private SurfaceViewWrapper debugSurfaceViewWrapper;
private final ColorInfo colorInfo;
private InputListener inputListener;
private OutputListener outputListener;
private ErrorListener errorListener;
private Executor errorListenerExecutor;
private @MonotonicNonNull EGLDisplay eglDisplay;
public DebugViewShaderProgram(
Context context, DebugViewProvider debugViewProvider, ColorInfo colorInfo) {
this.context = context;
this.debugViewProvider = debugViewProvider;
this.colorInfo = colorInfo;
inputListener = new InputListener() {};
outputListener = new OutputListener() {};
errorListener =
(frameProcessingException) ->
Log.e(TAG, "Exception caught by errorListener.", frameProcessingException);
errorListenerExecutor = directExecutor();
}
@Override
public void setInputListener(InputListener inputListener) {
this.inputListener = inputListener;
inputListener.onReadyToAcceptInputFrame();
}
@Override
public void setOutputListener(OutputListener outputListener) {
this.outputListener = outputListener;
}
@Override
public void setErrorListener(Executor executor, ErrorListener errorListener) {
this.errorListener = errorListener;
this.errorListenerExecutor = executor;
}
@Override
public void queueInputFrame(
GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) {
try {
ensureConfigured(inputTexture.width, inputTexture.height);
DefaultShaderProgram defaultShaderProgram = checkNotNull(this.defaultShaderProgram);
checkNotNull(this.debugSurfaceViewWrapper)
.maybeRenderToSurfaceView(
() -> defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs),
glObjectsProvider);
outputListener.onOutputFrameAvailable(inputTexture, presentationTimeUs);
} catch (VideoFrameProcessingException | GlUtil.GlException e) {
errorListenerExecutor.execute(
() -> errorListener.onError(VideoFrameProcessingException.from(e, presentationTimeUs)));
}
}
@Override
public void releaseOutputFrame(GlTextureInfo outputTexture) {
inputListener.onInputFrameProcessed(outputTexture);
inputListener.onReadyToAcceptInputFrame();
}
@Override
public void signalEndOfCurrentInputStream() {
outputListener.onCurrentOutputStreamEnded();
}
@Override
public void flush() {
if (defaultShaderProgram != null) {
defaultShaderProgram.flush();
}
inputListener.onFlush();
inputListener.onReadyToAcceptInputFrame();
}
@Override
public void release() throws VideoFrameProcessingException {
if (defaultShaderProgram != null) {
defaultShaderProgram.release();
}
try {
GlUtil.checkGlError();
} catch (GlUtil.GlException e) {
throw new VideoFrameProcessingException(e);
}
}
private void ensureConfigured(int inputWidth, int inputHeight)
throws VideoFrameProcessingException, GlUtil.GlException {
if (eglDisplay == null) {
eglDisplay = getDefaultEglDisplay();
}
EGLContext eglContext = GlUtil.getCurrentContext();
@Nullable
SurfaceView debugSurfaceView =
debugViewProvider.getDebugPreviewSurfaceView(inputWidth, inputHeight);
if (debugSurfaceView != null && !Objects.equals(this.debugSurfaceView, debugSurfaceView)) {
debugSurfaceViewWrapper =
new SurfaceViewWrapper(eglDisplay, eglContext, debugSurfaceView, colorInfo.colorTransfer);
}
this.debugSurfaceView = debugSurfaceView;
if (defaultShaderProgram == null) {
defaultShaderProgram =
DefaultShaderProgram.createApplyingOetf(
context,
/* matrixTransformations= */ ImmutableList.of(),
/* rgbMatrices= */ ImmutableList.of(),
colorInfo,
colorInfo.colorTransfer == C.COLOR_TRANSFER_LINEAR
? WORKING_COLOR_SPACE_LINEAR
: WORKING_COLOR_SPACE_DEFAULT);
}
}
/**
* Wrapper around a {@link SurfaceView} that keeps track of whether the output surface is valid,
* and makes rendering a no-op if not.
*/
private static final class SurfaceViewWrapper implements SurfaceHolder.Callback {
public final @C.ColorTransfer int outputColorTransfer;
private final EGLDisplay eglDisplay;
private final EGLContext eglContext;
@GuardedBy("this")
@Nullable
private Surface surface;
@GuardedBy("this")
@Nullable
private EGLSurface eglSurface;
private int width;
private int height;
public SurfaceViewWrapper(
EGLDisplay eglDisplay,
EGLContext eglContext,
SurfaceView surfaceView,
@C.ColorTransfer int outputColorTransfer) {
this.eglDisplay = eglDisplay;
this.eglContext = eglContext;
// PQ SurfaceView output is supported from API 33, but HLG output is supported from API 34.
// Therefore, convert HLG to PQ below API 34, so that HLG input can be displayed properly on
// API 33.
this.outputColorTransfer =
outputColorTransfer == C.COLOR_TRANSFER_HLG && Util.SDK_INT < 34
? C.COLOR_TRANSFER_ST2084
: outputColorTransfer;
surfaceView.getHolder().addCallback(this);
surface = surfaceView.getHolder().getSurface();
width = surfaceView.getWidth();
height = surfaceView.getHeight();
}
@Override
public void surfaceCreated(SurfaceHolder holder) {}
@Override
public synchronized void surfaceChanged(
SurfaceHolder holder, int format, int width, int height) {
this.width = width;
this.height = height;
Surface newSurface = holder.getSurface();
if (!newSurface.equals(surface)) {
surface = newSurface;
eglSurface = null;
}
}
@Override
public synchronized void surfaceDestroyed(SurfaceHolder holder) {
surface = null;
eglSurface = null;
width = C.LENGTH_UNSET;
height = C.LENGTH_UNSET;
}
/**
* Focuses the wrapped surface view's surface as an {@link EGLSurface}, renders using {@code
* renderingTask} and swaps buffers, if the view's holder has a valid surface. Does nothing
* otherwise.
*
* <p>Must be called on the GL thread.
*/
public synchronized void maybeRenderToSurfaceView(
VideoFrameProcessingTaskExecutor.Task renderingTask, GlObjectsProvider glObjectsProvider)
throws GlUtil.GlException, VideoFrameProcessingException {
if (surface == null) {
return;
}
if (eglSurface == null) {
eglSurface =
glObjectsProvider.createEglSurface(
eglDisplay, surface, outputColorTransfer, /* isEncoderInputSurface= */ false);
}
EGLSurface eglSurface = this.eglSurface;
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, width, height);
renderingTask.run();
EGL14.eglSwapBuffers(eglDisplay, eglSurface);
// Prevents white flashing on the debug SurfaceView when frames are rendered too fast.
// TODO(b/393316699) - Investigate removing this to speed up transcoding.
GLES20.glFinish();
}
}
}

37
libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java
View File

@ -419,6 +419,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
() -> () ->
createOpenGlObjectsAndFrameProcessor( createOpenGlObjectsAndFrameProcessor(
context, context,
debugViewProvider,
outputColorInfo, outputColorInfo,
sdrWorkingColorSpace, sdrWorkingColorSpace,
renderFramesAutomatically, renderFramesAutomatically,
@ -478,10 +479,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
private final List<Effect> activeEffects; private final List<Effect> activeEffects;
private final Object lock; private final Object lock;
/** The {@link ColorInfo} that all {@link Effect effects} work in. */
private final ColorInfo workingColorInfo;
private final ColorInfo outputColorInfo; private final ColorInfo outputColorInfo;
private volatile @MonotonicNonNull FrameInfo nextInputFrameInfo; private volatile @MonotonicNonNull FrameInfo nextInputFrameInfo;
@ -497,7 +494,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
Executor listenerExecutor, Executor listenerExecutor,
FinalShaderProgramWrapper finalShaderProgramWrapper, FinalShaderProgramWrapper finalShaderProgramWrapper,
boolean renderFramesAutomatically, boolean renderFramesAutomatically,
ColorInfo workingColorInfo,
ColorInfo outputColorInfo) { ColorInfo outputColorInfo) {
this.context = context; this.context = context;
this.glObjectsProvider = glObjectsProvider; this.glObjectsProvider = glObjectsProvider;
@ -509,7 +505,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
this.renderFramesAutomatically = renderFramesAutomatically; this.renderFramesAutomatically = renderFramesAutomatically;
this.activeEffects = new ArrayList<>(); this.activeEffects = new ArrayList<>();
this.lock = new Object(); this.lock = new Object();
this.workingColorInfo = workingColorInfo;
this.outputColorInfo = outputColorInfo; this.outputColorInfo = outputColorInfo;
this.finalShaderProgramWrapper = finalShaderProgramWrapper; this.finalShaderProgramWrapper = finalShaderProgramWrapper;
this.intermediateGlShaderPrograms = new ArrayList<>(); this.intermediateGlShaderPrograms = new ArrayList<>();
@ -822,6 +817,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
*/ */
private static DefaultVideoFrameProcessor createOpenGlObjectsAndFrameProcessor( private static DefaultVideoFrameProcessor createOpenGlObjectsAndFrameProcessor(
Context context, Context context,
DebugViewProvider debugViewProvider,
ColorInfo outputColorInfo, ColorInfo outputColorInfo,
@WorkingColorSpace int sdrWorkingColorSpace, @WorkingColorSpace int sdrWorkingColorSpace,
boolean renderFramesAutomatically, boolean renderFramesAutomatically,
@ -849,13 +845,12 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
.setColorTransfer(C.COLOR_TRANSFER_LINEAR) .setColorTransfer(C.COLOR_TRANSFER_LINEAR)
.setHdrStaticInfo(null) .setHdrStaticInfo(null)
.build(); .build();
ColorInfo intermediateColorInfo; ColorInfo intermediateColorInfo =
if (ColorInfo.isTransferHdr(outputColorInfo) ColorInfo.isTransferHdr(outputColorInfo)
|| sdrWorkingColorSpace == WORKING_COLOR_SPACE_LINEAR) { ? linearColorInfo
intermediateColorInfo = linearColorInfo; : sdrWorkingColorSpace == WORKING_COLOR_SPACE_LINEAR
} else { ? linearColorInfo
intermediateColorInfo = outputColorInfo; : outputColorInfo;
}
InputSwitcher inputSwitcher = InputSwitcher inputSwitcher =
new InputSwitcher( new InputSwitcher(
context, context,
@ -875,6 +870,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
eglDisplay, eglDisplay,
eglContextAndPlaceholderSurface.first, eglContextAndPlaceholderSurface.first,
eglContextAndPlaceholderSurface.second, eglContextAndPlaceholderSurface.second,
debugViewProvider,
outputColorInfo, outputColorInfo,
videoFrameProcessingTaskExecutor, videoFrameProcessingTaskExecutor,
videoFrameProcessorListenerExecutor, videoFrameProcessorListenerExecutor,
@ -894,7 +890,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
videoFrameProcessorListenerExecutor, videoFrameProcessorListenerExecutor,
finalShaderProgramWrapper, finalShaderProgramWrapper,
renderFramesAutomatically, renderFramesAutomatically,
intermediateColorInfo,
outputColorInfo); outputColorInfo);
} }
@ -907,7 +902,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
* *
* @param context The {@link Context}. * @param context The {@link Context}.
* @param effects The list of {@link GlEffect effects}. * @param effects The list of {@link GlEffect effects}.
* @param workingColorInfo The {@link ColorInfo} the {@link List<Effect> effects} work in. * @param outputColorInfo The {@link ColorInfo} on {@code DefaultVideoFrameProcessor} output.
* @param finalShaderProgramWrapper The {@link FinalShaderProgramWrapper} to apply the {@link * @param finalShaderProgramWrapper The {@link FinalShaderProgramWrapper} to apply the {@link
* GlMatrixTransformation GlMatrixTransformations} and {@link RgbMatrix RgbMatrices} after all * GlMatrixTransformation GlMatrixTransformations} and {@link RgbMatrix RgbMatrices} after all
* other {@link GlEffect GlEffects}. * other {@link GlEffect GlEffects}.
@ -916,7 +911,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
private static ImmutableList<GlShaderProgram> createGlShaderPrograms( private static ImmutableList<GlShaderProgram> createGlShaderPrograms(
Context context, Context context,
List<Effect> effects, List<Effect> effects,
ColorInfo workingColorInfo, ColorInfo outputColorInfo,
FinalShaderProgramWrapper finalShaderProgramWrapper) FinalShaderProgramWrapper finalShaderProgramWrapper)
throws VideoFrameProcessingException { throws VideoFrameProcessingException {
ImmutableList.Builder<GlShaderProgram> shaderProgramListBuilder = new ImmutableList.Builder<>(); ImmutableList.Builder<GlShaderProgram> shaderProgramListBuilder = new ImmutableList.Builder<>();
@ -943,18 +938,16 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
ImmutableList<GlMatrixTransformation> matrixTransformations = ImmutableList<GlMatrixTransformation> matrixTransformations =
matrixTransformationListBuilder.build(); matrixTransformationListBuilder.build();
ImmutableList<RgbMatrix> rgbMatrices = rgbMatrixListBuilder.build(); ImmutableList<RgbMatrix> rgbMatrices = rgbMatrixListBuilder.build();
boolean isOutputTransferHdr = ColorInfo.isTransferHdr(outputColorInfo);
if (!matrixTransformations.isEmpty() || !rgbMatrices.isEmpty()) { if (!matrixTransformations.isEmpty() || !rgbMatrices.isEmpty()) {
DefaultShaderProgram defaultShaderProgram = DefaultShaderProgram defaultShaderProgram =
DefaultShaderProgram.create( DefaultShaderProgram.create(
context, context, matrixTransformations, rgbMatrices, isOutputTransferHdr);
matrixTransformations,
rgbMatrices,
ColorInfo.isTransferHdr(workingColorInfo));
shaderProgramListBuilder.add(defaultShaderProgram); shaderProgramListBuilder.add(defaultShaderProgram);
matrixTransformationListBuilder = new ImmutableList.Builder<>(); matrixTransformationListBuilder = new ImmutableList.Builder<>();
rgbMatrixListBuilder = new ImmutableList.Builder<>(); rgbMatrixListBuilder = new ImmutableList.Builder<>();
} }
shaderProgramListBuilder.add(glEffect.toGlShaderProgram(context, workingColorInfo)); shaderProgramListBuilder.add(glEffect.toGlShaderProgram(context, isOutputTransferHdr));
} }
finalShaderProgramWrapper.setMatrixTransformations( finalShaderProgramWrapper.setMatrixTransformations(
@ -1030,7 +1023,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
// FinalShaderProgramWrapper. // FinalShaderProgramWrapper.
intermediateGlShaderPrograms.addAll( intermediateGlShaderPrograms.addAll(
createGlShaderPrograms( createGlShaderPrograms(
context, inputStreamInfo.effects, workingColorInfo, finalShaderProgramWrapper)); context, inputStreamInfo.effects, outputColorInfo, finalShaderProgramWrapper));
inputSwitcher.setDownstreamShaderProgram( inputSwitcher.setDownstreamShaderProgram(
getFirst(intermediateGlShaderPrograms, /* defaultValue= */ finalShaderProgramWrapper)); getFirst(intermediateGlShaderPrograms, /* defaultValue= */ finalShaderProgramWrapper));
chainShaderProgramsWithListeners( chainShaderProgramsWithListeners(

143
libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java
View File

@ -21,6 +21,7 @@ import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.effect.DebugTraceUtil.COMPONENT_VFP; import static androidx.media3.effect.DebugTraceUtil.COMPONENT_VFP;
import static androidx.media3.effect.DebugTraceUtil.EVENT_RENDERED_TO_OUTPUT_SURFACE; import static androidx.media3.effect.DebugTraceUtil.EVENT_RENDERED_TO_OUTPUT_SURFACE;
import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_LINEAR;
import android.content.Context; import android.content.Context;
import android.opengl.EGL14; import android.opengl.EGL14;
@ -28,11 +29,16 @@ import android.opengl.EGLContext;
import android.opengl.EGLDisplay; import android.opengl.EGLDisplay;
import android.opengl.EGLExt; import android.opengl.EGLExt;
import android.opengl.EGLSurface; import android.opengl.EGLSurface;
import android.opengl.GLES20;
import android.util.Pair; import android.util.Pair;
import android.view.Surface; import android.view.Surface;
import android.view.SurfaceHolder;
import android.view.SurfaceView;
import androidx.annotation.GuardedBy;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import androidx.media3.common.C; import androidx.media3.common.C;
import androidx.media3.common.ColorInfo; import androidx.media3.common.ColorInfo;
import androidx.media3.common.DebugViewProvider;
import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlObjectsProvider;
import androidx.media3.common.GlTextureInfo; import androidx.media3.common.GlTextureInfo;
import androidx.media3.common.SurfaceInfo; import androidx.media3.common.SurfaceInfo;
@ -82,6 +88,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final EGLDisplay eglDisplay; private final EGLDisplay eglDisplay;
private final EGLContext eglContext; private final EGLContext eglContext;
private final EGLSurface placeholderSurface; private final EGLSurface placeholderSurface;
private final DebugViewProvider debugViewProvider;
private final ColorInfo outputColorInfo; private final ColorInfo outputColorInfo;
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor; private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
private final Executor videoFrameProcessorListenerExecutor; private final Executor videoFrameProcessorListenerExecutor;
@ -97,6 +104,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private int inputWidth; private int inputWidth;
private int inputHeight; private int inputHeight;
@Nullable private DefaultShaderProgram defaultShaderProgram; @Nullable private DefaultShaderProgram defaultShaderProgram;
@Nullable private SurfaceViewWrapper debugSurfaceViewWrapper;
// Whether the input stream has ended, but not all input has been released. This is relevant only // Whether the input stream has ended, but not all input has been released. This is relevant only
// when renderFramesAutomatically is false. Ensures all frames are rendered before reporting // when renderFramesAutomatically is false. Ensures all frames are rendered before reporting
// onInputStreamProcessed. // onInputStreamProcessed.
@ -104,6 +112,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private boolean isInputStreamEndedWithPendingAvailableFrames; private boolean isInputStreamEndedWithPendingAvailableFrames;
private InputListener inputListener; private InputListener inputListener;
private @MonotonicNonNull Size outputSizeBeforeSurfaceTransformation; private @MonotonicNonNull Size outputSizeBeforeSurfaceTransformation;
@Nullable private SurfaceView debugSurfaceView;
@Nullable private OnInputStreamProcessedListener onInputStreamProcessedListener; @Nullable private OnInputStreamProcessedListener onInputStreamProcessedListener;
private boolean matrixTransformationsChanged; private boolean matrixTransformationsChanged;
private boolean outputSurfaceInfoChanged; private boolean outputSurfaceInfoChanged;
@ -117,6 +126,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
EGLDisplay eglDisplay, EGLDisplay eglDisplay,
EGLContext eglContext, EGLContext eglContext,
EGLSurface placeholderSurface, EGLSurface placeholderSurface,
DebugViewProvider debugViewProvider,
ColorInfo outputColorInfo, ColorInfo outputColorInfo,
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor, VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
Executor videoFrameProcessorListenerExecutor, Executor videoFrameProcessorListenerExecutor,
@ -131,6 +141,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.eglDisplay = eglDisplay; this.eglDisplay = eglDisplay;
this.eglContext = eglContext; this.eglContext = eglContext;
this.placeholderSurface = placeholderSurface; this.placeholderSurface = placeholderSurface;
this.debugViewProvider = debugViewProvider;
this.outputColorInfo = outputColorInfo; this.outputColorInfo = outputColorInfo;
this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor; this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor;
this.videoFrameProcessorListenerExecutor = videoFrameProcessorListenerExecutor; this.videoFrameProcessorListenerExecutor = videoFrameProcessorListenerExecutor;
@ -411,6 +422,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
videoFrameProcessorListener.onError( videoFrameProcessorListener.onError(
VideoFrameProcessingException.from(e, presentationTimeUs))); VideoFrameProcessingException.from(e, presentationTimeUs)));
} }
if (debugSurfaceViewWrapper != null && defaultShaderProgram != null) {
renderFrameToDebugSurface(glObjectsProvider, inputTexture, presentationTimeUs);
}
inputListener.onInputFrameProcessed(inputTexture); inputListener.onInputFrameProcessed(inputTexture);
} }
@ -523,6 +537,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
outputTexturePool.ensureConfigured(glObjectsProvider, outputWidth, outputHeight); outputTexturePool.ensureConfigured(glObjectsProvider, outputWidth, outputHeight);
} }
@Nullable
SurfaceView debugSurfaceView =
debugViewProvider.getDebugPreviewSurfaceView(outputWidth, outputHeight);
if (debugSurfaceView != null && !Util.areEqual(this.debugSurfaceView, debugSurfaceView)) {
debugSurfaceViewWrapper =
new SurfaceViewWrapper(
eglDisplay, eglContext, debugSurfaceView, outputColorInfo.colorTransfer);
}
this.debugSurfaceView = debugSurfaceView;
if (defaultShaderProgram != null if (defaultShaderProgram != null
&& (outputSurfaceInfoChanged || inputSizeChanged || matrixTransformationsChanged)) { && (outputSurfaceInfoChanged || inputSizeChanged || matrixTransformationsChanged)) {
defaultShaderProgram.release(); defaultShaderProgram.release();
@ -576,4 +600,123 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
return defaultShaderProgram; return defaultShaderProgram;
} }
private void renderFrameToDebugSurface(
GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) {
DefaultShaderProgram defaultShaderProgram = checkNotNull(this.defaultShaderProgram);
SurfaceViewWrapper debugSurfaceViewWrapper = checkNotNull(this.debugSurfaceViewWrapper);
try {
checkNotNull(debugSurfaceViewWrapper)
.maybeRenderToSurfaceView(
() -> {
GlUtil.clearFocusedBuffers();
if (sdrWorkingColorSpace == WORKING_COLOR_SPACE_LINEAR) {
@C.ColorTransfer
int configuredColorTransfer = defaultShaderProgram.getOutputColorTransfer();
defaultShaderProgram.setOutputColorTransfer(
debugSurfaceViewWrapper.outputColorTransfer);
defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs);
defaultShaderProgram.setOutputColorTransfer(configuredColorTransfer);
} else {
defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs);
}
},
glObjectsProvider);
} catch (VideoFrameProcessingException | GlUtil.GlException e) {
Log.d(TAG, "Error rendering to debug preview", e);
}
}
/**
* Wrapper around a {@link SurfaceView} that keeps track of whether the output surface is valid,
* and makes rendering a no-op if not.
*
* <p>This class should only be used for displaying a debug preview.
*/
private static final class SurfaceViewWrapper implements SurfaceHolder.Callback {
public final @C.ColorTransfer int outputColorTransfer;
private final EGLDisplay eglDisplay;
private final EGLContext eglContext;
@GuardedBy("this")
@Nullable
private Surface surface;
@GuardedBy("this")
@Nullable
private EGLSurface eglSurface;
private int width;
private int height;
public SurfaceViewWrapper(
EGLDisplay eglDisplay,
EGLContext eglContext,
SurfaceView surfaceView,
@C.ColorTransfer int outputColorTransfer) {
this.eglDisplay = eglDisplay;
this.eglContext = eglContext;
// PQ SurfaceView output is supported from API 33, but HLG output is supported from API 34.
// Therefore, convert HLG to PQ below API 34, so that HLG input can be displayed properly on
// API 33.
this.outputColorTransfer =
outputColorTransfer == C.COLOR_TRANSFER_HLG && Util.SDK_INT < 34
? C.COLOR_TRANSFER_ST2084
: outputColorTransfer;
surfaceView.getHolder().addCallback(this);
surface = surfaceView.getHolder().getSurface();
width = surfaceView.getWidth();
height = surfaceView.getHeight();
}
@Override
public void surfaceCreated(SurfaceHolder holder) {}
@Override
public synchronized void surfaceChanged(
SurfaceHolder holder, int format, int width, int height) {
this.width = width;
this.height = height;
Surface newSurface = holder.getSurface();
if (surface == null || !surface.equals(newSurface)) {
surface = newSurface;
eglSurface = null;
}
}
@Override
public synchronized void surfaceDestroyed(SurfaceHolder holder) {
surface = null;
eglSurface = null;
width = C.LENGTH_UNSET;
height = C.LENGTH_UNSET;
}
/**
* Focuses the wrapped surface view's surface as an {@link EGLSurface}, renders using {@code
* renderingTask} and swaps buffers, if the view's holder has a valid surface. Does nothing
* otherwise.
*
* <p>Must be called on the GL thread.
*/
public synchronized void maybeRenderToSurfaceView(
VideoFrameProcessingTaskExecutor.Task renderingTask, GlObjectsProvider glObjectsProvider)
throws GlUtil.GlException, VideoFrameProcessingException {
if (surface == null) {
return;
}
if (eglSurface == null) {
eglSurface =
glObjectsProvider.createEglSurface(
eglDisplay, surface, outputColorTransfer, /* isEncoderInputSurface= */ false);
}
EGLSurface eglSurface = this.eglSurface;
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, width, height);
renderingTask.run();
EGL14.eglSwapBuffers(eglDisplay, eglSurface);
// Prevents white flashing on the debug SurfaceView when frames are rendered too fast.
GLES20.glFinish();
}
}
} }

14
libraries/effect/src/main/java/androidx/media3/effect/GlEffect.java
View File

@ -16,7 +16,6 @@
package androidx.media3.effect; package androidx.media3.effect;
import android.content.Context; import android.content.Context;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.Effect; import androidx.media3.common.Effect;
import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.UnstableApi;
@ -43,19 +42,6 @@ public interface GlEffect extends Effect {
GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) GlShaderProgram toGlShaderProgram(Context context, boolean useHdr)
throws VideoFrameProcessingException; throws VideoFrameProcessingException;
/**
* Returns a {@link GlShaderProgram} that applies the effect.
*
* @param context A {@link Context}.
* @param colorInfo The {@link ColorInfo} of the input.
* @throws VideoFrameProcessingException If an error occurs while creating the {@link
* GlShaderProgram}.
*/
default GlShaderProgram toGlShaderProgram(Context context, ColorInfo colorInfo)
throws VideoFrameProcessingException {
return toGlShaderProgram(context, /* useHdr= */ ColorInfo.isTransferHdr(colorInfo));
}
/** /**
* Returns whether a {@link GlEffect} applies no change at every timestamp. * Returns whether a {@link GlEffect} applies no change at every timestamp.
* *

16
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
View File

@ -57,7 +57,6 @@ import androidx.media3.common.util.ListenerSet;
import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util; import androidx.media3.common.util.Util;
import androidx.media3.effect.DebugTraceUtil; import androidx.media3.effect.DebugTraceUtil;
import androidx.media3.effect.DebugViewEffect;
import androidx.media3.effect.DefaultVideoFrameProcessor; import androidx.media3.effect.DefaultVideoFrameProcessor;
import androidx.media3.exoplayer.source.DefaultMediaSourceFactory; import androidx.media3.exoplayer.source.DefaultMediaSourceFactory;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
@ -493,10 +492,8 @@ public final class Transformer {
* *
* @param debugViewProvider Provider for debug views. * @param debugViewProvider Provider for debug views.
* @return This builder. * @return This builder.
* @deprecated Add a {@link DebugViewEffect} to the list of video effects instead.
*/ */
@CanIgnoreReturnValue @CanIgnoreReturnValue
@Deprecated
public Builder setDebugViewProvider(DebugViewProvider debugViewProvider) { public Builder setDebugViewProvider(DebugViewProvider debugViewProvider) {
this.debugViewProvider = debugViewProvider; this.debugViewProvider = debugViewProvider;
return this; return this;
@ -1593,19 +1590,6 @@ public final class Transformer {
} }
editingMetricsCollector = new EditingMetricsCollector(context, EXPORTER_NAME, muxerName); editingMetricsCollector = new EditingMetricsCollector(context, EXPORTER_NAME, muxerName);
} }
if (debugViewProvider != DebugViewProvider.NONE) {
ImmutableList<Effect> videoEffectsWithDebugView =
new ImmutableList.Builder<Effect>()
.addAll(composition.effects.videoEffects)
.add(new DebugViewEffect(debugViewProvider))
.build();
composition =
composition
.buildUpon()
.setEffects(
new Effects(composition.effects.audioProcessors, videoEffectsWithDebugView))
.build();
}
transformerInternal = transformerInternal =
new TransformerInternal( new TransformerInternal(
context, context,