diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java index 637a417c34..9b1ec9eb02 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java @@ -61,7 +61,6 @@ import androidx.media3.datasource.DataSourceBitmapLoader; import androidx.media3.effect.BitmapOverlay; import androidx.media3.effect.Contrast; import androidx.media3.effect.DebugTraceUtil; -import androidx.media3.effect.DebugViewEffect; import androidx.media3.effect.DrawableOverlay; import androidx.media3.effect.GlEffect; import androidx.media3.effect.GlShaderProgram; @@ -312,6 +311,10 @@ public final class TransformerActivity extends AppCompatActivity { transformerBuilder.setMuxerFactory(new InAppFragmentedMp4Muxer.Factory()); } + if (bundle.getBoolean(ConfigurationActivity.ENABLE_DEBUG_PREVIEW)) { + transformerBuilder.setDebugViewProvider(new DemoDebugViewProvider()); + } + if (bundle.getBoolean(ConfigurationActivity.ENABLE_ANALYZER_MODE)) { return ExperimentalAnalyzerModeFactory.buildAnalyzer( this.getApplicationContext(), transformerBuilder.build()); @@ -562,10 +565,6 @@ public final class TransformerActivity extends AppCompatActivity { effects.add(Presentation.createForHeight(resolutionHeight)); } - if (bundle.getBoolean(ConfigurationActivity.ENABLE_DEBUG_PREVIEW)) { - effects.add(new DebugViewEffect(new DemoDebugViewProvider())); - } - return effects.build(); } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DebugViewEffect.java b/libraries/effect/src/main/java/androidx/media3/effect/DebugViewEffect.java deleted file mode 100644 index 5741916612..0000000000 --- a/libraries/effect/src/main/java/androidx/media3/effect/DebugViewEffect.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2025 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package androidx.media3.effect; - -import android.content.Context; -import android.view.SurfaceView; -import androidx.media3.common.ColorInfo; -import androidx.media3.common.DebugViewProvider; -import androidx.media3.common.util.UnstableApi; - -/** {@link GlEffect} that renders to a {@link SurfaceView} provided by {@link DebugViewProvider}. */ -@UnstableApi -public final class DebugViewEffect implements GlEffect { - - private final DebugViewProvider debugViewProvider; - - /** - * Creates a new instance. - * - * @param debugViewProvider The class that provides the {@link SurfaceView} that the debug preview - * will be rendered to. - */ - public DebugViewEffect(DebugViewProvider debugViewProvider) { - this.debugViewProvider = debugViewProvider; - } - - /** - * Throws {@link UnsupportedOperationException} because {@link #toGlShaderProgram(Context, - * ColorInfo)} should be used instead. - */ - @Override - public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) { - throw new UnsupportedOperationException(); - } - - @Override - public GlShaderProgram toGlShaderProgram(Context context, ColorInfo colorInfo) { - return new DebugViewShaderProgram(context, debugViewProvider, colorInfo); - } -} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DebugViewShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/DebugViewShaderProgram.java deleted file mode 100644 index b3fcc7d78d..0000000000 --- a/libraries/effect/src/main/java/androidx/media3/effect/DebugViewShaderProgram.java +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Copyright 2025 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package androidx.media3.effect; - -import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.common.util.GlUtil.getDefaultEglDisplay; -import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_DEFAULT; -import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_LINEAR; -import static com.google.common.util.concurrent.MoreExecutors.directExecutor; - -import android.content.Context; -import android.opengl.EGL14; -import android.opengl.EGLContext; -import android.opengl.EGLDisplay; -import android.opengl.EGLSurface; -import android.opengl.GLES20; -import android.view.Surface; -import android.view.SurfaceHolder; -import android.view.SurfaceView; -import androidx.annotation.GuardedBy; -import androidx.annotation.Nullable; -import androidx.media3.common.C; -import androidx.media3.common.ColorInfo; -import androidx.media3.common.DebugViewProvider; -import androidx.media3.common.GlObjectsProvider; -import androidx.media3.common.GlTextureInfo; -import androidx.media3.common.VideoFrameProcessingException; -import androidx.media3.common.util.GlUtil; -import androidx.media3.common.util.Log; -import androidx.media3.common.util.UnstableApi; -import androidx.media3.common.util.Util; -import com.google.common.collect.ImmutableList; -import java.util.Objects; -import java.util.concurrent.Executor; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; - -/** - * {@link GlShaderProgram} that renders to a {@link SurfaceView} provided by {@link - * DebugViewProvider}. - */ -@UnstableApi -public final class DebugViewShaderProgram implements GlShaderProgram { - private static final String TAG = "DebugViewShaderProgram"; - - private final Context context; - private final DebugViewProvider debugViewProvider; - @Nullable private SurfaceView debugSurfaceView; - @Nullable private DefaultShaderProgram defaultShaderProgram; - @Nullable private SurfaceViewWrapper debugSurfaceViewWrapper; - - private final ColorInfo colorInfo; - private InputListener inputListener; - private OutputListener outputListener; - private ErrorListener errorListener; - private Executor errorListenerExecutor; - - private @MonotonicNonNull EGLDisplay eglDisplay; - - public DebugViewShaderProgram( - Context context, DebugViewProvider debugViewProvider, ColorInfo colorInfo) { - this.context = context; - this.debugViewProvider = debugViewProvider; - this.colorInfo = colorInfo; - inputListener = new InputListener() {}; - outputListener = new OutputListener() {}; - errorListener = - (frameProcessingException) -> - Log.e(TAG, "Exception caught by errorListener.", frameProcessingException); - errorListenerExecutor = directExecutor(); - } - - @Override - public void setInputListener(InputListener inputListener) { - this.inputListener = inputListener; - inputListener.onReadyToAcceptInputFrame(); - } - - @Override - public void setOutputListener(OutputListener outputListener) { - this.outputListener = outputListener; - } - - @Override - public void setErrorListener(Executor executor, ErrorListener errorListener) { - this.errorListener = errorListener; - this.errorListenerExecutor = executor; - } - - @Override - public void queueInputFrame( - GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) { - try { - ensureConfigured(inputTexture.width, inputTexture.height); - DefaultShaderProgram defaultShaderProgram = checkNotNull(this.defaultShaderProgram); - checkNotNull(this.debugSurfaceViewWrapper) - .maybeRenderToSurfaceView( - () -> defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs), - glObjectsProvider); - outputListener.onOutputFrameAvailable(inputTexture, presentationTimeUs); - } catch (VideoFrameProcessingException | GlUtil.GlException e) { - errorListenerExecutor.execute( - () -> errorListener.onError(VideoFrameProcessingException.from(e, presentationTimeUs))); - } - } - - @Override - public void releaseOutputFrame(GlTextureInfo outputTexture) { - inputListener.onInputFrameProcessed(outputTexture); - inputListener.onReadyToAcceptInputFrame(); - } - - @Override - public void signalEndOfCurrentInputStream() { - outputListener.onCurrentOutputStreamEnded(); - } - - @Override - public void flush() { - if (defaultShaderProgram != null) { - defaultShaderProgram.flush(); - } - inputListener.onFlush(); - inputListener.onReadyToAcceptInputFrame(); - } - - @Override - public void release() throws VideoFrameProcessingException { - if (defaultShaderProgram != null) { - defaultShaderProgram.release(); - } - try { - GlUtil.checkGlError(); - } catch (GlUtil.GlException e) { - throw new VideoFrameProcessingException(e); - } - } - - private void ensureConfigured(int inputWidth, int inputHeight) - throws VideoFrameProcessingException, GlUtil.GlException { - if (eglDisplay == null) { - eglDisplay = getDefaultEglDisplay(); - } - EGLContext eglContext = GlUtil.getCurrentContext(); - @Nullable - SurfaceView debugSurfaceView = - debugViewProvider.getDebugPreviewSurfaceView(inputWidth, inputHeight); - if (debugSurfaceView != null && !Objects.equals(this.debugSurfaceView, debugSurfaceView)) { - debugSurfaceViewWrapper = - new SurfaceViewWrapper(eglDisplay, eglContext, debugSurfaceView, colorInfo.colorTransfer); - } - this.debugSurfaceView = debugSurfaceView; - if (defaultShaderProgram == null) { - defaultShaderProgram = - DefaultShaderProgram.createApplyingOetf( - context, - /* matrixTransformations= */ ImmutableList.of(), - /* rgbMatrices= */ ImmutableList.of(), - colorInfo, - colorInfo.colorTransfer == C.COLOR_TRANSFER_LINEAR - ? WORKING_COLOR_SPACE_LINEAR - : WORKING_COLOR_SPACE_DEFAULT); - } - } - - /** - * Wrapper around a {@link SurfaceView} that keeps track of whether the output surface is valid, - * and makes rendering a no-op if not. - */ - private static final class SurfaceViewWrapper implements SurfaceHolder.Callback { - public final @C.ColorTransfer int outputColorTransfer; - private final EGLDisplay eglDisplay; - private final EGLContext eglContext; - - @GuardedBy("this") - @Nullable - private Surface surface; - - @GuardedBy("this") - @Nullable - private EGLSurface eglSurface; - - private int width; - private int height; - - public SurfaceViewWrapper( - EGLDisplay eglDisplay, - EGLContext eglContext, - SurfaceView surfaceView, - @C.ColorTransfer int outputColorTransfer) { - this.eglDisplay = eglDisplay; - this.eglContext = eglContext; - // PQ SurfaceView output is supported from API 33, but HLG output is supported from API 34. - // Therefore, convert HLG to PQ below API 34, so that HLG input can be displayed properly on - // API 33. - this.outputColorTransfer = - outputColorTransfer == C.COLOR_TRANSFER_HLG && Util.SDK_INT < 34 - ? C.COLOR_TRANSFER_ST2084 - : outputColorTransfer; - surfaceView.getHolder().addCallback(this); - surface = surfaceView.getHolder().getSurface(); - width = surfaceView.getWidth(); - height = surfaceView.getHeight(); - } - - @Override - public void surfaceCreated(SurfaceHolder holder) {} - - @Override - public synchronized void surfaceChanged( - SurfaceHolder holder, int format, int width, int height) { - this.width = width; - this.height = height; - Surface newSurface = holder.getSurface(); - if (!newSurface.equals(surface)) { - surface = newSurface; - eglSurface = null; - } - } - - @Override - public synchronized void surfaceDestroyed(SurfaceHolder holder) { - surface = null; - eglSurface = null; - width = C.LENGTH_UNSET; - height = C.LENGTH_UNSET; - } - - /** - * Focuses the wrapped surface view's surface as an {@link EGLSurface}, renders using {@code - * renderingTask} and swaps buffers, if the view's holder has a valid surface. Does nothing - * otherwise. - * - *

Must be called on the GL thread. - */ - public synchronized void maybeRenderToSurfaceView( - VideoFrameProcessingTaskExecutor.Task renderingTask, GlObjectsProvider glObjectsProvider) - throws GlUtil.GlException, VideoFrameProcessingException { - if (surface == null) { - return; - } - - if (eglSurface == null) { - eglSurface = - glObjectsProvider.createEglSurface( - eglDisplay, surface, outputColorTransfer, /* isEncoderInputSurface= */ false); - } - EGLSurface eglSurface = this.eglSurface; - GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, width, height); - renderingTask.run(); - EGL14.eglSwapBuffers(eglDisplay, eglSurface); - // Prevents white flashing on the debug SurfaceView when frames are rendered too fast. - // TODO(b/393316699) - Investigate removing this to speed up transcoding. - GLES20.glFinish(); - } - } -} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index 87ead5eeda..e850c381b3 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -419,6 +419,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { () -> createOpenGlObjectsAndFrameProcessor( context, + debugViewProvider, outputColorInfo, sdrWorkingColorSpace, renderFramesAutomatically, @@ -478,10 +479,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { private final List activeEffects; private final Object lock; - - /** The {@link ColorInfo} that all {@link Effect effects} work in. */ - private final ColorInfo workingColorInfo; - private final ColorInfo outputColorInfo; private volatile @MonotonicNonNull FrameInfo nextInputFrameInfo; @@ -497,7 +494,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { Executor listenerExecutor, FinalShaderProgramWrapper finalShaderProgramWrapper, boolean renderFramesAutomatically, - ColorInfo workingColorInfo, ColorInfo outputColorInfo) { this.context = context; this.glObjectsProvider = glObjectsProvider; @@ -509,7 +505,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { this.renderFramesAutomatically = renderFramesAutomatically; this.activeEffects = new ArrayList<>(); this.lock = new Object(); - this.workingColorInfo = workingColorInfo; this.outputColorInfo = outputColorInfo; this.finalShaderProgramWrapper = finalShaderProgramWrapper; this.intermediateGlShaderPrograms = new ArrayList<>(); @@ -822,6 +817,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { */ private static DefaultVideoFrameProcessor createOpenGlObjectsAndFrameProcessor( Context context, + DebugViewProvider debugViewProvider, ColorInfo outputColorInfo, @WorkingColorSpace int sdrWorkingColorSpace, boolean renderFramesAutomatically, @@ -849,13 +845,12 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { .setColorTransfer(C.COLOR_TRANSFER_LINEAR) .setHdrStaticInfo(null) .build(); - ColorInfo intermediateColorInfo; - if (ColorInfo.isTransferHdr(outputColorInfo) - || sdrWorkingColorSpace == WORKING_COLOR_SPACE_LINEAR) { - intermediateColorInfo = linearColorInfo; - } else { - intermediateColorInfo = outputColorInfo; - } + ColorInfo intermediateColorInfo = + ColorInfo.isTransferHdr(outputColorInfo) + ? linearColorInfo + : sdrWorkingColorSpace == WORKING_COLOR_SPACE_LINEAR + ? linearColorInfo + : outputColorInfo; InputSwitcher inputSwitcher = new InputSwitcher( context, @@ -875,6 +870,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { eglDisplay, eglContextAndPlaceholderSurface.first, eglContextAndPlaceholderSurface.second, + debugViewProvider, outputColorInfo, videoFrameProcessingTaskExecutor, videoFrameProcessorListenerExecutor, @@ -894,7 +890,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { videoFrameProcessorListenerExecutor, finalShaderProgramWrapper, renderFramesAutomatically, - intermediateColorInfo, outputColorInfo); } @@ -907,7 +902,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { * * @param context The {@link Context}. * @param effects The list of {@link GlEffect effects}. - * @param workingColorInfo The {@link ColorInfo} the {@link List effects} work in. + * @param outputColorInfo The {@link ColorInfo} on {@code DefaultVideoFrameProcessor} output. * @param finalShaderProgramWrapper The {@link FinalShaderProgramWrapper} to apply the {@link * GlMatrixTransformation GlMatrixTransformations} and {@link RgbMatrix RgbMatrices} after all * other {@link GlEffect GlEffects}. @@ -916,7 +911,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { private static ImmutableList createGlShaderPrograms( Context context, List effects, - ColorInfo workingColorInfo, + ColorInfo outputColorInfo, FinalShaderProgramWrapper finalShaderProgramWrapper) throws VideoFrameProcessingException { ImmutableList.Builder shaderProgramListBuilder = new ImmutableList.Builder<>(); @@ -943,18 +938,16 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ImmutableList matrixTransformations = matrixTransformationListBuilder.build(); ImmutableList rgbMatrices = rgbMatrixListBuilder.build(); + boolean isOutputTransferHdr = ColorInfo.isTransferHdr(outputColorInfo); if (!matrixTransformations.isEmpty() || !rgbMatrices.isEmpty()) { DefaultShaderProgram defaultShaderProgram = DefaultShaderProgram.create( - context, - matrixTransformations, - rgbMatrices, - ColorInfo.isTransferHdr(workingColorInfo)); + context, matrixTransformations, rgbMatrices, isOutputTransferHdr); shaderProgramListBuilder.add(defaultShaderProgram); matrixTransformationListBuilder = new ImmutableList.Builder<>(); rgbMatrixListBuilder = new ImmutableList.Builder<>(); } - shaderProgramListBuilder.add(glEffect.toGlShaderProgram(context, workingColorInfo)); + shaderProgramListBuilder.add(glEffect.toGlShaderProgram(context, isOutputTransferHdr)); } finalShaderProgramWrapper.setMatrixTransformations( @@ -1030,7 +1023,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { // FinalShaderProgramWrapper. intermediateGlShaderPrograms.addAll( createGlShaderPrograms( - context, inputStreamInfo.effects, workingColorInfo, finalShaderProgramWrapper)); + context, inputStreamInfo.effects, outputColorInfo, finalShaderProgramWrapper)); inputSwitcher.setDownstreamShaderProgram( getFirst(intermediateGlShaderPrograms, /* defaultValue= */ finalShaderProgramWrapper)); chainShaderProgramsWithListeners( diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java index 8b4868a922..c94bfe3981 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java @@ -21,6 +21,7 @@ import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.effect.DebugTraceUtil.COMPONENT_VFP; import static androidx.media3.effect.DebugTraceUtil.EVENT_RENDERED_TO_OUTPUT_SURFACE; +import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_LINEAR; import android.content.Context; import android.opengl.EGL14; @@ -28,11 +29,16 @@ import android.opengl.EGLContext; import android.opengl.EGLDisplay; import android.opengl.EGLExt; import android.opengl.EGLSurface; +import android.opengl.GLES20; import android.util.Pair; import android.view.Surface; +import android.view.SurfaceHolder; +import android.view.SurfaceView; +import androidx.annotation.GuardedBy; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.ColorInfo; +import androidx.media3.common.DebugViewProvider; import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.SurfaceInfo; @@ -82,6 +88,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final EGLDisplay eglDisplay; private final EGLContext eglContext; private final EGLSurface placeholderSurface; + private final DebugViewProvider debugViewProvider; private final ColorInfo outputColorInfo; private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor; private final Executor videoFrameProcessorListenerExecutor; @@ -97,6 +104,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private int inputWidth; private int inputHeight; @Nullable private DefaultShaderProgram defaultShaderProgram; + @Nullable private SurfaceViewWrapper debugSurfaceViewWrapper; // Whether the input stream has ended, but not all input has been released. This is relevant only // when renderFramesAutomatically is false. Ensures all frames are rendered before reporting // onInputStreamProcessed. @@ -104,6 +112,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private boolean isInputStreamEndedWithPendingAvailableFrames; private InputListener inputListener; private @MonotonicNonNull Size outputSizeBeforeSurfaceTransformation; + @Nullable private SurfaceView debugSurfaceView; @Nullable private OnInputStreamProcessedListener onInputStreamProcessedListener; private boolean matrixTransformationsChanged; private boolean outputSurfaceInfoChanged; @@ -117,6 +126,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; EGLDisplay eglDisplay, EGLContext eglContext, EGLSurface placeholderSurface, + DebugViewProvider debugViewProvider, ColorInfo outputColorInfo, VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor, Executor videoFrameProcessorListenerExecutor, @@ -131,6 +141,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.eglDisplay = eglDisplay; this.eglContext = eglContext; this.placeholderSurface = placeholderSurface; + this.debugViewProvider = debugViewProvider; this.outputColorInfo = outputColorInfo; this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor; this.videoFrameProcessorListenerExecutor = videoFrameProcessorListenerExecutor; @@ -411,6 +422,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; videoFrameProcessorListener.onError( VideoFrameProcessingException.from(e, presentationTimeUs))); } + if (debugSurfaceViewWrapper != null && defaultShaderProgram != null) { + renderFrameToDebugSurface(glObjectsProvider, inputTexture, presentationTimeUs); + } inputListener.onInputFrameProcessed(inputTexture); } @@ -523,6 +537,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; outputTexturePool.ensureConfigured(glObjectsProvider, outputWidth, outputHeight); } + @Nullable + SurfaceView debugSurfaceView = + debugViewProvider.getDebugPreviewSurfaceView(outputWidth, outputHeight); + if (debugSurfaceView != null && !Util.areEqual(this.debugSurfaceView, debugSurfaceView)) { + debugSurfaceViewWrapper = + new SurfaceViewWrapper( + eglDisplay, eglContext, debugSurfaceView, outputColorInfo.colorTransfer); + } + this.debugSurfaceView = debugSurfaceView; + if (defaultShaderProgram != null && (outputSurfaceInfoChanged || inputSizeChanged || matrixTransformationsChanged)) { defaultShaderProgram.release(); @@ -576,4 +600,123 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } return defaultShaderProgram; } + + private void renderFrameToDebugSurface( + GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) { + DefaultShaderProgram defaultShaderProgram = checkNotNull(this.defaultShaderProgram); + SurfaceViewWrapper debugSurfaceViewWrapper = checkNotNull(this.debugSurfaceViewWrapper); + try { + checkNotNull(debugSurfaceViewWrapper) + .maybeRenderToSurfaceView( + () -> { + GlUtil.clearFocusedBuffers(); + if (sdrWorkingColorSpace == WORKING_COLOR_SPACE_LINEAR) { + @C.ColorTransfer + int configuredColorTransfer = defaultShaderProgram.getOutputColorTransfer(); + defaultShaderProgram.setOutputColorTransfer( + debugSurfaceViewWrapper.outputColorTransfer); + defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs); + defaultShaderProgram.setOutputColorTransfer(configuredColorTransfer); + } else { + defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs); + } + }, + glObjectsProvider); + } catch (VideoFrameProcessingException | GlUtil.GlException e) { + Log.d(TAG, "Error rendering to debug preview", e); + } + } + + /** + * Wrapper around a {@link SurfaceView} that keeps track of whether the output surface is valid, + * and makes rendering a no-op if not. + * + *

This class should only be used for displaying a debug preview. + */ + private static final class SurfaceViewWrapper implements SurfaceHolder.Callback { + public final @C.ColorTransfer int outputColorTransfer; + private final EGLDisplay eglDisplay; + private final EGLContext eglContext; + + @GuardedBy("this") + @Nullable + private Surface surface; + + @GuardedBy("this") + @Nullable + private EGLSurface eglSurface; + + private int width; + private int height; + + public SurfaceViewWrapper( + EGLDisplay eglDisplay, + EGLContext eglContext, + SurfaceView surfaceView, + @C.ColorTransfer int outputColorTransfer) { + this.eglDisplay = eglDisplay; + this.eglContext = eglContext; + // PQ SurfaceView output is supported from API 33, but HLG output is supported from API 34. + // Therefore, convert HLG to PQ below API 34, so that HLG input can be displayed properly on + // API 33. + this.outputColorTransfer = + outputColorTransfer == C.COLOR_TRANSFER_HLG && Util.SDK_INT < 34 + ? C.COLOR_TRANSFER_ST2084 + : outputColorTransfer; + surfaceView.getHolder().addCallback(this); + surface = surfaceView.getHolder().getSurface(); + width = surfaceView.getWidth(); + height = surfaceView.getHeight(); + } + + @Override + public void surfaceCreated(SurfaceHolder holder) {} + + @Override + public synchronized void surfaceChanged( + SurfaceHolder holder, int format, int width, int height) { + this.width = width; + this.height = height; + Surface newSurface = holder.getSurface(); + if (surface == null || !surface.equals(newSurface)) { + surface = newSurface; + eglSurface = null; + } + } + + @Override + public synchronized void surfaceDestroyed(SurfaceHolder holder) { + surface = null; + eglSurface = null; + width = C.LENGTH_UNSET; + height = C.LENGTH_UNSET; + } + + /** + * Focuses the wrapped surface view's surface as an {@link EGLSurface}, renders using {@code + * renderingTask} and swaps buffers, if the view's holder has a valid surface. Does nothing + * otherwise. + * + *

Must be called on the GL thread. + */ + public synchronized void maybeRenderToSurfaceView( + VideoFrameProcessingTaskExecutor.Task renderingTask, GlObjectsProvider glObjectsProvider) + throws GlUtil.GlException, VideoFrameProcessingException { + if (surface == null) { + return; + } + + if (eglSurface == null) { + eglSurface = + glObjectsProvider.createEglSurface( + eglDisplay, surface, outputColorTransfer, /* isEncoderInputSurface= */ false); + } + EGLSurface eglSurface = this.eglSurface; + GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, width, height); + renderingTask.run(); + EGL14.eglSwapBuffers(eglDisplay, eglSurface); + // Prevents white flashing on the debug SurfaceView when frames are rendered too fast. + GLES20.glFinish(); + } + } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/GlEffect.java b/libraries/effect/src/main/java/androidx/media3/effect/GlEffect.java index 24b0365d98..86b4af59a4 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/GlEffect.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/GlEffect.java @@ -16,7 +16,6 @@ package androidx.media3.effect; import android.content.Context; -import androidx.media3.common.ColorInfo; import androidx.media3.common.Effect; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.util.UnstableApi; @@ -43,19 +42,6 @@ public interface GlEffect extends Effect { GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) throws VideoFrameProcessingException; - /** - * Returns a {@link GlShaderProgram} that applies the effect. - * - * @param context A {@link Context}. - * @param colorInfo The {@link ColorInfo} of the input. - * @throws VideoFrameProcessingException If an error occurs while creating the {@link - * GlShaderProgram}. - */ - default GlShaderProgram toGlShaderProgram(Context context, ColorInfo colorInfo) - throws VideoFrameProcessingException { - return toGlShaderProgram(context, /* useHdr= */ ColorInfo.isTransferHdr(colorInfo)); - } - /** * Returns whether a {@link GlEffect} applies no change at every timestamp. * diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index 10195edcd6..9a1fe0fdf0 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -57,7 +57,6 @@ import androidx.media3.common.util.ListenerSet; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import androidx.media3.effect.DebugTraceUtil; -import androidx.media3.effect.DebugViewEffect; import androidx.media3.effect.DefaultVideoFrameProcessor; import androidx.media3.exoplayer.source.DefaultMediaSourceFactory; import com.google.common.collect.ImmutableList; @@ -493,10 +492,8 @@ public final class Transformer { * * @param debugViewProvider Provider for debug views. * @return This builder. - * @deprecated Add a {@link DebugViewEffect} to the list of video effects instead. */ @CanIgnoreReturnValue - @Deprecated public Builder setDebugViewProvider(DebugViewProvider debugViewProvider) { this.debugViewProvider = debugViewProvider; return this; @@ -1593,19 +1590,6 @@ public final class Transformer { } editingMetricsCollector = new EditingMetricsCollector(context, EXPORTER_NAME, muxerName); } - if (debugViewProvider != DebugViewProvider.NONE) { - ImmutableList videoEffectsWithDebugView = - new ImmutableList.Builder() - .addAll(composition.effects.videoEffects) - .add(new DebugViewEffect(debugViewProvider)) - .build(); - composition = - composition - .buildUpon() - .setEffects( - new Effects(composition.effects.audioProcessors, videoEffectsWithDebugView)) - .build(); - } transformerInternal = new TransformerInternal( context,