diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorTest.java
index 4b5adb3386..f1c83863bd 100644
--- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorTest.java
+++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorTest.java
@@ -92,7 +92,8 @@ public final class FrameEditorTest {
width,
height,
identityMatrix,
- frameEditorOutputImageReader.getSurface());
+ frameEditorOutputImageReader.getSurface(),
+ Transformer.DebugViewProvider.NONE);
// Queue the first video frame from the extractor.
String mimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME));
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java
index dc82f565cb..9fd832fc33 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java
@@ -15,6 +15,8 @@
*/
package androidx.media3.transformer;
+import static androidx.media3.common.util.Assertions.checkNotNull;
+
import android.content.Context;
import android.graphics.Matrix;
import android.graphics.SurfaceTexture;
@@ -25,6 +27,9 @@ import android.opengl.EGLExt;
import android.opengl.EGLSurface;
import android.opengl.GLES20;
import android.view.Surface;
+import android.view.SurfaceView;
+import androidx.annotation.Nullable;
+import androidx.media3.common.Format;
import androidx.media3.common.util.GlUtil;
import java.io.IOException;
@@ -43,6 +48,8 @@ import java.io.IOException;
* @param outputHeight The output height in pixels.
* @param transformationMatrix The transformation matrix to apply to each frame.
* @param outputSurface The {@link Surface}.
+ * @param debugViewProvider Provider for optional debug views to show intermediate output, for
+ * debugging.
* @return A configured {@code FrameEditor}.
*/
public static FrameEditor create(
@@ -50,7 +57,8 @@ import java.io.IOException;
int outputWidth,
int outputHeight,
Matrix transformationMatrix,
- Surface outputSurface) {
+ Surface outputSurface,
+ Transformer.DebugViewProvider debugViewProvider) {
EGLDisplay eglDisplay = GlUtil.createEglDisplay();
EGLContext eglContext;
try {
@@ -93,7 +101,33 @@ import java.io.IOException;
float[] transformationMatrixArray = getGlMatrixArray(transformationMatrix);
glProgram.setFloatsUniform("transformation_matrix", transformationMatrixArray);
- return new FrameEditor(eglDisplay, eglContext, eglSurface, textureId, glProgram);
+ @Nullable
+ SurfaceView debugSurfaceView =
+ debugViewProvider.getDebugPreviewSurfaceView(outputWidth, outputHeight);
+ @Nullable EGLSurface debugPreviewEglSurface;
+ int debugPreviewWidth;
+ int debugPreviewHeight;
+ if (debugSurfaceView != null) {
+ debugPreviewEglSurface =
+ GlUtil.getEglSurface(eglDisplay, checkNotNull(debugSurfaceView.getHolder()));
+ debugPreviewWidth = debugSurfaceView.getWidth();
+ debugPreviewHeight = debugSurfaceView.getHeight();
+ } else {
+ debugPreviewEglSurface = null;
+ debugPreviewWidth = Format.NO_VALUE;
+ debugPreviewHeight = Format.NO_VALUE;
+ }
+ return new FrameEditor(
+ eglDisplay,
+ eglContext,
+ eglSurface,
+ textureId,
+ glProgram,
+ outputWidth,
+ outputHeight,
+ debugPreviewEglSurface,
+ debugPreviewWidth,
+ debugPreviewHeight);
}
/**
@@ -147,8 +181,12 @@ import java.io.IOException;
private final int textureId;
private final SurfaceTexture inputSurfaceTexture;
private final Surface inputSurface;
-
private final GlUtil.Program glProgram;
+ private final int outputWidth;
+ private final int outputHeight;
+ @Nullable private final EGLSurface debugPreviewEglSurface;
+ private final int debugPreviewWidth;
+ private final int debugPreviewHeight;
private volatile boolean hasInputData;
@@ -157,12 +195,22 @@ import java.io.IOException;
EGLContext eglContext,
EGLSurface eglSurface,
int textureId,
- GlUtil.Program glProgram) {
+ GlUtil.Program glProgram,
+ int outputWidth,
+ int outputHeight,
+ @Nullable EGLSurface debugPreviewEglSurface,
+ int debugPreviewWidth,
+ int debugPreviewHeight) {
this.eglDisplay = eglDisplay;
this.eglContext = eglContext;
this.eglSurface = eglSurface;
this.textureId = textureId;
this.glProgram = glProgram;
+ this.outputWidth = outputWidth;
+ this.outputHeight = outputHeight;
+ this.debugPreviewEglSurface = debugPreviewEglSurface;
+ this.debugPreviewWidth = debugPreviewWidth;
+ this.debugPreviewHeight = debugPreviewHeight;
textureTransformMatrix = new float[16];
inputSurfaceTexture = new SurfaceTexture(textureId);
inputSurfaceTexture.setOnFrameAvailableListener(surfaceTexture -> hasInputData = true);
@@ -188,10 +236,17 @@ import java.io.IOException;
inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
glProgram.setFloatsUniform("tex_transform", textureTransformMatrix);
glProgram.bindAttributesAndUniforms();
- GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
+
+ focusAndDrawQuad(eglSurface, outputWidth, outputHeight);
long surfaceTextureTimestampNs = inputSurfaceTexture.getTimestamp();
EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, surfaceTextureTimestampNs);
EGL14.eglSwapBuffers(eglDisplay, eglSurface);
+
+ if (debugPreviewEglSurface != null) {
+ focusAndDrawQuad(debugPreviewEglSurface, debugPreviewWidth, debugPreviewHeight);
+ EGL14.eglSwapBuffers(eglDisplay, debugPreviewEglSurface);
+ }
+
hasInputData = false;
}
@@ -203,4 +258,13 @@ import java.io.IOException;
inputSurfaceTexture.release();
inputSurface.release();
}
+
+ /**
+ * Focuses the specified surface with the specified width and height, then draws a four-vertex
+ * triangle strip (which is a quadrilateral).
+ */
+ private void focusAndDrawQuad(EGLSurface eglSurface, int width, int height) {
+ GlUtil.focusSurface(eglDisplay, eglContext, eglSurface, width, height);
+ GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
+ }
}
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
index f22288c02c..a65a39beee 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
@@ -29,6 +29,7 @@ import android.graphics.Matrix;
import android.os.Handler;
import android.os.Looper;
import android.os.ParcelFileDescriptor;
+import android.view.SurfaceView;
import androidx.annotation.IntDef;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
@@ -106,6 +107,7 @@ public final class Transformer {
@Nullable private String audioMimeType;
@Nullable private String videoMimeType;
private Transformer.Listener listener;
+ private DebugViewProvider debugViewProvider;
private Looper looper;
private Clock clock;
@@ -119,6 +121,7 @@ public final class Transformer {
listener = new Listener() {};
looper = Util.getCurrentOrMainLooper();
clock = Clock.DEFAULT;
+ debugViewProvider = DebugViewProvider.NONE;
}
/**
@@ -135,6 +138,7 @@ public final class Transformer {
listener = new Listener() {};
looper = Util.getCurrentOrMainLooper();
clock = Clock.DEFAULT;
+ debugViewProvider = DebugViewProvider.NONE;
}
/** Creates a builder with the values of the provided {@link Transformer}. */
@@ -152,6 +156,7 @@ public final class Transformer {
this.videoMimeType = transformer.transformation.videoMimeType;
this.listener = transformer.listener;
this.looper = transformer.looper;
+ this.debugViewProvider = transformer.debugViewProvider;
this.clock = transformer.clock;
}
@@ -358,6 +363,21 @@ public final class Transformer {
return this;
}
+ /**
+ * Sets a provider for views to show diagnostic information (if available) during
+ * transformation. This is intended for debugging. The default value is {@link
+ * DebugViewProvider#NONE}, which doesn't show any debug info.
+ *
+ * Not all transformations will result in debug views being populated.
+ *
+ * @param debugViewProvider Provider for debug views.
+ * @return This builder.
+ */
+ public Builder setDebugViewProvider(DebugViewProvider debugViewProvider) {
+ this.debugViewProvider = debugViewProvider;
+ return this;
+ }
+
/**
* Sets the {@link Clock} that will be used by the transformer. The default value is {@link
* Clock#DEFAULT}.
@@ -424,7 +444,14 @@ public final class Transformer {
audioMimeType,
videoMimeType);
return new Transformer(
- context, mediaSourceFactory, muxerFactory, transformation, listener, looper, clock);
+ context,
+ mediaSourceFactory,
+ muxerFactory,
+ transformation,
+ listener,
+ looper,
+ clock,
+ debugViewProvider);
}
private void checkSampleMimeType(String sampleMimeType) {
@@ -456,6 +483,22 @@ public final class Transformer {
default void onTransformationError(MediaItem inputMediaItem, Exception exception) {}
}
+ /** Provider for views to show diagnostic information during transformation, for debugging. */
+ public interface DebugViewProvider {
+
+ /** Debug view provider that doesn't show any debug info. */
+ DebugViewProvider NONE = (int width, int height) -> null;
+
+ /**
+ * Returns a new surface view to show a preview of transformer output with the given
+ * width/height in pixels, or {@code null} if no debug information should be shown.
+ *
+ *
This method may be called on an arbitrary thread.
+ */
+ @Nullable
+ SurfaceView getDebugPreviewSurfaceView(int width, int height);
+ }
+
/**
* Progress state. One of {@link #PROGRESS_STATE_WAITING_FOR_AVAILABILITY}, {@link
* #PROGRESS_STATE_AVAILABLE}, {@link #PROGRESS_STATE_UNAVAILABLE}, {@link
@@ -489,6 +532,7 @@ public final class Transformer {
private final Transformation transformation;
private final Looper looper;
private final Clock clock;
+ private final Transformer.DebugViewProvider debugViewProvider;
private Transformer.Listener listener;
@Nullable private MuxerWrapper muxerWrapper;
@@ -502,7 +546,8 @@ public final class Transformer {
Transformation transformation,
Transformer.Listener listener,
Looper looper,
- Clock clock) {
+ Clock clock,
+ Transformer.DebugViewProvider debugViewProvider) {
checkState(
!transformation.removeAudio || !transformation.removeVideo,
"Audio and video cannot both be removed.");
@@ -513,6 +558,7 @@ public final class Transformer {
this.listener = listener;
this.looper = looper;
this.clock = clock;
+ this.debugViewProvider = debugViewProvider;
progressState = PROGRESS_STATE_NO_TRANSFORMATION;
}
@@ -610,7 +656,9 @@ public final class Transformer {
.build();
player =
new ExoPlayer.Builder(
- context, new TransformerRenderersFactory(context, muxerWrapper, transformation))
+ context,
+ new TransformerRenderersFactory(
+ context, muxerWrapper, transformation, debugViewProvider))
.setMediaSourceFactory(mediaSourceFactory)
.setTrackSelector(trackSelector)
.setLoadControl(loadControl)
@@ -699,12 +747,17 @@ public final class Transformer {
private final MuxerWrapper muxerWrapper;
private final TransformerMediaClock mediaClock;
private final Transformation transformation;
+ private final Transformer.DebugViewProvider debugViewProvider;
public TransformerRenderersFactory(
- Context context, MuxerWrapper muxerWrapper, Transformation transformation) {
+ Context context,
+ MuxerWrapper muxerWrapper,
+ Transformation transformation,
+ Transformer.DebugViewProvider debugViewProvider) {
this.context = context;
this.muxerWrapper = muxerWrapper;
this.transformation = transformation;
+ this.debugViewProvider = debugViewProvider;
mediaClock = new TransformerMediaClock();
}
@@ -724,7 +777,8 @@ public final class Transformer {
}
if (!transformation.removeVideo) {
renderers[index] =
- new TransformerVideoRenderer(context, muxerWrapper, mediaClock, transformation);
+ new TransformerVideoRenderer(
+ context, muxerWrapper, mediaClock, transformation, debugViewProvider);
index++;
}
return renderers;
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java
index a5de1ce6ef..8e3fe1609d 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java
@@ -35,6 +35,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private static final String TAG = "TVideoRenderer";
private final Context context;
+ private final Transformer.DebugViewProvider debugViewProvider;
private final DecoderInputBuffer decoderInputBuffer;
private @MonotonicNonNull SefSlowMotionFlattener sefSlowMotionFlattener;
@@ -43,9 +44,11 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
Context context,
MuxerWrapper muxerWrapper,
TransformerMediaClock mediaClock,
- Transformation transformation) {
+ Transformation transformation,
+ Transformer.DebugViewProvider debugViewProvider) {
super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformation);
this.context = context;
+ this.debugViewProvider = debugViewProvider;
decoderInputBuffer =
new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
}
@@ -69,7 +72,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
}
Format inputFormat = checkNotNull(formatHolder.format);
if (shouldTranscode(inputFormat)) {
- samplePipeline = new VideoSamplePipeline(context, inputFormat, transformation, getIndex());
+ samplePipeline =
+ new VideoSamplePipeline(
+ context, inputFormat, transformation, getIndex(), debugViewProvider);
} else {
samplePipeline = new PassthroughSamplePipeline(inputFormat);
}
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java
index b6fc1a119b..8fe846e9ab 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java
@@ -48,7 +48,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private boolean waitingForFrameEditorInput;
public VideoSamplePipeline(
- Context context, Format inputFormat, Transformation transformation, int rendererIndex)
+ Context context,
+ Format inputFormat,
+ Transformation transformation,
+ int rendererIndex,
+ Transformer.DebugViewProvider debugViewProvider)
throws ExoPlaybackException {
decoderInputBuffer =
new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
@@ -87,7 +91,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
outputWidth,
outputHeight,
transformation.transformationMatrix,
- /* outputSurface= */ checkNotNull(encoder.getInputSurface()));
+ /* outputSurface= */ checkNotNull(encoder.getInputSurface()),
+ debugViewProvider);
}
try {
decoder =