loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for showing debug info during transformation

Browse Source 
Being able to see the output of the GL pipeline is useful for debugging. For
example, when we previously saw flakiness it would have been useful to be able
to tell quickly whether the output looked wrong without needing to run a
transformation to the end then inspect the output file, and when working on
support for HDR editing it's useful to be able to do manual testing on devices
that don't support HDR encoding (but do support decoding/processing it with
GL).

Also change the progress indicator to be linear as this looks better in the
demo app when shown next to the debug preview.

PiperOrigin-RevId: 414999491
This commit is contained in:
andrewlewis 2021-12-08 15:41:38 +00:00 committed by Ian Baker
parent d7867800dc
commit 174120a7cf
5 changed files with 144 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorTest.java
View File

@ -92,7 +92,8 @@ public final class FrameEditorTest {
width,
height,
identityMatrix,
frameEditorOutputImageReader.getSurface());
frameEditorOutputImageReader.getSurface(),
Transformer.DebugViewProvider.NONE);
// Queue the first video frame from the extractor.
String mimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME));

74
libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java
View File

@ -15,6 +15,8 @@
*/
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkNotNull;
import android.content.Context;
import android.graphics.Matrix;
import android.graphics.SurfaceTexture;
@ -25,6 +27,9 @@ import android.opengl.EGLExt;
import android.opengl.EGLSurface;
import android.opengl.GLES20;
import android.view.Surface;
import android.view.SurfaceView;
import androidx.annotation.Nullable;
import androidx.media3.common.Format;
import androidx.media3.common.util.GlUtil;
import java.io.IOException;
@ -43,6 +48,8 @@ import java.io.IOException;
* @param outputHeight The output height in pixels.
* @param transformationMatrix The transformation matrix to apply to each frame.
* @param outputSurface The {@link Surface}.
* @param debugViewProvider Provider for optional debug views to show intermediate output, for
* debugging.
* @return A configured {@code FrameEditor}.
*/
public static FrameEditor create(
@ -50,7 +57,8 @@ import java.io.IOException;
int outputWidth,
int outputHeight,
Matrix transformationMatrix,
Surface outputSurface) {
Surface outputSurface,
Transformer.DebugViewProvider debugViewProvider) {
EGLDisplay eglDisplay = GlUtil.createEglDisplay();
EGLContext eglContext;
try {
@ -93,7 +101,33 @@ import java.io.IOException;
float[] transformationMatrixArray = getGlMatrixArray(transformationMatrix);
glProgram.setFloatsUniform("transformation_matrix", transformationMatrixArray);
return new FrameEditor(eglDisplay, eglContext, eglSurface, textureId, glProgram);
@Nullable
SurfaceView debugSurfaceView =
debugViewProvider.getDebugPreviewSurfaceView(outputWidth, outputHeight);
@Nullable EGLSurface debugPreviewEglSurface;
int debugPreviewWidth;
int debugPreviewHeight;
if (debugSurfaceView != null) {
debugPreviewEglSurface =
GlUtil.getEglSurface(eglDisplay, checkNotNull(debugSurfaceView.getHolder()));
debugPreviewWidth = debugSurfaceView.getWidth();
debugPreviewHeight = debugSurfaceView.getHeight();
} else {
debugPreviewEglSurface = null;
debugPreviewWidth = Format.NO_VALUE;
debugPreviewHeight = Format.NO_VALUE;
}
return new FrameEditor(
eglDisplay,
eglContext,
eglSurface,
textureId,
glProgram,
outputWidth,
outputHeight,
debugPreviewEglSurface,
debugPreviewWidth,
debugPreviewHeight);
}
/**
@ -147,8 +181,12 @@ import java.io.IOException;
private final int textureId;
private final SurfaceTexture inputSurfaceTexture;
private final Surface inputSurface;
private final GlUtil.Program glProgram;
private final int outputWidth;
private final int outputHeight;
@Nullable private final EGLSurface debugPreviewEglSurface;
private final int debugPreviewWidth;
private final int debugPreviewHeight;
private volatile boolean hasInputData;
@ -157,12 +195,22 @@ import java.io.IOException;
EGLContext eglContext,
EGLSurface eglSurface,
int textureId,
GlUtil.Program glProgram) {
GlUtil.Program glProgram,
int outputWidth,
int outputHeight,
@Nullable EGLSurface debugPreviewEglSurface,
int debugPreviewWidth,
int debugPreviewHeight) {
this.eglDisplay = eglDisplay;
this.eglContext = eglContext;
this.eglSurface = eglSurface;
this.textureId = textureId;
this.glProgram = glProgram;
this.outputWidth = outputWidth;
this.outputHeight = outputHeight;
this.debugPreviewEglSurface = debugPreviewEglSurface;
this.debugPreviewWidth = debugPreviewWidth;
this.debugPreviewHeight = debugPreviewHeight;
textureTransformMatrix = new float[16];
inputSurfaceTexture = new SurfaceTexture(textureId);
inputSurfaceTexture.setOnFrameAvailableListener(surfaceTexture -> hasInputData = true);
@ -188,10 +236,17 @@ import java.io.IOException;
inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
glProgram.setFloatsUniform("tex_transform", textureTransformMatrix);
glProgram.bindAttributesAndUniforms();
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
focusAndDrawQuad(eglSurface, outputWidth, outputHeight);
long surfaceTextureTimestampNs = inputSurfaceTexture.getTimestamp();
EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, surfaceTextureTimestampNs);
EGL14.eglSwapBuffers(eglDisplay, eglSurface);
if (debugPreviewEglSurface != null) {
focusAndDrawQuad(debugPreviewEglSurface, debugPreviewWidth, debugPreviewHeight);
EGL14.eglSwapBuffers(eglDisplay, debugPreviewEglSurface);
}
hasInputData = false;
}
@ -203,4 +258,13 @@ import java.io.IOException;
inputSurfaceTexture.release();
inputSurface.release();
}
/**
* Focuses the specified surface with the specified width and height, then draws a four-vertex
* triangle strip (which is a quadrilateral).
*/
private void focusAndDrawQuad(EGLSurface eglSurface, int width, int height) {
GlUtil.focusSurface(eglDisplay, eglContext, eglSurface, width, height);
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
}
}

64
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
View File

@ -29,6 +29,7 @@ import android.graphics.Matrix;
import android.os.Handler;
import android.os.Looper;
import android.os.ParcelFileDescriptor;
import android.view.SurfaceView;
import androidx.annotation.IntDef;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
@ -106,6 +107,7 @@ public final class Transformer {
@Nullable private String audioMimeType;
@Nullable private String videoMimeType;
private Transformer.Listener listener;
private DebugViewProvider debugViewProvider;
private Looper looper;
private Clock clock;
@ -119,6 +121,7 @@ public final class Transformer {
listener = new Listener() {};
looper = Util.getCurrentOrMainLooper();
clock = Clock.DEFAULT;
debugViewProvider = DebugViewProvider.NONE;
}
/**
@ -135,6 +138,7 @@ public final class Transformer {
listener = new Listener() {};
looper = Util.getCurrentOrMainLooper();
clock = Clock.DEFAULT;
debugViewProvider = DebugViewProvider.NONE;
}
/** Creates a builder with the values of the provided {@link Transformer}. */
@ -152,6 +156,7 @@ public final class Transformer {
this.videoMimeType = transformer.transformation.videoMimeType;
this.listener = transformer.listener;
this.looper = transformer.looper;
this.debugViewProvider = transformer.debugViewProvider;
this.clock = transformer.clock;
}
@ -358,6 +363,21 @@ public final class Transformer {
return this;
}
/**
* Sets a provider for views to show diagnostic information (if available) during
* transformation. This is intended for debugging. The default value is {@link
* DebugViewProvider#NONE}, which doesn't show any debug info.
*
* <p>Not all transformations will result in debug views being populated.
*
* @param debugViewProvider Provider for debug views.
* @return This builder.
*/
public Builder setDebugViewProvider(DebugViewProvider debugViewProvider) {
this.debugViewProvider = debugViewProvider;
return this;
}
/**
* Sets the {@link Clock} that will be used by the transformer. The default value is {@link
* Clock#DEFAULT}.
@ -424,7 +444,14 @@ public final class Transformer {
audioMimeType,
videoMimeType);
return new Transformer(
context, mediaSourceFactory, muxerFactory, transformation, listener, looper, clock);
context,
mediaSourceFactory,
muxerFactory,
transformation,
listener,
looper,
clock,
debugViewProvider);
}
private void checkSampleMimeType(String sampleMimeType) {
@ -456,6 +483,22 @@ public final class Transformer {
default void onTransformationError(MediaItem inputMediaItem, Exception exception) {}
}
/** Provider for views to show diagnostic information during transformation, for debugging. */
public interface DebugViewProvider {
/** Debug view provider that doesn't show any debug info. */
DebugViewProvider NONE = (int width, int height) -> null;
/**
* Returns a new surface view to show a preview of transformer output with the given
* width/height in pixels, or {@code null} if no debug information should be shown.
*
* <p>This method may be called on an arbitrary thread.
*/
@Nullable
SurfaceView getDebugPreviewSurfaceView(int width, int height);
}
/**
* Progress state. One of {@link #PROGRESS_STATE_WAITING_FOR_AVAILABILITY}, {@link
* #PROGRESS_STATE_AVAILABLE}, {@link #PROGRESS_STATE_UNAVAILABLE}, {@link
@ -489,6 +532,7 @@ public final class Transformer {
private final Transformation transformation;
private final Looper looper;
private final Clock clock;
private final Transformer.DebugViewProvider debugViewProvider;
private Transformer.Listener listener;
@Nullable private MuxerWrapper muxerWrapper;
@ -502,7 +546,8 @@ public final class Transformer {
Transformation transformation,
Transformer.Listener listener,
Looper looper,
Clock clock) {
Clock clock,
Transformer.DebugViewProvider debugViewProvider) {
checkState(
!transformation.removeAudio || !transformation.removeVideo,
"Audio and video cannot both be removed.");
@ -513,6 +558,7 @@ public final class Transformer {
this.listener = listener;
this.looper = looper;
this.clock = clock;
this.debugViewProvider = debugViewProvider;
progressState = PROGRESS_STATE_NO_TRANSFORMATION;
}
@ -610,7 +656,9 @@ public final class Transformer {
.build();
player =
new ExoPlayer.Builder(
context, new TransformerRenderersFactory(context, muxerWrapper, transformation))
context,
new TransformerRenderersFactory(
context, muxerWrapper, transformation, debugViewProvider))
.setMediaSourceFactory(mediaSourceFactory)
.setTrackSelector(trackSelector)
.setLoadControl(loadControl)
@ -699,12 +747,17 @@ public final class Transformer {
private final MuxerWrapper muxerWrapper;
private final TransformerMediaClock mediaClock;
private final Transformation transformation;
private final Transformer.DebugViewProvider debugViewProvider;
public TransformerRenderersFactory(
Context context, MuxerWrapper muxerWrapper, Transformation transformation) {
Context context,
MuxerWrapper muxerWrapper,
Transformation transformation,
Transformer.DebugViewProvider debugViewProvider) {
this.context = context;
this.muxerWrapper = muxerWrapper;
this.transformation = transformation;
this.debugViewProvider = debugViewProvider;
mediaClock = new TransformerMediaClock();
}
@ -724,7 +777,8 @@ public final class Transformer {
}
if (!transformation.removeVideo) {
renderers[index] =
new TransformerVideoRenderer(context, muxerWrapper, mediaClock, transformation);
new TransformerVideoRenderer(
context, muxerWrapper, mediaClock, transformation, debugViewProvider);
index++;
}
return renderers;

9
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java
View File

@ -35,6 +35,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private static final String TAG = "TVideoRenderer";
private final Context context;
private final Transformer.DebugViewProvider debugViewProvider;
private final DecoderInputBuffer decoderInputBuffer;
private @MonotonicNonNull SefSlowMotionFlattener sefSlowMotionFlattener;
@ -43,9 +44,11 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
Context context,
MuxerWrapper muxerWrapper,
TransformerMediaClock mediaClock,
Transformation transformation) {
Transformation transformation,
Transformer.DebugViewProvider debugViewProvider) {
super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformation);
this.context = context;
this.debugViewProvider = debugViewProvider;
decoderInputBuffer =
new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
}
@ -69,7 +72,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
}
Format inputFormat = checkNotNull(formatHolder.format);
if (shouldTranscode(inputFormat)) {
samplePipeline = new VideoSamplePipeline(context, inputFormat, transformation, getIndex());
samplePipeline =
new VideoSamplePipeline(
context, inputFormat, transformation, getIndex(), debugViewProvider);
} else {
samplePipeline = new PassthroughSamplePipeline(inputFormat);
}

9
libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java
View File

@ -48,7 +48,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private boolean waitingForFrameEditorInput;
public VideoSamplePipeline(
Context context, Format inputFormat, Transformation transformation, int rendererIndex)
Context context,
Format inputFormat,
Transformation transformation,
int rendererIndex,
Transformer.DebugViewProvider debugViewProvider)
throws ExoPlaybackException {
decoderInputBuffer =
new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
@ -87,7 +91,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
outputWidth,
outputHeight,
transformation.transformationMatrix,
/* outputSurface= */ checkNotNull(encoder.getInputSurface()));
/* outputSurface= */ checkNotNull(encoder.getInputSurface()),
debugViewProvider);
}
try {
decoder =