loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

FrameProcessor: Replace SurfaceInfo.Provider with setter.

Browse Source 
The FinalMatrixTransformationProcessorWrapper ensures that the
surface is only replaced when it is not being rendered to and vice
versa.

PiperOrigin-RevId: 458007639
This commit is contained in:
hschlueter 2022-06-29 17:27:05 +00:00 committed by Marc Baechinger
parent 3fc6a66527
commit 234015cb95
7 changed files with 236 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java
View File

@ -356,6 +356,16 @@ public final class GlEffectsFrameProcessorPixelTest {
GlEffectsFrameProcessor.create(
context,
new FrameProcessor.Listener() {
@Override
public void onOutputSizeChanged(int width, int height) {
outputImageReader =
ImageReader.newInstance(
width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
checkNotNull(glEffectsFrameProcessor)
.setOutputSurfaceInfo(
new SurfaceInfo(outputImageReader.getSurface(), width, height));
}
@Override
public void onFrameProcessingError(FrameProcessingException exception) {
frameProcessingException.set(exception);
@ -368,16 +378,6 @@ public final class GlEffectsFrameProcessorPixelTest {
},
/* streamOffsetUs= */ 0L,
effects,
/* outputSurfaceProvider= */ (requestedWidth, requestedHeight) -> {
outputImageReader =
ImageReader.newInstance(
requestedWidth,
requestedHeight,
PixelFormat.RGBA_8888,
/* maxImages= */ 1);
return new SurfaceInfo(
outputImageReader.getSurface(), requestedWidth, requestedHeight);
},
Transformer.DebugViewProvider.NONE,
/* enableExperimentalHdrEditing= */ false));
glEffectsFrameProcessor.setInputFrameInfo(

158
libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java
View File

@ -34,6 +34,7 @@ import androidx.annotation.WorkerThread;
import androidx.media3.common.C;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.Log;
import androidx.media3.common.util.Util;
import com.google.common.collect.ImmutableList;
import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@ -57,7 +58,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final ImmutableList<GlMatrixTransformation> matrixTransformations;
private final EGLDisplay eglDisplay;
private final EGLContext eglContext;
private final SurfaceInfo.Provider outputSurfaceProvider;
private final long streamOffsetUs;
private final Transformer.DebugViewProvider debugViewProvider;
private final FrameProcessor.Listener frameProcessorListener;
@ -66,17 +66,23 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private int inputWidth;
private int inputHeight;
@Nullable private MatrixTransformationProcessor matrixTransformationProcessor;
@Nullable private SurfaceInfo outputSurfaceInfo;
@Nullable private EGLSurface outputEglSurface;
@Nullable private SurfaceViewWrapper debugSurfaceViewWrapper;
private @MonotonicNonNull Listener listener;
private @MonotonicNonNull Size outputSizeBeforeSurfaceTransformation;
@GuardedBy("this")
@Nullable
private SurfaceInfo outputSurfaceInfo;
@GuardedBy("this")
@Nullable
private EGLSurface outputEglSurface;
public FinalMatrixTransformationProcessorWrapper(
Context context,
EGLDisplay eglDisplay,
EGLContext eglContext,
ImmutableList<GlMatrixTransformation> matrixTransformations,
SurfaceInfo.Provider outputSurfaceProvider,
long streamOffsetUs,
FrameProcessor.Listener frameProcessorListener,
Transformer.DebugViewProvider debugViewProvider,
@ -85,7 +91,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.matrixTransformations = matrixTransformations;
this.eglDisplay = eglDisplay;
this.eglContext = eglContext;
this.outputSurfaceProvider = outputSurfaceProvider;
this.streamOffsetUs = streamOffsetUs;
this.debugViewProvider = debugViewProvider;
this.frameProcessorListener = frameProcessorListener;
@ -107,28 +112,30 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Override
public boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
try {
if (!ensureConfigured(inputTexture.width, inputTexture.height)) {
return false;
synchronized (this) {
if (!ensureConfigured(inputTexture.width, inputTexture.height)) {
return false;
}
EGLSurface outputEglSurface = this.outputEglSurface;
SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo;
MatrixTransformationProcessor matrixTransformationProcessor =
this.matrixTransformationProcessor;
GlUtil.focusEglSurface(
eglDisplay,
eglContext,
outputEglSurface,
outputSurfaceInfo.width,
outputSurfaceInfo.height);
GlUtil.clearOutputFrame();
matrixTransformationProcessor.drawFrame(inputTexture.texId, presentationTimeUs);
EGLExt.eglPresentationTimeANDROID(
eglDisplay,
outputEglSurface,
/* presentationTimeNs= */ (presentationTimeUs + streamOffsetUs) * 1000);
EGL14.eglSwapBuffers(eglDisplay, outputEglSurface);
}
EGLSurface outputEglSurface = this.outputEglSurface;
SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo;
MatrixTransformationProcessor matrixTransformationProcessor =
this.matrixTransformationProcessor;
GlUtil.focusEglSurface(
eglDisplay,
eglContext,
outputEglSurface,
outputSurfaceInfo.width,
outputSurfaceInfo.height);
GlUtil.clearOutputFrame();
matrixTransformationProcessor.drawFrame(inputTexture.texId, presentationTimeUs);
EGLExt.eglPresentationTimeANDROID(
eglDisplay,
outputEglSurface,
/* presentationTimeNs= */ (presentationTimeUs + streamOffsetUs) * 1000);
EGL14.eglSwapBuffers(eglDisplay, outputEglSurface);
} catch (FrameProcessingException | GlUtil.GlException e) {
frameProcessorListener.onFrameProcessingError(
FrameProcessingException.from(e, presentationTimeUs));
@ -156,24 +163,25 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@EnsuresNonNullIf(
expression = {"outputSurfaceInfo", "outputEglSurface", "matrixTransformationProcessor"},
result = true)
private boolean ensureConfigured(int inputWidth, int inputHeight)
private synchronized boolean ensureConfigured(int inputWidth, int inputHeight)
throws FrameProcessingException, GlUtil.GlException {
if (inputWidth == this.inputWidth
&& inputHeight == this.inputHeight
&& outputSurfaceInfo != null
&& outputEglSurface != null
&& matrixTransformationProcessor != null) {
return true;
if (this.inputWidth != inputWidth
|| this.inputHeight != inputHeight
|| this.outputSizeBeforeSurfaceTransformation == null) {
this.inputWidth = inputWidth;
this.inputHeight = inputHeight;
Size outputSizeBeforeSurfaceTransformation =
MatrixUtils.configureAndGetOutputSize(inputWidth, inputHeight, matrixTransformations);
if (!Util.areEqual(
this.outputSizeBeforeSurfaceTransformation, outputSizeBeforeSurfaceTransformation)) {
this.outputSizeBeforeSurfaceTransformation = outputSizeBeforeSurfaceTransformation;
frameProcessorListener.onOutputSizeChanged(
outputSizeBeforeSurfaceTransformation.getWidth(),
outputSizeBeforeSurfaceTransformation.getHeight());
}
}
this.inputWidth = inputWidth;
this.inputHeight = inputHeight;
Size requestedOutputSize =
MatrixUtils.configureAndGetOutputSize(inputWidth, inputHeight, matrixTransformations);
@Nullable
SurfaceInfo outputSurfaceInfo =
outputSurfaceProvider.getSurfaceInfo(
requestedOutputSize.getWidth(), requestedOutputSize.getHeight());
if (outputSurfaceInfo == null) {
if (matrixTransformationProcessor != null) {
matrixTransformationProcessor.release();
@ -182,40 +190,44 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
outputEglSurface = null;
return false;
}
if (outputSurfaceInfo == this.outputSurfaceInfo
&& outputEglSurface != null
&& matrixTransformationProcessor != null) {
return true;
SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo;
@Nullable EGLSurface outputEglSurface = this.outputEglSurface;
if (outputEglSurface == null) { // This means that outputSurfaceInfo changed.
if (enableExperimentalHdrEditing) {
// TODO(b/227624622): Don't assume BT.2020 PQ input/output.
outputEglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurfaceInfo.surface);
} else {
outputEglSurface = GlUtil.getEglSurface(eglDisplay, outputSurfaceInfo.surface);
}
@Nullable
SurfaceView debugSurfaceView =
debugViewProvider.getDebugPreviewSurfaceView(
outputSurfaceInfo.width, outputSurfaceInfo.height);
if (debugSurfaceView != null) {
debugSurfaceViewWrapper =
new SurfaceViewWrapper(
eglDisplay, eglContext, enableExperimentalHdrEditing, debugSurfaceView);
}
if (matrixTransformationProcessor != null) {
matrixTransformationProcessor.release();
matrixTransformationProcessor = null;
}
}
EGLSurface outputEglSurface;
if (enableExperimentalHdrEditing) {
// TODO(b/227624622): Don't assume BT.2020 PQ input/output.
outputEglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurfaceInfo.surface);
} else {
outputEglSurface = GlUtil.getEglSurface(eglDisplay, outputSurfaceInfo.surface);
if (matrixTransformationProcessor == null) {
matrixTransformationProcessor =
createMatrixTransformationProcessorForOutputSurface(outputSurfaceInfo);
}
@Nullable
SurfaceView debugSurfaceView =
debugViewProvider.getDebugPreviewSurfaceView(
outputSurfaceInfo.width, outputSurfaceInfo.height);
if (debugSurfaceView != null) {
debugSurfaceViewWrapper =
new SurfaceViewWrapper(
eglDisplay, eglContext, enableExperimentalHdrEditing, debugSurfaceView);
}
matrixTransformationProcessor =
createMatrixTransformationProcessorForOutputSurface(requestedOutputSize, outputSurfaceInfo);
this.outputSurfaceInfo = outputSurfaceInfo;
this.outputEglSurface = outputEglSurface;
return true;
}
private MatrixTransformationProcessor createMatrixTransformationProcessorForOutputSurface(
Size requestedOutputSize, SurfaceInfo outputSurfaceInfo) throws FrameProcessingException {
SurfaceInfo outputSurfaceInfo) throws FrameProcessingException {
ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder =
new ImmutableList.Builder<GlMatrixTransformation>().addAll(matrixTransformations);
if (outputSurfaceInfo.orientationDegrees != 0) {
@ -224,12 +236,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
.setRotationDegrees(outputSurfaceInfo.orientationDegrees)
.build());
}
if (outputSurfaceInfo.width != requestedOutputSize.getWidth()
|| outputSurfaceInfo.height != requestedOutputSize.getHeight()) {
matrixTransformationListBuilder.add(
Presentation.createForWidthAndHeight(
outputSurfaceInfo.width, outputSurfaceInfo.height, Presentation.LAYOUT_SCALE_TO_FIT));
}
matrixTransformationListBuilder.add(
Presentation.createForWidthAndHeight(
outputSurfaceInfo.width, outputSurfaceInfo.height, Presentation.LAYOUT_SCALE_TO_FIT));
MatrixTransformationProcessor matrixTransformationProcessor =
new MatrixTransformationProcessor(context, matrixTransformationListBuilder.build());
@ -258,6 +267,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
}
public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
if (!Util.areEqual(this.outputSurfaceInfo, outputSurfaceInfo)) {
this.outputSurfaceInfo = outputSurfaceInfo;
this.outputEglSurface = null;
}
}
/**
* Wrapper around a {@link SurfaceView} that keeps track of whether the output surface is valid,
* and makes rendering a no-op if not.

26
libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java
View File

@ -16,6 +16,7 @@
package androidx.media3.transformer;
import android.view.Surface;
import androidx.annotation.Nullable;
/** Interface for a frame processor that applies changes to individual video frames. */
/* package */ interface FrameProcessor {
@ -26,6 +27,14 @@ import android.view.Surface;
*/
interface Listener {
/**
* Called when the output size after applying the final effect changes.
*
* <p>The output size after applying the final effect can differ from the size specified using
* {@link #setOutputSurfaceInfo(SurfaceInfo)}.
*/
void onOutputSizeChanged(int width, int height);
/**
* Called when an exception occurs during asynchronous frame processing.
*
@ -68,6 +77,23 @@ import android.view.Surface;
*/
int getPendingInputFrameCount();
/**
* Sets the output surface and supporting information.
*
* <p>The new output {@link SurfaceInfo} is applied from the next output frame rendered onwards.
* If the output {@link SurfaceInfo} is {@code null}, the {@code FrameProcessor} will stop
* rendering and resume rendering pending frames once a non-null {@link SurfaceInfo} is set.
*
* <p>If the dimensions given in {@link SurfaceInfo} do not match the {@linkplain
* Listener#onOutputSizeChanged(int,int) output size after applying the final effect} the frames
* are resized before rendering to the surface and letter/pillar-boxing is applied.
*
* <p>The caller is responsible for tracking the lifecycle of the {@link SurfaceInfo#surface}
* including calling this method with a new surface if it is destroyed. When this method returns,
* the previous output surface is no longer being used and can safely be released by the caller.
*/
void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo);
/**
* Informs the {@code FrameProcessor} that no further input frames should be accepted.
*

114
libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java
View File

@ -23,6 +23,7 @@ import android.graphics.SurfaceTexture;
import android.opengl.EGL14;
import android.opengl.EGLContext;
import android.opengl.EGLDisplay;
import android.util.Pair;
import android.view.Surface;
import androidx.annotation.Nullable;
import androidx.annotation.WorkerThread;
@ -51,8 +52,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* @param context A {@link Context}.
* @param listener A {@link Listener}.
* @param effects The {@link GlEffect GlEffects} to apply to each frame.
* @param outputSurfaceProvider A {@link SurfaceInfo.Provider} managing the output {@link
* Surface}.
* @param debugViewProvider A {@link Transformer.DebugViewProvider}.
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
* @return A new instance.
@ -64,7 +63,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
FrameProcessor.Listener listener,
long streamOffsetUs,
List<GlEffect> effects,
SurfaceInfo.Provider outputSurfaceProvider,
Transformer.DebugViewProvider debugViewProvider,
boolean enableExperimentalHdrEditing)
throws FrameProcessingException {
@ -79,7 +77,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
listener,
streamOffsetUs,
effects,
outputSurfaceProvider,
debugViewProvider,
enableExperimentalHdrEditing,
singleThreadExecutorService));
@ -108,7 +105,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
FrameProcessor.Listener listener,
long streamOffsetUs,
List<GlEffect> effects,
SurfaceInfo.Provider outputSurfaceProvider,
Transformer.DebugViewProvider debugViewProvider,
boolean enableExperimentalHdrEditing,
ExecutorService singleThreadExecutorService)
@ -131,24 +127,31 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
GlUtil.focusPlaceholderEglSurface(eglContext, eglDisplay);
}
ImmutableList<GlTextureProcessor> textureProcessors =
getGlTextureProcessorsForGlEffects(
context,
effects,
eglDisplay,
eglContext,
outputSurfaceProvider,
streamOffsetUs,
listener,
debugViewProvider,
enableExperimentalHdrEditing);
Pair<ImmutableList<GlTextureProcessor>, FinalMatrixTransformationProcessorWrapper>
textureProcessors =
getGlTextureProcessorsForGlEffects(
context,
effects,
eglDisplay,
eglContext,
streamOffsetUs,
listener,
debugViewProvider,
enableExperimentalHdrEditing);
ImmutableList<GlTextureProcessor> intermediateTextureProcessors = textureProcessors.first;
FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper =
textureProcessors.second;
ExternalTextureProcessor externalTextureProcessor =
new ExternalTextureProcessor(context, enableExperimentalHdrEditing);
FrameProcessingTaskExecutor frameProcessingTaskExecutor =
new FrameProcessingTaskExecutor(singleThreadExecutorService, listener);
chainTextureProcessorsWithListeners(
externalTextureProcessor, textureProcessors, frameProcessingTaskExecutor, listener);
externalTextureProcessor,
intermediateTextureProcessors,
finalTextureProcessorWrapper,
frameProcessingTaskExecutor,
listener);
return new GlEffectsFrameProcessor(
eglDisplay,
@ -157,7 +160,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
streamOffsetUs,
/* inputExternalTextureId= */ GlUtil.createExternalTexture(),
externalTextureProcessor,
textureProcessors);
intermediateTextureProcessors,
finalTextureProcessorWrapper);
}
/**
@ -165,21 +169,21 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* MatrixTransformationProcessor} and converts all other {@link GlEffect} instances to separate
* {@link GlTextureProcessor} instances.
*
* <p>The final {@link GlTextureProcessor} is wrapped in a {@link
* FinalMatrixTransformationProcessorWrapper} so that it can write directly to the {@linkplain
* SurfaceInfo.Provider provided output surface}.
* @return A {@link Pair} containing a list of {@link GlTextureProcessor} instances to apply in
* the given order and a {@link FinalMatrixTransformationProcessorWrapper} to apply after
* them.
*/
private static ImmutableList<GlTextureProcessor> getGlTextureProcessorsForGlEffects(
Context context,
List<GlEffect> effects,
EGLDisplay eglDisplay,
EGLContext eglContext,
SurfaceInfo.Provider outputSurfaceProvider,
long streamOffsetUs,
FrameProcessor.Listener listener,
Transformer.DebugViewProvider debugViewProvider,
boolean enableExperimentalHdrEditing)
throws FrameProcessingException {
private static Pair<ImmutableList<GlTextureProcessor>, FinalMatrixTransformationProcessorWrapper>
getGlTextureProcessorsForGlEffects(
Context context,
List<GlEffect> effects,
EGLDisplay eglDisplay,
EGLContext eglContext,
long streamOffsetUs,
FrameProcessor.Listener listener,
Transformer.DebugViewProvider debugViewProvider,
boolean enableExperimentalHdrEditing)
throws FrameProcessingException {
ImmutableList.Builder<GlTextureProcessor> textureProcessorListBuilder =
new ImmutableList.Builder<>();
ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder =
@ -199,18 +203,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
textureProcessorListBuilder.add(effect.toGlTextureProcessor(context));
}
textureProcessorListBuilder.add(
return Pair.create(
textureProcessorListBuilder.build(),
new FinalMatrixTransformationProcessorWrapper(
context,
eglDisplay,
eglContext,
matrixTransformationListBuilder.build(),
outputSurfaceProvider,
streamOffsetUs,
listener,
debugViewProvider,
enableExperimentalHdrEditing));
return textureProcessorListBuilder.build();
}
/**
@ -221,21 +224,26 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
*/
private static void chainTextureProcessorsWithListeners(
ExternalTextureProcessor externalTextureProcessor,
ImmutableList<GlTextureProcessor> textureProcessors,
ImmutableList<GlTextureProcessor> intermediateTextureProcessors,
FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper,
FrameProcessingTaskExecutor frameProcessingTaskExecutor,
FrameProcessor.Listener listener) {
externalTextureProcessor.setListener(
new ChainingGlTextureProcessorListener(
/* previousGlTextureProcessor= */ null,
textureProcessors.get(0),
/* nextGlTextureProcessor= */ intermediateTextureProcessors.size() > 0
? intermediateTextureProcessors.get(0)
: finalTextureProcessorWrapper,
frameProcessingTaskExecutor,
listener));
GlTextureProcessor previousGlTextureProcessor = externalTextureProcessor;
for (int i = 0; i < textureProcessors.size(); i++) {
GlTextureProcessor glTextureProcessor = textureProcessors.get(i);
for (int i = 0; i < intermediateTextureProcessors.size(); i++) {
GlTextureProcessor glTextureProcessor = intermediateTextureProcessors.get(i);
@Nullable
GlTextureProcessor nextGlTextureProcessor =
i + 1 < textureProcessors.size() ? textureProcessors.get(i + 1) : null;
i + 1 < intermediateTextureProcessors.size()
? intermediateTextureProcessors.get(i + 1)
: finalTextureProcessorWrapper;
glTextureProcessor.setListener(
new ChainingGlTextureProcessorListener(
previousGlTextureProcessor,
@ -244,6 +252,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
listener));
previousGlTextureProcessor = glTextureProcessor;
}
finalTextureProcessorWrapper.setListener(
new ChainingGlTextureProcessorListener(
previousGlTextureProcessor,
/* nextGlTextureProcessor= */ null,
frameProcessingTaskExecutor,
listener));
}
private static final String TAG = "GlEffectsFrameProcessor";
@ -267,7 +281,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final float[] inputSurfaceTextureTransformMatrix;
private final int inputExternalTextureId;
private final ExternalTextureProcessor inputExternalTextureProcessor;
private final ImmutableList<GlTextureProcessor> textureProcessors;
private final ImmutableList<GlTextureProcessor> intermediateTextureProcessors;
private final FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper;
private final ConcurrentLinkedQueue<FrameInfo> pendingInputFrames;
private @MonotonicNonNull FrameInfo nextInputFrameInfo;
@ -280,8 +295,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
long streamOffsetUs,
int inputExternalTextureId,
ExternalTextureProcessor inputExternalTextureProcessor,
ImmutableList<GlTextureProcessor> textureProcessors) {
checkState(!textureProcessors.isEmpty());
ImmutableList<GlTextureProcessor> intermediateTextureProcessors,
FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper) {
this.eglDisplay = eglDisplay;
this.eglContext = eglContext;
@ -289,7 +304,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.streamOffsetUs = streamOffsetUs;
this.inputExternalTextureId = inputExternalTextureId;
this.inputExternalTextureProcessor = inputExternalTextureProcessor;
this.textureProcessors = textureProcessors;
this.intermediateTextureProcessors = intermediateTextureProcessors;
this.finalTextureProcessorWrapper = finalTextureProcessorWrapper;
inputSurfaceTexture = new SurfaceTexture(inputExternalTextureId);
inputSurface = new Surface(inputSurfaceTexture);
@ -323,6 +339,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
return pendingInputFrames.size();
}
@Override
public void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
finalTextureProcessorWrapper.setOutputSurfaceInfo(outputSurfaceInfo);
}
@Override
public void signalEndOfInputStream() {
checkState(!inputStreamEnded);
@ -423,9 +444,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private void releaseTextureProcessorsAndDestroyGlContext()
throws GlUtil.GlException, FrameProcessingException {
inputExternalTextureProcessor.release();
for (int i = 0; i < textureProcessors.size(); i++) {
textureProcessors.get(i).release();
for (int i = 0; i < intermediateTextureProcessors.size(); i++) {
intermediateTextureProcessors.get(i).release();
}
finalTextureProcessorWrapper.release();
GlUtil.destroyEglContext(eglDisplay, eglContext);
}
}

33
libraries/transformer/src/main/java/androidx/media3/transformer/SurfaceInfo.java
View File

@ -55,16 +55,27 @@ import androidx.annotation.Nullable;
this.orientationDegrees = orientationDegrees;
}
/** A provider for a {@link SurfaceInfo} instance. */
public interface Provider {
/**
* Provides a {@linkplain SurfaceInfo surface} for the requested dimensions.
*
* <p>The dimensions given in the provided {@link SurfaceInfo} may differ from the requested
* dimensions. It is up to the caller to transform frames from the requested dimensions to the
* provided dimensions before rendering them to the {@link SurfaceInfo#surface}.
*/
@Nullable
SurfaceInfo getSurfaceInfo(int requestedWidth, int requestedHeight);
@Override
public boolean equals(@Nullable Object o) {
if (this == o) {
return true;
}
if (!(o instanceof SurfaceInfo)) {
return false;
}
SurfaceInfo that = (SurfaceInfo) o;
return width == that.width
&& height == that.height
&& orientationDegrees == that.orientationDegrees
&& surface.equals(that.surface);
}
@Override
public int hashCode() {
int result = surface.hashCode();
result = 31 * result + width;
result = 31 * result + height;
result = 31 * result + orientationDegrees;
return result;
}
}

34
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
View File

@ -95,14 +95,23 @@ import org.checkerframework.dataflow.qual.Pure;
inputFormat,
allowedOutputMimeTypes,
transformationRequest,
fallbackListener,
asyncErrorListener);
fallbackListener);
try {
frameProcessor =
GlEffectsFrameProcessor.create(
context,
new FrameProcessor.Listener() {
@Override
public void onOutputSizeChanged(int width, int height) {
try {
checkNotNull(frameProcessor)
.setOutputSurfaceInfo(encoderWrapper.getSurfaceInfo(width, height));
} catch (TransformationException exception) {
asyncErrorListener.onTransformationException(exception);
}
}
@Override
public void onFrameProcessingError(FrameProcessingException exception) {
asyncErrorListener.onTransformationException(
@ -121,7 +130,6 @@ import org.checkerframework.dataflow.qual.Pure;
},
streamOffsetUs,
effectsListBuilder.build(),
/* outputSurfaceProvider= */ encoderWrapper,
debugViewProvider,
transformationRequest.enableHdrEditing);
} catch (FrameProcessingException e) {
@ -284,14 +292,13 @@ import org.checkerframework.dataflow.qual.Pure;
* dimensions, the same encoder is used and the provided dimensions stay fixed.
*/
@VisibleForTesting
/* package */ static final class EncoderWrapper implements SurfaceInfo.Provider {
/* package */ static final class EncoderWrapper {
private final Codec.EncoderFactory encoderFactory;
private final Format inputFormat;
private final List<String> allowedOutputMimeTypes;
private final TransformationRequest transformationRequest;
private final FallbackListener fallbackListener;
private final Transformer.AsyncErrorListener asyncErrorListener;
private @MonotonicNonNull SurfaceInfo encoderSurfaceInfo;
@ -304,20 +311,18 @@ import org.checkerframework.dataflow.qual.Pure;
Format inputFormat,
List<String> allowedOutputMimeTypes,
TransformationRequest transformationRequest,
FallbackListener fallbackListener,
Transformer.AsyncErrorListener asyncErrorListener) {
FallbackListener fallbackListener) {
this.encoderFactory = encoderFactory;
this.inputFormat = inputFormat;
this.allowedOutputMimeTypes = allowedOutputMimeTypes;
this.transformationRequest = transformationRequest;
this.fallbackListener = fallbackListener;
this.asyncErrorListener = asyncErrorListener;
}
@Override
@Nullable
public SurfaceInfo getSurfaceInfo(int requestedWidth, int requestedHeight) {
public SurfaceInfo getSurfaceInfo(int requestedWidth, int requestedHeight)
throws TransformationException {
if (releaseEncoder) {
return null;
}
@ -349,13 +354,8 @@ import org.checkerframework.dataflow.qual.Pure;
: inputFormat.sampleMimeType)
.build();
try {
encoder =
encoderFactory.createForVideoEncoding(requestedEncoderFormat, allowedOutputMimeTypes);
} catch (TransformationException e) {
asyncErrorListener.onTransformationException(e);
return null;
}
encoder =
encoderFactory.createForVideoEncoding(requestedEncoderFormat, allowedOutputMimeTypes);
Format encoderSupportedFormat = encoder.getConfigurationFormat();
fallbackListener.onTransformationRequestFinalized(
createFallbackTransformationRequest(

12
libraries/transformer/src/test/java/androidx/media3/transformer/VideoEncoderWrapperTest.java
View File

@ -50,8 +50,7 @@ public final class VideoEncoderWrapperTest {
/* inputFormat= */ new Format.Builder().build(),
/* allowedOutputMimeTypes= */ ImmutableList.of(),
emptyTransformationRequest,
fallbackListener,
mock(Transformer.AsyncErrorListener.class));
fallbackListener);
@Before
public void registerTrack() {
@ -59,7 +58,7 @@ public final class VideoEncoderWrapperTest {
}
@Test
public void getSurfaceInfo_landscape_leavesOrientationUnchanged() {
public void getSurfaceInfo_landscape_leavesOrientationUnchanged() throws Exception {
int inputWidth = 200;
int inputHeight = 150;
@ -71,7 +70,7 @@ public final class VideoEncoderWrapperTest {
}
@Test
public void getSurfaceInfo_square_leavesOrientationUnchanged() {
public void getSurfaceInfo_square_leavesOrientationUnchanged() throws Exception {
int inputWidth = 150;
int inputHeight = 150;
@ -83,7 +82,7 @@ public final class VideoEncoderWrapperTest {
}
@Test
public void getSurfaceInfo_portrait_flipsOrientation() {
public void getSurfaceInfo_portrait_flipsOrientation() throws Exception {
int inputWidth = 150;
int inputHeight = 200;
@ -95,7 +94,8 @@ public final class VideoEncoderWrapperTest {
}
@Test
public void getSurfaceInfo_withEncoderFallback_usesFallbackResolution() {
public void getSurfaceInfo_withEncoderFallback_usesFallbackResolution()
throws TransformationException {
int inputWidth = 200;
int inputHeight = 150;
int fallbackWidth = 100;