diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java
index e77ae2dfa2..910e724981 100644
--- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java
+++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java
@@ -367,8 +367,6 @@ public final class GlEffectsFrameProcessorPixelTest {
}
},
pixelWidthHeightRatio,
- inputWidth,
- inputHeight,
/* streamOffsetUs= */ 0L,
effects,
/* outputSurfaceProvider= */ (requestedWidth, requestedHeight) -> {
@@ -383,6 +381,7 @@ public final class GlEffectsFrameProcessorPixelTest {
},
Transformer.DebugViewProvider.NONE,
/* enableExperimentalHdrEditing= */ false));
+ glEffectsFrameProcessor.setInputFrameInfo(new FrameInfo(inputWidth, inputHeight));
glEffectsFrameProcessor.registerInputFrame();
// Queue the first video frame from the extractor.
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameInfo.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameInfo.java
new file mode 100644
index 0000000000..0b0af7821b
--- /dev/null
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameInfo.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.transformer;
+
+import static androidx.media3.common.util.Assertions.checkArgument;
+
+/** Value class specifying information about a decoded video frame. */
+/* package */ class FrameInfo {
+ /** The width of the frame, in pixels. */
+ public final int width;
+ /** The height of the frame, in pixels. */
+ public final int height;
+
+ // TODO(b/227625423): Add pixelWidthHeightRatio.
+ // TODO(b/227624622): Add color space information for HDR.
+
+ public FrameInfo(int width, int height) {
+ checkArgument(width > 0, "width must be positive, but is: " + width);
+ checkArgument(height > 0, "height must be positive, but is: " + height);
+
+ this.width = width;
+ this.height = height;
+ }
+}
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java
index a4338cd6a2..db0a3bec7b 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java
@@ -41,12 +41,21 @@ import android.view.Surface;
/** Returns the input {@link Surface}. */
Surface getInputSurface();
+ /**
+ * Sets information about the input frames.
+ *
+ * The new input information is applied from the next frame {@linkplain #registerInputFrame()
+ * registered} onwards.
+ */
+ void setInputFrameInfo(FrameInfo inputFrameInfo);
+
/**
* Informs the {@code FrameProcessor} that a frame will be queued to its input surface.
*
*
Must be called before rendering a frame to the frame processor's input surface.
*
- * @throws IllegalStateException If called after {@link #signalEndOfInputStream()}.
+ * @throws IllegalStateException If called after {@link #signalEndOfInputStream()} or before
+ * {@link #setInputFrameInfo(FrameInfo)}.
*/
void registerInputFrame();
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java
index fc73daeb36..75b9b80d25 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java
@@ -15,8 +15,8 @@
*/
package androidx.media3.transformer;
-import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkState;
+import static androidx.media3.common.util.Assertions.checkStateNotNull;
import android.content.Context;
import android.graphics.SurfaceTexture;
@@ -31,10 +31,11 @@ import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.Util;
import com.google.common.collect.ImmutableList;
import java.util.List;
+import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicInteger;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* A {@link FrameProcessor} implementation that applies {@link GlEffect} instances using OpenGL on a
@@ -51,8 +52,6 @@ import java.util.concurrent.atomic.AtomicInteger;
* @param listener A {@link Listener}.
* @param pixelWidthHeightRatio The ratio of width over height for each pixel. Pixels are expanded
* by this ratio so that the output frame's pixels have a ratio of 1.
- * @param inputWidth The input frame width, in pixels.
- * @param inputHeight The input frame height, in pixels.
* @param effects The {@link GlEffect GlEffects} to apply to each frame.
* @param outputSurfaceProvider A {@link SurfaceInfo.Provider} managing the output {@link
* Surface}.
@@ -66,16 +65,12 @@ import java.util.concurrent.atomic.AtomicInteger;
Context context,
FrameProcessor.Listener listener,
float pixelWidthHeightRatio,
- int inputWidth,
- int inputHeight,
long streamOffsetUs,
List effects,
SurfaceInfo.Provider outputSurfaceProvider,
Transformer.DebugViewProvider debugViewProvider,
boolean enableExperimentalHdrEditing)
throws FrameProcessingException {
- checkArgument(inputWidth > 0, "inputWidth must be positive");
- checkArgument(inputHeight > 0, "inputHeight must be positive");
ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME);
@@ -86,8 +81,6 @@ import java.util.concurrent.atomic.AtomicInteger;
context,
listener,
pixelWidthHeightRatio,
- inputWidth,
- inputHeight,
streamOffsetUs,
effects,
outputSurfaceProvider,
@@ -118,8 +111,6 @@ import java.util.concurrent.atomic.AtomicInteger;
Context context,
FrameProcessor.Listener listener,
float pixelWidthHeightRatio,
- int inputWidth,
- int inputHeight,
long streamOffsetUs,
List effects,
SurfaceInfo.Provider outputSurfaceProvider,
@@ -177,8 +168,7 @@ import java.util.concurrent.atomic.AtomicInteger;
eglContext,
frameProcessingTaskExecutor,
streamOffsetUs,
- /* inputExternalTexture= */ new TextureInfo(
- GlUtil.createExternalTexture(), /* fboId= */ C.INDEX_UNSET, inputWidth, inputHeight),
+ /* inputExternalTextureId= */ GlUtil.createExternalTexture(),
externalTextureProcessor,
textureProcessors);
}
@@ -300,21 +290,19 @@ import java.util.concurrent.atomic.AtomicInteger;
* timestamps, in microseconds.
*/
private final long streamOffsetUs;
- /**
- * Number of frames {@linkplain #registerInputFrame() registered} but not processed off the {@link
- * #inputSurfaceTexture} yet.
- */
- private final AtomicInteger pendingInputFrameCount;
+
/** Associated with an OpenGL external texture. */
private final SurfaceTexture inputSurfaceTexture;
/** Wraps the {@link #inputSurfaceTexture}. */
private final Surface inputSurface;
private final float[] inputSurfaceTextureTransformMatrix;
- private final TextureInfo inputExternalTexture;
+ private final int inputExternalTextureId;
private final ExternalTextureProcessor inputExternalTextureProcessor;
private final ImmutableList textureProcessors;
+ private final ConcurrentLinkedQueue pendingInputFrames;
+ private @MonotonicNonNull FrameInfo nextInputFrameInfo;
private boolean inputStreamEnded;
private GlEffectsFrameProcessor(
@@ -322,7 +310,7 @@ import java.util.concurrent.atomic.AtomicInteger;
EGLContext eglContext,
FrameProcessingTaskExecutor frameProcessingTaskExecutor,
long streamOffsetUs,
- TextureInfo inputExternalTexture,
+ int inputExternalTextureId,
ExternalTextureProcessor inputExternalTextureProcessor,
ImmutableList textureProcessors) {
checkState(!textureProcessors.isEmpty());
@@ -331,33 +319,40 @@ import java.util.concurrent.atomic.AtomicInteger;
this.eglContext = eglContext;
this.frameProcessingTaskExecutor = frameProcessingTaskExecutor;
this.streamOffsetUs = streamOffsetUs;
- this.inputExternalTexture = inputExternalTexture;
+ this.inputExternalTextureId = inputExternalTextureId;
this.inputExternalTextureProcessor = inputExternalTextureProcessor;
this.textureProcessors = textureProcessors;
- pendingInputFrameCount = new AtomicInteger();
- inputSurfaceTexture = new SurfaceTexture(inputExternalTexture.texId);
+ inputSurfaceTexture = new SurfaceTexture(inputExternalTextureId);
inputSurface = new Surface(inputSurfaceTexture);
inputSurfaceTextureTransformMatrix = new float[16];
+ pendingInputFrames = new ConcurrentLinkedQueue<>();
}
@Override
public Surface getInputSurface() {
- // TODO(b/227625423): Allow input surface to be recreated for input size change.
inputSurfaceTexture.setOnFrameAvailableListener(
surfaceTexture -> frameProcessingTaskExecutor.submit(this::processInputFrame));
return inputSurface;
}
+ @Override
+ public void setInputFrameInfo(FrameInfo inputFrameInfo) {
+ nextInputFrameInfo = inputFrameInfo;
+ }
+
@Override
public void registerInputFrame() {
checkState(!inputStreamEnded);
- pendingInputFrameCount.incrementAndGet();
+ checkStateNotNull(
+ nextInputFrameInfo, "setInputFrameInfo must be called before registering input frames");
+
+ pendingInputFrames.add(nextInputFrameInfo);
}
@Override
public int getPendingInputFrameCount() {
- return pendingInputFrameCount.get();
+ return pendingInputFrames.size();
}
@Override
@@ -401,10 +396,15 @@ import java.util.concurrent.atomic.AtomicInteger;
long presentationTimeUs = inputFrameTimeNs / 1000 - streamOffsetUs;
inputSurfaceTexture.getTransformMatrix(inputSurfaceTextureTransformMatrix);
inputExternalTextureProcessor.setTextureTransformMatrix(inputSurfaceTextureTransformMatrix);
+ FrameInfo inputFrameInfo = pendingInputFrames.remove();
checkState(
inputExternalTextureProcessor.maybeQueueInputFrame(
- inputExternalTexture, presentationTimeUs));
- checkState(pendingInputFrameCount.getAndDecrement() > 0);
+ new TextureInfo(
+ inputExternalTextureId,
+ /* fboId= */ C.INDEX_UNSET,
+ inputFrameInfo.width,
+ inputFrameInfo.height),
+ presentationTimeUs));
// After the inputExternalTextureProcessor has produced an output frame, it is processed
// asynchronously by the texture processors chained after it.
}
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
index aab980b4a5..5454e35147 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
@@ -120,8 +120,6 @@ import org.checkerframework.dataflow.qual.Pure;
}
},
inputFormat.pixelWidthHeightRatio,
- /* inputWidth= */ decodedWidth,
- /* inputHeight= */ decodedHeight,
streamOffsetUs,
effectsListBuilder.build(),
/* outputSurfaceProvider= */ encoderWrapper,
@@ -131,6 +129,7 @@ import org.checkerframework.dataflow.qual.Pure;
throw TransformationException.createForFrameProcessingException(
e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
}
+ frameProcessor.setInputFrameInfo(new FrameInfo(decodedWidth, decodedHeight));
decoder =
decoderFactory.createForVideoDecoding(