Allow FrameProcessor input surface size changes.

This will be useful for downgrading to a lower resolution during a slow preview and for processing slide-shows once sequential multi-asset editing is supported. PiperOrigin-RevId: 457017255
2025-04-30 06:46:50 +08:00 · 2022-06-24 16:31:31 +01:00 · 2022-06-24 16:31:31 +01:00 · 30c52c58f0
commit 30c52c58f0
parent 638c758721
5 changed files with 78 additions and 34 deletions
--- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java
+++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/GlEffectsFrameProcessorPixelTest.java
@ -367,8 +367,6 @@ public final class GlEffectsFrameProcessorPixelTest {
                    }
                  },
                  pixelWidthHeightRatio,
-                  inputWidth,
-                  inputHeight,
                  /* streamOffsetUs= */ 0L,
                  effects,
                  /* outputSurfaceProvider= */ (requestedWidth, requestedHeight) -> {
@ -383,6 +381,7 @@ public final class GlEffectsFrameProcessorPixelTest {
                  },
                  Transformer.DebugViewProvider.NONE,
                  /* enableExperimentalHdrEditing= */ false));
+      glEffectsFrameProcessor.setInputFrameInfo(new FrameInfo(inputWidth, inputHeight));
      glEffectsFrameProcessor.registerInputFrame();

      // Queue the first video frame from the extractor.
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameInfo.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameInfo.java
@ -0,0 +1,37 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.transformer;
+
+import static androidx.media3.common.util.Assertions.checkArgument;
+
+/** Value class specifying information about a decoded video frame. */
+/* package */ class FrameInfo {
+  /** The width of the frame, in pixels. */
+  public final int width;
+  /** The height of the frame, in pixels. */
+  public final int height;
+
+  // TODO(b/227625423): Add pixelWidthHeightRatio.
+  // TODO(b/227624622): Add color space information for HDR.
+
+  public FrameInfo(int width, int height) {
+    checkArgument(width > 0, "width must be positive, but is: " + width);
+    checkArgument(height > 0, "height must be positive, but is: " + height);
+
+    this.width = width;
+    this.height = height;
+  }
+}
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessor.java
@ -41,12 +41,21 @@ import android.view.Surface;
  /** Returns the input {@link Surface}. */
  Surface getInputSurface();

+  /**
+   * Sets information about the input frames.
+   *
+   * <p>The new input information is applied from the next frame {@linkplain #registerInputFrame()
+   * registered} onwards.
+   */
+  void setInputFrameInfo(FrameInfo inputFrameInfo);
+
  /**
   * Informs the {@code FrameProcessor} that a frame will be queued to its input surface.
   *
   * <p>Must be called before rendering a frame to the frame processor's input surface.
   *
-   * @throws IllegalStateException If called after {@link #signalEndOfInputStream()}.
+   * @throws IllegalStateException If called after {@link #signalEndOfInputStream()} or before
+   *     {@link #setInputFrameInfo(FrameInfo)}.
   */
  void registerInputFrame();

--- a/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java
@ -15,8 +15,8 @@
 */
 package androidx.media3.transformer;

-import static androidx.media3.common.util.Assertions.checkArgument;
 import static androidx.media3.common.util.Assertions.checkState;
+import static androidx.media3.common.util.Assertions.checkStateNotNull;

 import android.content.Context;
 import android.graphics.SurfaceTexture;
@ -31,10 +31,11 @@ import androidx.media3.common.util.GlUtil;
 import androidx.media3.common.util.Util;
 import com.google.common.collect.ImmutableList;
 import java.util.List;
+import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicInteger;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;

 /**
 * A {@link FrameProcessor} implementation that applies {@link GlEffect} instances using OpenGL on a
@ -51,8 +52,6 @@ import java.util.concurrent.atomic.AtomicInteger;
   * @param listener A {@link Listener}.
   * @param pixelWidthHeightRatio The ratio of width over height for each pixel. Pixels are expanded
   *     by this ratio so that the output frame's pixels have a ratio of 1.
-   * @param inputWidth The input frame width, in pixels.
-   * @param inputHeight The input frame height, in pixels.
   * @param effects The {@link GlEffect GlEffects} to apply to each frame.
   * @param outputSurfaceProvider A {@link SurfaceInfo.Provider} managing the output {@link
   *     Surface}.
@ -66,16 +65,12 @@ import java.util.concurrent.atomic.AtomicInteger;
      Context context,
      FrameProcessor.Listener listener,
      float pixelWidthHeightRatio,
-      int inputWidth,
-      int inputHeight,
      long streamOffsetUs,
      List<GlEffect> effects,
      SurfaceInfo.Provider outputSurfaceProvider,
      Transformer.DebugViewProvider debugViewProvider,
      boolean enableExperimentalHdrEditing)
      throws FrameProcessingException {
-    checkArgument(inputWidth > 0, "inputWidth must be positive");
-    checkArgument(inputHeight > 0, "inputHeight must be positive");

    ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME);

@ -86,8 +81,6 @@ import java.util.concurrent.atomic.AtomicInteger;
                    context,
                    listener,
                    pixelWidthHeightRatio,
-                    inputWidth,
-                    inputHeight,
                    streamOffsetUs,
                    effects,
                    outputSurfaceProvider,
@ -118,8 +111,6 @@ import java.util.concurrent.atomic.AtomicInteger;
      Context context,
      FrameProcessor.Listener listener,
      float pixelWidthHeightRatio,
-      int inputWidth,
-      int inputHeight,
      long streamOffsetUs,
      List<GlEffect> effects,
      SurfaceInfo.Provider outputSurfaceProvider,
@ -177,8 +168,7 @@ import java.util.concurrent.atomic.AtomicInteger;
        eglContext,
        frameProcessingTaskExecutor,
        streamOffsetUs,
-        /* inputExternalTexture= */ new TextureInfo(
-            GlUtil.createExternalTexture(), /* fboId= */ C.INDEX_UNSET, inputWidth, inputHeight),
+        /* inputExternalTextureId= */ GlUtil.createExternalTexture(),
        externalTextureProcessor,
        textureProcessors);
  }
@ -300,21 +290,19 @@ import java.util.concurrent.atomic.AtomicInteger;
   * timestamps, in microseconds.
   */
  private final long streamOffsetUs;
-  /**
-   * Number of frames {@linkplain #registerInputFrame() registered} but not processed off the {@link
-   * #inputSurfaceTexture} yet.
-   */
-  private final AtomicInteger pendingInputFrameCount;
+
  /** Associated with an OpenGL external texture. */
  private final SurfaceTexture inputSurfaceTexture;
  /** Wraps the {@link #inputSurfaceTexture}. */
  private final Surface inputSurface;

  private final float[] inputSurfaceTextureTransformMatrix;
-  private final TextureInfo inputExternalTexture;
+  private final int inputExternalTextureId;
  private final ExternalTextureProcessor inputExternalTextureProcessor;
  private final ImmutableList<GlTextureProcessor> textureProcessors;
+  private final ConcurrentLinkedQueue<FrameInfo> pendingInputFrames;

+  private @MonotonicNonNull FrameInfo nextInputFrameInfo;
  private boolean inputStreamEnded;

  private GlEffectsFrameProcessor(
@ -322,7 +310,7 @@ import java.util.concurrent.atomic.AtomicInteger;
      EGLContext eglContext,
      FrameProcessingTaskExecutor frameProcessingTaskExecutor,
      long streamOffsetUs,
-      TextureInfo inputExternalTexture,
+      int inputExternalTextureId,
      ExternalTextureProcessor inputExternalTextureProcessor,
      ImmutableList<GlTextureProcessor> textureProcessors) {
    checkState(!textureProcessors.isEmpty());
@ -331,33 +319,40 @@ import java.util.concurrent.atomic.AtomicInteger;
    this.eglContext = eglContext;
    this.frameProcessingTaskExecutor = frameProcessingTaskExecutor;
    this.streamOffsetUs = streamOffsetUs;
-    this.inputExternalTexture = inputExternalTexture;
+    this.inputExternalTextureId = inputExternalTextureId;
    this.inputExternalTextureProcessor = inputExternalTextureProcessor;
    this.textureProcessors = textureProcessors;

-    pendingInputFrameCount = new AtomicInteger();
-    inputSurfaceTexture = new SurfaceTexture(inputExternalTexture.texId);
+    inputSurfaceTexture = new SurfaceTexture(inputExternalTextureId);
    inputSurface = new Surface(inputSurfaceTexture);
    inputSurfaceTextureTransformMatrix = new float[16];
+    pendingInputFrames = new ConcurrentLinkedQueue<>();
  }

  @Override
  public Surface getInputSurface() {
-    // TODO(b/227625423): Allow input surface to be recreated for input size change.
    inputSurfaceTexture.setOnFrameAvailableListener(
        surfaceTexture -> frameProcessingTaskExecutor.submit(this::processInputFrame));
    return inputSurface;
  }

+  @Override
+  public void setInputFrameInfo(FrameInfo inputFrameInfo) {
+    nextInputFrameInfo = inputFrameInfo;
+  }
+
  @Override
  public void registerInputFrame() {
    checkState(!inputStreamEnded);
-    pendingInputFrameCount.incrementAndGet();
+    checkStateNotNull(
+        nextInputFrameInfo, "setInputFrameInfo must be called before registering input frames");
+
+    pendingInputFrames.add(nextInputFrameInfo);
  }

  @Override
  public int getPendingInputFrameCount() {
-    return pendingInputFrameCount.get();
+    return pendingInputFrames.size();
  }

  @Override
@ -401,10 +396,15 @@ import java.util.concurrent.atomic.AtomicInteger;
    long presentationTimeUs = inputFrameTimeNs / 1000 - streamOffsetUs;
    inputSurfaceTexture.getTransformMatrix(inputSurfaceTextureTransformMatrix);
    inputExternalTextureProcessor.setTextureTransformMatrix(inputSurfaceTextureTransformMatrix);
+    FrameInfo inputFrameInfo = pendingInputFrames.remove();
    checkState(
        inputExternalTextureProcessor.maybeQueueInputFrame(
-            inputExternalTexture, presentationTimeUs));
-    checkState(pendingInputFrameCount.getAndDecrement() > 0);
+            new TextureInfo(
+                inputExternalTextureId,
+                /* fboId= */ C.INDEX_UNSET,
+                inputFrameInfo.width,
+                inputFrameInfo.height),
+            presentationTimeUs));
    // After the inputExternalTextureProcessor has produced an output frame, it is processed
    // asynchronously by the texture processors chained after it.
  }
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
@ -120,8 +120,6 @@ import org.checkerframework.dataflow.qual.Pure;
                }
              },
              inputFormat.pixelWidthHeightRatio,
-              /* inputWidth= */ decodedWidth,
-              /* inputHeight= */ decodedHeight,
              streamOffsetUs,
              effectsListBuilder.build(),
              /* outputSurfaceProvider= */ encoderWrapper,
@ -131,6 +129,7 @@ import org.checkerframework.dataflow.qual.Pure;
      throw TransformationException.createForFrameProcessingException(
          e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
    }
+    frameProcessor.setInputFrameInfo(new FrameInfo(decodedWidth, decodedHeight));

    decoder =
        decoderFactory.createForVideoDecoding(