loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add input type for automatic frame registration

Browse Source 
Deprecate `setInputDefaultBufferSize` and `setRequireRegisteringAllInputFrames`
as the new input stream type replaces these (as far as we know they are always
used together).

This is in preparation for supporting asset loaders signaling that they require
these features, specifically for recording from a surface.

PiperOrigin-RevId: 648686087
This commit is contained in:
andrewlewis 2024-07-02 05:38:56 -07:00 committed by Copybara-Service
parent a2087ba5bb
commit 95260e28a5
8 changed files with 160 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
RELEASENOTES.md
View File

@ -31,6 +31,10 @@
* Image:
* DRM:
* Effect:
* Deprecate `DefaultVideoFrameProcessor#setInputDefaultBufferSize` and
`DefaultVideoFrameProcessor.Builder#setRequireRegisteringAllInputFrames`.
Use the new frame processor input type
`INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION` instead.
* Muxers:
* IMA extension:
* Session:

20
libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java
View File

@ -48,12 +48,18 @@ import java.util.concurrent.Executor;
public interface VideoFrameProcessor {
/**
* Specifies how the input frames are made available to the {@link VideoFrameProcessor}. One of
* {@link #INPUT_TYPE_SURFACE}, {@link #INPUT_TYPE_BITMAP} or {@link #INPUT_TYPE_TEXTURE_ID}.
* {@link #INPUT_TYPE_SURFACE}, {@link #INPUT_TYPE_BITMAP}, {@link #INPUT_TYPE_TEXTURE_ID} or
* {@link #INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION}.
*/
@Documented
@Retention(RetentionPolicy.SOURCE)
@Target(TYPE_USE)
@IntDef({INPUT_TYPE_SURFACE, INPUT_TYPE_BITMAP, INPUT_TYPE_TEXTURE_ID})
@IntDef({
INPUT_TYPE_SURFACE,
INPUT_TYPE_BITMAP,
INPUT_TYPE_TEXTURE_ID,
INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION,
})
@interface InputType {}
/**
@ -73,6 +79,16 @@ public interface VideoFrameProcessor {
*/
int INPUT_TYPE_TEXTURE_ID = 3;
/**
* Input frames come from the {@linkplain #getInputSurface input surface} and don't need to be
* {@linkplain #registerInputFrame registered} (unlike with {@link #INPUT_TYPE_SURFACE}).
*
* <p>Every frame must use the {@linkplain #registerInputStream(int, List, FrameInfo) input
* stream's registered} frame info. Also sets the surface's {@linkplain
* android.graphics.SurfaceTexture#setDefaultBufferSize(int, int) default buffer size}.
*/
int INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION = 4;
/** A factory for {@link VideoFrameProcessor} instances. */
interface Factory {

74
libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorTest.java
View File

@ -16,12 +16,15 @@
package androidx.media3.effect;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmapUnpremultipliedAlpha;
import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import static com.google.common.truth.Truth.assertThat;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.view.Surface;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.ColorInfo;
@ -32,8 +35,10 @@ import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.util.ConditionVariable;
import androidx.media3.common.util.ConstantRateTimestampIterator;
import androidx.media3.common.util.NullableType;
import androidx.media3.common.util.SystemClock;
import androidx.media3.common.util.Util;
import androidx.media3.test.utils.BitmapPixelTestUtil;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.MoreExecutors;
@ -41,6 +46,7 @@ import java.util.List;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@ -288,6 +294,74 @@ public class DefaultVideoFrameProcessorTest {
.isAtLeast(firstStreamLastFrameAvailableTimeMs.get());
}
@Test
public void registerInputStreamWithAutomaticFrameRegistration_succeeds() throws Exception {
CountDownLatch inputStreamRegisteredCountDownLatch = new CountDownLatch(1);
AtomicInteger outputFrameCount = new AtomicInteger();
AtomicReference<@NullableType Exception> error = new AtomicReference<>();
CountDownLatch endedCountDownLatch = new CountDownLatch(1);
defaultVideoFrameProcessor =
createDefaultVideoFrameProcessor(
new VideoFrameProcessor.Listener() {
@Override
public void onInputStreamRegistered(
@VideoFrameProcessor.InputType int inputType,
List<Effect> effects,
FrameInfo frameInfo) {
inputStreamRegisteredCountDownLatch.countDown();
}
@Override
public void onOutputSizeChanged(int width, int height) {}
@Override
public void onOutputFrameAvailableForRendering(long presentationTimeUs) {
outputFrameCount.incrementAndGet();
}
@Override
public void onError(VideoFrameProcessingException exception) {
error.set(exception);
}
@Override
public void onEnded() {
endedCountDownLatch.countDown();
}
});
Bitmap bitmap = BitmapPixelTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH);
defaultVideoFrameProcessor.registerInputStream(
VideoFrameProcessor.INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION,
/* effects= */ ImmutableList.of(),
new FrameInfo.Builder(ColorInfo.SRGB_BT709_FULL, bitmap.getWidth(), bitmap.getHeight())
.build());
inputStreamRegisteredCountDownLatch.await();
checkState(defaultVideoFrameProcessor.registerInputFrame());
int inputFrameCount = 2;
Surface surface = defaultVideoFrameProcessor.getInputSurface();
for (int i = 0; i < inputFrameCount; i++) {
Canvas canvas = surface.lockCanvas(/* inOutDirty= */ null);
// Load the bitmap each time, as it's recycled after each use.
canvas.drawBitmap(
BitmapPixelTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH),
/* left= */ 0f,
/* top= */ 0f,
/* paint= */ null);
// This causes a frame to become available on the input surface, which is processed by the
// video frame processor.
surface.unlockCanvasAndPost(canvas);
}
defaultVideoFrameProcessor.signalEndOfInput();
if (!endedCountDownLatch.await(TEST_TIMEOUT_MS, MILLISECONDS)) {
throw new IllegalStateException("Test timeout", error.get());
}
assertThat(error.get()).isNull();
assertThat(outputFrameCount.get()).isEqualTo(inputFrameCount);
}
private DefaultVideoFrameProcessor createDefaultVideoFrameProcessor(
VideoFrameProcessor.Listener listener) throws Exception {
return checkNotNull(factory)

13
libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java
View File

@ -199,7 +199,14 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
* <p>Regardless of the value set, {@link #registerInputStream(int, List, FrameInfo)} must be
* called for each input stream to specify the format for upcoming frames before calling
* {@link #registerInputFrame()}.
*
* @param requireRegisteringAllInputFrames Whether registering every input frame is required.
* @deprecated For automatic frame registration ({@code
* setRequireRegisteringAllInputFrames(false)}), use {@link
* VideoFrameProcessor#INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION} instead. This call
* can be removed otherwise.
*/
@Deprecated
@CanIgnoreReturnValue
public Builder setRequireRegisteringAllInputFrames(boolean requireRegisteringAllInputFrames) {
this.requireRegisteringAllInputFrames = requireRegisteringAllInputFrames;
@ -514,7 +521,11 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
*
* @param width The default width for input buffers, in pixels.
* @param height The default height for input buffers, in pixels.
* @deprecated Set the input type to {@link
* VideoFrameProcessor#INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION} instead, which sets
* the default buffer size automatically based on the registered frame info.
*/
@Deprecated
public void setInputDefaultBufferSize(int width, int height) {
inputSwitcher.setInputDefaultBufferSize(width, height);
}
@ -933,6 +944,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
return "Bitmap";
case INPUT_TYPE_TEXTURE_ID:
return "Texture ID";
case INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION:
return "Surface with automatic frame registration";
default:
throw new IllegalArgumentException(String.valueOf(inputType));
}

13
libraries/effect/src/main/java/androidx/media3/effect/ExternalTextureManager.java
View File

@ -87,7 +87,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final float[] textureTransformMatrix;
private final Queue<FrameInfo> pendingFrames;
private final ScheduledExecutorService scheduledExecutorService;
private final boolean repeatLastRegisteredFrame;
private final boolean experimentalAdjustSurfaceTextureTransformationMatrix;
// Must be accessed on the GL thread.
@ -98,6 +97,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
// The frame that is sent downstream and is not done processing yet.
@Nullable private FrameInfo currentFrame;
@Nullable private FrameInfo lastRegisteredFrame;
private boolean repeatLastRegisteredFrame;
@Nullable private Future<?> forceSignalEndOfStreamFuture;
private boolean shouldRejectIncomingFrames;
@ -234,6 +234,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
});
}
@Override
public void setInputFrameInfo(FrameInfo inputFrameInfo, boolean automaticReregistration) {
// Ignore inputFrameInfo when not automatically re-registering frames because it's also passed
// to registerInputFrame.
repeatLastRegisteredFrame = automaticReregistration;
if (repeatLastRegisteredFrame) {
lastRegisteredFrame = inputFrameInfo;
surfaceTexture.setDefaultBufferSize(inputFrameInfo.width, inputFrameInfo.height);
}
}
/**
* Notifies the {@code ExternalTextureManager} that a frame with the given {@link FrameInfo} will
* become available via the {@link SurfaceTexture} eventually.

56
libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java
View File

@ -19,6 +19,7 @@ package androidx.media3.effect;
import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP;
import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE;
import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION;
import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
@ -81,15 +82,16 @@ import org.checkerframework.checker.nullness.qual.Nullable;
this.experimentalAdjustSurfaceTextureTransformationMatrix =
experimentalAdjustSurfaceTextureTransformationMatrix;
// TODO(b/274109008): Investigate lazy instantiating the texture managers.
inputs.put(
INPUT_TYPE_SURFACE,
// TODO(b/274109008): Investigate lazily instantiating the texture managers.
Input surfaceInput =
new Input(
new ExternalTextureManager(
glObjectsProvider,
videoFrameProcessingTaskExecutor,
repeatLastRegisteredFrame,
experimentalAdjustSurfaceTextureTransformationMatrix)));
experimentalAdjustSurfaceTextureTransformationMatrix));
inputs.put(INPUT_TYPE_SURFACE, surfaceInput);
inputs.put(INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION, surfaceInput);
inputs.put(
INPUT_TYPE_BITMAP,
new Input(
@ -109,6 +111,7 @@ import org.checkerframework.checker.nullness.qual.Nullable;
DefaultShaderProgram samplingShaderProgram;
switch (inputType) {
case INPUT_TYPE_SURFACE:
case INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION:
samplingShaderProgram =
DefaultShaderProgram.createWithExternalSampler(
context,
@ -152,29 +155,28 @@ import org.checkerframework.checker.nullness.qual.Nullable;
checkState(contains(inputs, newInputType), "Input type not registered: " + newInputType);
for (int i = 0; i < inputs.size(); i++) {
@VideoFrameProcessor.InputType int inputType = inputs.keyAt(i);
Input input = inputs.get(inputType);
if (inputType == newInputType) {
if (input.getInputColorInfo() == null
|| !newInputFrameInfo.colorInfo.equals(input.getInputColorInfo())) {
input.setSamplingGlShaderProgram(
createSamplingShaderProgram(newInputFrameInfo.colorInfo, newInputType));
input.setInputColorInfo(newInputFrameInfo.colorInfo);
}
input.setChainingListener(
new GatedChainingListenerWrapper(
glObjectsProvider,
checkNotNull(input.getSamplingGlShaderProgram()),
this.downstreamShaderProgram,
videoFrameProcessingTaskExecutor));
input.setActive(true);
downstreamShaderProgram.setInputListener(checkNotNull(input.gatedChainingListenerWrapper));
activeTextureManager = input.textureManager;
} else {
input.setActive(false);
}
inputs.get(inputs.keyAt(i)).setActive(false);
}
checkNotNull(activeTextureManager).setInputFrameInfo(newInputFrameInfo);
// Activate the relevant input for the new input type.
Input input = inputs.get(newInputType);
if (input.getInputColorInfo() == null
|| !newInputFrameInfo.colorInfo.equals(input.getInputColorInfo())) {
input.setSamplingGlShaderProgram(
createSamplingShaderProgram(newInputFrameInfo.colorInfo, newInputType));
input.setInputColorInfo(newInputFrameInfo.colorInfo);
}
input.setChainingListener(
new GatedChainingListenerWrapper(
glObjectsProvider,
checkNotNull(input.getSamplingGlShaderProgram()),
this.downstreamShaderProgram,
videoFrameProcessingTaskExecutor));
input.setActive(true);
downstreamShaderProgram.setInputListener(checkNotNull(input.gatedChainingListenerWrapper));
activeTextureManager = input.textureManager;
boolean automaticRegistration = newInputType == INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION;
checkNotNull(activeTextureManager).setInputFrameInfo(newInputFrameInfo, automaticRegistration);
}
/** Returns whether the {@code InputSwitcher} is connected to an active input. */
@ -203,7 +205,7 @@ import org.checkerframework.checker.nullness.qual.Nullable;
/**
* Returns the input {@link Surface}.
*
* @return The input {@link Surface}, regardless if the current input is {@linkplain
* @return The input {@link Surface}, regardless of whether the current input is {@linkplain
* #switchToInput set} to {@link VideoFrameProcessor#INPUT_TYPE_SURFACE}.
*/
public Surface getInputSurface() {

2
libraries/effect/src/main/java/androidx/media3/effect/TexIdTextureManager.java
View File

@ -106,7 +106,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
@Override
public void setInputFrameInfo(FrameInfo inputFrameInfo) {
public void setInputFrameInfo(FrameInfo inputFrameInfo, boolean automaticReregistration) {
this.inputFrameInfo = inputFrameInfo;
}

11
libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java
View File

@ -107,12 +107,19 @@ import androidx.media3.common.util.TimestampIterator;
* Sets information about the input frames.
*
* <p>The new input information is applied from the next frame {@linkplain #registerInputFrame
* registered} or {@linkplain #queueInputTexture queued} onwards.
* registered} or {@linkplain #queueInputTexture queued} onwards. If the implementation requires
* frames to be registered, it may use the {@link FrameInfo} passed to {@link
* #registerInputFrame(FrameInfo)} instead of the one passed here.
*
* <p>Pixels are expanded using the {@link FrameInfo#pixelWidthHeightRatio} so that the output
* frames' pixels have a ratio of 1.
*
* @param inputFrameInfo Information about the next input frame.
* @param automaticReregistration Whether the frames should be re-registered automatically, if
* using an input surface. Pass {@code false} if every frame will be registered before it is
* rendered to the surface.
*/
public void setInputFrameInfo(FrameInfo inputFrameInfo) {
public void setInputFrameInfo(FrameInfo inputFrameInfo, boolean automaticReregistration) {
// Do nothing.
}