Compositor: Implement OverlaySettings and custom in/out size support.
Implement VideoCompositor support of: * Different input and output sizes * CompositorSettings, to customize output size based on input texture sizes * OverlaySettings, to place an input frame in an arbitrary position on the output frame. Also, refactor Overlay's matrix logic to make it more reusable between Compositor and Overlays PiperOrigin-RevId: 561931854
This commit is contained in:
parent
6c2713f153
commit
57bc215210
@ -36,6 +36,7 @@ import androidx.media3.common.util.GlProgram;
|
||||
import androidx.media3.common.util.GlUtil;
|
||||
import androidx.media3.common.util.Log;
|
||||
import androidx.media3.common.util.LongArrayQueue;
|
||||
import androidx.media3.common.util.Size;
|
||||
import androidx.media3.common.util.UnstableApi;
|
||||
import androidx.media3.common.util.Util;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
@ -50,8 +51,8 @@ import java.util.concurrent.ExecutorService;
|
||||
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
|
||||
/**
|
||||
* A basic {@link VideoCompositor} implementation that takes in frames from exactly 2 input sources'
|
||||
* streams and combines them into one output stream.
|
||||
* A basic {@link VideoCompositor} implementation that takes in frames from exactly 2 SDR input
|
||||
* sources' streams and combines them into one output stream.
|
||||
*
|
||||
* <p>The first {@linkplain #registerInputSource registered source} will be the primary stream,
|
||||
* which is used to determine the output frames' timestamps and dimensions.
|
||||
@ -62,17 +63,33 @@ public final class DefaultVideoCompositor implements VideoCompositor {
|
||||
// * Use a lock to synchronize inputFrameInfos more narrowly, to reduce blocking.
|
||||
// * If the primary stream ends, consider setting the secondary stream as the new primary stream,
|
||||
// so that secondary stream frames aren't dropped.
|
||||
// * Add support for HDR input.
|
||||
|
||||
/** A default implementation of {@link VideoCompositor.Settings}. */
|
||||
public static final class Settings implements VideoCompositor.Settings {
|
||||
@Override
|
||||
public Size getOutputSize(List<Size> inputSizes) {
|
||||
return inputSizes.get(PRIMARY_INPUT_ID);
|
||||
}
|
||||
|
||||
@Override
|
||||
public OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs) {
|
||||
return new OverlaySettings.Builder().build();
|
||||
}
|
||||
}
|
||||
|
||||
private static final String THREAD_NAME = "Effect:DefaultVideoCompositor:GlThread";
|
||||
private static final String TAG = "DefaultVideoCompositor";
|
||||
private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl";
|
||||
private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_copy_es2.glsl";
|
||||
private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_alpha_scale_es2.glsl";
|
||||
private static final int PRIMARY_INPUT_ID = 0;
|
||||
|
||||
private final Context context;
|
||||
private final VideoCompositor.Listener listener;
|
||||
private final GlTextureProducer.Listener textureOutputListener;
|
||||
private final GlObjectsProvider glObjectsProvider;
|
||||
private final VideoCompositor.Settings settings;
|
||||
private final OverlayMatrixProvider overlayMatrixProvider;
|
||||
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
|
||||
|
||||
@GuardedBy("this")
|
||||
@ -100,6 +117,7 @@ public final class DefaultVideoCompositor implements VideoCompositor {
|
||||
public DefaultVideoCompositor(
|
||||
Context context,
|
||||
GlObjectsProvider glObjectsProvider,
|
||||
VideoCompositor.Settings settings,
|
||||
@Nullable ExecutorService executorService,
|
||||
VideoCompositor.Listener listener,
|
||||
GlTextureProducer.Listener textureOutputListener,
|
||||
@ -108,6 +126,8 @@ public final class DefaultVideoCompositor implements VideoCompositor {
|
||||
this.listener = listener;
|
||||
this.textureOutputListener = textureOutputListener;
|
||||
this.glObjectsProvider = glObjectsProvider;
|
||||
this.settings = settings;
|
||||
this.overlayMatrixProvider = new OverlayMatrixProvider();
|
||||
|
||||
inputSources = new ArrayList<>();
|
||||
outputTexturePool =
|
||||
@ -180,7 +200,11 @@ public final class DefaultVideoCompositor implements VideoCompositor {
|
||||
checkState(!inputSource.isInputEnded);
|
||||
|
||||
InputFrameInfo inputFrameInfo =
|
||||
new InputFrameInfo(textureProducer, inputTexture, presentationTimeUs);
|
||||
new InputFrameInfo(
|
||||
textureProducer,
|
||||
inputTexture,
|
||||
presentationTimeUs,
|
||||
settings.getOverlaySettings(inputId, presentationTimeUs));
|
||||
inputSource.frameInfos.add(inputFrameInfo);
|
||||
|
||||
if (inputId == PRIMARY_INPUT_ID) {
|
||||
@ -277,17 +301,17 @@ public final class DefaultVideoCompositor implements VideoCompositor {
|
||||
|
||||
ensureGlProgramConfigured();
|
||||
|
||||
// TODO: b/262694346 - Allow different input frame dimensions.
|
||||
InputFrameInfo primaryInputFrame = framesToComposite.get(PRIMARY_INPUT_ID);
|
||||
GlTextureInfo primaryInputTexture = primaryInputFrame.texture;
|
||||
outputTexturePool.ensureConfigured(
|
||||
glObjectsProvider, primaryInputTexture.width, primaryInputTexture.height);
|
||||
|
||||
for (int i = 1; i < framesToComposite.size(); i++) {
|
||||
GlTextureInfo textureToComposite = framesToComposite.get(i).texture;
|
||||
checkState(primaryInputTexture.width == textureToComposite.width);
|
||||
checkState(primaryInputTexture.height == textureToComposite.height);
|
||||
ImmutableList.Builder<Size> inputSizes = new ImmutableList.Builder<>();
|
||||
for (int i = 0; i < framesToComposite.size(); i++) {
|
||||
GlTextureInfo texture = framesToComposite.get(i).texture;
|
||||
inputSizes.add(new Size(texture.width, texture.height));
|
||||
}
|
||||
Size outputSize = settings.getOutputSize(inputSizes.build());
|
||||
outputTexturePool.ensureConfigured(
|
||||
glObjectsProvider, outputSize.getWidth(), outputSize.getHeight());
|
||||
|
||||
GlTextureInfo outputTexture = outputTexturePool.useTexture();
|
||||
long outputPresentationTimestampUs = primaryInputFrame.presentationTimeUs;
|
||||
outputTextureTimestamps.add(outputPresentationTimestampUs);
|
||||
@ -394,16 +418,18 @@ public final class DefaultVideoCompositor implements VideoCompositor {
|
||||
GlUtil.getNormalizedCoordinateBounds(),
|
||||
GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
|
||||
glProgram.setFloatsUniform("uTexTransformationMatrix", GlUtil.create4x4IdentityMatrix());
|
||||
glProgram.setFloatsUniform("uTransformationMatrix", GlUtil.create4x4IdentityMatrix());
|
||||
} catch (IOException e) {
|
||||
throw new VideoFrameProcessingException(e);
|
||||
}
|
||||
}
|
||||
|
||||
// Enhanced for-loops are discouraged in media3.effect due to short-lived allocations.
|
||||
@SuppressWarnings("ListReverse")
|
||||
private void drawFrame(List<InputFrameInfo> framesToComposite, GlTextureInfo outputTexture)
|
||||
throws GlUtil.GlException {
|
||||
GlUtil.focusFramebufferUsingCurrentContext(
|
||||
outputTexture.fboId, outputTexture.width, outputTexture.height);
|
||||
overlayMatrixProvider.configure(new Size(outputTexture.width, outputTexture.height));
|
||||
GlUtil.clearFocusedBuffers();
|
||||
|
||||
GlProgram glProgram = checkNotNull(this.glProgram);
|
||||
@ -423,7 +449,7 @@ public final class DefaultVideoCompositor implements VideoCompositor {
|
||||
|
||||
// Draw textures from back to front.
|
||||
for (int i = framesToComposite.size() - 1; i >= 0; i--) {
|
||||
blendOntoFocusedTexture(framesToComposite.get(i).texture.texId);
|
||||
blendOntoFocusedTexture(framesToComposite.get(i));
|
||||
}
|
||||
|
||||
GLES20.glDisable(GLES20.GL_BLEND);
|
||||
@ -431,9 +457,16 @@ public final class DefaultVideoCompositor implements VideoCompositor {
|
||||
GlUtil.checkGlError();
|
||||
}
|
||||
|
||||
private void blendOntoFocusedTexture(int texId) throws GlUtil.GlException {
|
||||
private void blendOntoFocusedTexture(InputFrameInfo inputFrameInfo) throws GlUtil.GlException {
|
||||
GlProgram glProgram = checkNotNull(this.glProgram);
|
||||
glProgram.setSamplerTexIdUniform("uTexSampler", texId, /* texUnitIndex= */ 0);
|
||||
GlTextureInfo inputTexture = inputFrameInfo.texture;
|
||||
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexture.texId, /* texUnitIndex= */ 0);
|
||||
float[] transformationMatrix =
|
||||
overlayMatrixProvider.getTransformationMatrix(
|
||||
/* overlaySize= */ new Size(inputTexture.width, inputTexture.height),
|
||||
inputFrameInfo.overlaySettings);
|
||||
glProgram.setFloatsUniform("uTransformationMatrix", transformationMatrix);
|
||||
glProgram.setFloatUniform("uAlphaScale", inputFrameInfo.overlaySettings.alphaScale);
|
||||
glProgram.bindAttributesAndUniforms();
|
||||
|
||||
// The four-vertex triangle strip forms a quad.
|
||||
@ -480,12 +513,17 @@ public final class DefaultVideoCompositor implements VideoCompositor {
|
||||
public final GlTextureProducer textureProducer;
|
||||
public final GlTextureInfo texture;
|
||||
public final long presentationTimeUs;
|
||||
public final OverlaySettings overlaySettings;
|
||||
|
||||
public InputFrameInfo(
|
||||
GlTextureProducer textureProducer, GlTextureInfo texture, long presentationTimeUs) {
|
||||
GlTextureProducer textureProducer,
|
||||
GlTextureInfo texture,
|
||||
long presentationTimeUs,
|
||||
OverlaySettings overlaySettings) {
|
||||
this.textureProducer = textureProducer;
|
||||
this.texture = texture;
|
||||
this.presentationTimeUs = presentationTimeUs;
|
||||
this.overlaySettings = overlaySettings;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -15,7 +15,7 @@
|
||||
*/
|
||||
package androidx.media3.effect;
|
||||
|
||||
import static androidx.media3.common.util.Assertions.checkNotNull;
|
||||
import static androidx.media3.common.util.Assertions.checkStateNotNull;
|
||||
|
||||
import android.opengl.Matrix;
|
||||
import android.util.Pair;
|
||||
@ -23,15 +23,14 @@ import androidx.media3.common.util.GlUtil;
|
||||
import androidx.media3.common.util.Size;
|
||||
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
|
||||
/* package */ final class OverlayMatrixProvider {
|
||||
private static final int MATRIX_OFFSET = 0;
|
||||
/** Provides a matrix for {@link OverlaySettings}, to be applied on a vertex. */
|
||||
/* package */ class OverlayMatrixProvider {
|
||||
protected static final int MATRIX_OFFSET = 0;
|
||||
private final float[] videoFrameAnchorMatrix;
|
||||
private final float[] videoFrameAnchorMatrixInv;
|
||||
private final float[] aspectRatioMatrix;
|
||||
private final float[] scaleMatrix;
|
||||
private final float[] scaleMatrixInv;
|
||||
private final float[] overlayAnchorMatrix;
|
||||
private final float[] overlayAnchorMatrixInv;
|
||||
private final float[] rotateMatrix;
|
||||
private final float[] overlayAspectRatioMatrix;
|
||||
private final float[] overlayAspectRatioMatrixInv;
|
||||
@ -41,9 +40,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
public OverlayMatrixProvider() {
|
||||
aspectRatioMatrix = GlUtil.create4x4IdentityMatrix();
|
||||
videoFrameAnchorMatrix = GlUtil.create4x4IdentityMatrix();
|
||||
videoFrameAnchorMatrixInv = GlUtil.create4x4IdentityMatrix();
|
||||
overlayAnchorMatrix = GlUtil.create4x4IdentityMatrix();
|
||||
overlayAnchorMatrixInv = GlUtil.create4x4IdentityMatrix();
|
||||
rotateMatrix = GlUtil.create4x4IdentityMatrix();
|
||||
scaleMatrix = GlUtil.create4x4IdentityMatrix();
|
||||
scaleMatrixInv = GlUtil.create4x4IdentityMatrix();
|
||||
@ -56,6 +53,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
this.backgroundSize = backgroundSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the transformation matrix.
|
||||
*
|
||||
* <p>This instance must be {@linkplain #configure configured} before this method is called.
|
||||
*/
|
||||
public float[] getTransformationMatrix(Size overlaySize, OverlaySettings overlaySettings) {
|
||||
reset();
|
||||
|
||||
@ -67,44 +69,38 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
videoFrameAnchor.first,
|
||||
videoFrameAnchor.second,
|
||||
/* z= */ 0f);
|
||||
Matrix.invertM(videoFrameAnchorMatrixInv, MATRIX_OFFSET, videoFrameAnchorMatrix, MATRIX_OFFSET);
|
||||
|
||||
checkStateNotNull(backgroundSize);
|
||||
Matrix.scaleM(
|
||||
aspectRatioMatrix,
|
||||
MATRIX_OFFSET,
|
||||
checkNotNull(backgroundSize).getWidth() / (float) overlaySize.getWidth(),
|
||||
checkNotNull(backgroundSize).getHeight() / (float) overlaySize.getHeight(),
|
||||
(float) overlaySize.getWidth() / backgroundSize.getWidth(),
|
||||
(float) overlaySize.getHeight() / backgroundSize.getHeight(),
|
||||
/* z= */ 1f);
|
||||
|
||||
// Scale the image.
|
||||
Pair<Float, Float> scale = overlaySettings.scale;
|
||||
Matrix.scaleM(
|
||||
scaleMatrix,
|
||||
MATRIX_OFFSET,
|
||||
scaleMatrix,
|
||||
MATRIX_OFFSET,
|
||||
scale.first,
|
||||
scale.second,
|
||||
/* z= */ 1f);
|
||||
Matrix.scaleM(scaleMatrix, MATRIX_OFFSET, scale.first, scale.second, /* z= */ 1f);
|
||||
Matrix.invertM(scaleMatrixInv, MATRIX_OFFSET, scaleMatrix, MATRIX_OFFSET);
|
||||
|
||||
// Translate the overlay within its frame.
|
||||
// Translate the overlay within its frame. To position the overlay's anchor at the correct
|
||||
// position, it must be translated the opposite direction by the same magnitude.
|
||||
Pair<Float, Float> overlayAnchor = overlaySettings.overlayAnchor;
|
||||
Matrix.translateM(
|
||||
overlayAnchorMatrix, MATRIX_OFFSET, overlayAnchor.first, overlayAnchor.second, /* z= */ 0f);
|
||||
Matrix.invertM(overlayAnchorMatrixInv, MATRIX_OFFSET, overlayAnchorMatrix, MATRIX_OFFSET);
|
||||
overlayAnchorMatrix,
|
||||
MATRIX_OFFSET,
|
||||
-1 * overlayAnchor.first,
|
||||
-1 * overlayAnchor.second,
|
||||
/* z= */ 0f);
|
||||
|
||||
// Rotate the image.
|
||||
Matrix.rotateM(
|
||||
rotateMatrix,
|
||||
MATRIX_OFFSET,
|
||||
rotateMatrix,
|
||||
MATRIX_OFFSET,
|
||||
overlaySettings.rotationDegrees,
|
||||
/* x= */ 0f,
|
||||
/* y= */ 0f,
|
||||
/* z= */ 1f);
|
||||
Matrix.invertM(rotateMatrix, MATRIX_OFFSET, rotateMatrix, MATRIX_OFFSET);
|
||||
|
||||
// Rotation matrix needs to account for overlay aspect ratio to prevent stretching.
|
||||
Matrix.scaleM(
|
||||
@ -116,67 +112,18 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
Matrix.invertM(
|
||||
overlayAspectRatioMatrixInv, MATRIX_OFFSET, overlayAspectRatioMatrix, MATRIX_OFFSET);
|
||||
|
||||
// Rotation needs to be agnostic of the scaling matrix and the aspect ratios.
|
||||
// transformationMatrix = scaleMatrixInv * overlayAspectRatioMatrix * rotateMatrix *
|
||||
// overlayAspectRatioInv * scaleMatrix * overlayAnchorMatrixInv * scaleMatrixInv *
|
||||
// aspectRatioMatrix * videoFrameAnchorMatrixInv
|
||||
Matrix.multiplyMM(
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
scaleMatrixInv,
|
||||
MATRIX_OFFSET);
|
||||
// transformationMatrix = videoFrameAnchorMatrix * aspectRatioMatrix
|
||||
// * scaleMatrix * overlayAnchorMatrix * scaleMatrixInv
|
||||
// * overlayAspectRatioMatrix * rotateMatrix * overlayAspectRatioMatrixInv
|
||||
// * scaleMatrix.
|
||||
|
||||
// Anchor position in output frame.
|
||||
Matrix.multiplyMM(
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
overlayAspectRatioMatrix,
|
||||
MATRIX_OFFSET);
|
||||
|
||||
// Rotation matrix.
|
||||
Matrix.multiplyMM(
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
rotateMatrix,
|
||||
MATRIX_OFFSET);
|
||||
|
||||
Matrix.multiplyMM(
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
overlayAspectRatioMatrixInv,
|
||||
MATRIX_OFFSET);
|
||||
|
||||
Matrix.multiplyMM(
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
scaleMatrix,
|
||||
MATRIX_OFFSET);
|
||||
|
||||
// Translate image.
|
||||
Matrix.multiplyMM(
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
overlayAnchorMatrixInv,
|
||||
MATRIX_OFFSET);
|
||||
|
||||
// Scale image.
|
||||
Matrix.multiplyMM(
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
scaleMatrixInv,
|
||||
videoFrameAnchorMatrix,
|
||||
MATRIX_OFFSET);
|
||||
|
||||
// Correct for aspect ratio of image in output frame.
|
||||
@ -188,23 +135,67 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
aspectRatioMatrix,
|
||||
MATRIX_OFFSET);
|
||||
|
||||
// Anchor position in output frame.
|
||||
Matrix.multiplyMM(
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
videoFrameAnchorMatrixInv,
|
||||
scaleMatrix,
|
||||
MATRIX_OFFSET);
|
||||
Matrix.multiplyMM(
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
overlayAnchorMatrix,
|
||||
MATRIX_OFFSET);
|
||||
Matrix.multiplyMM(
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
scaleMatrixInv,
|
||||
MATRIX_OFFSET);
|
||||
|
||||
// Rotation needs to be agnostic of the scaling matrix and the aspect ratios.
|
||||
Matrix.multiplyMM(
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
overlayAspectRatioMatrix,
|
||||
MATRIX_OFFSET);
|
||||
Matrix.multiplyMM(
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
rotateMatrix,
|
||||
MATRIX_OFFSET);
|
||||
Matrix.multiplyMM(
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
overlayAspectRatioMatrixInv,
|
||||
MATRIX_OFFSET);
|
||||
|
||||
// Scale image.
|
||||
Matrix.multiplyMM(
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
transformationMatrix,
|
||||
MATRIX_OFFSET,
|
||||
scaleMatrix,
|
||||
MATRIX_OFFSET);
|
||||
|
||||
return transformationMatrix;
|
||||
}
|
||||
|
||||
private void reset() {
|
||||
GlUtil.setToIdentity(aspectRatioMatrix);
|
||||
GlUtil.setToIdentity(videoFrameAnchorMatrix);
|
||||
GlUtil.setToIdentity(videoFrameAnchorMatrixInv);
|
||||
GlUtil.setToIdentity(overlayAnchorMatrix);
|
||||
GlUtil.setToIdentity(overlayAnchorMatrixInv);
|
||||
GlUtil.setToIdentity(scaleMatrix);
|
||||
GlUtil.setToIdentity(scaleMatrixInv);
|
||||
GlUtil.setToIdentity(rotateMatrix);
|
||||
|
@ -22,7 +22,10 @@ import androidx.annotation.FloatRange;
|
||||
import androidx.media3.common.util.UnstableApi;
|
||||
import com.google.errorprone.annotations.CanIgnoreReturnValue;
|
||||
|
||||
/** Contains information to control how a {@link TextureOverlay} is displayed on the screen. */
|
||||
/**
|
||||
* Contains information to control how an input texture (for example, a {@link VideoCompositor} or
|
||||
* {@link TextureOverlay}) is displayed on a background.
|
||||
*/
|
||||
@UnstableApi
|
||||
public final class OverlaySettings {
|
||||
public final boolean useHdr;
|
||||
@ -47,6 +50,11 @@ public final class OverlaySettings {
|
||||
this.rotationDegrees = rotationDegrees;
|
||||
}
|
||||
|
||||
/** Returns a new {@link Builder} initialized with the values of this instance. */
|
||||
/* package */ Builder buildUpon() {
|
||||
return new Builder(this);
|
||||
}
|
||||
|
||||
/** A builder for {@link OverlaySettings} instances. */
|
||||
public static final class Builder {
|
||||
private boolean useHdr;
|
||||
@ -65,6 +73,15 @@ public final class OverlaySettings {
|
||||
rotationDegrees = 0f;
|
||||
}
|
||||
|
||||
private Builder(OverlaySettings overlaySettings) {
|
||||
this.useHdr = overlaySettings.useHdr;
|
||||
this.alphaScale = overlaySettings.alphaScale;
|
||||
this.videoFrameAnchor = overlaySettings.videoFrameAnchor;
|
||||
this.overlayAnchor = overlaySettings.overlayAnchor;
|
||||
this.scale = overlaySettings.scale;
|
||||
this.rotationDegrees = overlaySettings.rotationDegrees;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets whether input overlay comes from an HDR source. If {@code true}, colors will be in
|
||||
* linear RGB BT.2020. If {@code false}, colors will be in linear RGB BT.709.
|
||||
@ -92,6 +109,7 @@ public final class OverlaySettings {
|
||||
return this;
|
||||
}
|
||||
|
||||
// TODO: b/262694346 - Rename this method to setBackgroundAnchor in a follow-up CL.
|
||||
/**
|
||||
* Sets the coordinates for the anchor point of the overlay within the video frame.
|
||||
*
|
||||
|
@ -29,7 +29,7 @@ import com.google.common.collect.ImmutableList;
|
||||
/* package */ final class OverlayShaderProgram extends BaseGlShaderProgram {
|
||||
|
||||
private final GlProgram glProgram;
|
||||
private final OverlayMatrixProvider overlayMatrixProvider;
|
||||
private final SamplerOverlayMatrixProvider samplerOverlayMatrixProvider;
|
||||
private final ImmutableList<TextureOverlay> overlays;
|
||||
|
||||
/**
|
||||
@ -49,7 +49,7 @@ import com.google.common.collect.ImmutableList;
|
||||
overlays.size() <= 15,
|
||||
"OverlayShaderProgram does not support more than 15 overlays in the same instance.");
|
||||
this.overlays = overlays;
|
||||
this.overlayMatrixProvider = new OverlayMatrixProvider();
|
||||
this.samplerOverlayMatrixProvider = new SamplerOverlayMatrixProvider();
|
||||
try {
|
||||
glProgram =
|
||||
new GlProgram(createVertexShader(overlays.size()), createFragmentShader(overlays.size()));
|
||||
@ -66,7 +66,7 @@ import com.google.common.collect.ImmutableList;
|
||||
@Override
|
||||
public Size configure(int inputWidth, int inputHeight) {
|
||||
Size videoSize = new Size(inputWidth, inputHeight);
|
||||
overlayMatrixProvider.configure(/* backgroundSize= */ videoSize);
|
||||
samplerOverlayMatrixProvider.configure(/* backgroundSize= */ videoSize);
|
||||
for (TextureOverlay overlay : overlays) {
|
||||
overlay.configure(videoSize);
|
||||
}
|
||||
@ -91,7 +91,7 @@ import com.google.common.collect.ImmutableList;
|
||||
|
||||
glProgram.setFloatsUniform(
|
||||
Util.formatInvariant("uTransformationMatrix%d", texUnitIndex),
|
||||
overlayMatrixProvider.getTransformationMatrix(overlaySize, overlaySettings));
|
||||
samplerOverlayMatrixProvider.getTransformationMatrix(overlaySize, overlaySettings));
|
||||
|
||||
glProgram.setFloatUniform(
|
||||
Util.formatInvariant("uOverlayAlphaScale%d", texUnitIndex),
|
||||
|
@ -0,0 +1,56 @@
|
||||
/*
|
||||
* Copyright 2023 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package androidx.media3.effect;
|
||||
|
||||
import android.opengl.Matrix;
|
||||
import androidx.media3.common.util.GlUtil;
|
||||
import androidx.media3.common.util.Size;
|
||||
|
||||
/**
|
||||
* Provides a matrix based on {@link OverlaySettings} to be applied on a texture sampling
|
||||
* coordinate.
|
||||
*/
|
||||
/* package */ final class SamplerOverlayMatrixProvider extends OverlayMatrixProvider {
|
||||
private final float[] transformationMatrixInv;
|
||||
|
||||
public SamplerOverlayMatrixProvider() {
|
||||
super();
|
||||
transformationMatrixInv = GlUtil.create4x4IdentityMatrix();
|
||||
}
|
||||
|
||||
@Override
|
||||
public float[] getTransformationMatrix(Size overlaySize, OverlaySettings overlaySettings) {
|
||||
// When sampling from a (for example, texture) sampler, the overlay anchor's x and y coordinates
|
||||
// are flipped.
|
||||
OverlaySettings samplerOverlaySettings =
|
||||
overlaySettings
|
||||
.buildUpon()
|
||||
.setOverlayAnchor(
|
||||
/* x= */ -1 * overlaySettings.overlayAnchor.first,
|
||||
/* y= */ -1 * overlaySettings.overlayAnchor.second)
|
||||
.build();
|
||||
|
||||
// When sampling from a (for example, texture) sampler, the transformation matrix applied to a
|
||||
// sampler's coordinate should be the inverse of the transformation matrix that would otherwise
|
||||
// be applied to a vertex.
|
||||
Matrix.invertM(
|
||||
transformationMatrixInv,
|
||||
MATRIX_OFFSET,
|
||||
super.getTransformationMatrix(overlaySize, samplerOverlaySettings),
|
||||
MATRIX_OFFSET);
|
||||
return transformationMatrixInv;
|
||||
}
|
||||
}
|
@ -17,10 +17,12 @@ package androidx.media3.effect;
|
||||
|
||||
import androidx.media3.common.GlTextureInfo;
|
||||
import androidx.media3.common.VideoFrameProcessingException;
|
||||
import androidx.media3.common.util.Size;
|
||||
import androidx.media3.common.util.UnstableApi;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Interface for a video compositor that combines frames from mutliple input sources to produce
|
||||
* Interface for a video compositor that combines frames from multiple input sources to produce
|
||||
* output frames.
|
||||
*
|
||||
* <p>Input and output are provided via OpenGL textures.
|
||||
@ -41,6 +43,23 @@ public interface VideoCompositor extends GlTextureProducer {
|
||||
void onEnded();
|
||||
}
|
||||
|
||||
/** Settings for the {@link VideoCompositor}. */
|
||||
interface Settings {
|
||||
// TODO: b/262694346 - Consider adding more features, like selecting a:
|
||||
// * custom order for drawing (instead of primary stream on top), and
|
||||
// * different primary source.
|
||||
|
||||
/**
|
||||
* Returns an output texture {@link Size}, based on {@code inputSizes}.
|
||||
*
|
||||
* @param inputSizes The {@link Size} of each input frame, ordered by {@code inputId}.
|
||||
*/
|
||||
Size getOutputSize(List<Size> inputSizes);
|
||||
|
||||
/** Returns {@link OverlaySettings} for {@code inputId} at time {@code presentationTimeUs}. */
|
||||
OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Registers a new input source, and returns a unique {@code inputId} corresponding to this
|
||||
* source, to be used in {@link #queueInputTexture}.
|
||||
|
Binary file not shown.
After Width: | Height: | Size: 10 KiB |
Binary file not shown.
After Width: | Height: | Size: 12 KiB |
Binary file not shown.
After Width: | Height: | Size: 27 KiB |
@ -24,6 +24,7 @@ import static androidx.media3.test.utils.VideoFrameProcessorTestRunner.createTim
|
||||
import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.common.truth.Truth.assertWithMessage;
|
||||
import static java.lang.Math.max;
|
||||
import static java.util.concurrent.TimeUnit.MILLISECONDS;
|
||||
|
||||
import android.graphics.Bitmap;
|
||||
@ -44,6 +45,7 @@ import androidx.media3.common.Effect;
|
||||
import androidx.media3.common.GlObjectsProvider;
|
||||
import androidx.media3.common.VideoFrameProcessingException;
|
||||
import androidx.media3.common.util.GlUtil;
|
||||
import androidx.media3.common.util.Size;
|
||||
import androidx.media3.common.util.Util;
|
||||
import androidx.media3.effect.AlphaScale;
|
||||
import androidx.media3.effect.DefaultGlObjectsProvider;
|
||||
@ -51,6 +53,7 @@ import androidx.media3.effect.DefaultVideoCompositor;
|
||||
import androidx.media3.effect.DefaultVideoFrameProcessor;
|
||||
import androidx.media3.effect.OverlayEffect;
|
||||
import androidx.media3.effect.OverlaySettings;
|
||||
import androidx.media3.effect.Presentation;
|
||||
import androidx.media3.effect.RgbFilter;
|
||||
import androidx.media3.effect.ScaleAndRotateTransformation;
|
||||
import androidx.media3.effect.TextOverlay;
|
||||
@ -60,6 +63,7 @@ import androidx.media3.test.utils.TextureBitmapReader;
|
||||
import androidx.media3.test.utils.VideoFrameProcessorTestRunner;
|
||||
import com.google.common.base.Ascii;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.Iterables;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
@ -103,15 +107,15 @@ public final class DefaultVideoCompositorPixelTest {
|
||||
private static final String ORIGINAL_PNG_ASSET_PATH =
|
||||
"media/bitmap/input_images/media3test_srgb.png";
|
||||
private static final String TEST_DIRECTORY = "media/bitmap/CompositorTestTimestamps/";
|
||||
|
||||
private @MonotonicNonNull String testId;
|
||||
private @MonotonicNonNull VideoCompositorTestRunner compositorTestRunner;
|
||||
private static final ImmutableList<ImmutableList<Effect>> TWO_INPUT_COMPOSITOR_EFFECT_LISTS =
|
||||
ImmutableList.of(
|
||||
ImmutableList.of(RgbFilter.createGrayscaleFilter(), new AlphaScale(0.7f)),
|
||||
ImmutableList.of(
|
||||
new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build()));
|
||||
|
||||
private @MonotonicNonNull String testId;
|
||||
private @MonotonicNonNull VideoCompositorTestRunner compositorTestRunner;
|
||||
|
||||
@Before
|
||||
@EnsuresNonNull("testId")
|
||||
public void setUpTestId() {
|
||||
@ -125,6 +129,8 @@ public final class DefaultVideoCompositorPixelTest {
|
||||
}
|
||||
}
|
||||
|
||||
// Tests for alpha and frame alpha/occlusion.
|
||||
|
||||
@Test
|
||||
@RequiresNonNull("testId")
|
||||
public void compositeTwoInputs_withOneFrameFromEach_differentTimestamp_matchesExpectedBitmap()
|
||||
@ -155,12 +161,13 @@ public final class DefaultVideoCompositorPixelTest {
|
||||
@RequiresNonNull("testId")
|
||||
public void compositeTwoInputs_withPrimaryTransparent_differentTimestamp_matchesExpectedBitmap()
|
||||
throws Exception {
|
||||
ImmutableList<ImmutableList<Effect>> inputEffects =
|
||||
ImmutableList<ImmutableList<Effect>> inputEffectLists =
|
||||
ImmutableList.of(
|
||||
ImmutableList.of(new AlphaScale(0f)),
|
||||
ImmutableList.of(
|
||||
new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build()));
|
||||
compositorTestRunner = new VideoCompositorTestRunner(testId, useSharedExecutor, inputEffects);
|
||||
compositorTestRunner =
|
||||
new VideoCompositorTestRunner(testId, useSharedExecutor, inputEffectLists);
|
||||
|
||||
compositorTestRunner.queueBitmapToInput(
|
||||
/* inputId= */ 0, /* timestamps= */ ImmutableList.of(0L));
|
||||
@ -186,12 +193,13 @@ public final class DefaultVideoCompositorPixelTest {
|
||||
@RequiresNonNull("testId")
|
||||
public void compositeTwoInputs_withPrimaryOpaque_differentTimestamp_matchesExpectedBitmap()
|
||||
throws Exception {
|
||||
ImmutableList<ImmutableList<Effect>> inputEffects =
|
||||
ImmutableList<ImmutableList<Effect>> inputEffectLists =
|
||||
ImmutableList.of(
|
||||
ImmutableList.of(RgbFilter.createGrayscaleFilter(), new AlphaScale(100f)),
|
||||
ImmutableList.of(
|
||||
new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build()));
|
||||
compositorTestRunner = new VideoCompositorTestRunner(testId, useSharedExecutor, inputEffects);
|
||||
compositorTestRunner =
|
||||
new VideoCompositorTestRunner(testId, useSharedExecutor, inputEffectLists);
|
||||
|
||||
compositorTestRunner.queueBitmapToInput(
|
||||
/* inputId= */ 0, /* timestamps= */ ImmutableList.of(0L));
|
||||
@ -217,11 +225,12 @@ public final class DefaultVideoCompositorPixelTest {
|
||||
@RequiresNonNull("testId")
|
||||
public void compositeTwoInputs_withSecondaryTransparent_differentTimestamp_matchesExpectedBitmap()
|
||||
throws Exception {
|
||||
ImmutableList<ImmutableList<Effect>> inputEffects =
|
||||
ImmutableList<ImmutableList<Effect>> inputEffectLists =
|
||||
ImmutableList.of(
|
||||
ImmutableList.of(RgbFilter.createGrayscaleFilter(), new AlphaScale(0.7f)),
|
||||
ImmutableList.of(new AlphaScale(0f)));
|
||||
compositorTestRunner = new VideoCompositorTestRunner(testId, useSharedExecutor, inputEffects);
|
||||
compositorTestRunner =
|
||||
new VideoCompositorTestRunner(testId, useSharedExecutor, inputEffectLists);
|
||||
|
||||
compositorTestRunner.queueBitmapToInput(
|
||||
/* inputId= */ 0, /* timestamps= */ ImmutableList.of(0L));
|
||||
@ -243,6 +252,8 @@ public final class DefaultVideoCompositorPixelTest {
|
||||
ImmutableList.of("0s_1s_transparent"));
|
||||
}
|
||||
|
||||
// Tests for mixing different frame rates and timestamps.
|
||||
|
||||
@Test
|
||||
@RequiresNonNull("testId")
|
||||
public void compositeTwoInputs_withFiveFramesFromEach_matchesExpectedTimestamps()
|
||||
@ -428,6 +439,8 @@ public final class DefaultVideoCompositorPixelTest {
|
||||
ImmutableList.of("0s_1s", "1s_1s", "2s_1s", "3s_3s", "4s_4s"));
|
||||
}
|
||||
|
||||
// Tests for "many" inputs/frames.
|
||||
|
||||
@Test
|
||||
@RequiresNonNull("testId")
|
||||
public void compositeTwoInputs_withTenFramesFromEach_matchesExpectedFrameCount()
|
||||
@ -468,6 +481,8 @@ public final class DefaultVideoCompositorPixelTest {
|
||||
assertThat(compositorTestRunner.getCompositedTimestamps()).hasSize(numberOfFramesToQueue);
|
||||
}
|
||||
|
||||
// Tests for different amounts of inputs.
|
||||
|
||||
@Test
|
||||
@RequiresNonNull("testId")
|
||||
public void compositeOneInput_matchesExpectedBitmap() throws Exception {
|
||||
@ -526,6 +541,120 @@ public final class DefaultVideoCompositorPixelTest {
|
||||
ImmutableList.of("0s_1s_0s", "1s_1s_0s", "2s_1s_2s"));
|
||||
}
|
||||
|
||||
// Tests for different layouts.
|
||||
|
||||
@Test
|
||||
@RequiresNonNull("testId")
|
||||
public void compositeTwoInputs_pictureInPicture_matchesExpectedBitmap() throws Exception {
|
||||
ImmutableList<ImmutableList<Effect>> inputEffectLists =
|
||||
ImmutableList.of(ImmutableList.of(), ImmutableList.of(RgbFilter.createGrayscaleFilter()));
|
||||
VideoCompositor.Settings pictureInPictureSettings =
|
||||
new VideoCompositor.Settings() {
|
||||
@Override
|
||||
public Size getOutputSize(List<Size> inputSizes) {
|
||||
return inputSizes.get(0);
|
||||
}
|
||||
|
||||
@Override
|
||||
public OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs) {
|
||||
if (inputId == 0) {
|
||||
// This tests all OverlaySettings builder variables.
|
||||
return new OverlaySettings.Builder()
|
||||
.setScale(.25f, .5f)
|
||||
.setOverlayAnchor(1, -1)
|
||||
.setVideoFrameAnchor(.9f, -.7f)
|
||||
.setRotationDegrees(20)
|
||||
.setAlphaScale(.5f)
|
||||
.build();
|
||||
} else {
|
||||
return new OverlaySettings.Builder().build();
|
||||
}
|
||||
}
|
||||
};
|
||||
compositorTestRunner =
|
||||
new VideoCompositorTestRunner(
|
||||
testId, useSharedExecutor, inputEffectLists, pictureInPictureSettings);
|
||||
|
||||
compositorTestRunner.queueBitmapToAllInputs(1);
|
||||
compositorTestRunner.endCompositing();
|
||||
|
||||
compositorTestRunner.saveAndAssertCompositedBitmapsMatchExpected(
|
||||
ImmutableList.of("picture_in_picture"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@RequiresNonNull("testId")
|
||||
public void compositeTwoInputs_differentDimensions_matchesExpectedBitmap() throws Exception {
|
||||
ImmutableList<ImmutableList<Effect>> inputEffectLists =
|
||||
ImmutableList.of(
|
||||
ImmutableList.of(
|
||||
Presentation.createForWidthAndHeight(100, 100, Presentation.LAYOUT_STRETCH_TO_FIT)),
|
||||
ImmutableList.of(RgbFilter.createGrayscaleFilter()));
|
||||
VideoCompositor.Settings secondStreamAsOutputSizeSettings =
|
||||
new VideoCompositor.Settings() {
|
||||
@Override
|
||||
public Size getOutputSize(List<Size> inputSizes) {
|
||||
return Iterables.getLast(inputSizes);
|
||||
}
|
||||
|
||||
@Override
|
||||
public OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs) {
|
||||
return new OverlaySettings.Builder().build();
|
||||
}
|
||||
};
|
||||
compositorTestRunner =
|
||||
new VideoCompositorTestRunner(
|
||||
testId, useSharedExecutor, inputEffectLists, secondStreamAsOutputSizeSettings);
|
||||
|
||||
compositorTestRunner.queueBitmapToAllInputs(1);
|
||||
compositorTestRunner.endCompositing();
|
||||
|
||||
compositorTestRunner.saveAndAssertCompositedBitmapsMatchExpected(
|
||||
ImmutableList.of("different_dimensions"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@RequiresNonNull("testId")
|
||||
public void compositeTwoInputs_stacked_matchesExpectedBitmap() throws Exception {
|
||||
ImmutableList<ImmutableList<Effect>> inputEffectLists =
|
||||
ImmutableList.of(
|
||||
ImmutableList.of(RgbFilter.createGrayscaleFilter()),
|
||||
ImmutableList.of(),
|
||||
ImmutableList.of(RgbFilter.createInvertedFilter()));
|
||||
VideoCompositor.Settings stackedFrameSettings =
|
||||
new VideoCompositor.Settings() {
|
||||
private static final int NUMBER_OF_INPUT_STREAMS = 3;
|
||||
|
||||
@Override
|
||||
public Size getOutputSize(List<Size> inputSizes) {
|
||||
// Return the maximum width and sum of all heights.
|
||||
int width = 0;
|
||||
int height = 0;
|
||||
for (int i = 0; i < inputSizes.size(); i++) {
|
||||
width = max(width, inputSizes.get(i).getWidth());
|
||||
height += inputSizes.get(i).getHeight();
|
||||
}
|
||||
return new Size(width, height);
|
||||
}
|
||||
|
||||
@Override
|
||||
public OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs) {
|
||||
return new OverlaySettings.Builder()
|
||||
.setOverlayAnchor(-1, -1)
|
||||
.setVideoFrameAnchor(-1, -1 + 2f * inputId / NUMBER_OF_INPUT_STREAMS)
|
||||
.build();
|
||||
}
|
||||
};
|
||||
compositorTestRunner =
|
||||
new VideoCompositorTestRunner(
|
||||
testId, useSharedExecutor, inputEffectLists, stackedFrameSettings);
|
||||
|
||||
compositorTestRunner.queueBitmapToAllInputs(1);
|
||||
compositorTestRunner.endCompositing();
|
||||
|
||||
compositorTestRunner.saveAndAssertCompositedBitmapsMatchExpected(ImmutableList.of("stacked"));
|
||||
}
|
||||
|
||||
/**
|
||||
* A test runner for {@link DefaultVideoCompositor} tests.
|
||||
*
|
||||
@ -544,7 +673,7 @@ public final class DefaultVideoCompositorPixelTest {
|
||||
private final String testId;
|
||||
|
||||
/**
|
||||
* Creates an instance.
|
||||
* Creates an instance using {@link DefaultVideoCompositor.Settings}.
|
||||
*
|
||||
* @param testId The {@link String} identifier for the test, used to name output files.
|
||||
* @param useSharedExecutor Whether to use a shared executor for {@link
|
||||
@ -559,6 +688,27 @@ public final class DefaultVideoCompositorPixelTest {
|
||||
boolean useSharedExecutor,
|
||||
ImmutableList<ImmutableList<Effect>> inputEffectLists)
|
||||
throws GlUtil.GlException, VideoFrameProcessingException {
|
||||
this(testId, useSharedExecutor, inputEffectLists, new DefaultVideoCompositor.Settings());
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an instance.
|
||||
*
|
||||
* @param testId The {@link String} identifier for the test, used to name output files.
|
||||
* @param useSharedExecutor Whether to use a shared executor for {@link
|
||||
* VideoFrameProcessorTestRunner} and {@link VideoCompositor} instances.
|
||||
* @param inputEffectLists {@link Effect}s to apply for {@link VideoCompositor} input sources.
|
||||
* The size of this outer {@link List} is the amount of inputs. One inner list of {@link
|
||||
* Effect}s is used for each input. For each input, the frame timestamp and {@code inputId}
|
||||
* are overlaid via {@link TextOverlay} prior to its effects being applied.
|
||||
* @param settings The {@link VideoCompositor.Settings}.
|
||||
*/
|
||||
public VideoCompositorTestRunner(
|
||||
String testId,
|
||||
boolean useSharedExecutor,
|
||||
ImmutableList<ImmutableList<Effect>> inputEffectLists,
|
||||
VideoCompositor.Settings settings)
|
||||
throws GlUtil.GlException, VideoFrameProcessingException {
|
||||
this.testId = testId;
|
||||
timeoutMs = inputEffectLists.size() * VIDEO_FRAME_PROCESSING_WAIT_MS;
|
||||
sharedExecutorService =
|
||||
@ -575,6 +725,7 @@ public final class DefaultVideoCompositorPixelTest {
|
||||
new DefaultVideoCompositor(
|
||||
getApplicationContext(),
|
||||
glObjectsProvider,
|
||||
settings,
|
||||
sharedExecutorService,
|
||||
new VideoCompositor.Listener() {
|
||||
@Override
|
||||
|
Loading…
x
Reference in New Issue
Block a user