loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use OverlayEffect for ‘Overlay logo & timer' in transformer demo.

Browse Source 
PiperOrigin-RevId: 497112875
This commit is contained in:
tofunmi 2022-12-22 10:05:44 +00:00 committed by Marc Baechinger
parent e8572be6d5
commit bea32abefc
4 changed files with 113 additions and 197 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

181
demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayProcessor.java
View File

@ -1,181 +0,0 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.demo.transformer;
import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import android.content.Context;
import android.content.pm.PackageManager;
import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.Matrix;
import android.graphics.Paint;
import android.graphics.drawable.BitmapDrawable;
import android.opengl.GLES20;
import android.opengl.GLUtils;
import androidx.media3.common.C;
import androidx.media3.common.FrameProcessingException;
import androidx.media3.common.util.GlProgram;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.Size;
import androidx.media3.effect.SingleFrameGlTextureProcessor;
import java.io.IOException;
import java.util.Locale;
/**
* A {@link SingleFrameGlTextureProcessor} that overlays a bitmap with a logo and timer on each
* frame.
*
* <p>The bitmap is drawn using an Android {@link Canvas}.
*/
// TODO(b/227625365): Delete this class and use a texture processor from the Transformer library,
// once overlaying a bitmap and text is supported in Transformer.
/* package */ final class BitmapOverlayProcessor extends SingleFrameGlTextureProcessor {
private static final String VERTEX_SHADER_PATH = "vertex_shader_copy_es2.glsl";
private static final String FRAGMENT_SHADER_PATH = "fragment_shader_bitmap_overlay_es2.glsl";
private static final int BITMAP_WIDTH_HEIGHT = 512;
private final Paint paint;
private final Bitmap overlayBitmap;
private final Bitmap logoBitmap;
private final Canvas overlayCanvas;
private final GlProgram glProgram;
private float bitmapScaleX;
private float bitmapScaleY;
private int bitmapTexId;
/**
* Creates a new instance.
*
* @param context The {@link Context}.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in linear RGB BT.2020. If {@code false}, colors will be in linear RGB BT.709.
* @throws FrameProcessingException If a problem occurs while reading shader files.
*/
public BitmapOverlayProcessor(Context context, boolean useHdr) throws FrameProcessingException {
super(useHdr);
checkArgument(!useHdr, "BitmapOverlayProcessor does not support HDR colors.");
paint = new Paint();
paint.setTextSize(64);
paint.setAntiAlias(true);
paint.setARGB(0xFF, 0xFF, 0xFF, 0xFF);
paint.setColor(Color.GRAY);
overlayBitmap =
Bitmap.createBitmap(BITMAP_WIDTH_HEIGHT, BITMAP_WIDTH_HEIGHT, Bitmap.Config.ARGB_8888);
overlayCanvas = new Canvas(overlayBitmap);
try {
logoBitmap =
((BitmapDrawable)
context.getPackageManager().getApplicationIcon(context.getPackageName()))
.getBitmap();
} catch (PackageManager.NameNotFoundException e) {
throw new IllegalStateException(e);
}
try {
bitmapTexId =
GlUtil.createTexture(
BITMAP_WIDTH_HEIGHT,
BITMAP_WIDTH_HEIGHT,
/* useHighPrecisionColorComponents= */ false);
GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, /* level= */ 0, overlayBitmap, /* border= */ 0);
glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH);
} catch (GlUtil.GlException | IOException e) {
throw new FrameProcessingException(e);
}
// Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
glProgram.setBufferAttribute(
"aFramePosition",
GlUtil.getNormalizedCoordinateBounds(),
GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
glProgram.setSamplerTexIdUniform("uTexSampler1", bitmapTexId, /* texUnitIndex= */ 1);
}
@Override
public Size configure(int inputWidth, int inputHeight) {
if (inputWidth > inputHeight) {
bitmapScaleX = inputWidth / (float) inputHeight;
bitmapScaleY = 1f;
} else {
bitmapScaleX = 1f;
bitmapScaleY = inputHeight / (float) inputWidth;
}
glProgram.setFloatUniform("uScaleX", bitmapScaleX);
glProgram.setFloatUniform("uScaleY", bitmapScaleY);
return new Size(inputWidth, inputHeight);
}
@Override
public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
try {
glProgram.use();
// Draw to the canvas and store it in a texture.
String text =
String.format(Locale.US, "%.02f", presentationTimeUs / (float) C.MICROS_PER_SECOND);
overlayBitmap.eraseColor(Color.TRANSPARENT);
overlayCanvas.drawBitmap(checkStateNotNull(logoBitmap), /* left= */ 3, /* top= */ 378, paint);
overlayCanvas.drawText(text, /* x= */ 160, /* y= */ 466, paint);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, bitmapTexId);
GLUtils.texSubImage2D(
GLES20.GL_TEXTURE_2D,
/* level= */ 0,
/* xoffset= */ 0,
/* yoffset= */ 0,
flipBitmapVertically(overlayBitmap));
GlUtil.checkGlError();
glProgram.setSamplerTexIdUniform("uTexSampler0", inputTexId, /* texUnitIndex= */ 0);
glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e, presentationTimeUs);
}
}
@Override
public void release() throws FrameProcessingException {
super.release();
try {
glProgram.delete();
} catch (GlUtil.GlException e) {
throw new FrameProcessingException(e);
}
}
private static Bitmap flipBitmapVertically(Bitmap bitmap) {
Matrix flip = new Matrix();
flip.postScale(1f, -1f);
return Bitmap.createBitmap(
bitmap,
/* x= */ 0,
/* y= */ 0,
bitmap.getWidth(),
bitmap.getHeight(),
flip,
/* filter= */ true);
}
}

6
demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java
View File

@ -104,8 +104,8 @@ public final class ConfigurationActivity extends AppCompatActivity {
public static final int CONTRAST_INDEX = 6;
public static final int PERIODIC_VIGNETTE_INDEX = 7;
public static final int SPIN_3D_INDEX = 8;
public static final int OVERLAY_LOGO_AND_TIMER_INDEX = 9;
public static final int ZOOM_IN_INDEX = 10;
public static final int ZOOM_IN_INDEX = 9;
public static final int OVERLAY_LOGO_AND_TIMER_INDEX = 10;
public static final int BITMAP_OVERLAY_INDEX = 11;
public static final int TEXT_OVERLAY_INDEX = 12;
@ -167,8 +167,8 @@ public final class ConfigurationActivity extends AppCompatActivity {
"Contrast",
"Periodic vignette",
"3D spin",
"Overlay logo & timer",
"Zoom in start",
"Overlay logo & timer",
"Custom Bitmap Overlay",
"Custom Text Overlay",
};

66
demos/transformer/src/main/java/androidx/media3/demo/transformer/TimerOverlay.java Normal file
View File

@ -0,0 +1,66 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.demo.transformer;
import android.graphics.Color;
import android.opengl.Matrix;
import android.text.Spannable;
import android.text.SpannableString;
import android.text.style.ForegroundColorSpan;
import androidx.media3.common.C;
import androidx.media3.common.util.GlUtil;
import androidx.media3.effect.OverlaySettings;
import androidx.media3.effect.TextOverlay;
import androidx.media3.effect.TextureOverlay;
import java.util.Locale;
/**
* A {@link TextureOverlay} that displays a "time elapsed" timer in the bottom left corner of the
* frame.
*/
/* package */ final class TimerOverlay extends TextOverlay {
private final OverlaySettings overlaySettings;
public TimerOverlay() {
float[] positioningMatrix = GlUtil.create4x4IdentityMatrix();
Matrix.translateM(
positioningMatrix, /* mOffset= */ 0, /* x= */ -0.7f, /* y= */ -0.95f, /* z= */ 1);
overlaySettings =
new OverlaySettings.Builder()
.setAnchor(/* x= */ -1f, /* y= */ -1f)
.setMatrix(positioningMatrix)
.build();
}
@Override
public SpannableString getText(long presentationTimeUs) {
SpannableString text =
new SpannableString(
String.format(Locale.US, "%.02f", presentationTimeUs / (float) C.MICROS_PER_SECOND));
text.setSpan(
new ForegroundColorSpan(Color.WHITE),
/* start= */ 0,
text.length(),
Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
return text;
}
@Override
public OverlaySettings getOverlaySettings(long presentationTimeUs) {
return overlaySettings;
}
}

51
demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java
View File

@ -25,7 +25,9 @@ import android.content.Context;
import android.content.Intent;
import android.content.pm.PackageManager;
import android.graphics.Color;
import android.graphics.drawable.Drawable;
import android.net.Uri;
import android.opengl.Matrix;
import android.os.Bundle;
import android.os.Handler;
import android.text.Spannable;
@ -46,10 +48,12 @@ import androidx.media3.common.DebugViewProvider;
import androidx.media3.common.Effect;
import androidx.media3.common.MediaItem;
import androidx.media3.common.audio.AudioProcessor;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.Log;
import androidx.media3.common.util.Util;
import androidx.media3.effect.BitmapOverlay;
import androidx.media3.effect.Contrast;
import androidx.media3.effect.DrawableOverlay;
import androidx.media3.effect.GlEffect;
import androidx.media3.effect.GlTextureProcessor;
import androidx.media3.effect.HslAdjustment;
@ -60,6 +64,7 @@ import androidx.media3.effect.RgbFilter;
import androidx.media3.effect.RgbMatrix;
import androidx.media3.effect.SingleColorLut;
import androidx.media3.effect.TextOverlay;
import androidx.media3.effect.TextureOverlay;
import androidx.media3.exoplayer.ExoPlayer;
import androidx.media3.exoplayer.audio.SilenceSkippingAudioProcessor;
import androidx.media3.exoplayer.audio.SonicAudioProcessor;
@ -191,14 +196,18 @@ public final class TransformerActivity extends AppCompatActivity {
Uri uri = checkNotNull(intent.getData());
try {
externalCacheFile = createExternalCacheFile("transformer-output.mp4");
} catch (IOException e) {
throw new IllegalStateException(e);
}
String filePath = externalCacheFile.getAbsolutePath();
@Nullable Bundle bundle = intent.getExtras();
MediaItem mediaItem = createMediaItem(bundle, uri);
try {
Transformer transformer = createTransformer(bundle, filePath);
transformationStopwatch.start();
transformer.startTransformation(mediaItem, filePath);
this.transformer = transformer;
} catch (IOException e) {
} catch (PackageManager.NameNotFoundException e) {
throw new IllegalStateException(e);
}
inputCardView.setVisibility(View.GONE);
@ -253,7 +262,8 @@ public final class TransformerActivity extends AppCompatActivity {
"progressViewGroup",
"debugFrame",
})
private Transformer createTransformer(@Nullable Bundle bundle, String filePath) {
private Transformer createTransformer(@Nullable Bundle bundle, String filePath)
throws PackageManager.NameNotFoundException {
Transformer.Builder transformerBuilder = new Transformer.Builder(/* context= */ this);
if (bundle != null) {
TransformationRequest.Builder requestBuilder = new TransformationRequest.Builder();
@ -371,7 +381,8 @@ public final class TransformerActivity extends AppCompatActivity {
return processors.build();
}
private ImmutableList<Effect> createVideoEffectsFromBundle(Bundle bundle) {
private ImmutableList<Effect> createVideoEffectsFromBundle(Bundle bundle)
throws PackageManager.NameNotFoundException {
@Nullable
boolean[] selectedEffects =
bundle.getBooleanArray(ConfigurationActivity.VIDEO_EFFECTS_SELECTIONS);
@ -492,12 +503,33 @@ public final class TransformerActivity extends AppCompatActivity {
if (selectedEffects[ConfigurationActivity.SPIN_3D_INDEX]) {
effects.add(MatrixTransformationFactory.createSpin3dEffect());
}
if (selectedEffects[ConfigurationActivity.OVERLAY_LOGO_AND_TIMER_INDEX]) {
effects.add((GlEffect) BitmapOverlayProcessor::new);
}
if (selectedEffects[ConfigurationActivity.ZOOM_IN_INDEX]) {
effects.add(MatrixTransformationFactory.createZoomInTransition());
}
effects.add(createOverlayEffectFromBundle(bundle, selectedEffects));
return effects.build();
}
private OverlayEffect createOverlayEffectFromBundle(Bundle bundle, boolean[] selectedEffects)
throws PackageManager.NameNotFoundException {
ImmutableList.Builder<TextureOverlay> overlays = new ImmutableList.Builder<>();
if (selectedEffects[ConfigurationActivity.OVERLAY_LOGO_AND_TIMER_INDEX]) {
float[] logoPositioningMatrix = GlUtil.create4x4IdentityMatrix();
Matrix.translateM(
logoPositioningMatrix, /* mOffset= */ 0, /* x= */ -0.95f, /* y= */ -0.95f, /* z= */ 1);
OverlaySettings logoSettings =
new OverlaySettings.Builder()
.setMatrix(logoPositioningMatrix)
.setAnchor(/* x= */ -1f, /* y= */ -1f)
.build();
Drawable logo = getPackageManager().getApplicationIcon(getPackageName());
logo.setBounds(
/* left= */ 0, /* top= */ 0, logo.getIntrinsicWidth(), logo.getIntrinsicHeight());
TextureOverlay logoOverlay = DrawableOverlay.createStaticDrawableOverlay(logo, logoSettings);
TextureOverlay timerOverlay = new TimerOverlay();
overlays.add(logoOverlay, timerOverlay);
}
if (selectedEffects[ConfigurationActivity.BITMAP_OVERLAY_INDEX]) {
OverlaySettings overlaySettings =
new OverlaySettings.Builder()
@ -509,7 +541,7 @@ public final class TransformerActivity extends AppCompatActivity {
BitmapOverlay.createStaticBitmapOverlay(
Uri.parse(checkNotNull(bundle.getString(ConfigurationActivity.BITMAP_OVERLAY_URI))),
overlaySettings);
effects.add(new OverlayEffect(ImmutableList.of(bitmapOverlay)));
overlays.add(bitmapOverlay);
}
if (selectedEffects[ConfigurationActivity.TEXT_OVERLAY_INDEX]) {
OverlaySettings overlaySettings =
@ -526,10 +558,9 @@ public final class TransformerActivity extends AppCompatActivity {
overlayText.length(),
Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
TextOverlay textOverlay = TextOverlay.createStaticTextOverlay(overlayText, overlaySettings);
// TODO(227625365): use the same OverlayEffect object for bitmap and text overlays.
effects.add(new OverlayEffect(ImmutableList.of(textOverlay)));
overlays.add(textOverlay);
}
return effects.build();
return new OverlayEffect(overlays.build());
}
@RequiresNonNull({