diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/OverlayTextureProcessorPixelTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/OverlayTextureProcessorPixelTest.java index 6054b7db41..97b9e32c9e 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/OverlayTextureProcessorPixelTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/OverlayTextureProcessorPixelTest.java @@ -27,10 +27,14 @@ import static com.google.common.truth.Truth.assertThat; import android.content.Context; import android.graphics.Bitmap; +import android.graphics.Color; import android.opengl.EGLContext; import android.opengl.EGLDisplay; import android.opengl.EGLSurface; import android.opengl.Matrix; +import android.text.Spannable; +import android.text.SpannableString; +import android.text.style.ForegroundColorSpan; import android.util.Pair; import androidx.media3.common.FrameProcessingException; import androidx.media3.common.util.GlUtil; @@ -60,6 +64,10 @@ public class OverlayTextureProcessorPixelTest { "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_default.png"; public static final String OVERLAY_BITMAP_SCALED = "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_scaled.png"; + public static final String OVERLAY_TEXT_DEFAULT = + "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_text_default.png"; + public static final String OVERLAY_TEXT_TRANSLATE = + "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_text_translate.png"; private final Context context = getApplicationContext(); @@ -159,6 +167,65 @@ public class OverlayTextureProcessorPixelTest { assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); } + @Test + public void drawFrame_textOverlay_blendsTextIntoFrame() throws Exception { + String testId = "drawFrame_textOverlay"; + SpannableString overlayText = new SpannableString(/* source= */ "Text styling"); + overlayText.setSpan( + new ForegroundColorSpan(Color.GRAY), + /* start= */ 0, + /* end= */ 4, + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + TextOverlay staticTextOverlay = TextOverlay.createStaticTextOverlay(overlayText); + overlayTextureProcessor = + new OverlayEffect(ImmutableList.of(staticTextOverlay)) + .toGlTextureProcessor(context, /* useHdr= */ false); + Pair outputSize = overlayTextureProcessor.configure(inputWidth, inputHeight); + setupOutputTexture(outputSize.first, outputSize.second); + Bitmap expectedBitmap = readBitmap(OVERLAY_TEXT_DEFAULT); + + overlayTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + createArgb8888BitmapFromCurrentGlFramebuffer(outputSize.first, outputSize.second); + + maybeSaveTestBitmapToCacheDirectory(testId, /* bitmapLabel= */ "actual", actualBitmap); + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void drawFrame_translatedTextOverlay_blendsTextIntoFrame() throws Exception { + String testId = "drawFrame_translatedTextOverlay"; + float[] translateMatrix = GlUtil.create4x4IdentityMatrix(); + Matrix.translateM(translateMatrix, /* mOffset= */ 0, /* x= */ 0.5f, /* y= */ 0.5f, /* z= */ 1); + SpannableString overlayText = new SpannableString(/* source= */ "Text styling"); + overlayText.setSpan( + new ForegroundColorSpan(Color.GRAY), + /* start= */ 0, + /* end= */ 4, + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + OverlaySettings overlaySettings = + new OverlaySettings.Builder().setMatrix(translateMatrix).build(); + TextOverlay staticTextOverlay = + TextOverlay.createStaticTextOverlay(overlayText, overlaySettings); + overlayTextureProcessor = + new OverlayEffect(ImmutableList.of(staticTextOverlay)) + .toGlTextureProcessor(context, /* useHdr= */ false); + Pair outputSize = overlayTextureProcessor.configure(inputWidth, inputHeight); + setupOutputTexture(outputSize.first, outputSize.second); + Bitmap expectedBitmap = readBitmap(OVERLAY_TEXT_TRANSLATE); + + overlayTextureProcessor.drawFrame(inputTexId, /* presentationTimeUs= */ 0); + Bitmap actualBitmap = + createArgb8888BitmapFromCurrentGlFramebuffer(outputSize.first, outputSize.second); + + maybeSaveTestBitmapToCacheDirectory(testId, /* bitmapLabel= */ "actual", actualBitmap); + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + private void setupOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException { int outputTexId = GlUtil.createTexture( diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DrawableOverlay.java b/libraries/effect/src/main/java/androidx/media3/effect/DrawableOverlay.java new file mode 100644 index 0000000000..c11a368097 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/DrawableOverlay.java @@ -0,0 +1,79 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkNotNull; + +import android.graphics.Bitmap; +import android.graphics.Canvas; +import android.graphics.drawable.Drawable; +import androidx.media3.common.util.UnstableApi; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * Creates a {@link TextureOverlay} from {@link Drawable}. + * + *

Uses a canvas to draw {@link DrawableOverlay} onto {@link BitmapOverlay}, which is then + * displayed on each frame. + */ +@UnstableApi +public abstract class DrawableOverlay extends BitmapOverlay { + private @MonotonicNonNull Bitmap lastBitmap; + private @MonotonicNonNull Drawable lastDrawable; + + /** + * Returns the overlay {@link Drawable} displayed at the specified timestamp. + * + * @param presentationTimeUs The presentation timestamp of the current frame, in microseconds. + */ + public abstract Drawable getDrawable(long presentationTimeUs); + + @Override + public Bitmap getBitmap(long presentationTimeUs) { + Drawable overlayDrawable = getDrawable(presentationTimeUs); + // TODO(b/227625365): Drawable doesn't implement the equals method, so investigate other methods + // of + // detecting the need to redraw the bitmap. + if (!overlayDrawable.equals(lastDrawable)) { + lastDrawable = overlayDrawable; + lastBitmap = + Bitmap.createBitmap( + lastDrawable.getIntrinsicWidth(), + lastDrawable.getIntrinsicHeight(), + Bitmap.Config.ARGB_8888); + Canvas canvas = new Canvas(lastBitmap); + lastDrawable.draw(canvas); + } + return checkNotNull(lastBitmap); + } + + /** + * Creates a {@link TextOverlay} that shows the {@code overlayDrawable} with the same {@link + * OverlaySettings} throughout the whole video. + * + * @param overlaySettings The {@link OverlaySettings} configuring how the overlay is displayed on + * the frames. + */ + public static DrawableOverlay createStaticDrawableOverlay( + Drawable drawable, OverlaySettings overlaySettings) { + return new DrawableOverlay() { + @Override + public Drawable getDrawable(long presentationTimeUs) { + return drawable; + } + }; + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/TextOverlay.java b/libraries/effect/src/main/java/androidx/media3/effect/TextOverlay.java new file mode 100644 index 0000000000..00bbf44bf2 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/TextOverlay.java @@ -0,0 +1,129 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Util.SDK_INT; + +import android.annotation.SuppressLint; +import android.graphics.Bitmap; +import android.graphics.Canvas; +import android.text.Layout; +import android.text.SpannableString; +import android.text.StaticLayout; +import android.text.TextPaint; +import androidx.annotation.DoNotInline; +import androidx.annotation.RequiresApi; +import androidx.media3.common.util.UnstableApi; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * Creates a {@link TextureOverlay} from text. + * + *

Uses a {@link SpannableString} store the text and support advanced per-character text styling. + */ +@UnstableApi +public abstract class TextOverlay extends BitmapOverlay { + public static final int TEXT_SIZE_PIXELS = 100; + + private @MonotonicNonNull Bitmap lastBitmap; + private @MonotonicNonNull SpannableString lastText; + + /** + * Returns the overlay text displayed at the specified timestamp. + * + * @param presentationTimeUs The presentation timestamp of the current frame, in microseconds. + */ + public abstract SpannableString getText(long presentationTimeUs); + + @SuppressLint("InlinedApi") // Inlined Layout constants. + @Override + public Bitmap getBitmap(long presentationTimeUs) { + SpannableString overlayText = getText(presentationTimeUs); + if (!overlayText.equals(lastText)) { + lastText = overlayText; + TextPaint textPaint = new TextPaint(); + textPaint.setTextSize(TEXT_SIZE_PIXELS); + StaticLayout staticLayout; + int width = (int) textPaint.measureText(overlayText, /* start= */ 0, overlayText.length()); + if (SDK_INT >= 23) { + staticLayout = Api23.getStaticLayout(overlayText, textPaint, width); + } else { + staticLayout = + new StaticLayout( + overlayText, + textPaint, + width, + Layout.Alignment.ALIGN_NORMAL, + Layout.DEFAULT_LINESPACING_MULTIPLIER, + Layout.DEFAULT_LINESPACING_ADDITION, + /* includepad= */ true); + } + lastBitmap = + Bitmap.createBitmap( + staticLayout.getWidth(), staticLayout.getHeight(), Bitmap.Config.ARGB_8888); + Canvas canvas = new Canvas(checkNotNull(lastBitmap)); + staticLayout.draw(canvas); + } + return checkNotNull(lastBitmap); + } + + /** + * Creates a {@link TextOverlay} that shows the {@code overlayText} with the same default settings + * in {@link OverlaySettings} throughout the whole video. + */ + public static TextOverlay createStaticTextOverlay(SpannableString overlayText) { + return new TextOverlay() { + @Override + public SpannableString getText(long presentationTimeUs) { + return overlayText; + } + }; + } + + /** + * Creates a {@link TextOverlay} that shows the {@code overlayText} with the same {@link + * OverlaySettings} throughout the whole video. + * + * @param overlaySettings The {@link OverlaySettings} configuring how the overlay is displayed on + * the frames. + */ + public static TextOverlay createStaticTextOverlay( + SpannableString overlayText, OverlaySettings overlaySettings) { + return new TextOverlay() { + @Override + public SpannableString getText(long presentationTimeUs) { + return overlayText; + } + + @Override + public OverlaySettings getOverlaySettings(long presentationTimeUs) { + return overlaySettings; + } + }; + } + + @RequiresApi(23) + private static final class Api23 { + @DoNotInline + public static StaticLayout getStaticLayout( + SpannableString text, TextPaint textPaint, int width) { + return StaticLayout.Builder.obtain( + text, /* start= */ 0, /* end= */ text.length(), textPaint, width) + .build(); + } + } +} diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_text_default.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_text_default.png new file mode 100644 index 0000000000..2789065a15 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_text_default.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_text_translate.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_text_translate.png new file mode 100644 index 0000000000..ba6992ecc5 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_text_translate.png differ