diff --git a/libraries/common/src/main/java/androidx/media3/common/text/Cue.java b/libraries/common/src/main/java/androidx/media3/common/text/Cue.java index e38f715176..bd06e22b81 100644 --- a/libraries/common/src/main/java/androidx/media3/common/text/Cue.java +++ b/libraries/common/src/main/java/androidx/media3/common/text/Cue.java @@ -15,6 +15,7 @@ */ package androidx.media3.common.text; +import static androidx.media3.common.text.CustomSpanBundler.bundleCustomSpans; import static java.lang.annotation.ElementType.FIELD; import static java.lang.annotation.ElementType.LOCAL_VARIABLE; import static java.lang.annotation.ElementType.METHOD; @@ -26,6 +27,7 @@ import android.graphics.Color; import android.os.Bundle; import android.text.Layout; import android.text.Layout.Alignment; +import android.text.SpannableString; import android.text.Spanned; import android.text.SpannedString; import android.text.TextUtils; @@ -42,6 +44,7 @@ import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import java.util.ArrayList; import org.checkerframework.dataflow.qual.Pure; /** Contains information about a specific cue, including textual content and formatting data. */ @@ -826,6 +829,7 @@ public final class Cue implements Bundleable { // Bundleable implementation. private static final String FIELD_TEXT = Util.intToStringMaxRadix(0); + private static final String FIELD_CUSTOM_SPANS = Util.intToStringMaxRadix(17); private static final String FIELD_TEXT_ALIGNMENT = Util.intToStringMaxRadix(1); private static final String FIELD_MULTI_ROW_ALIGNMENT = Util.intToStringMaxRadix(2); private static final String FIELD_BITMAP = Util.intToStringMaxRadix(3); @@ -848,6 +852,12 @@ public final class Cue implements Bundleable { public Bundle toBundle() { Bundle bundle = new Bundle(); bundle.putCharSequence(FIELD_TEXT, text); + if (text instanceof Spanned) { + ArrayList customSpanBundles = bundleCustomSpans((Spanned) text); + if (!customSpanBundles.isEmpty()) { + bundle.putParcelableArrayList(FIELD_CUSTOM_SPANS, customSpanBundles); + } + } bundle.putSerializable(FIELD_TEXT_ALIGNMENT, textAlignment); bundle.putSerializable(FIELD_MULTI_ROW_ALIGNMENT, multiRowAlignment); bundle.putParcelable(FIELD_BITMAP, bitmap); @@ -882,6 +892,15 @@ public final class Cue implements Bundleable { @Nullable CharSequence text = bundle.getCharSequence(FIELD_TEXT); if (text != null) { builder.setText(text); + @Nullable + ArrayList customSpanBundles = bundle.getParcelableArrayList(FIELD_CUSTOM_SPANS); + if (customSpanBundles != null) { + SpannableString textWithCustomSpans = SpannableString.valueOf(text); + for (Bundle customSpanBundle : customSpanBundles) { + CustomSpanBundler.unbundleAndApplyCustomSpan(customSpanBundle, textWithCustomSpans); + } + builder.setText(textWithCustomSpans); + } } @Nullable Alignment textAlignment = (Alignment) bundle.getSerializable(FIELD_TEXT_ALIGNMENT); if (textAlignment != null) { diff --git a/libraries/common/src/main/java/androidx/media3/common/text/CustomSpanBundler.java b/libraries/common/src/main/java/androidx/media3/common/text/CustomSpanBundler.java new file mode 100644 index 0000000000..edcda586d2 --- /dev/null +++ b/libraries/common/src/main/java/androidx/media3/common/text/CustomSpanBundler.java @@ -0,0 +1,135 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.common.text; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static java.lang.annotation.ElementType.TYPE_USE; + +import android.os.Bundle; +import android.text.Spannable; +import android.text.Spanned; +import android.text.style.BackgroundColorSpan; +import android.text.style.StrikethroughSpan; +import android.text.style.UnderlineSpan; +import androidx.annotation.IntDef; +import androidx.annotation.Nullable; +import androidx.media3.common.util.Util; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.util.ArrayList; + +/** + * Provides serialization support for custom Media3 styling spans. + * + *

Custom Media3 spans are not serialized by {@link Bundle#putCharSequence}, unlike + * platform-provided spans such as {@link StrikethroughSpan}, {@link UnderlineSpan}, {@link + * BackgroundColorSpan} etc. + * + *

{@link Cue#text} might contain custom spans, there is a need for serialization support. + */ +/* package */ final class CustomSpanBundler { + + /** + * Media3 custom span implementations. One of the following: + * + *

+ */ + @Documented + @Retention(RetentionPolicy.SOURCE) + @Target({TYPE_USE}) + @IntDef({UNKNOWN, RUBY, TEXT_EMPHASIS, HORIZONTAL_TEXT_IN_VERTICAL_CONTEXT}) + private @interface CustomSpanType {} + + private static final int UNKNOWN = -1; + + private static final int RUBY = 1; + + private static final int TEXT_EMPHASIS = 2; + + private static final int HORIZONTAL_TEXT_IN_VERTICAL_CONTEXT = 3; + + private static final String FIELD_START_INDEX = Util.intToStringMaxRadix(0); + private static final String FIELD_END_INDEX = Util.intToStringMaxRadix(1); + private static final String FIELD_FLAGS = Util.intToStringMaxRadix(2); + private static final String FIELD_TYPE = Util.intToStringMaxRadix(3); + private static final String FIELD_PARAMS = Util.intToStringMaxRadix(4); + + @SuppressWarnings("NonApiType") // Intentionally using ArrayList for putParcelableArrayList. + public static ArrayList bundleCustomSpans(Spanned text) { + ArrayList bundledCustomSpans = new ArrayList<>(); + for (RubySpan span : text.getSpans(0, text.length(), RubySpan.class)) { + Bundle bundle = spanToBundle(text, span, /* spanType= */ RUBY, /* params= */ span.toBundle()); + bundledCustomSpans.add(bundle); + } + for (TextEmphasisSpan span : text.getSpans(0, text.length(), TextEmphasisSpan.class)) { + Bundle bundle = + spanToBundle(text, span, /* spanType= */ TEXT_EMPHASIS, /* params= */ span.toBundle()); + bundledCustomSpans.add(bundle); + } + for (HorizontalTextInVerticalContextSpan span : + text.getSpans(0, text.length(), HorizontalTextInVerticalContextSpan.class)) { + Bundle bundle = + spanToBundle( + text, span, /* spanType= */ HORIZONTAL_TEXT_IN_VERTICAL_CONTEXT, /* params= */ null); + bundledCustomSpans.add(bundle); + } + return bundledCustomSpans; + } + + public static void unbundleAndApplyCustomSpan(Bundle customSpanBundle, Spannable text) { + int start = customSpanBundle.getInt(FIELD_START_INDEX); + int end = customSpanBundle.getInt(FIELD_END_INDEX); + int flags = customSpanBundle.getInt(FIELD_FLAGS); + int customSpanType = customSpanBundle.getInt(FIELD_TYPE, UNKNOWN); + @Nullable Bundle span = customSpanBundle.getBundle(FIELD_PARAMS); + switch (customSpanType) { + case RUBY: + text.setSpan(RubySpan.fromBundle(checkNotNull(span)), start, end, flags); + break; + case TEXT_EMPHASIS: + text.setSpan(TextEmphasisSpan.fromBundle(checkNotNull(span)), start, end, flags); + break; + case HORIZONTAL_TEXT_IN_VERTICAL_CONTEXT: + text.setSpan(new HorizontalTextInVerticalContextSpan(), start, end, flags); + break; + default: + break; + } + } + + private static Bundle spanToBundle( + Spanned spanned, Object span, @CustomSpanType int spanType, @Nullable Bundle params) { + Bundle bundle = new Bundle(); + bundle.putInt(FIELD_START_INDEX, spanned.getSpanStart(span)); + bundle.putInt(FIELD_END_INDEX, spanned.getSpanEnd(span)); + bundle.putInt(FIELD_FLAGS, spanned.getSpanFlags(span)); + bundle.putInt(FIELD_TYPE, spanType); + if (params != null) { + bundle.putBundle(FIELD_PARAMS, params); + } + return bundle; + } + + private CustomSpanBundler() {} +} diff --git a/libraries/common/src/main/java/androidx/media3/common/text/RubySpan.java b/libraries/common/src/main/java/androidx/media3/common/text/RubySpan.java index 482cb7a92d..260812a33b 100644 --- a/libraries/common/src/main/java/androidx/media3/common/text/RubySpan.java +++ b/libraries/common/src/main/java/androidx/media3/common/text/RubySpan.java @@ -16,7 +16,11 @@ */ package androidx.media3.common.text; +import static androidx.media3.common.util.Assertions.checkNotNull; + +import android.os.Bundle; import androidx.media3.common.util.UnstableApi; +import androidx.media3.common.util.Util; /** * A styling span for ruby text. @@ -41,8 +45,24 @@ public final class RubySpan implements LanguageFeatureSpan { /** The position of the ruby text relative to the base text. */ public final @TextAnnotation.Position int position; + private static final String FIELD_TEXT = Util.intToStringMaxRadix(0); + private static final String FIELD_POSITION = Util.intToStringMaxRadix(1); + public RubySpan(String rubyText, @TextAnnotation.Position int position) { this.rubyText = rubyText; this.position = position; } + + public Bundle toBundle() { + Bundle bundle = new Bundle(); + bundle.putString(FIELD_TEXT, rubyText); + bundle.putInt(FIELD_POSITION, position); + return bundle; + } + + public static RubySpan fromBundle(Bundle bundle) { + return new RubySpan( + /* rubyText= */ checkNotNull(bundle.getString(FIELD_TEXT)), + /* position= */ bundle.getInt(FIELD_POSITION)); + } } diff --git a/libraries/common/src/main/java/androidx/media3/common/text/TextEmphasisSpan.java b/libraries/common/src/main/java/androidx/media3/common/text/TextEmphasisSpan.java index 11595c4f84..4a39694a1c 100644 --- a/libraries/common/src/main/java/androidx/media3/common/text/TextEmphasisSpan.java +++ b/libraries/common/src/main/java/androidx/media3/common/text/TextEmphasisSpan.java @@ -18,8 +18,10 @@ package androidx.media3.common.text; import static java.lang.annotation.ElementType.TYPE_USE; import static java.lang.annotation.RetentionPolicy.SOURCE; +import android.os.Bundle; import androidx.annotation.IntDef; import androidx.media3.common.util.UnstableApi; +import androidx.media3.common.util.Util; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.Target; @@ -91,10 +93,29 @@ public final class TextEmphasisSpan implements LanguageFeatureSpan { /** The position of the text emphasis relative to the base text. */ public final @TextAnnotation.Position int position; + private static final String FIELD_MARK_SHAPE = Util.intToStringMaxRadix(0); + private static final String FIELD_MARK_FILL = Util.intToStringMaxRadix(1); + private static final String FIELD_POSITION = Util.intToStringMaxRadix(2); + public TextEmphasisSpan( @MarkShape int shape, @MarkFill int fill, @TextAnnotation.Position int position) { this.markShape = shape; this.markFill = fill; this.position = position; } + + public Bundle toBundle() { + Bundle bundle = new Bundle(); + bundle.putInt(FIELD_MARK_SHAPE, markShape); + bundle.putInt(FIELD_MARK_FILL, markFill); + bundle.putInt(FIELD_POSITION, position); + return bundle; + } + + public static TextEmphasisSpan fromBundle(Bundle bundle) { + return new TextEmphasisSpan( + /* shape= */ bundle.getInt(FIELD_MARK_SHAPE), + /* fill= */ bundle.getInt(FIELD_MARK_FILL), + /* position= */ bundle.getInt(FIELD_POSITION)); + } } diff --git a/libraries/common/src/test/java/androidx/media3/common/text/CustomCueBundlerTest.java b/libraries/common/src/test/java/androidx/media3/common/text/CustomCueBundlerTest.java new file mode 100644 index 0000000000..45a8d2ab1e --- /dev/null +++ b/libraries/common/src/test/java/androidx/media3/common/text/CustomCueBundlerTest.java @@ -0,0 +1,114 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.common.text; + +import static com.google.common.truth.Truth.assertThat; +import static java.util.stream.Collectors.toCollection; + +import android.os.Bundle; +import android.text.SpannableString; +import android.util.Pair; +import androidx.media3.extractor.text.CueDecoder; +import androidx.media3.extractor.text.CueEncoder; +import androidx.media3.test.utils.truth.SpannedSubject; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableMap; +import com.google.common.reflect.ClassPath; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Test of {@link Cue} serialization and deserialization using {@link CueEncoder} and {@link + * CueDecoder} with a {@code text} that contains custom (i.e. non-Android native) spans. + */ +@RunWith(AndroidJUnit4.class) +public class CustomCueBundlerTest { + + private static final RubySpan RUBY_SPAN = + new RubySpan("ruby text", TextAnnotation.POSITION_AFTER); + private static final TextEmphasisSpan TEXT_EMPHASIS_SPAN = + new TextEmphasisSpan( + TextEmphasisSpan.MARK_SHAPE_CIRCLE, + TextEmphasisSpan.MARK_FILL_FILLED, + TextAnnotation.POSITION_AFTER); + private static final HorizontalTextInVerticalContextSpan + HORIZONTAL_TEXT_IN_VERTICAL_CONTEXT_SPAN = new HorizontalTextInVerticalContextSpan(); + private static final ImmutableMap> ALL_SPANS_TO_START_END_INDEX = + ImmutableMap.of( + RUBY_SPAN, new Pair<>(1, 2), + TEXT_EMPHASIS_SPAN, new Pair<>(2, 3), + HORIZONTAL_TEXT_IN_VERTICAL_CONTEXT_SPAN, new Pair<>(5, 7)); + + @Test + public void serializingSpannableWithAllCustomSpans() { + SpannableString spannableString = new SpannableString("test string"); + for (Map.Entry> spanToStartEndIndex : + ALL_SPANS_TO_START_END_INDEX.entrySet()) { + spannableString.setSpan( + spanToStartEndIndex.getKey(), + spanToStartEndIndex.getValue().first, + spanToStartEndIndex.getValue().second, + /* flags= */ 0); + } + + ArrayList bundles = CustomSpanBundler.bundleCustomSpans(spannableString); + + assertThat(bundles).hasSize(ALL_SPANS_TO_START_END_INDEX.size()); + + SpannableString result = new SpannableString("test string"); + for (Bundle bundle : bundles) { + CustomSpanBundler.unbundleAndApplyCustomSpan(bundle, result); + } + SpannedSubject.assertThat(result) + .hasRubySpanBetween( + ALL_SPANS_TO_START_END_INDEX.get(RUBY_SPAN).first, + ALL_SPANS_TO_START_END_INDEX.get(RUBY_SPAN).second) + .withTextAndPosition(RUBY_SPAN.rubyText, RUBY_SPAN.position); + SpannedSubject.assertThat(result) + .hasTextEmphasisSpanBetween( + ALL_SPANS_TO_START_END_INDEX.get(TEXT_EMPHASIS_SPAN).first, + ALL_SPANS_TO_START_END_INDEX.get(TEXT_EMPHASIS_SPAN).second) + .withMarkAndPosition( + TEXT_EMPHASIS_SPAN.markShape, TEXT_EMPHASIS_SPAN.markFill, TEXT_EMPHASIS_SPAN.position); + SpannedSubject.assertThat(result) + .hasHorizontalTextInVerticalContextSpanBetween( + ALL_SPANS_TO_START_END_INDEX.get(HORIZONTAL_TEXT_IN_VERTICAL_CONTEXT_SPAN).first, + ALL_SPANS_TO_START_END_INDEX.get(HORIZONTAL_TEXT_IN_VERTICAL_CONTEXT_SPAN).second); + } + + @Test + public void noUnsupportedCustomSpanTypes() throws Exception { + Set supportedSpanClassNames = + ALL_SPANS_TO_START_END_INDEX.keySet().stream() + .map(s -> s.getClass().getName()) + .collect(toCollection(HashSet::new)); + ClassPath classPath = ClassPath.from(getClass().getClassLoader()); + for (ClassPath.ClassInfo classInfo : classPath.getAllClasses()) { + if (classInfo.getPackageName().equals("androidx.media3.common.text") + && classInfo.getName().endsWith("Span")) { + Class clazz = classInfo.load(); + if (!clazz.isInterface() && !Modifier.isAbstract(clazz.getModifiers())) { + assertThat(supportedSpanClassNames).contains(classInfo.getName()); + } + } + } + } +} diff --git a/libraries/extractor/src/test/java/androidx/media3/extractor/text/CueSerializationTest.java b/libraries/extractor/src/test/java/androidx/media3/extractor/text/CueSerializationTest.java index ed7ab221c2..75b1ef155c 100644 --- a/libraries/extractor/src/test/java/androidx/media3/extractor/text/CueSerializationTest.java +++ b/libraries/extractor/src/test/java/androidx/media3/extractor/text/CueSerializationTest.java @@ -19,16 +19,15 @@ import static com.google.common.truth.Truth.assertThat; import android.graphics.Bitmap; import android.graphics.Color; -import android.graphics.Typeface; import android.text.Layout; import android.text.Spannable; import android.text.SpannableString; import android.text.Spanned; import android.text.SpannedString; import android.text.style.StrikethroughSpan; -import android.text.style.StyleSpan; -import android.text.style.UnderlineSpan; import androidx.media3.common.text.Cue; +import androidx.media3.common.text.RubySpan; +import androidx.media3.common.text.TextAnnotation; import androidx.media3.test.utils.truth.SpannedSubject; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; @@ -87,48 +86,52 @@ public class CueSerializationTest { } @Test - public void serializingBitmapCueAndCueWithAndroidSpans() { + public void serializingBitmapCue() { CueEncoder encoder = new CueEncoder(); CueDecoder decoder = new CueDecoder(); - Spannable spannable = SpannableString.valueOf("text text"); - spannable.setSpan( - new StrikethroughSpan(), 0, "text".length(), Spanned.SPAN_EXCLUSIVE_EXCLUSIVE); - spannable.setSpan( - new StyleSpan(Typeface.BOLD), 0, "text text".length(), Spanned.SPAN_EXCLUSIVE_EXCLUSIVE); - spannable.setSpan( - new StyleSpan(Typeface.ITALIC), 0, "text text".length(), Spanned.SPAN_EXCLUSIVE_EXCLUSIVE); - spannable.setSpan( - new UnderlineSpan(), - "text ".length(), - "text text".length(), - Spanned.SPAN_EXCLUSIVE_EXCLUSIVE); - Cue textCue = new Cue.Builder().setText(spannable).build(); Bitmap bitmap = Bitmap.createBitmap(1, 1, Bitmap.Config.ARGB_8888); Cue bitmapCue = new Cue.Builder().setBitmap(bitmap).build(); // encoding and decoding - byte[] encodedCues = - encoder.encode(ImmutableList.of(textCue, bitmapCue), /* durationUs= */ 2000); + byte[] encodedCues = encoder.encode(ImmutableList.of(bitmapCue), /* durationUs= */ 2000); CuesWithTiming cuesAfterDecoding = decoder.decode(/* startTimeUs= */ 1000, encodedCues); assertThat(cuesAfterDecoding.startTimeUs).isEqualTo(1000); assertThat(cuesAfterDecoding.durationUs).isEqualTo(2000); assertThat(cuesAfterDecoding.endTimeUs).isEqualTo(3000); - assertThat(cuesAfterDecoding.cues).hasSize(2); - Cue textCueAfterDecoding = cuesAfterDecoding.cues.get(0); - Cue bitmapCueAfterDecoding = cuesAfterDecoding.cues.get(1); - - assertThat(textCueAfterDecoding.text.toString()).isEqualTo(textCue.text.toString()); - SpannedSubject.assertThat((Spanned) textCueAfterDecoding.text) - .hasStrikethroughSpanBetween(0, "text".length()); - SpannedSubject.assertThat((Spanned) textCueAfterDecoding.text) - .hasBoldSpanBetween(0, "text text".length()); - SpannedSubject.assertThat((Spanned) textCueAfterDecoding.text) - .hasItalicSpanBetween(0, "text text".length()); - SpannedSubject.assertThat((Spanned) textCueAfterDecoding.text) - .hasUnderlineSpanBetween("text ".length(), "text text".length()); - + Cue bitmapCueAfterDecoding = cuesAfterDecoding.cues.get(0); assertThat(bitmapCueAfterDecoding.bitmap.sameAs(bitmap)).isTrue(); } + + @Test + public void serializingCueWithAndroidAndCustomSpans() { + CueEncoder encoder = new CueEncoder(); + CueDecoder decoder = new CueDecoder(); + Spannable spannable = SpannableString.valueOf("The Player"); + spannable.setSpan(new StrikethroughSpan(), 0, "The".length(), Spanned.SPAN_EXCLUSIVE_EXCLUSIVE); + spannable.setSpan( + new RubySpan("small ruby", TextAnnotation.POSITION_AFTER), + "The ".length(), + "The Player".length(), + Spanned.SPAN_EXCLUSIVE_EXCLUSIVE); + Cue mixedSpansCue = new Cue.Builder().setText(spannable).build(); + + // encoding and decoding + byte[] encodedCues = encoder.encode(ImmutableList.of(mixedSpansCue), /* durationUs= */ 2000); + CuesWithTiming cuesAfterDecoding = decoder.decode(/* startTimeUs= */ 1000, encodedCues); + + assertThat(cuesAfterDecoding.startTimeUs).isEqualTo(1000); + assertThat(cuesAfterDecoding.durationUs).isEqualTo(2000); + assertThat(cuesAfterDecoding.endTimeUs).isEqualTo(3000); + + Cue mixedSpansCueAfterDecoding = cuesAfterDecoding.cues.get(0); + + assertThat(mixedSpansCueAfterDecoding.text.toString()).isEqualTo(mixedSpansCue.text.toString()); + Spanned mixedSpans = (Spanned) mixedSpansCueAfterDecoding.text; + SpannedSubject.assertThat(mixedSpans).hasStrikethroughSpanBetween(0, "The".length()); + SpannedSubject.assertThat(mixedSpans) + .hasRubySpanBetween("The ".length(), "The Player".length()) + .withTextAndPosition("small ruby", TextAnnotation.POSITION_AFTER); + } }