From aeb2fefe483323ed07ab1e0881ae351fedda13c9 Mon Sep 17 00:00:00 2001 From: tonihei Date: Thu, 18 Jul 2019 16:18:49 +0100 Subject: [PATCH] Further language normalization tweaks for API < 21. 1. Using the Locale on API<21 doesn't make any sense because it's a no-op anyway. Slightly restructured the code to avoid that. 2. API<21 often reports languages with non-standard underscores instead of dashes. Normalize that too. 3. Some invalid language tags on API>21 get normalized to "und". Use original tag in such a case. Issue:#6153 PiperOrigin-RevId: 258773463 --- RELEASENOTES.md | 3 + .../google/android/exoplayer2/util/Util.java | 57 +++++++++---------- .../android/exoplayer2/util/UtilTest.java | 15 +++++ 3 files changed, 46 insertions(+), 29 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 7194882758..30098e01df 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -29,6 +29,9 @@ ([#6192](https://github.com/google/ExoPlayer/issues/6192)). * Switch normalized BCP-47 language codes to use 2-letter ISO 639-1 language tags instead of 3-letter ISO 639-2 language tags. +* Fix issue where invalid language tags were normalized to "und" instead of + keeping the original + ([#6153](https://github.com/google/ExoPlayer/issues/6153)). ### 2.10.3 ### diff --git a/library/core/src/main/java/com/google/android/exoplayer2/util/Util.java b/library/core/src/main/java/com/google/android/exoplayer2/util/Util.java index a8aa2c630b..e700fc6751 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/util/Util.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/util/Util.java @@ -470,25 +470,31 @@ public final class Util { if (language == null) { return null; } - Locale locale = getLocaleForLanguageTag(language); - String localeLanguage = locale.getLanguage(); - int localeLanguageLength = localeLanguage.length(); - if (localeLanguageLength == 0) { - // Return original language for invalid language tags. - return toLowerInvariant(language); - } else if (localeLanguageLength == 3) { - // Locale.toLanguageTag will ensure a normalized well-formed output. However, 3-letter - // ISO 639-2 language codes will not be converted to 2-letter ISO 639-1 codes automatically. + // Locale data (especially for API < 21) may produce tags with '_' instead of the + // standard-conformant '-'. + String normalizedTag = language.replace('_', '-'); + if (Util.SDK_INT >= 21) { + // Filters out ill-formed sub-tags, replaces deprecated tags and normalizes all valid tags. + normalizedTag = normalizeLanguageCodeSyntaxV21(normalizedTag); + } + if (normalizedTag.isEmpty() || "und".equals(normalizedTag)) { + // Tag isn't valid, keep using the original. + normalizedTag = language; + } + normalizedTag = Util.toLowerInvariant(normalizedTag); + String mainLanguage = Util.splitAtFirst(normalizedTag, "-")[0]; + if (mainLanguage.length() == 3) { + // 3-letter ISO 639-2/B or ISO 639-2/T language codes will not be converted to 2-letter ISO + // 639-1 codes automatically. if (languageTagIso3ToIso2 == null) { languageTagIso3ToIso2 = createIso3ToIso2Map(); } - String iso2Language = languageTagIso3ToIso2.get(localeLanguage); + String iso2Language = languageTagIso3ToIso2.get(mainLanguage); if (iso2Language != null) { - localeLanguage = iso2Language; + normalizedTag = iso2Language + normalizedTag.substring(/* beginIndex= */ 3); } } - String normTag = getLocaleLanguageTag(locale); - return toLowerInvariant(localeLanguage + normTag.substring(localeLanguageLength)); + return normalizedTag; } /** @@ -1982,32 +1988,25 @@ public final class Util { } private static String[] getSystemLocales() { + Configuration config = Resources.getSystem().getConfiguration(); return SDK_INT >= 24 - ? getSystemLocalesV24() - : new String[] {getLocaleLanguageTag(Resources.getSystem().getConfiguration().locale)}; + ? getSystemLocalesV24(config) + : SDK_INT >= 21 ? getSystemLocaleV21(config) : new String[] {config.locale.toString()}; } @TargetApi(24) - private static String[] getSystemLocalesV24() { - return Util.split(Resources.getSystem().getConfiguration().getLocales().toLanguageTags(), ","); - } - - private static Locale getLocaleForLanguageTag(String languageTag) { - return Util.SDK_INT >= 21 ? getLocaleForLanguageTagV21(languageTag) : new Locale(languageTag); + private static String[] getSystemLocalesV24(Configuration config) { + return Util.split(config.getLocales().toLanguageTags(), ","); } @TargetApi(21) - private static Locale getLocaleForLanguageTagV21(String languageTag) { - return Locale.forLanguageTag(languageTag); - } - - private static String getLocaleLanguageTag(Locale locale) { - return SDK_INT >= 21 ? getLocaleLanguageTagV21(locale) : locale.toString(); + private static String[] getSystemLocaleV21(Configuration config) { + return new String[] {config.locale.toLanguageTag()}; } @TargetApi(21) - private static String getLocaleLanguageTagV21(Locale locale) { - return locale.toLanguageTag(); + private static String normalizeLanguageCodeSyntaxV21(String languageTag) { + return Locale.forLanguageTag(languageTag).toLanguageTag(); } private static @C.NetworkType int getMobileNetworkType(NetworkInfo networkInfo) { diff --git a/library/core/src/test/java/com/google/android/exoplayer2/util/UtilTest.java b/library/core/src/test/java/com/google/android/exoplayer2/util/UtilTest.java index f85ee37c07..5a13ed0dd8 100644 --- a/library/core/src/test/java/com/google/android/exoplayer2/util/UtilTest.java +++ b/library/core/src/test/java/com/google/android/exoplayer2/util/UtilTest.java @@ -268,10 +268,14 @@ public class UtilTest { @Test @Config(sdk = 21) public void testNormalizeLanguageCodeV21() { + assertThat(Util.normalizeLanguageCode(null)).isNull(); + assertThat(Util.normalizeLanguageCode("")).isEmpty(); assertThat(Util.normalizeLanguageCode("es")).isEqualTo("es"); assertThat(Util.normalizeLanguageCode("spa")).isEqualTo("es"); assertThat(Util.normalizeLanguageCode("es-AR")).isEqualTo("es-ar"); assertThat(Util.normalizeLanguageCode("SpA-ar")).isEqualTo("es-ar"); + assertThat(Util.normalizeLanguageCode("es_AR")).isEqualTo("es-ar"); + assertThat(Util.normalizeLanguageCode("spa_ar")).isEqualTo("es-ar"); assertThat(Util.normalizeLanguageCode("es-AR-dialect")).isEqualTo("es-ar-dialect"); assertThat(Util.normalizeLanguageCode("ES-419")).isEqualTo("es-419"); assertThat(Util.normalizeLanguageCode("zh-hans-tw")).isEqualTo("zh-hans-tw"); @@ -284,9 +288,20 @@ public class UtilTest { @Test @Config(sdk = 16) public void testNormalizeLanguageCode() { + assertThat(Util.normalizeLanguageCode(null)).isNull(); + assertThat(Util.normalizeLanguageCode("")).isEmpty(); assertThat(Util.normalizeLanguageCode("es")).isEqualTo("es"); assertThat(Util.normalizeLanguageCode("spa")).isEqualTo("es"); assertThat(Util.normalizeLanguageCode("es-AR")).isEqualTo("es-ar"); + assertThat(Util.normalizeLanguageCode("SpA-ar")).isEqualTo("es-ar"); + assertThat(Util.normalizeLanguageCode("es_AR")).isEqualTo("es-ar"); + assertThat(Util.normalizeLanguageCode("spa_ar")).isEqualTo("es-ar"); + assertThat(Util.normalizeLanguageCode("es-AR-dialect")).isEqualTo("es-ar-dialect"); + assertThat(Util.normalizeLanguageCode("ES-419")).isEqualTo("es-419"); + assertThat(Util.normalizeLanguageCode("zh-hans-tw")).isEqualTo("zh-hans-tw"); + // Doesn't work on API < 21 because we can't use Locale syntax verification. + // assertThat(Util.normalizeLanguageCode("zh-tw-hans")).isEqualTo("zh-tw"); + assertThat(Util.normalizeLanguageCode("zho-hans-tw")).isEqualTo("zh-hans-tw"); assertThat(Util.normalizeLanguageCode("und")).isEqualTo("und"); assertThat(Util.normalizeLanguageCode("DoesNotExist")).isEqualTo("doesnotexist"); }