Further language normalization tweaks for API < 21.

1. Using the Locale on API<21 doesn't make any sense because it's a no-op
   anyway. Slightly restructured the code to avoid that.
2. API<21 often reports languages with non-standard underscores instead of
   dashes. Normalize that too.
3. Some invalid language tags on API>21 get normalized to "und". Use original
   tag in such a case.

Issue:#6153
PiperOrigin-RevId: 258773463

RELEASENOTES.md

@@ -29,6 +29,9 @@
   ([#6192](https://github.com/google/ExoPlayer/issues/6192)).
 * Switch normalized BCP-47 language codes to use 2-letter ISO 639-1 language
   tags instead of 3-letter ISO 639-2 language tags.
+* Fix issue where invalid language tags were normalized to "und" instead of
+  keeping the original
+  ([#6153](https://github.com/google/ExoPlayer/issues/6153)).
 
 ### 2.10.3 ###
 

library/core/src/main/java/com/google/android/exoplayer2/util/Util.java

@@ -470,25 +470,31 @@ public final class Util {
     if (language == null) {
       return null;
     }
-    Locale locale = getLocaleForLanguageTag(language);
+    // Locale data (especially for API < 21) may produce tags with '_' instead of the
-    String localeLanguage = locale.getLanguage();
+    // standard-conformant '-'.
-    int localeLanguageLength = localeLanguage.length();
+    String normalizedTag = language.replace('_', '-');
-    if (localeLanguageLength == 0) {
+    if (Util.SDK_INT >= 21) {
-      // Return original language for invalid language tags.
+      // Filters out ill-formed sub-tags, replaces deprecated tags and normalizes all valid tags.
-      return toLowerInvariant(language);
+      normalizedTag = normalizeLanguageCodeSyntaxV21(normalizedTag);
-    } else if (localeLanguageLength == 3) {
+    }
-      // Locale.toLanguageTag will ensure a normalized well-formed output. However, 3-letter
+    if (normalizedTag.isEmpty() || "und".equals(normalizedTag)) {
-      // ISO 639-2 language codes will not be converted to 2-letter ISO 639-1 codes automatically.
+      // Tag isn't valid, keep using the original.
+      normalizedTag = language;
+    }
+    normalizedTag = Util.toLowerInvariant(normalizedTag);
+    String mainLanguage = Util.splitAtFirst(normalizedTag, "-")[0];
+    if (mainLanguage.length() == 3) {
+      // 3-letter ISO 639-2/B or ISO 639-2/T language codes will not be converted to 2-letter ISO
+      // 639-1 codes automatically.
       if (languageTagIso3ToIso2 == null) {
         languageTagIso3ToIso2 = createIso3ToIso2Map();
       }
-      String iso2Language = languageTagIso3ToIso2.get(localeLanguage);
+      String iso2Language = languageTagIso3ToIso2.get(mainLanguage);
       if (iso2Language != null) {
-        localeLanguage = iso2Language;
+        normalizedTag = iso2Language + normalizedTag.substring(/* beginIndex= */ 3);
       }
     }
-    String normTag = getLocaleLanguageTag(locale);
+    return normalizedTag;
-    return toLowerInvariant(localeLanguage + normTag.substring(localeLanguageLength));
   }
 
   /**
@@ -1982,32 +1988,25 @@ public final class Util {
   }
 
   private static String[] getSystemLocales() {
+    Configuration config = Resources.getSystem().getConfiguration();
     return SDK_INT >= 24
-        ? getSystemLocalesV24()
+        ? getSystemLocalesV24(config)
-        : new String[] {getLocaleLanguageTag(Resources.getSystem().getConfiguration().locale)};
+        : SDK_INT >= 21 ? getSystemLocaleV21(config) : new String[] {config.locale.toString()};
   }
 
   @TargetApi(24)
-  private static String[] getSystemLocalesV24() {
+  private static String[] getSystemLocalesV24(Configuration config) {
-    return Util.split(Resources.getSystem().getConfiguration().getLocales().toLanguageTags(), ",");
+    return Util.split(config.getLocales().toLanguageTags(), ",");
-  }
-
-  private static Locale getLocaleForLanguageTag(String languageTag) {
-    return Util.SDK_INT >= 21 ? getLocaleForLanguageTagV21(languageTag) : new Locale(languageTag);
   }
 
   @TargetApi(21)
-  private static Locale getLocaleForLanguageTagV21(String languageTag) {
+  private static String[] getSystemLocaleV21(Configuration config) {
-    return Locale.forLanguageTag(languageTag);
+    return new String[] {config.locale.toLanguageTag()};
-  }
-
-  private static String getLocaleLanguageTag(Locale locale) {
-    return SDK_INT >= 21 ? getLocaleLanguageTagV21(locale) : locale.toString();
   }
 
   @TargetApi(21)
-  private static String getLocaleLanguageTagV21(Locale locale) {
+  private static String normalizeLanguageCodeSyntaxV21(String languageTag) {
-    return locale.toLanguageTag();
+    return Locale.forLanguageTag(languageTag).toLanguageTag();
   }
 
   private static @C.NetworkType int getMobileNetworkType(NetworkInfo networkInfo) {

library/core/src/test/java/com/google/android/exoplayer2/util/UtilTest.java

@@ -268,10 +268,14 @@ public class UtilTest {
   @Test
   @Config(sdk = 21)
   public void testNormalizeLanguageCodeV21() {
+    assertThat(Util.normalizeLanguageCode(null)).isNull();
+    assertThat(Util.normalizeLanguageCode("")).isEmpty();
     assertThat(Util.normalizeLanguageCode("es")).isEqualTo("es");
     assertThat(Util.normalizeLanguageCode("spa")).isEqualTo("es");
     assertThat(Util.normalizeLanguageCode("es-AR")).isEqualTo("es-ar");
     assertThat(Util.normalizeLanguageCode("SpA-ar")).isEqualTo("es-ar");
+    assertThat(Util.normalizeLanguageCode("es_AR")).isEqualTo("es-ar");
+    assertThat(Util.normalizeLanguageCode("spa_ar")).isEqualTo("es-ar");
     assertThat(Util.normalizeLanguageCode("es-AR-dialect")).isEqualTo("es-ar-dialect");
     assertThat(Util.normalizeLanguageCode("ES-419")).isEqualTo("es-419");
     assertThat(Util.normalizeLanguageCode("zh-hans-tw")).isEqualTo("zh-hans-tw");
@@ -284,9 +288,20 @@ public class UtilTest {
   @Test
   @Config(sdk = 16)
   public void testNormalizeLanguageCode() {
+    assertThat(Util.normalizeLanguageCode(null)).isNull();
+    assertThat(Util.normalizeLanguageCode("")).isEmpty();
     assertThat(Util.normalizeLanguageCode("es")).isEqualTo("es");
     assertThat(Util.normalizeLanguageCode("spa")).isEqualTo("es");
     assertThat(Util.normalizeLanguageCode("es-AR")).isEqualTo("es-ar");
+    assertThat(Util.normalizeLanguageCode("SpA-ar")).isEqualTo("es-ar");
+    assertThat(Util.normalizeLanguageCode("es_AR")).isEqualTo("es-ar");
+    assertThat(Util.normalizeLanguageCode("spa_ar")).isEqualTo("es-ar");
+    assertThat(Util.normalizeLanguageCode("es-AR-dialect")).isEqualTo("es-ar-dialect");
+    assertThat(Util.normalizeLanguageCode("ES-419")).isEqualTo("es-419");
+    assertThat(Util.normalizeLanguageCode("zh-hans-tw")).isEqualTo("zh-hans-tw");
+    // Doesn't work on API < 21 because we can't use Locale syntax verification.
+    // assertThat(Util.normalizeLanguageCode("zh-tw-hans")).isEqualTo("zh-tw");
+    assertThat(Util.normalizeLanguageCode("zho-hans-tw")).isEqualTo("zh-hans-tw");
     assertThat(Util.normalizeLanguageCode("und")).isEqualTo("und");
     assertThat(Util.normalizeLanguageCode("DoesNotExist")).isEqualTo("doesnotexist");
   }