diff --git a/library/core/src/main/java/com/google/android/exoplayer2/util/Util.java b/library/core/src/main/java/com/google/android/exoplayer2/util/Util.java index 87209bbd19..6214e231ee 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/util/Util.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/util/Util.java @@ -134,9 +134,8 @@ public final class Util { + "(T(([0-9]*)H)?(([0-9]*)M)?(([0-9.]*)S)?)?$"); private static final Pattern ESCAPED_CHARACTER_PATTERN = Pattern.compile("%([A-Fa-f0-9]{2})"); - // Android standardizes to ISO 639-1 2-letter codes and provides no way to map a 3-letter - // ISO 639-2 code back to the corresponding 2-letter code. - @Nullable private static HashMap languageTagIso3ToIso2; + // Replacement map of ISO language codes used for normalization. + @Nullable private static HashMap languageTagReplacementMap; private Util() {} @@ -510,26 +509,23 @@ public final class Util { // Locale data (especially for API < 21) may produce tags with '_' instead of the // standard-conformant '-'. String normalizedTag = language.replace('_', '-'); - if (Util.SDK_INT >= 21) { - // Filters out ill-formed sub-tags, replaces deprecated tags and normalizes all valid tags. - normalizedTag = normalizeLanguageCodeSyntaxV21(normalizedTag); - } if (normalizedTag.isEmpty() || "und".equals(normalizedTag)) { // Tag isn't valid, keep using the original. normalizedTag = language; } normalizedTag = Util.toLowerInvariant(normalizedTag); String mainLanguage = Util.splitAtFirst(normalizedTag, "-")[0]; - if (mainLanguage.length() == 3) { - // 3-letter ISO 639-2/B or ISO 639-2/T language codes will not be converted to 2-letter ISO - // 639-1 codes automatically. - if (languageTagIso3ToIso2 == null) { - languageTagIso3ToIso2 = createIso3ToIso2Map(); - } - String iso2Language = languageTagIso3ToIso2.get(mainLanguage); - if (iso2Language != null) { - normalizedTag = iso2Language + normalizedTag.substring(/* beginIndex= */ 3); - } + if (languageTagReplacementMap == null) { + languageTagReplacementMap = createIsoLanguageReplacementMap(); + } + @Nullable String replacedLanguage = languageTagReplacementMap.get(mainLanguage); + if (replacedLanguage != null) { + normalizedTag = + replacedLanguage + normalizedTag.substring(/* beginIndex= */ mainLanguage.length()); + mainLanguage = replacedLanguage; + } + if ("no".equals(mainLanguage) || "i".equals(mainLanguage) || "zh".equals(mainLanguage)) { + normalizedTag = maybeReplaceGrandfatheredLanguageTags(normalizedTag); } return normalizedTag; } @@ -2082,11 +2078,6 @@ public final class Util { return locale.toLanguageTag(); } - @TargetApi(21) - private static String normalizeLanguageCodeSyntaxV21(String languageTag) { - return Locale.forLanguageTag(languageTag).toLanguageTag(); - } - private static @C.NetworkType int getMobileNetworkType(NetworkInfo networkInfo) { switch (networkInfo.getSubtype()) { case TelephonyManager.NETWORK_TYPE_EDGE: @@ -2117,32 +2108,45 @@ public final class Util { } } - private static HashMap createIso3ToIso2Map() { + private static HashMap createIsoLanguageReplacementMap() { String[] iso2Languages = Locale.getISOLanguages(); - HashMap iso3ToIso2 = + HashMap replacedLanguages = new HashMap<>( - /* initialCapacity= */ iso2Languages.length + iso3BibliographicalToIso2.length); + /* initialCapacity= */ iso2Languages.length + additionalIsoLanguageReplacements.length); for (String iso2 : iso2Languages) { try { // This returns the ISO 639-2/T code for the language. String iso3 = new Locale(iso2).getISO3Language(); if (!TextUtils.isEmpty(iso3)) { - iso3ToIso2.put(iso3, iso2); + replacedLanguages.put(iso3, iso2); } } catch (MissingResourceException e) { // Shouldn't happen for list of known languages, but we don't want to throw either. } } - // Add additional ISO 639-2/B codes to mapping. - for (int i = 0; i < iso3BibliographicalToIso2.length; i += 2) { - iso3ToIso2.put(iso3BibliographicalToIso2[i], iso3BibliographicalToIso2[i + 1]); + // Add additional replacement mappings. + for (int i = 0; i < additionalIsoLanguageReplacements.length; i += 2) { + replacedLanguages.put( + additionalIsoLanguageReplacements[i], additionalIsoLanguageReplacements[i + 1]); } - return iso3ToIso2; + return replacedLanguages; } - // See https://en.wikipedia.org/wiki/List_of_ISO_639-2_codes. - private static final String[] iso3BibliographicalToIso2 = + private static String maybeReplaceGrandfatheredLanguageTags(String languageTag) { + for (int i = 0; i < isoGrandfatheredTagReplacements.length; i += 2) { + if (languageTag.startsWith(isoGrandfatheredTagReplacements[i])) { + return isoGrandfatheredTagReplacements[i + 1] + + languageTag.substring(/* beginIndex= */ isoGrandfatheredTagReplacements[i].length()); + } + } + return languageTag; + } + + // Additional mapping from ISO3 to ISO2 language codes. + private static final String[] additionalIsoLanguageReplacements = new String[] { + // Bibliographical codes defined in ISO 639-2/B, replaced by terminological code defined in + // ISO 639-2/T. See https://en.wikipedia.org/wiki/List_of_ISO_639-2_codes. "alb", "sq", "arm", "hy", "baq", "eu", @@ -2161,8 +2165,50 @@ public final class Util { "may", "ms", "per", "fa", "rum", "ro", + "scc", "hbs-srp", "slo", "sk", - "wel", "cy" + "wel", "cy", + // Deprecated 2-letter codes, replaced by modern equivalent (including macrolanguage) + // See https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes, "ISO 639:1988" + "id", "ms-ind", + "iw", "he", + "heb", "he", + "ji", "yi", + // Individual macrolanguage codes mapped back to full macrolanguage code. + // See https://en.wikipedia.org/wiki/ISO_639_macrolanguage + "in", "ms-ind", + "ind", "ms-ind", + "nb", "no-nob", + "nob", "no-nob", + "nn", "no-nno", + "nno", "no-nno", + "tw", "ak-twi", + "twi", "ak-twi", + "bs", "hbs-bos", + "bos", "hbs-bos", + "hr", "hbs-hrv", + "hrv", "hbs-hrv", + "sr", "hbs-srp", + "srp", "hbs-srp", + "cmn", "zh-cmn", + "hak", "zh-hak", + "nan", "zh-nan", + "hsn", "zh-hsn" + }; + + // "Grandfathered tags", replaced by modern equivalents (including macrolanguage) + // See https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry. + private static final String[] isoGrandfatheredTagReplacements = + new String[] { + "i-lux", "lb", + "i-hak", "zh-hak", + "i-navajo", "nv", + "no-bok", "no-nob", + "no-nyn", "no-nno", + "zh-guoyu", "zh-cmn", + "zh-hakka", "zh-hak", + "zh-min-nan", "zh-nan", + "zh-xiang", "zh-hsn" }; /** diff --git a/library/core/src/test/java/com/google/android/exoplayer2/offline/DownloadHelperTest.java b/library/core/src/test/java/com/google/android/exoplayer2/offline/DownloadHelperTest.java index 49c8302d30..7c2cbf6a52 100644 --- a/library/core/src/test/java/com/google/android/exoplayer2/offline/DownloadHelperTest.java +++ b/library/core/src/test/java/com/google/android/exoplayer2/offline/DownloadHelperTest.java @@ -50,6 +50,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.robolectric.annotation.LooperMode; @@ -69,41 +70,59 @@ public class DownloadHelperTest { private static final Format VIDEO_FORMAT_LOW = createVideoFormat(/* bitrate= */ 200_000); private static final Format VIDEO_FORMAT_HIGH = createVideoFormat(/* bitrate= */ 800_000); - private static final Format AUDIO_FORMAT_US = createAudioFormat(/* language= */ "US"); - private static final Format AUDIO_FORMAT_ZH = createAudioFormat(/* language= */ "ZH"); - private static final Format TEXT_FORMAT_US = createTextFormat(/* language= */ "US"); - private static final Format TEXT_FORMAT_ZH = createTextFormat(/* language= */ "ZH"); + private static Format audioFormatUs; + private static Format audioFormatZh; + private static Format textFormatUs; + private static Format textFormatZh; private static final TrackGroup TRACK_GROUP_VIDEO_BOTH = new TrackGroup(VIDEO_FORMAT_LOW, VIDEO_FORMAT_HIGH); private static final TrackGroup TRACK_GROUP_VIDEO_SINGLE = new TrackGroup(VIDEO_FORMAT_LOW); - private static final TrackGroup TRACK_GROUP_AUDIO_US = new TrackGroup(AUDIO_FORMAT_US); - private static final TrackGroup TRACK_GROUP_AUDIO_ZH = new TrackGroup(AUDIO_FORMAT_ZH); - private static final TrackGroup TRACK_GROUP_TEXT_US = new TrackGroup(TEXT_FORMAT_US); - private static final TrackGroup TRACK_GROUP_TEXT_ZH = new TrackGroup(TEXT_FORMAT_ZH); - private static final TrackGroupArray TRACK_GROUP_ARRAY_ALL = - new TrackGroupArray( - TRACK_GROUP_VIDEO_BOTH, - TRACK_GROUP_AUDIO_US, - TRACK_GROUP_AUDIO_ZH, - TRACK_GROUP_TEXT_US, - TRACK_GROUP_TEXT_ZH); - private static final TrackGroupArray TRACK_GROUP_ARRAY_SINGLE = - new TrackGroupArray(TRACK_GROUP_VIDEO_SINGLE, TRACK_GROUP_AUDIO_US); - private static final TrackGroupArray[] TRACK_GROUP_ARRAYS = - new TrackGroupArray[] {TRACK_GROUP_ARRAY_ALL, TRACK_GROUP_ARRAY_SINGLE}; + private static TrackGroup trackGroupAudioUs; + private static TrackGroup trackGroupAudioZh; + private static TrackGroup trackGroupTextUs; + private static TrackGroup trackGroupTextZh; - private Uri testUri; + private static TrackGroupArray trackGroupArrayAll; + private static TrackGroupArray trackGroupArraySingle; + private static TrackGroupArray[] trackGroupArrays; + + private static Uri testUri; private DownloadHelper downloadHelper; + @BeforeClass + public static void staticSetUp() { + audioFormatUs = createAudioFormat(/* language= */ "US"); + audioFormatZh = createAudioFormat(/* language= */ "ZH"); + textFormatUs = createTextFormat(/* language= */ "US"); + textFormatZh = createTextFormat(/* language= */ "ZH"); + + trackGroupAudioUs = new TrackGroup(audioFormatUs); + trackGroupAudioZh = new TrackGroup(audioFormatZh); + trackGroupTextUs = new TrackGroup(textFormatUs); + trackGroupTextZh = new TrackGroup(textFormatZh); + + trackGroupArrayAll = + new TrackGroupArray( + TRACK_GROUP_VIDEO_BOTH, + trackGroupAudioUs, + trackGroupAudioZh, + trackGroupTextUs, + trackGroupTextZh); + trackGroupArraySingle = + new TrackGroupArray(TRACK_GROUP_VIDEO_SINGLE, trackGroupAudioUs); + trackGroupArrays = + new TrackGroupArray[] {trackGroupArrayAll, trackGroupArraySingle}; + + testUri = Uri.parse("http://test.uri"); + } + @Before public void setUp() { - testUri = Uri.parse("http://test.uri"); - FakeRenderer videoRenderer = new FakeRenderer(VIDEO_FORMAT_LOW, VIDEO_FORMAT_HIGH); - FakeRenderer audioRenderer = new FakeRenderer(AUDIO_FORMAT_US, AUDIO_FORMAT_ZH); - FakeRenderer textRenderer = new FakeRenderer(TEXT_FORMAT_US, TEXT_FORMAT_ZH); + FakeRenderer audioRenderer = new FakeRenderer(audioFormatUs, audioFormatZh); + FakeRenderer textRenderer = new FakeRenderer(textFormatUs, textFormatZh); RenderersFactory renderersFactory = (handler, videoListener, audioListener, metadata, text, drm) -> new Renderer[] {textRenderer, audioRenderer, videoRenderer}; @@ -143,8 +162,8 @@ public class DownloadHelperTest { TrackGroupArray trackGroupArrayPeriod0 = downloadHelper.getTrackGroups(/* periodIndex= */ 0); TrackGroupArray trackGroupArrayPeriod1 = downloadHelper.getTrackGroups(/* periodIndex= */ 1); - assertThat(trackGroupArrayPeriod0).isEqualTo(TRACK_GROUP_ARRAYS[0]); - assertThat(trackGroupArrayPeriod1).isEqualTo(TRACK_GROUP_ARRAYS[1]); + assertThat(trackGroupArrayPeriod0).isEqualTo(trackGroupArrays[0]); + assertThat(trackGroupArrayPeriod1).isEqualTo(trackGroupArrays[1]); } @Test @@ -162,13 +181,13 @@ public class DownloadHelperTest { assertThat(mappedTracks0.getTrackGroups(/* rendererIndex= */ 1).length).isEqualTo(2); assertThat(mappedTracks0.getTrackGroups(/* rendererIndex= */ 2).length).isEqualTo(1); assertThat(mappedTracks0.getTrackGroups(/* rendererIndex= */ 0).get(/* index= */ 0)) - .isEqualTo(TRACK_GROUP_TEXT_US); + .isEqualTo(trackGroupTextUs); assertThat(mappedTracks0.getTrackGroups(/* rendererIndex= */ 0).get(/* index= */ 1)) - .isEqualTo(TRACK_GROUP_TEXT_ZH); + .isEqualTo(trackGroupTextZh); assertThat(mappedTracks0.getTrackGroups(/* rendererIndex= */ 1).get(/* index= */ 0)) - .isEqualTo(TRACK_GROUP_AUDIO_US); + .isEqualTo(trackGroupAudioUs); assertThat(mappedTracks0.getTrackGroups(/* rendererIndex= */ 1).get(/* index= */ 1)) - .isEqualTo(TRACK_GROUP_AUDIO_ZH); + .isEqualTo(trackGroupAudioZh); assertThat(mappedTracks0.getTrackGroups(/* rendererIndex= */ 2).get(/* index= */ 0)) .isEqualTo(TRACK_GROUP_VIDEO_BOTH); @@ -180,7 +199,7 @@ public class DownloadHelperTest { assertThat(mappedTracks1.getTrackGroups(/* rendererIndex= */ 1).length).isEqualTo(1); assertThat(mappedTracks1.getTrackGroups(/* rendererIndex= */ 2).length).isEqualTo(1); assertThat(mappedTracks1.getTrackGroups(/* rendererIndex= */ 1).get(/* index= */ 0)) - .isEqualTo(TRACK_GROUP_AUDIO_US); + .isEqualTo(trackGroupAudioUs); assertThat(mappedTracks1.getTrackGroups(/* rendererIndex= */ 2).get(/* index= */ 0)) .isEqualTo(TRACK_GROUP_VIDEO_SINGLE); } @@ -202,12 +221,12 @@ public class DownloadHelperTest { List selectedVideo1 = downloadHelper.getTrackSelections(/* periodIndex= */ 1, /* rendererIndex= */ 2); - assertSingleTrackSelectionEquals(selectedText0, TRACK_GROUP_TEXT_US, 0); - assertSingleTrackSelectionEquals(selectedAudio0, TRACK_GROUP_AUDIO_US, 0); + assertSingleTrackSelectionEquals(selectedText0, trackGroupTextUs, 0); + assertSingleTrackSelectionEquals(selectedAudio0, trackGroupAudioUs, 0); assertSingleTrackSelectionEquals(selectedVideo0, TRACK_GROUP_VIDEO_BOTH, 1); assertThat(selectedText1).isEmpty(); - assertSingleTrackSelectionEquals(selectedAudio1, TRACK_GROUP_AUDIO_US, 0); + assertSingleTrackSelectionEquals(selectedAudio1, trackGroupAudioUs, 0); assertSingleTrackSelectionEquals(selectedVideo1, TRACK_GROUP_VIDEO_SINGLE, 0); } @@ -236,7 +255,7 @@ public class DownloadHelperTest { // Verify assertThat(selectedText1).isEmpty(); - assertSingleTrackSelectionEquals(selectedAudio1, TRACK_GROUP_AUDIO_US, 0); + assertSingleTrackSelectionEquals(selectedAudio1, trackGroupAudioUs, 0); assertSingleTrackSelectionEquals(selectedVideo1, TRACK_GROUP_VIDEO_SINGLE, 0); } @@ -266,12 +285,12 @@ public class DownloadHelperTest { List selectedVideo1 = downloadHelper.getTrackSelections(/* periodIndex= */ 1, /* rendererIndex= */ 2); - assertSingleTrackSelectionEquals(selectedText0, TRACK_GROUP_TEXT_ZH, 0); - assertSingleTrackSelectionEquals(selectedAudio0, TRACK_GROUP_AUDIO_ZH, 0); + assertSingleTrackSelectionEquals(selectedText0, trackGroupTextZh, 0); + assertSingleTrackSelectionEquals(selectedAudio0, trackGroupAudioZh, 0); assertThat(selectedVideo0).isEmpty(); assertThat(selectedText1).isEmpty(); - assertSingleTrackSelectionEquals(selectedAudio1, TRACK_GROUP_AUDIO_US, 0); + assertSingleTrackSelectionEquals(selectedAudio1, trackGroupAudioUs, 0); assertSingleTrackSelectionEquals(selectedVideo1, TRACK_GROUP_VIDEO_SINGLE, 0); } @@ -302,14 +321,14 @@ public class DownloadHelperTest { List selectedVideo1 = downloadHelper.getTrackSelections(/* periodIndex= */ 1, /* rendererIndex= */ 2); - assertSingleTrackSelectionEquals(selectedText0, TRACK_GROUP_TEXT_US, 0); + assertSingleTrackSelectionEquals(selectedText0, trackGroupTextUs, 0); assertThat(selectedAudio0).hasSize(2); - assertTrackSelectionEquals(selectedAudio0.get(0), TRACK_GROUP_AUDIO_US, 0); - assertTrackSelectionEquals(selectedAudio0.get(1), TRACK_GROUP_AUDIO_ZH, 0); + assertTrackSelectionEquals(selectedAudio0.get(0), trackGroupAudioUs, 0); + assertTrackSelectionEquals(selectedAudio0.get(1), trackGroupAudioZh, 0); assertSingleTrackSelectionEquals(selectedVideo0, TRACK_GROUP_VIDEO_BOTH, 0, 1); assertThat(selectedText1).isEmpty(); - assertSingleTrackSelectionEquals(selectedAudio1, TRACK_GROUP_AUDIO_US, 0); + assertSingleTrackSelectionEquals(selectedAudio1, trackGroupAudioUs, 0); assertSingleTrackSelectionEquals(selectedVideo1, TRACK_GROUP_VIDEO_SINGLE, 0); } @@ -338,12 +357,12 @@ public class DownloadHelperTest { assertThat(selectedVideo0).isEmpty(); assertThat(selectedText0).isEmpty(); assertThat(selectedAudio0).hasSize(2); - assertTrackSelectionEquals(selectedAudio0.get(0), TRACK_GROUP_AUDIO_ZH, 0); - assertTrackSelectionEquals(selectedAudio0.get(1), TRACK_GROUP_AUDIO_US, 0); + assertTrackSelectionEquals(selectedAudio0.get(0), trackGroupAudioZh, 0); + assertTrackSelectionEquals(selectedAudio0.get(1), trackGroupAudioUs, 0); assertThat(selectedVideo1).isEmpty(); assertThat(selectedText1).isEmpty(); - assertSingleTrackSelectionEquals(selectedAudio1, TRACK_GROUP_AUDIO_US, 0); + assertSingleTrackSelectionEquals(selectedAudio1, trackGroupAudioUs, 0); } @Test @@ -372,8 +391,8 @@ public class DownloadHelperTest { assertThat(selectedVideo0).isEmpty(); assertThat(selectedAudio0).isEmpty(); assertThat(selectedText0).hasSize(2); - assertTrackSelectionEquals(selectedText0.get(0), TRACK_GROUP_TEXT_ZH, 0); - assertTrackSelectionEquals(selectedText0.get(1), TRACK_GROUP_TEXT_US, 0); + assertTrackSelectionEquals(selectedText0.get(0), trackGroupTextZh, 0); + assertTrackSelectionEquals(selectedText0.get(1), trackGroupTextUs, 0); assertThat(selectedVideo1).isEmpty(); assertThat(selectedAudio1).isEmpty(); @@ -501,7 +520,7 @@ public class DownloadHelperTest { public MediaPeriod createPeriod(MediaPeriodId id, Allocator allocator, long startPositionUs) { int periodIndex = TEST_TIMELINE.getIndexOfPeriod(id.periodUid); return new FakeMediaPeriod( - TRACK_GROUP_ARRAYS[periodIndex], + trackGroupArrays[periodIndex], new EventDispatcher() .withParameters(/* windowIndex= */ 0, id, /* mediaTimeOffsetMs= */ 0)) { @Override @@ -509,7 +528,7 @@ public class DownloadHelperTest { List result = new ArrayList<>(); for (TrackSelection trackSelection : trackSelections) { int groupIndex = - TRACK_GROUP_ARRAYS[periodIndex].indexOf(trackSelection.getTrackGroup()); + trackGroupArrays[periodIndex].indexOf(trackSelection.getTrackGroup()); for (int i = 0; i < trackSelection.length(); i++) { result.add( new StreamKey(periodIndex, groupIndex, trackSelection.getIndexInTrackGroup(i))); diff --git a/library/core/src/test/java/com/google/android/exoplayer2/util/UtilTest.java b/library/core/src/test/java/com/google/android/exoplayer2/util/UtilTest.java index 172fdf31ad..3a972db4b5 100644 --- a/library/core/src/test/java/com/google/android/exoplayer2/util/UtilTest.java +++ b/library/core/src/test/java/com/google/android/exoplayer2/util/UtilTest.java @@ -308,10 +308,17 @@ public class UtilTest { } @Test - @Config(sdk = 21) - public void testNormalizeLanguageCodeV21() { + @Config(sdk = Config.ALL_SDKS) + public void normalizeLanguageCode_keepsUndefinedTagsUnchanged() { assertThat(Util.normalizeLanguageCode(null)).isNull(); assertThat(Util.normalizeLanguageCode("")).isEmpty(); + assertThat(Util.normalizeLanguageCode("und")).isEqualTo("und"); + assertThat(Util.normalizeLanguageCode("DoesNotExist")).isEqualTo("doesnotexist"); + } + + @Test + @Config(sdk = Config.ALL_SDKS) + public void normalizeLanguageCode_normalizesCodeToTwoLetterISOAndLowerCase_keepingAllSubtags() { assertThat(Util.normalizeLanguageCode("es")).isEqualTo("es"); assertThat(Util.normalizeLanguageCode("spa")).isEqualTo("es"); assertThat(Util.normalizeLanguageCode("es-AR")).isEqualTo("es-ar"); @@ -319,37 +326,18 @@ public class UtilTest { assertThat(Util.normalizeLanguageCode("es_AR")).isEqualTo("es-ar"); assertThat(Util.normalizeLanguageCode("spa_ar")).isEqualTo("es-ar"); assertThat(Util.normalizeLanguageCode("es-AR-dialect")).isEqualTo("es-ar-dialect"); + // Regional subtag (South America) assertThat(Util.normalizeLanguageCode("ES-419")).isEqualTo("es-419"); + // Script subtag (Simplified Taiwanese) assertThat(Util.normalizeLanguageCode("zh-hans-tw")).isEqualTo("zh-hans-tw"); - assertThat(Util.normalizeLanguageCode("zh-tw-hans")).isEqualTo("zh-tw"); assertThat(Util.normalizeLanguageCode("zho-hans-tw")).isEqualTo("zh-hans-tw"); - assertThat(Util.normalizeLanguageCode("und")).isEqualTo("und"); - assertThat(Util.normalizeLanguageCode("DoesNotExist")).isEqualTo("doesnotexist"); + // Non-spec compliant subtags. + assertThat(Util.normalizeLanguageCode("sv-illegalSubtag")).isEqualTo("sv-illegalsubtag"); } @Test - @Config(sdk = 16) - public void testNormalizeLanguageCode() { - assertThat(Util.normalizeLanguageCode(null)).isNull(); - assertThat(Util.normalizeLanguageCode("")).isEmpty(); - assertThat(Util.normalizeLanguageCode("es")).isEqualTo("es"); - assertThat(Util.normalizeLanguageCode("spa")).isEqualTo("es"); - assertThat(Util.normalizeLanguageCode("es-AR")).isEqualTo("es-ar"); - assertThat(Util.normalizeLanguageCode("SpA-ar")).isEqualTo("es-ar"); - assertThat(Util.normalizeLanguageCode("es_AR")).isEqualTo("es-ar"); - assertThat(Util.normalizeLanguageCode("spa_ar")).isEqualTo("es-ar"); - assertThat(Util.normalizeLanguageCode("es-AR-dialect")).isEqualTo("es-ar-dialect"); - assertThat(Util.normalizeLanguageCode("ES-419")).isEqualTo("es-419"); - assertThat(Util.normalizeLanguageCode("zh-hans-tw")).isEqualTo("zh-hans-tw"); - // Doesn't work on API < 21 because we can't use Locale syntax verification. - // assertThat(Util.normalizeLanguageCode("zh-tw-hans")).isEqualTo("zh-tw"); - assertThat(Util.normalizeLanguageCode("zho-hans-tw")).isEqualTo("zh-hans-tw"); - assertThat(Util.normalizeLanguageCode("und")).isEqualTo("und"); - assertThat(Util.normalizeLanguageCode("DoesNotExist")).isEqualTo("doesnotexist"); - } - - @Test - public void testNormalizeIso6392BibliographicalAndTextualCodes() { + @Config(sdk = Config.ALL_SDKS) + public void normalizeLanguageCode_iso6392BibliographicalAndTextualCodes_areNormalizedToSameTag() { // See https://en.wikipedia.org/wiki/List_of_ISO_639-2_codes. assertThat(Util.normalizeLanguageCode("alb")).isEqualTo(Util.normalizeLanguageCode("sqi")); assertThat(Util.normalizeLanguageCode("arm")).isEqualTo(Util.normalizeLanguageCode("hye")); @@ -369,10 +357,79 @@ public class UtilTest { assertThat(Util.normalizeLanguageCode("per")).isEqualTo(Util.normalizeLanguageCode("fas")); assertThat(Util.normalizeLanguageCode("rum")).isEqualTo(Util.normalizeLanguageCode("ron")); assertThat(Util.normalizeLanguageCode("slo")).isEqualTo(Util.normalizeLanguageCode("slk")); + assertThat(Util.normalizeLanguageCode("scc")).isEqualTo(Util.normalizeLanguageCode("srp")); assertThat(Util.normalizeLanguageCode("tib")).isEqualTo(Util.normalizeLanguageCode("bod")); assertThat(Util.normalizeLanguageCode("wel")).isEqualTo(Util.normalizeLanguageCode("cym")); } + @Test + @Config(sdk = Config.ALL_SDKS) + public void + normalizeLanguageCode_deprecatedLanguageTagsAndModernReplacement_areNormalizedToSameTag() { + // See https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes, "ISO 639:1988" + assertThat(Util.normalizeLanguageCode("in")).isEqualTo(Util.normalizeLanguageCode("id")); + assertThat(Util.normalizeLanguageCode("in")).isEqualTo(Util.normalizeLanguageCode("ind")); + assertThat(Util.normalizeLanguageCode("iw")).isEqualTo(Util.normalizeLanguageCode("he")); + assertThat(Util.normalizeLanguageCode("iw")).isEqualTo(Util.normalizeLanguageCode("heb")); + assertThat(Util.normalizeLanguageCode("ji")).isEqualTo(Util.normalizeLanguageCode("yi")); + assertThat(Util.normalizeLanguageCode("ji")).isEqualTo(Util.normalizeLanguageCode("yid")); + + // Grandfathered tags + assertThat(Util.normalizeLanguageCode("i-lux")).isEqualTo(Util.normalizeLanguageCode("lb")); + assertThat(Util.normalizeLanguageCode("i-lux")).isEqualTo(Util.normalizeLanguageCode("ltz")); + assertThat(Util.normalizeLanguageCode("i-hak")).isEqualTo(Util.normalizeLanguageCode("hak")); + assertThat(Util.normalizeLanguageCode("i-hak")).isEqualTo(Util.normalizeLanguageCode("zh-hak")); + assertThat(Util.normalizeLanguageCode("i-navajo")).isEqualTo(Util.normalizeLanguageCode("nv")); + assertThat(Util.normalizeLanguageCode("i-navajo")).isEqualTo(Util.normalizeLanguageCode("nav")); + assertThat(Util.normalizeLanguageCode("no-bok")).isEqualTo(Util.normalizeLanguageCode("nb")); + assertThat(Util.normalizeLanguageCode("no-bok")).isEqualTo(Util.normalizeLanguageCode("nob")); + assertThat(Util.normalizeLanguageCode("no-nyn")).isEqualTo(Util.normalizeLanguageCode("nn")); + assertThat(Util.normalizeLanguageCode("no-nyn")).isEqualTo(Util.normalizeLanguageCode("nno")); + assertThat(Util.normalizeLanguageCode("zh-guoyu")).isEqualTo(Util.normalizeLanguageCode("cmn")); + assertThat(Util.normalizeLanguageCode("zh-guoyu")) + .isEqualTo(Util.normalizeLanguageCode("zh-cmn")); + assertThat(Util.normalizeLanguageCode("zh-hakka")).isEqualTo(Util.normalizeLanguageCode("hak")); + assertThat(Util.normalizeLanguageCode("zh-hakka")) + .isEqualTo(Util.normalizeLanguageCode("zh-hak")); + assertThat(Util.normalizeLanguageCode("zh-min-nan")) + .isEqualTo(Util.normalizeLanguageCode("nan")); + assertThat(Util.normalizeLanguageCode("zh-min-nan")) + .isEqualTo(Util.normalizeLanguageCode("zh-nan")); + assertThat(Util.normalizeLanguageCode("zh-xiang")).isEqualTo(Util.normalizeLanguageCode("hsn")); + assertThat(Util.normalizeLanguageCode("zh-xiang")) + .isEqualTo(Util.normalizeLanguageCode("zh-hsn")); + } + + @Test + @Config(sdk = Config.ALL_SDKS) + public void normalizeLanguageCode_macrolanguageTags_areFullyMaintained() { + // See https://en.wikipedia.org/wiki/ISO_639_macrolanguage + assertThat(Util.normalizeLanguageCode("zh-cmn")).isEqualTo("zh-cmn"); + assertThat(Util.normalizeLanguageCode("zho-cmn")).isEqualTo("zh-cmn"); + assertThat(Util.normalizeLanguageCode("ar-ayl")).isEqualTo("ar-ayl"); + assertThat(Util.normalizeLanguageCode("ara-ayl")).isEqualTo("ar-ayl"); + + // Special case of short codes that are actually part of a macrolanguage. + assertThat(Util.normalizeLanguageCode("nb")).isEqualTo("no-nob"); + assertThat(Util.normalizeLanguageCode("nn")).isEqualTo("no-nno"); + assertThat(Util.normalizeLanguageCode("nob")).isEqualTo("no-nob"); + assertThat(Util.normalizeLanguageCode("nno")).isEqualTo("no-nno"); + assertThat(Util.normalizeLanguageCode("tw")).isEqualTo("ak-twi"); + assertThat(Util.normalizeLanguageCode("twi")).isEqualTo("ak-twi"); + assertThat(Util.normalizeLanguageCode("bs")).isEqualTo("hbs-bos"); + assertThat(Util.normalizeLanguageCode("bos")).isEqualTo("hbs-bos"); + assertThat(Util.normalizeLanguageCode("hr")).isEqualTo("hbs-hrv"); + assertThat(Util.normalizeLanguageCode("hrv")).isEqualTo("hbs-hrv"); + assertThat(Util.normalizeLanguageCode("sr")).isEqualTo("hbs-srp"); + assertThat(Util.normalizeLanguageCode("srp")).isEqualTo("hbs-srp"); + assertThat(Util.normalizeLanguageCode("id")).isEqualTo("ms-ind"); + assertThat(Util.normalizeLanguageCode("ind")).isEqualTo("ms-ind"); + assertThat(Util.normalizeLanguageCode("cmn")).isEqualTo("zh-cmn"); + assertThat(Util.normalizeLanguageCode("hak")).isEqualTo("zh-hak"); + assertThat(Util.normalizeLanguageCode("nan")).isEqualTo("zh-nan"); + assertThat(Util.normalizeLanguageCode("hsn")).isEqualTo("zh-hsn"); + } + private static void assertEscapeUnescapeFileName(String fileName, String escapedFileName) { assertThat(escapeFileName(fileName)).isEqualTo(escapedFileName); assertThat(unescapeFileName(escapedFileName)).isEqualTo(fileName);