Skip to content

Commit 537dabc

Browse files
miss-islingtonserhiy-storchaka
miss-islington
and
serhiy-storchaka
authored
[3.9] gh-91575: Update case-insensitive matching in re to the latest Unicode version (GH-91580). (GH-91661) (GH-91837)
(cherry picked from commit 1c2fceb) (cherry picked from commit 1748816) Co-authored-by: Serhiy Storchaka <[email protected]>
1 parent 76ff686 commit 537dabc

File tree

3 files changed

+78
-9
lines changed

3 files changed

+78
-9
lines changed

‎Lib/sre_compile.py

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,22 @@
5252
(0x3c2, 0x3c3), # ςσ
5353
# GREEK SMALL LETTER PHI, GREEK PHI SYMBOL
5454
(0x3c6, 0x3d5), # φϕ
55+
# CYRILLIC SMALL LETTER VE, CYRILLIC SMALL LETTER ROUNDED VE
56+
(0x432, 0x1c80), # вᲀ
57+
# CYRILLIC SMALL LETTER DE, CYRILLIC SMALL LETTER LONG-LEGGED DE
58+
(0x434, 0x1c81), # дᲁ
59+
# CYRILLIC SMALL LETTER O, CYRILLIC SMALL LETTER NARROW O
60+
(0x43e, 0x1c82), # оᲂ
61+
# CYRILLIC SMALL LETTER ES, CYRILLIC SMALL LETTER WIDE ES
62+
(0x441, 0x1c83), # сᲃ
63+
# CYRILLIC SMALL LETTER TE, CYRILLIC SMALL LETTER TALL TE, CYRILLIC SMALL LETTER THREE-LEGGED TE
64+
(0x442, 0x1c84, 0x1c85), # тᲄᲅ
65+
# CYRILLIC SMALL LETTER HARD SIGN, CYRILLIC SMALL LETTER TALL HARD SIGN
66+
(0x44a, 0x1c86), # ъᲆ
67+
# CYRILLIC SMALL LETTER YAT, CYRILLIC SMALL LETTER TALL YAT
68+
(0x463, 0x1c87), # ѣᲇ
69+
# CYRILLIC SMALL LETTER UNBLENDED UK, CYRILLIC SMALL LETTER MONOGRAPH UK
70+
(0x1c88, 0xa64b), # ᲈꙋ
5571
# LATIN SMALL LETTER S WITH DOT ABOVE, LATIN SMALL LETTER LONG S WITH DOT ABOVE
5672
(0x1e61, 0x1e9b), # ṡẛ
5773
# LATIN SMALL LIGATURE LONG S T, LATIN SMALL LIGATURE ST
@@ -320,11 +336,19 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None):
320336
charmap += b'\0' * 0xff00
321337
continue
322338
# Character set contains non-BMP character codes.
339+
# For range, all BMP characters in the range are already
340+
# proceeded.
323341
if fixup:
324342
hascased = True
325-
# There are only two ranges of cased non-BMP characters:
326-
# 10400-1044F (Deseret) and 118A0-118DF (Warang Citi),
327-
# and for both ranges RANGE_UNI_IGNORE works.
343+
# For now, IN_UNI_IGNORE+LITERAL and
344+
# IN_UNI_IGNORE+RANGE_UNI_IGNORE work for all non-BMP
345+
# characters, because two characters (at least one of
346+
# which is not in the BMP) match case-insensitively
347+
# if and only if:
348+
# 1) c1.lower() == c2.lower()
349+
# 2) c1.lower() == c2 or c1.lower().upper() == c2
350+
# Also, both c.lower() and c.lower().upper() are single
351+
# characters for every non-BMP character.
328352
if op is RANGE:
329353
op = RANGE_UNI_IGNORE
330354
tail.append((op, av))

‎Lib/test/test_re.py

Lines changed: 49 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -872,16 +872,30 @@ def test_ignore_case(self):
872872
self.assertEqual(re.match(r"((a)\s(abc|a))", "a a", re.I).group(1), "a a")
873873
self.assertEqual(re.match(r"((a)\s(abc|a)*)", "a aa", re.I).group(1), "a aa")
874874

875-
assert '\u212a'.lower() == 'k' # 'K'
875+
# Two different characters have the same lowercase.
876+
assert 'K'.lower() == '\u212a'.lower() == 'k' # 'K'
876877
self.assertTrue(re.match(r'K', '\u212a', re.I))
877878
self.assertTrue(re.match(r'k', '\u212a', re.I))
878879
self.assertTrue(re.match(r'\u212a', 'K', re.I))
879880
self.assertTrue(re.match(r'\u212a', 'k', re.I))
880-
assert '\u017f'.upper() == 'S' # 'ſ'
881+
882+
# Two different characters have the same uppercase.
883+
assert 's'.upper() == '\u017f'.upper() == 'S' # 'ſ'
881884
self.assertTrue(re.match(r'S', '\u017f', re.I))
882885
self.assertTrue(re.match(r's', '\u017f', re.I))
883886
self.assertTrue(re.match(r'\u017f', 'S', re.I))
884887
self.assertTrue(re.match(r'\u017f', 's', re.I))
888+
889+
# Two different characters have the same uppercase. Unicode 9.0+.
890+
assert '\u0432'.upper() == '\u1c80'.upper() == '\u0412' # 'в', 'ᲀ', 'В'
891+
self.assertTrue(re.match(r'\u0412', '\u0432', re.I))
892+
self.assertTrue(re.match(r'\u0412', '\u1c80', re.I))
893+
self.assertTrue(re.match(r'\u0432', '\u0412', re.I))
894+
self.assertTrue(re.match(r'\u0432', '\u1c80', re.I))
895+
self.assertTrue(re.match(r'\u1c80', '\u0412', re.I))
896+
self.assertTrue(re.match(r'\u1c80', '\u0432', re.I))
897+
898+
# Two different characters have the same multicharacter uppercase.
885899
assert '\ufb05'.upper() == '\ufb06'.upper() == 'ST' # 'ſt', 'st'
886900
self.assertTrue(re.match(r'\ufb05', '\ufb06', re.I))
887901
self.assertTrue(re.match(r'\ufb06', '\ufb05', re.I))
@@ -895,16 +909,31 @@ def test_ignore_case_set(self):
895909
self.assertTrue(re.match(br'[19a]', b'a', re.I))
896910
self.assertTrue(re.match(br'[19a]', b'A', re.I))
897911
self.assertTrue(re.match(br'[19A]', b'a', re.I))
898-
assert '\u212a'.lower() == 'k' # 'K'
912+
913+
# Two different characters have the same lowercase.
914+
assert 'K'.lower() == '\u212a'.lower() == 'k' # 'K'
899915
self.assertTrue(re.match(r'[19K]', '\u212a', re.I))
900916
self.assertTrue(re.match(r'[19k]', '\u212a', re.I))
901917
self.assertTrue(re.match(r'[19\u212a]', 'K', re.I))
902918
self.assertTrue(re.match(r'[19\u212a]', 'k', re.I))
903-
assert '\u017f'.upper() == 'S' # 'ſ'
919+
920+
# Two different characters have the same uppercase.
921+
assert 's'.upper() == '\u017f'.upper() == 'S' # 'ſ'
904922
self.assertTrue(re.match(r'[19S]', '\u017f', re.I))
905923
self.assertTrue(re.match(r'[19s]', '\u017f', re.I))
906924
self.assertTrue(re.match(r'[19\u017f]', 'S', re.I))
907925
self.assertTrue(re.match(r'[19\u017f]', 's', re.I))
926+
927+
# Two different characters have the same uppercase. Unicode 9.0+.
928+
assert '\u0432'.upper() == '\u1c80'.upper() == '\u0412' # 'в', 'ᲀ', 'В'
929+
self.assertTrue(re.match(r'[19\u0412]', '\u0432', re.I))
930+
self.assertTrue(re.match(r'[19\u0412]', '\u1c80', re.I))
931+
self.assertTrue(re.match(r'[19\u0432]', '\u0412', re.I))
932+
self.assertTrue(re.match(r'[19\u0432]', '\u1c80', re.I))
933+
self.assertTrue(re.match(r'[19\u1c80]', '\u0412', re.I))
934+
self.assertTrue(re.match(r'[19\u1c80]', '\u0432', re.I))
935+
936+
# Two different characters have the same multicharacter uppercase.
908937
assert '\ufb05'.upper() == '\ufb06'.upper() == 'ST' # 'ſt', 'st'
909938
self.assertTrue(re.match(r'[19\ufb05]', '\ufb06', re.I))
910939
self.assertTrue(re.match(r'[19\ufb06]', '\ufb05', re.I))
@@ -928,16 +957,30 @@ def test_ignore_case_range(self):
928957
self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010428', re.I))
929958
self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010400', re.I))
930959

931-
assert '\u212a'.lower() == 'k' # 'K'
960+
# Two different characters have the same lowercase.
961+
assert 'K'.lower() == '\u212a'.lower() == 'k' # 'K'
932962
self.assertTrue(re.match(r'[J-M]', '\u212a', re.I))
933963
self.assertTrue(re.match(r'[j-m]', '\u212a', re.I))
934964
self.assertTrue(re.match(r'[\u2129-\u212b]', 'K', re.I))
935965
self.assertTrue(re.match(r'[\u2129-\u212b]', 'k', re.I))
936-
assert '\u017f'.upper() == 'S' # 'ſ'
966+
967+
# Two different characters have the same uppercase.
968+
assert 's'.upper() == '\u017f'.upper() == 'S' # 'ſ'
937969
self.assertTrue(re.match(r'[R-T]', '\u017f', re.I))
938970
self.assertTrue(re.match(r'[r-t]', '\u017f', re.I))
939971
self.assertTrue(re.match(r'[\u017e-\u0180]', 'S', re.I))
940972
self.assertTrue(re.match(r'[\u017e-\u0180]', 's', re.I))
973+
974+
# Two different characters have the same uppercase. Unicode 9.0+.
975+
assert '\u0432'.upper() == '\u1c80'.upper() == '\u0412' # 'в', 'ᲀ', 'В'
976+
self.assertTrue(re.match(r'[\u0411-\u0413]', '\u0432', re.I))
977+
self.assertTrue(re.match(r'[\u0411-\u0413]', '\u1c80', re.I))
978+
self.assertTrue(re.match(r'[\u0431-\u0433]', '\u0412', re.I))
979+
self.assertTrue(re.match(r'[\u0431-\u0433]', '\u1c80', re.I))
980+
self.assertTrue(re.match(r'[\u1c80-\u1c82]', '\u0412', re.I))
981+
self.assertTrue(re.match(r'[\u1c80-\u1c82]', '\u0432', re.I))
982+
983+
# Two different characters have the same multicharacter uppercase.
941984
assert '\ufb05'.upper() == '\ufb06'.upper() == 'ST' # 'ſt', 'st'
942985
self.assertTrue(re.match(r'[\ufb04-\ufb05]', '\ufb06', re.I))
943986
self.assertTrue(re.match(r'[\ufb06-\ufb07]', '\ufb05', re.I))

‎Misc/NEWS.d/next/Library/2022-04-15-18-38-21.gh-issue-91575.fSyAxS.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Update case-insensitive matching in the :mod:`re` module to the latest
2+
Unicode version.

0 commit comments

Comments
 (0)