* enc/unicode/case-folding.rb, casefold.h: Fixed bug that avoided inclusion

of compatibility characters in uppper-/lower-case mappings.
  (with Kimihito Matsui)


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@53890 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
This commit is contained in:
duerst 2016-02-22 09:17:43 +00:00
Родитель cdc5ac2e1e
Коммит 5470ce8206
3 изменённых файлов: 162 добавлений и 157 удалений

Просмотреть файл

@ -1,3 +1,9 @@
Mon Feb 22 18:17:03 2016 Martin Duerst <duerst@it.aoyama.ac.jp>
* enc/unicode/case-folding.rb, casefold.h: Fixed bug that avoided inclusion
of compatibility characters in uppper-/lower-case mappings.
(with Kimihito Matsui)
Sun Feb 21 13:57:18 2016 Nobuyoshi Nakada <nobu@ruby-lang.org>
* cgi/escape/escape.c: Optimize CGI.unescape performance by C ext

Просмотреть файл

@ -205,12 +205,11 @@ class CaseMapping
def initialize (mapping_directory)
@mappings = {}
IO.readlines(File.expand_path('UnicodeData.txt', mapping_directory), encoding: Encoding::ASCII_8BIT).each do |line|
next if line =~ /</
next if line =~ /^</
code, _1,_2,_3,_4,_5,_6,_7,_8,_9,_10,_11, upper, lower, title = line.chomp.split ';'
unless upper and lower and title and (upper+lower+title)==''
@mappings[code] = MapItem.new(code, upper, lower, title)
end
end
# IO.readlines(File.expand_path('SpecialCasing.txt', mapping_directory))

Просмотреть файл

@ -84,13 +84,13 @@ static const CaseFold_11_Type CaseFold_11_Table[] = {
{0x012a, {1|F|D, {0x012b}}},
{0x012c, {1|F|D, {0x012d}}},
{0x012e, {1|F|D, {0x012f}}},
{0x0132, {1|F, {0x0133}}},
{0x0132, {1|F|D, {0x0133}}},
{0x0134, {1|F|D, {0x0135}}},
{0x0136, {1|F|D, {0x0137}}},
{0x0139, {1|F|D, {0x013a}}},
{0x013b, {1|F|D, {0x013c}}},
{0x013d, {1|F|D, {0x013e}}},
{0x013f, {1|F, {0x0140}}},
{0x013f, {1|F|D, {0x0140}}},
{0x0141, {1|F|D, {0x0142}}},
{0x0143, {1|F|D, {0x0144}}},
{0x0145, {1|F|D, {0x0146}}},
@ -160,12 +160,12 @@ static const CaseFold_11_Type CaseFold_11_Table[] = {
{0x01b7, {1|F|D, {0x0292}}},
{0x01b8, {1|F|D, {0x01b9}}},
{0x01bc, {1|F|D, {0x01bd}}},
{0x01c4, {1|F, {0x01c6}}},
{0x01c5, {1|F, {0x01c6}}},
{0x01c7, {1|F, {0x01c9}}},
{0x01c8, {1|F, {0x01c9}}},
{0x01ca, {1|F, {0x01cc}}},
{0x01cb, {1|F, {0x01cc}}},
{0x01c4, {1|F|D, {0x01c6}}},
{0x01c5, {1|F|D, {0x01c6}}},
{0x01c7, {1|F|D, {0x01c9}}},
{0x01c8, {1|F|D, {0x01c9}}},
{0x01ca, {1|F|D, {0x01cc}}},
{0x01cb, {1|F|D, {0x01cc}}},
{0x01cd, {1|F|D, {0x01ce}}},
{0x01cf, {1|F|D, {0x01d0}}},
{0x01d1, {1|F|D, {0x01d2}}},
@ -184,8 +184,8 @@ static const CaseFold_11_Type CaseFold_11_Table[] = {
{0x01ec, {1|F|D, {0x01ed}}},
{0x01ee, {1|F|D, {0x01ef}}},
{0x01f0, {2|F, {0x006a, 0x030c}}},
{0x01f1, {1|F, {0x01f3}}},
{0x01f2, {1|F, {0x01f3}}},
{0x01f1, {1|F|D, {0x01f3}}},
{0x01f2, {1|F|D, {0x01f3}}},
{0x01f4, {1|F|D, {0x01f5}}},
{0x01f6, {1|F|D, {0x0195}}},
{0x01f7, {1|F|D, {0x01bf}}},
@ -292,10 +292,10 @@ static const CaseFold_11_Type CaseFold_11_Table[] = {
{0x03ee, {1|F|D, {0x03ef}}},
{0x03f0, {1|F, {0x03ba}}},
{0x03f1, {1|F, {0x03c1}}},
{0x03f4, {1|F, {0x03b8}}},
{0x03f4, {1|F|D, {0x03b8}}},
{0x03f5, {1|F, {0x03b5}}},
{0x03f7, {1|F|D, {0x03f8}}},
{0x03f9, {1|F, {0x03f2}}},
{0x03f9, {1|F|D, {0x03f2}}},
{0x03fa, {1|F|D, {0x03fb}}},
{0x03fd, {1|F|D, {0x037b}}},
{0x03fe, {1|F|D, {0x037c}}},
@ -816,49 +816,49 @@ static const CaseFold_11_Type CaseFold_11_Table[] = {
{0x212a, {1|F|D, {0x006b}}},
{0x212b, {1|F|D, {0x00e5}}},
{0x2132, {1|F|D, {0x214e}}},
{0x2160, {1|F, {0x2170}}},
{0x2161, {1|F, {0x2171}}},
{0x2162, {1|F, {0x2172}}},
{0x2163, {1|F, {0x2173}}},
{0x2164, {1|F, {0x2174}}},
{0x2165, {1|F, {0x2175}}},
{0x2166, {1|F, {0x2176}}},
{0x2167, {1|F, {0x2177}}},
{0x2168, {1|F, {0x2178}}},
{0x2169, {1|F, {0x2179}}},
{0x216a, {1|F, {0x217a}}},
{0x216b, {1|F, {0x217b}}},
{0x216c, {1|F, {0x217c}}},
{0x216d, {1|F, {0x217d}}},
{0x216e, {1|F, {0x217e}}},
{0x216f, {1|F, {0x217f}}},
{0x2160, {1|F|D, {0x2170}}},
{0x2161, {1|F|D, {0x2171}}},
{0x2162, {1|F|D, {0x2172}}},
{0x2163, {1|F|D, {0x2173}}},
{0x2164, {1|F|D, {0x2174}}},
{0x2165, {1|F|D, {0x2175}}},
{0x2166, {1|F|D, {0x2176}}},
{0x2167, {1|F|D, {0x2177}}},
{0x2168, {1|F|D, {0x2178}}},
{0x2169, {1|F|D, {0x2179}}},
{0x216a, {1|F|D, {0x217a}}},
{0x216b, {1|F|D, {0x217b}}},
{0x216c, {1|F|D, {0x217c}}},
{0x216d, {1|F|D, {0x217d}}},
{0x216e, {1|F|D, {0x217e}}},
{0x216f, {1|F|D, {0x217f}}},
{0x2183, {1|F|D, {0x2184}}},
{0x24b6, {1|F, {0x24d0}}},
{0x24b7, {1|F, {0x24d1}}},
{0x24b8, {1|F, {0x24d2}}},
{0x24b9, {1|F, {0x24d3}}},
{0x24ba, {1|F, {0x24d4}}},
{0x24bb, {1|F, {0x24d5}}},
{0x24bc, {1|F, {0x24d6}}},
{0x24bd, {1|F, {0x24d7}}},
{0x24be, {1|F, {0x24d8}}},
{0x24bf, {1|F, {0x24d9}}},
{0x24c0, {1|F, {0x24da}}},
{0x24c1, {1|F, {0x24db}}},
{0x24c2, {1|F, {0x24dc}}},
{0x24c3, {1|F, {0x24dd}}},
{0x24c4, {1|F, {0x24de}}},
{0x24c5, {1|F, {0x24df}}},
{0x24c6, {1|F, {0x24e0}}},
{0x24c7, {1|F, {0x24e1}}},
{0x24c8, {1|F, {0x24e2}}},
{0x24c9, {1|F, {0x24e3}}},
{0x24ca, {1|F, {0x24e4}}},
{0x24cb, {1|F, {0x24e5}}},
{0x24cc, {1|F, {0x24e6}}},
{0x24cd, {1|F, {0x24e7}}},
{0x24ce, {1|F, {0x24e8}}},
{0x24cf, {1|F, {0x24e9}}},
{0x24b6, {1|F|D, {0x24d0}}},
{0x24b7, {1|F|D, {0x24d1}}},
{0x24b8, {1|F|D, {0x24d2}}},
{0x24b9, {1|F|D, {0x24d3}}},
{0x24ba, {1|F|D, {0x24d4}}},
{0x24bb, {1|F|D, {0x24d5}}},
{0x24bc, {1|F|D, {0x24d6}}},
{0x24bd, {1|F|D, {0x24d7}}},
{0x24be, {1|F|D, {0x24d8}}},
{0x24bf, {1|F|D, {0x24d9}}},
{0x24c0, {1|F|D, {0x24da}}},
{0x24c1, {1|F|D, {0x24db}}},
{0x24c2, {1|F|D, {0x24dc}}},
{0x24c3, {1|F|D, {0x24dd}}},
{0x24c4, {1|F|D, {0x24de}}},
{0x24c5, {1|F|D, {0x24df}}},
{0x24c6, {1|F|D, {0x24e0}}},
{0x24c7, {1|F|D, {0x24e1}}},
{0x24c8, {1|F|D, {0x24e2}}},
{0x24c9, {1|F|D, {0x24e3}}},
{0x24ca, {1|F|D, {0x24e4}}},
{0x24cb, {1|F|D, {0x24e5}}},
{0x24cc, {1|F|D, {0x24e6}}},
{0x24cd, {1|F|D, {0x24e7}}},
{0x24ce, {1|F|D, {0x24e8}}},
{0x24cf, {1|F|D, {0x24e9}}},
{0x2c00, {1|F|D, {0x2c30}}},
{0x2c01, {1|F|D, {0x2c31}}},
{0x2c02, {1|F|D, {0x2c32}}},
@ -1173,32 +1173,32 @@ static const CaseFold_11_Type CaseFold_11_Table[] = {
{0xfb15, {2|F, {0x0574, 0x056b}}},
{0xfb16, {2|F, {0x057e, 0x0576}}},
{0xfb17, {2|F, {0x0574, 0x056d}}},
{0xff21, {1|F, {0xff41}}},
{0xff22, {1|F, {0xff42}}},
{0xff23, {1|F, {0xff43}}},
{0xff24, {1|F, {0xff44}}},
{0xff25, {1|F, {0xff45}}},
{0xff26, {1|F, {0xff46}}},
{0xff27, {1|F, {0xff47}}},
{0xff28, {1|F, {0xff48}}},
{0xff29, {1|F, {0xff49}}},
{0xff2a, {1|F, {0xff4a}}},
{0xff2b, {1|F, {0xff4b}}},
{0xff2c, {1|F, {0xff4c}}},
{0xff2d, {1|F, {0xff4d}}},
{0xff2e, {1|F, {0xff4e}}},
{0xff2f, {1|F, {0xff4f}}},
{0xff30, {1|F, {0xff50}}},
{0xff31, {1|F, {0xff51}}},
{0xff32, {1|F, {0xff52}}},
{0xff33, {1|F, {0xff53}}},
{0xff34, {1|F, {0xff54}}},
{0xff35, {1|F, {0xff55}}},
{0xff36, {1|F, {0xff56}}},
{0xff37, {1|F, {0xff57}}},
{0xff38, {1|F, {0xff58}}},
{0xff39, {1|F, {0xff59}}},
{0xff3a, {1|F, {0xff5a}}},
{0xff21, {1|F|D, {0xff41}}},
{0xff22, {1|F|D, {0xff42}}},
{0xff23, {1|F|D, {0xff43}}},
{0xff24, {1|F|D, {0xff44}}},
{0xff25, {1|F|D, {0xff45}}},
{0xff26, {1|F|D, {0xff46}}},
{0xff27, {1|F|D, {0xff47}}},
{0xff28, {1|F|D, {0xff48}}},
{0xff29, {1|F|D, {0xff49}}},
{0xff2a, {1|F|D, {0xff4a}}},
{0xff2b, {1|F|D, {0xff4b}}},
{0xff2c, {1|F|D, {0xff4c}}},
{0xff2d, {1|F|D, {0xff4d}}},
{0xff2e, {1|F|D, {0xff4e}}},
{0xff2f, {1|F|D, {0xff4f}}},
{0xff30, {1|F|D, {0xff50}}},
{0xff31, {1|F|D, {0xff51}}},
{0xff32, {1|F|D, {0xff52}}},
{0xff33, {1|F|D, {0xff53}}},
{0xff34, {1|F|D, {0xff54}}},
{0xff35, {1|F|D, {0xff55}}},
{0xff36, {1|F|D, {0xff56}}},
{0xff37, {1|F|D, {0xff57}}},
{0xff38, {1|F|D, {0xff58}}},
{0xff39, {1|F|D, {0xff59}}},
{0xff3a, {1|F|D, {0xff5a}}},
{0x10400, {1|F|D, {0x10428}}},
{0x10401, {1|F|D, {0x10429}}},
{0x10402, {1|F|D, {0x1042a}}},
@ -3240,13 +3240,13 @@ static const CaseUnfold_11_Type CaseUnfold_11_Table[] = {
{0x012b, {1|U, {0x012a}}},
{0x012d, {1|U, {0x012c}}},
{0x012f, {1|U, {0x012e}}},
{0x0133, {1, {0x0132}}},
{0x0133, {1|U, {0x0132}}},
{0x0135, {1|U, {0x0134}}},
{0x0137, {1|U, {0x0136}}},
{0x013a, {1|U, {0x0139}}},
{0x013c, {1|U, {0x013b}}},
{0x013e, {1|U, {0x013d}}},
{0x0140, {1, {0x013f}}},
{0x0140, {1|U, {0x013f}}},
{0x0142, {1|U, {0x0141}}},
{0x0144, {1|U, {0x0143}}},
{0x0146, {1|U, {0x0145}}},
@ -3298,9 +3298,9 @@ static const CaseUnfold_11_Type CaseUnfold_11_Table[] = {
{0x01b9, {1|U, {0x01b8}}},
{0x01bd, {1|U, {0x01bc}}},
{0x01bf, {1|U, {0x01f7}}},
{0x01c6, {2, {0x01c4, 0x01c5}}},
{0x01c9, {2, {0x01c7, 0x01c8}}},
{0x01cc, {2, {0x01ca, 0x01cb}}},
{0x01c6, {2|U, {0x01c4, 0x01c5}}},
{0x01c9, {2|U, {0x01c7, 0x01c8}}},
{0x01cc, {2|U, {0x01ca, 0x01cb}}},
{0x01ce, {1|U, {0x01cd}}},
{0x01d0, {1|U, {0x01cf}}},
{0x01d2, {1|U, {0x01d1}}},
@ -3319,7 +3319,7 @@ static const CaseUnfold_11_Type CaseUnfold_11_Table[] = {
{0x01eb, {1|U, {0x01ea}}},
{0x01ed, {1|U, {0x01ec}}},
{0x01ef, {1|U, {0x01ee}}},
{0x01f3, {2, {0x01f1, 0x01f2}}},
{0x01f3, {2|U, {0x01f1, 0x01f2}}},
{0x01f5, {1|U, {0x01f4}}},
{0x01f9, {1|U, {0x01f8}}},
{0x01fb, {1|U, {0x01fa}}},
@ -3446,7 +3446,7 @@ static const CaseUnfold_11_Type CaseUnfold_11_Table[] = {
{0x03eb, {1|U, {0x03ea}}},
{0x03ed, {1|U, {0x03ec}}},
{0x03ef, {1|U, {0x03ee}}},
{0x03f2, {1, {0x03f9}}},
{0x03f2, {1|U, {0x03f9}}},
{0x03f3, {1|U, {0x037f}}},
{0x03f8, {1|U, {0x03f7}}},
{0x03fb, {1|U, {0x03fa}}},
@ -3917,49 +3917,49 @@ static const CaseUnfold_11_Type CaseUnfold_11_Table[] = {
{0x1fe1, {1|U, {0x1fe9}}},
{0x1fe5, {1|U, {0x1fec}}},
{0x214e, {1|U, {0x2132}}},
{0x2170, {1, {0x2160}}},
{0x2171, {1, {0x2161}}},
{0x2172, {1, {0x2162}}},
{0x2173, {1, {0x2163}}},
{0x2174, {1, {0x2164}}},
{0x2175, {1, {0x2165}}},
{0x2176, {1, {0x2166}}},
{0x2177, {1, {0x2167}}},
{0x2178, {1, {0x2168}}},
{0x2179, {1, {0x2169}}},
{0x217a, {1, {0x216a}}},
{0x217b, {1, {0x216b}}},
{0x217c, {1, {0x216c}}},
{0x217d, {1, {0x216d}}},
{0x217e, {1, {0x216e}}},
{0x217f, {1, {0x216f}}},
{0x2170, {1|U, {0x2160}}},
{0x2171, {1|U, {0x2161}}},
{0x2172, {1|U, {0x2162}}},
{0x2173, {1|U, {0x2163}}},
{0x2174, {1|U, {0x2164}}},
{0x2175, {1|U, {0x2165}}},
{0x2176, {1|U, {0x2166}}},
{0x2177, {1|U, {0x2167}}},
{0x2178, {1|U, {0x2168}}},
{0x2179, {1|U, {0x2169}}},
{0x217a, {1|U, {0x216a}}},
{0x217b, {1|U, {0x216b}}},
{0x217c, {1|U, {0x216c}}},
{0x217d, {1|U, {0x216d}}},
{0x217e, {1|U, {0x216e}}},
{0x217f, {1|U, {0x216f}}},
{0x2184, {1|U, {0x2183}}},
{0x24d0, {1, {0x24b6}}},
{0x24d1, {1, {0x24b7}}},
{0x24d2, {1, {0x24b8}}},
{0x24d3, {1, {0x24b9}}},
{0x24d4, {1, {0x24ba}}},
{0x24d5, {1, {0x24bb}}},
{0x24d6, {1, {0x24bc}}},
{0x24d7, {1, {0x24bd}}},
{0x24d8, {1, {0x24be}}},
{0x24d9, {1, {0x24bf}}},
{0x24da, {1, {0x24c0}}},
{0x24db, {1, {0x24c1}}},
{0x24dc, {1, {0x24c2}}},
{0x24dd, {1, {0x24c3}}},
{0x24de, {1, {0x24c4}}},
{0x24df, {1, {0x24c5}}},
{0x24e0, {1, {0x24c6}}},
{0x24e1, {1, {0x24c7}}},
{0x24e2, {1, {0x24c8}}},
{0x24e3, {1, {0x24c9}}},
{0x24e4, {1, {0x24ca}}},
{0x24e5, {1, {0x24cb}}},
{0x24e6, {1, {0x24cc}}},
{0x24e7, {1, {0x24cd}}},
{0x24e8, {1, {0x24ce}}},
{0x24e9, {1, {0x24cf}}},
{0x24d0, {1|U, {0x24b6}}},
{0x24d1, {1|U, {0x24b7}}},
{0x24d2, {1|U, {0x24b8}}},
{0x24d3, {1|U, {0x24b9}}},
{0x24d4, {1|U, {0x24ba}}},
{0x24d5, {1|U, {0x24bb}}},
{0x24d6, {1|U, {0x24bc}}},
{0x24d7, {1|U, {0x24bd}}},
{0x24d8, {1|U, {0x24be}}},
{0x24d9, {1|U, {0x24bf}}},
{0x24da, {1|U, {0x24c0}}},
{0x24db, {1|U, {0x24c1}}},
{0x24dc, {1|U, {0x24c2}}},
{0x24dd, {1|U, {0x24c3}}},
{0x24de, {1|U, {0x24c4}}},
{0x24df, {1|U, {0x24c5}}},
{0x24e0, {1|U, {0x24c6}}},
{0x24e1, {1|U, {0x24c7}}},
{0x24e2, {1|U, {0x24c8}}},
{0x24e3, {1|U, {0x24c9}}},
{0x24e4, {1|U, {0x24ca}}},
{0x24e5, {1|U, {0x24cb}}},
{0x24e6, {1|U, {0x24cc}}},
{0x24e7, {1|U, {0x24cd}}},
{0x24e8, {1|U, {0x24ce}}},
{0x24e9, {1|U, {0x24cf}}},
{0x2c30, {1|U, {0x2c00}}},
{0x2c31, {1|U, {0x2c01}}},
{0x2c32, {1|U, {0x2c02}}},
@ -4206,32 +4206,32 @@ static const CaseUnfold_11_Type CaseUnfold_11_Table[] = {
{0xa7b5, {1|U, {0xa7b4}}},
{0xa7b7, {1|U, {0xa7b6}}},
{0xab53, {1|U, {0xa7b3}}},
{0xff41, {1, {0xff21}}},
{0xff42, {1, {0xff22}}},
{0xff43, {1, {0xff23}}},
{0xff44, {1, {0xff24}}},
{0xff45, {1, {0xff25}}},
{0xff46, {1, {0xff26}}},
{0xff47, {1, {0xff27}}},
{0xff48, {1, {0xff28}}},
{0xff49, {1, {0xff29}}},
{0xff4a, {1, {0xff2a}}},
{0xff4b, {1, {0xff2b}}},
{0xff4c, {1, {0xff2c}}},
{0xff4d, {1, {0xff2d}}},
{0xff4e, {1, {0xff2e}}},
{0xff4f, {1, {0xff2f}}},
{0xff50, {1, {0xff30}}},
{0xff51, {1, {0xff31}}},
{0xff52, {1, {0xff32}}},
{0xff53, {1, {0xff33}}},
{0xff54, {1, {0xff34}}},
{0xff55, {1, {0xff35}}},
{0xff56, {1, {0xff36}}},
{0xff57, {1, {0xff37}}},
{0xff58, {1, {0xff38}}},
{0xff59, {1, {0xff39}}},
{0xff5a, {1, {0xff3a}}},
{0xff41, {1|U, {0xff21}}},
{0xff42, {1|U, {0xff22}}},
{0xff43, {1|U, {0xff23}}},
{0xff44, {1|U, {0xff24}}},
{0xff45, {1|U, {0xff25}}},
{0xff46, {1|U, {0xff26}}},
{0xff47, {1|U, {0xff27}}},
{0xff48, {1|U, {0xff28}}},
{0xff49, {1|U, {0xff29}}},
{0xff4a, {1|U, {0xff2a}}},
{0xff4b, {1|U, {0xff2b}}},
{0xff4c, {1|U, {0xff2c}}},
{0xff4d, {1|U, {0xff2d}}},
{0xff4e, {1|U, {0xff2e}}},
{0xff4f, {1|U, {0xff2f}}},
{0xff50, {1|U, {0xff30}}},
{0xff51, {1|U, {0xff31}}},
{0xff52, {1|U, {0xff32}}},
{0xff53, {1|U, {0xff33}}},
{0xff54, {1|U, {0xff34}}},
{0xff55, {1|U, {0xff35}}},
{0xff56, {1|U, {0xff36}}},
{0xff57, {1|U, {0xff37}}},
{0xff58, {1|U, {0xff38}}},
{0xff59, {1|U, {0xff39}}},
{0xff5a, {1|U, {0xff3a}}},
{0x10428, {1|U, {0x10400}}},
{0x10429, {1|U, {0x10401}}},
{0x1042a, {1|U, {0x10402}}},