From 2f455ceca40dcddb0f92c2918796673187cd8ed4 Mon Sep 17 00:00:00 2001 From: duerst Date: Tue, 22 Mar 2016 12:08:30 +0000 Subject: * include/ruby/oniguruma.h: Additional flag for characters that are titlecase. * enc/unicode/case-folding.rb, casefold.h: Using above flag in data. * enc/unicode.c: Marking capitalized character as unmodified if it is already titlecase. * test/ruby/enc/test_case_mapping.rb: Tests for above functionality. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@54229 b2dd03c8-39d4-4d8f-98ff-823fe69b080e --- ChangeLog | 11 +++++++ enc/unicode.c | 7 ++++- enc/unicode/case-folding.rb | 12 +++++--- enc/unicode/casefold.h | 62 +++++++++++++++++++------------------- include/ruby/oniguruma.h | 1 + test/ruby/enc/test_case_mapping.rb | 6 ++++ 6 files changed, 63 insertions(+), 36 deletions(-) diff --git a/ChangeLog b/ChangeLog index 94e824b2dd..37cf8e10e0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,14 @@ +Tue Mar 22 21:08:30 2016 Martin Duerst + + * include/ruby/oniguruma.h: Additional flag for characters that are titlecase. + + * enc/unicode/case-folding.rb, casefold.h: Using above flag in data. + + * enc/unicode.c: Marking capitalized character as unmodified if it is + already titlecase. + + * test/ruby/enc/test_case_mapping.rb: Tests for above functionality. + Tue Mar 22 14:18:59 2016 Nobuyoshi Nakada * parse.y (lambda_body, parser_yylex): warn mismatched indentation diff --git a/enc/unicode.c b/enc/unicode.c index 796eb94c19..c6df689ef5 100644 --- a/enc/unicode.c +++ b/enc/unicode.c @@ -158,6 +158,7 @@ code3_equal(const OnigCodePoint *x, const OnigCodePoint *y) #define ST ONIGENC_CASE_TITLECASE #define SU ONIGENC_CASE_UP_SPECIAL #define SL ONIGENC_CASE_DOWN_SPECIAL +#define IT ONIGENC_CASE_IS_TITLECASE #define I(n) OnigSpecialIndexEncode(n) #define L(n) SpecialsLengthEncode(n) @@ -169,6 +170,7 @@ code3_equal(const OnigCodePoint *x, const OnigCodePoint *y) #undef ST #undef SU #undef SL +#undef IT #undef I #undef L @@ -718,8 +720,11 @@ onigenc_unicode_case_map(OnigCaseFoldType* flagP, OnigCodePoint *SpecialsStart = CaseMappingSpecials + OnigSpecialIndexDecode(folded->n); if (OnigCaseFoldFlags(folded->n)&ONIGENC_CASE_TITLECASE) { - if (flags&ONIGENC_CASE_TITLECASE) + if (flags&ONIGENC_CASE_TITLECASE) { + if (OnigCaseFoldFlags(folded->n)&ONIGENC_CASE_IS_TITLECASE) + flags ^= ONIGENC_CASE_MODIFIED; goto SpecialsCopy; + } else SpecialsStart += SpecialsLengthExtract(*SpecialsStart); } diff --git a/enc/unicode/case-folding.rb b/enc/unicode/case-folding.rb index d309dd6d15..00258cde6a 100755 --- a/enc/unicode/case-folding.rb +++ b/enc/unicode/case-folding.rb @@ -190,7 +190,7 @@ class CaseFolding end class MapItem - attr_accessor :upper, :lower, :title + attr_accessor :upper, :lower, :title, :code def initialize(code, upper, lower, title) @code = code @@ -242,15 +242,19 @@ class CaseMapping specials = [] unless item.upper == item.title specials << item.title - flags += "|ST" + flags += '|ST' + if item.code == item.title + flags += '|IT' + end + end unless item.lower.nil? or item.lower==from or item.lower==to specials << item.lower - flags += "|SL" + flags += '|SL' end unless item.upper.nil? or item.upper==from or item.upper==to specials << item.upper - flags += "|SU" + flags += '|SU' end if specials.first flags += "|I(#{@specials_length})" diff --git a/enc/unicode/casefold.h b/enc/unicode/casefold.h index fcc912035d..c324102769 100644 --- a/enc/unicode/casefold.h +++ b/enc/unicode/casefold.h @@ -161,11 +161,11 @@ static const CaseFold_11_Type CaseFold_11_Table[] = { {0x01b8, {1|F|D, {0x01b9}}}, {0x01bc, {1|F|D, {0x01bd}}}, {0x01c4, {1|F|D|ST|I(8), {0x01c6}}}, - {0x01c5, {1|F|D|ST|SU|I(9), {0x01c6}}}, + {0x01c5, {1|F|D|ST|IT|SU|I(9), {0x01c6}}}, {0x01c7, {1|F|D|ST|I(11), {0x01c9}}}, - {0x01c8, {1|F|D|ST|SU|I(12), {0x01c9}}}, + {0x01c8, {1|F|D|ST|IT|SU|I(12), {0x01c9}}}, {0x01ca, {1|F|D|ST|I(14), {0x01cc}}}, - {0x01cb, {1|F|D|ST|SU|I(15), {0x01cc}}}, + {0x01cb, {1|F|D|ST|IT|SU|I(15), {0x01cc}}}, {0x01cd, {1|F|D, {0x01ce}}}, {0x01cf, {1|F|D, {0x01d0}}}, {0x01d1, {1|F|D, {0x01d2}}}, @@ -185,7 +185,7 @@ static const CaseFold_11_Type CaseFold_11_Table[] = { {0x01ee, {1|F|D, {0x01ef}}}, {0x01f0, {2|F|SU|I(17), {0x006a, 0x030c}}}, {0x01f1, {1|F|D|ST|I(19), {0x01f3}}}, - {0x01f2, {1|F|D|ST|SU|I(20), {0x01f3}}}, + {0x01f2, {1|F|D|ST|IT|SU|I(20), {0x01f3}}}, {0x01f4, {1|F|D, {0x01f5}}}, {0x01f6, {1|F|D, {0x0195}}}, {0x01f7, {1|F|D, {0x01bf}}}, @@ -723,14 +723,14 @@ static const CaseFold_11_Type CaseFold_11_Table[] = { {0x1f85, {2|F|ST|SU|I(79), {0x1f05, 0x03b9}}}, {0x1f86, {2|F|ST|SU|I(82), {0x1f06, 0x03b9}}}, {0x1f87, {2|F|ST|SU|I(85), {0x1f07, 0x03b9}}}, - {0x1f88, {2|F|ST|SL|SU|I(88), {0x1f00, 0x03b9}}}, - {0x1f89, {2|F|ST|SL|SU|I(92), {0x1f01, 0x03b9}}}, - {0x1f8a, {2|F|ST|SL|SU|I(96), {0x1f02, 0x03b9}}}, - {0x1f8b, {2|F|ST|SL|SU|I(100), {0x1f03, 0x03b9}}}, - {0x1f8c, {2|F|ST|SL|SU|I(104), {0x1f04, 0x03b9}}}, - {0x1f8d, {2|F|ST|SL|SU|I(108), {0x1f05, 0x03b9}}}, - {0x1f8e, {2|F|ST|SL|SU|I(112), {0x1f06, 0x03b9}}}, - {0x1f8f, {2|F|ST|SL|SU|I(116), {0x1f07, 0x03b9}}}, + {0x1f88, {2|F|ST|IT|SL|SU|I(88), {0x1f00, 0x03b9}}}, + {0x1f89, {2|F|ST|IT|SL|SU|I(92), {0x1f01, 0x03b9}}}, + {0x1f8a, {2|F|ST|IT|SL|SU|I(96), {0x1f02, 0x03b9}}}, + {0x1f8b, {2|F|ST|IT|SL|SU|I(100), {0x1f03, 0x03b9}}}, + {0x1f8c, {2|F|ST|IT|SL|SU|I(104), {0x1f04, 0x03b9}}}, + {0x1f8d, {2|F|ST|IT|SL|SU|I(108), {0x1f05, 0x03b9}}}, + {0x1f8e, {2|F|ST|IT|SL|SU|I(112), {0x1f06, 0x03b9}}}, + {0x1f8f, {2|F|ST|IT|SL|SU|I(116), {0x1f07, 0x03b9}}}, {0x1f90, {2|F|ST|SU|I(120), {0x1f20, 0x03b9}}}, {0x1f91, {2|F|ST|SU|I(123), {0x1f21, 0x03b9}}}, {0x1f92, {2|F|ST|SU|I(126), {0x1f22, 0x03b9}}}, @@ -739,14 +739,14 @@ static const CaseFold_11_Type CaseFold_11_Table[] = { {0x1f95, {2|F|ST|SU|I(135), {0x1f25, 0x03b9}}}, {0x1f96, {2|F|ST|SU|I(138), {0x1f26, 0x03b9}}}, {0x1f97, {2|F|ST|SU|I(141), {0x1f27, 0x03b9}}}, - {0x1f98, {2|F|ST|SL|SU|I(144), {0x1f20, 0x03b9}}}, - {0x1f99, {2|F|ST|SL|SU|I(148), {0x1f21, 0x03b9}}}, - {0x1f9a, {2|F|ST|SL|SU|I(152), {0x1f22, 0x03b9}}}, - {0x1f9b, {2|F|ST|SL|SU|I(156), {0x1f23, 0x03b9}}}, - {0x1f9c, {2|F|ST|SL|SU|I(160), {0x1f24, 0x03b9}}}, - {0x1f9d, {2|F|ST|SL|SU|I(164), {0x1f25, 0x03b9}}}, - {0x1f9e, {2|F|ST|SL|SU|I(168), {0x1f26, 0x03b9}}}, - {0x1f9f, {2|F|ST|SL|SU|I(172), {0x1f27, 0x03b9}}}, + {0x1f98, {2|F|ST|IT|SL|SU|I(144), {0x1f20, 0x03b9}}}, + {0x1f99, {2|F|ST|IT|SL|SU|I(148), {0x1f21, 0x03b9}}}, + {0x1f9a, {2|F|ST|IT|SL|SU|I(152), {0x1f22, 0x03b9}}}, + {0x1f9b, {2|F|ST|IT|SL|SU|I(156), {0x1f23, 0x03b9}}}, + {0x1f9c, {2|F|ST|IT|SL|SU|I(160), {0x1f24, 0x03b9}}}, + {0x1f9d, {2|F|ST|IT|SL|SU|I(164), {0x1f25, 0x03b9}}}, + {0x1f9e, {2|F|ST|IT|SL|SU|I(168), {0x1f26, 0x03b9}}}, + {0x1f9f, {2|F|ST|IT|SL|SU|I(172), {0x1f27, 0x03b9}}}, {0x1fa0, {2|F|ST|SU|I(176), {0x1f60, 0x03b9}}}, {0x1fa1, {2|F|ST|SU|I(179), {0x1f61, 0x03b9}}}, {0x1fa2, {2|F|ST|SU|I(182), {0x1f62, 0x03b9}}}, @@ -755,14 +755,14 @@ static const CaseFold_11_Type CaseFold_11_Table[] = { {0x1fa5, {2|F|ST|SU|I(191), {0x1f65, 0x03b9}}}, {0x1fa6, {2|F|ST|SU|I(194), {0x1f66, 0x03b9}}}, {0x1fa7, {2|F|ST|SU|I(197), {0x1f67, 0x03b9}}}, - {0x1fa8, {2|F|ST|SL|SU|I(200), {0x1f60, 0x03b9}}}, - {0x1fa9, {2|F|ST|SL|SU|I(204), {0x1f61, 0x03b9}}}, - {0x1faa, {2|F|ST|SL|SU|I(208), {0x1f62, 0x03b9}}}, - {0x1fab, {2|F|ST|SL|SU|I(212), {0x1f63, 0x03b9}}}, - {0x1fac, {2|F|ST|SL|SU|I(216), {0x1f64, 0x03b9}}}, - {0x1fad, {2|F|ST|SL|SU|I(220), {0x1f65, 0x03b9}}}, - {0x1fae, {2|F|ST|SL|SU|I(224), {0x1f66, 0x03b9}}}, - {0x1faf, {2|F|ST|SL|SU|I(228), {0x1f67, 0x03b9}}}, + {0x1fa8, {2|F|ST|IT|SL|SU|I(200), {0x1f60, 0x03b9}}}, + {0x1fa9, {2|F|ST|IT|SL|SU|I(204), {0x1f61, 0x03b9}}}, + {0x1faa, {2|F|ST|IT|SL|SU|I(208), {0x1f62, 0x03b9}}}, + {0x1fab, {2|F|ST|IT|SL|SU|I(212), {0x1f63, 0x03b9}}}, + {0x1fac, {2|F|ST|IT|SL|SU|I(216), {0x1f64, 0x03b9}}}, + {0x1fad, {2|F|ST|IT|SL|SU|I(220), {0x1f65, 0x03b9}}}, + {0x1fae, {2|F|ST|IT|SL|SU|I(224), {0x1f66, 0x03b9}}}, + {0x1faf, {2|F|ST|IT|SL|SU|I(228), {0x1f67, 0x03b9}}}, {0x1fb2, {2|F|ST|SU|I(232), {0x1f70, 0x03b9}}}, {0x1fb3, {2|F|ST|SU|I(236), {0x03b1, 0x03b9}}}, {0x1fb4, {2|F|ST|SU|I(239), {0x03ac, 0x03b9}}}, @@ -772,7 +772,7 @@ static const CaseFold_11_Type CaseFold_11_Table[] = { {0x1fb9, {1|F|D, {0x1fb1}}}, {0x1fba, {1|F|D, {0x1f70}}}, {0x1fbb, {1|F|D, {0x1f71}}}, - {0x1fbc, {2|F|ST|SL|SU|I(251), {0x03b1, 0x03b9}}}, + {0x1fbc, {2|F|ST|IT|SL|SU|I(251), {0x03b1, 0x03b9}}}, {0x1fbe, {1|F|SU|I(255), {0x03b9}}}, {0x1fc2, {2|F|ST|SU|I(256), {0x1f74, 0x03b9}}}, {0x1fc3, {2|F|ST|SU|I(260), {0x03b7, 0x03b9}}}, @@ -783,7 +783,7 @@ static const CaseFold_11_Type CaseFold_11_Table[] = { {0x1fc9, {1|F|D, {0x1f73}}}, {0x1fca, {1|F|D, {0x1f74}}}, {0x1fcb, {1|F|D, {0x1f75}}}, - {0x1fcc, {2|F|ST|SL|SU|I(275), {0x03b7, 0x03b9}}}, + {0x1fcc, {2|F|ST|IT|SL|SU|I(275), {0x03b7, 0x03b9}}}, {0x1fd2, {3|F|SU|I(279), {0x03b9, 0x0308, 0x0300}}}, {0x1fd3, {3|F|SU|I(282), {0x03b9, 0x0308, 0x0301}}}, {0x1fd6, {2|F|SU|I(285), {0x03b9, 0x0342}}}, @@ -811,7 +811,7 @@ static const CaseFold_11_Type CaseFold_11_Table[] = { {0x1ff9, {1|F|D, {0x1f79}}}, {0x1ffa, {1|F|D, {0x1f7c}}}, {0x1ffb, {1|F|D, {0x1f7d}}}, - {0x1ffc, {2|F|ST|SL|SU|I(322), {0x03c9, 0x03b9}}}, + {0x1ffc, {2|F|ST|IT|SL|SU|I(322), {0x03c9, 0x03b9}}}, {0x2126, {1|F|D, {0x03c9}}}, {0x212a, {1|F|D, {0x006b}}}, {0x212b, {1|F|D, {0x00e5}}}, diff --git a/include/ruby/oniguruma.h b/include/ruby/oniguruma.h index b087ef829f..defa3917e5 100644 --- a/include/ruby/oniguruma.h +++ b/include/ruby/oniguruma.h @@ -152,6 +152,7 @@ ONIG_EXTERN OnigCaseFoldType OnigDefaultCaseFoldFlag; #define ONIGENC_CASE_FOLD_TURKISH_AZERI (1<<20) /* needs mapping specific to Turkic languages; better not change original value! */ #define ONIGENC_CASE_FOLD_LITHUANIAN (1<<21) /* needs Lithuanian-specific mapping */ #define ONIGENC_CASE_ASCII_ONLY (1<<22) /* only modify ASCII range */ +#define ONIGENC_CASE_IS_TITLECASE (1<<23) /* character itself is already titlecase */ #define INTERNAL_ONIGENC_CASE_FOLD_MULTI_CHAR (1<<30) /* better not change original value! */ #define ONIGENC_CASE_FOLD_MIN INTERNAL_ONIGENC_CASE_FOLD_MULTI_CHAR diff --git a/test/ruby/enc/test_case_mapping.rb b/test/ruby/enc/test_case_mapping.rb index 56d43b97c1..0f77ad3f1d 100644 --- a/test/ruby/enc/test_case_mapping.rb +++ b/test/ruby/enc/test_case_mapping.rb @@ -78,6 +78,12 @@ class TestCaseMappingPreliminary < Test::Unit::TestCase assert_equal 'ᎠᎡᎢᎣᎤᎥᎦᎧᎨᎩ', "\uab70\uab71\uab72\uab73\uab74\uab75\uab76\uab77\uab78\uab79".downcase(:fold) end + def test_titlecase + check_downcase_properties 'dz dž lj nj', 'Dz Dž Lj Nj', :lithuanian + check_upcase_properties 'DZ DŽ LJ NJ', 'Dz Dž Lj Nj', :lithuanian + check_capitalize_properties 'Dz', 'DZ', :lithuanian + end + def test_ascii_option check_downcase_properties 'yukihiro matsumoto (matz)', 'Yukihiro MATSUMOTO (MATZ)', :ascii check_upcase_properties 'YUKIHIRO MATSUMOTO (MATZ)', 'yukihiro matsumoto (matz)', :ascii -- cgit v1.2.3