diff options
author | duerst <duerst@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> | 2014-10-19 00:38:40 +0000 |
---|---|---|
committer | duerst <duerst@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> | 2014-10-19 00:38:40 +0000 |
commit | 6017de0314b5115183c04b81562bbe7fdb71959a (patch) | |
tree | e1656a203723663406bb8fbc46f7a5e635fe54cb | |
parent | bb49cd8a6ca89e985d3918931b67cda358b26ad3 (diff) | |
download | ruby-6017de0314b5115183c04b81562bbe7fdb71959a.tar.gz |
lib/unicode_normalize/normalize.rb: Importing from
https://github.com/duerst/eprun/blob/master/lib/normalize.rb.
git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@48005 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
-rw-r--r-- | ChangeLog | 5 | ||||
-rw-r--r-- | lib/unicode_normalize/normalize.rb | 176 |
2 files changed, 181 insertions, 0 deletions
@@ -1,3 +1,8 @@ +Sun Oct 19 09:38:38 2014 Martin Duerst <duerst@it.aoyama.ac.jp> + + * lib/unicode_normalize/normalize.rb: Importing from + https://github.com/duerst/eprun/blob/master/lib/normalize.rb. + Sat Oct 18 20:40:52 2014 Kazuki Tsujimoto <kazuki@callcc.net> * vm_core.h, proc.c, vm_backtrace.c, vm_trace.c: diff --git a/lib/unicode_normalize/normalize.rb b/lib/unicode_normalize/normalize.rb new file mode 100644 index 0000000000..4f0fdbc78a --- /dev/null +++ b/lib/unicode_normalize/normalize.rb @@ -0,0 +1,176 @@ +# coding: utf-8
+
+# Copyright 2010-2013 Ayumu Nojima (野島 歩) and Martin J. Dürst (duerst@it.aoyama.ac.jp)
+# available under the same licence as Ruby itself
+# (see http://www.ruby-lang.org/en/LICENSE.txt)
+
+require_relative 'normalize_tables'
+
+
+module Normalize
+ ## Constant for max hash capacity to avoid DoS attack
+ MAX_HASH_LENGTH = 18000 # enough for all test cases, otherwise tests get slow
+
+ ## Regular Expressions and Hash Constants
+ REGEXP_D = Regexp.compile(REGEXP_D_STRING, Regexp::EXTENDED)
+ REGEXP_C = Regexp.compile(REGEXP_C_STRING, Regexp::EXTENDED)
+ REGEXP_K = Regexp.compile(REGEXP_K_STRING, Regexp::EXTENDED)
+ NF_HASH_D = Hash.new do |hash, key|
+ hash.delete hash.first[0] if hash.length>MAX_HASH_LENGTH # prevent DoS attack
+ hash[key] = Normalize.nfd_one(key)
+ end
+ NF_HASH_C = Hash.new do |hash, key|
+ hash.delete hash.first[0] if hash.length>MAX_HASH_LENGTH # prevent DoS attack
+ hash[key] = Normalize.nfc_one(key)
+ end
+ NF_HASH_K = Hash.new do |hash, key|
+ hash.delete hash.first[0] if hash.length>MAX_HASH_LENGTH # prevent DoS attack
+ hash[key] = Normalize.nfkd_one(key)
+ end
+
+ ## Constants For Hangul
+ SBASE = 0xAC00
+ LBASE = 0x1100
+ VBASE = 0x1161
+ TBASE = 0x11A7
+ LCOUNT = 19
+ VCOUNT = 21
+ TCOUNT = 28
+ NCOUNT = VCOUNT * TCOUNT
+ SCOUNT = LCOUNT * NCOUNT
+
+ # Unicode-based encodings (except UTF-8)
+ UNICODE_ENCODINGS = [Encoding::UTF_16BE, Encoding::UTF_16LE, Encoding::UTF_32BE, Encoding::UTF_32LE,
+ Encoding::GB18030, Encoding::UCS_2BE, Encoding::UCS_4BE]
+
+ ## Hangul Algorithm
+ def Normalize.hangul_decomp_one(target)
+ sIndex = target.ord - SBASE
+ return target if sIndex < 0 || sIndex >= SCOUNT
+ l = LBASE + sIndex / NCOUNT
+ v = VBASE + (sIndex % NCOUNT) / TCOUNT
+ t = TBASE + sIndex % TCOUNT
+ (t==TBASE ? [l, v] : [l, v, t]).pack('U*') + target[1..-1]
+ end
+
+ def Normalize.hangul_comp_one(string)
+ length = string.length
+ if length>1 and 0 <= (lead =string[0].ord-LBASE) and lead < LCOUNT and
+ 0 <= (vowel=string[1].ord-VBASE) and vowel < VCOUNT
+ lead_vowel = SBASE + (lead * VCOUNT + vowel) * TCOUNT
+ if length>2 and 0 <= (trail=string[2].ord-TBASE) and trail < TCOUNT
+ (lead_vowel + trail).chr(Encoding::UTF_8) + string[3..-1]
+ else
+ lead_vowel.chr(Encoding::UTF_8) + string[2..-1]
+ end
+ else
+ string
+ end
+ end
+
+ ## Canonical Ordering
+ def Normalize.canonical_ordering_one(string)
+ sorting = string.each_char.collect { |c| [c, CLASS_TABLE[c]] }
+ (sorting.length-2).downto(0) do |i| # bubble sort
+ (0..i).each do |j|
+ later_class = sorting[j+1].last
+ if 0<later_class and later_class<sorting[j].last
+ sorting[j], sorting[j+1] = sorting[j+1], sorting[j]
+ end
+ end
+ end
+ return sorting.collect(&:first).join
+ end
+
+ ## Normalization Forms for Patterns (not whole Strings)
+ def Normalize.nfd_one(string)
+ string = string.dup
+ (0...string.length).each do |position|
+ if decomposition = DECOMPOSITION_TABLE[string[position]]
+ string[position] = decomposition
+ end
+ end
+ canonical_ordering_one(hangul_decomp_one(string))
+ end
+
+ def Normalize.nfkd_one(string)
+ string = string.dup
+ position = 0
+ while position < string.length
+ if decomposition = KOMPATIBLE_TABLE[string[position]]
+ string[position] = decomposition
+ else
+ position += 1
+ end
+ end
+ string
+ end
+
+ def Normalize.nfc_one (string)
+ nfd_string = nfd_one string
+ start = nfd_string[0]
+ last_class = CLASS_TABLE[start]-1
+ accents = ''
+ nfd_string[1..-1].each_char do |accent|
+ accent_class = CLASS_TABLE[accent]
+ if last_class<accent_class and composite = COMPOSITION_TABLE[start+accent]
+ start = composite
+ else
+ accents += accent
+ last_class = accent_class
+ end
+ end
+ hangul_comp_one(start+accents)
+ end
+
+ def Normalize.normalize(string, form = :nfc)
+ encoding = string.encoding
+ if encoding == Encoding::UTF_8
+ case form
+ when :nfc then
+ string.gsub REGEXP_C, NF_HASH_C
+ when :nfd then
+ string.gsub REGEXP_D, NF_HASH_D
+ when :nfkc then
+ string.gsub(REGEXP_K, NF_HASH_K).gsub REGEXP_C, NF_HASH_C
+ when :nfkd then
+ string.gsub(REGEXP_K, NF_HASH_K).gsub REGEXP_D, NF_HASH_D
+ else
+ raise ArgumentError, "Invalid normalization form #{form}."
+ end
+ elsif UNICODE_ENCODINGS.include? encoding
+ normalize(string.encode(Encoding::UTF_8), form).encode(encoding)
+ else
+ raise Encoding::CompatibilityError, "Unicode Normalization not appropriate for #{encoding}"
+ end
+ end
+
+ def Normalize.normalized?(string, form = :nfc)
+ encoding = string.encoding
+ if encoding == Encoding::UTF_8
+ case form
+ when :nfc then
+ string.scan REGEXP_C do |match|
+ return false if NF_HASH_C[match] != match
+ end
+ true
+ when :nfd then
+ string.scan REGEXP_D do |match|
+ return false if NF_HASH_D[match] != match
+ end
+ true
+ when :nfkc then
+ normalized?(string, :nfc) and string !~ REGEXP_K
+ when :nfkd then
+ normalized?(string, :nfd) and string !~ REGEXP_K
+ else
+ raise ArgumentError, "Invalid normalization form #{form}."
+ end
+ elsif UNICODE_ENCODINGS.include? encoding
+ normalized? string.encode(Encoding::UTF_8), form
+ else
+ raise Encoding::CompatibilityError, "Unicode Normalization not appropriate for #{encoding}"
+ end
+ end
+
+end # module
|