From 33447b80d52f395b26c31a907648503129b1d077 Mon Sep 17 00:00:00 2001 From: duerst Date: Mon, 6 Oct 2014 01:27:34 +0000 Subject: tool/unicode_norm_gen.rb: Data generation script imported from https://github.com/duerst/eprun/blob/master/lib/generate.rb git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@47808 b2dd03c8-39d4-4d8f-98ff-823fe69b080e --- ChangeLog | 5 ++ tool/unicode_norm_gen.rb | 178 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 183 insertions(+) create mode 100644 tool/unicode_norm_gen.rb diff --git a/ChangeLog b/ChangeLog index b9c076f2f7..f81cd64a04 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,8 @@ +Mon Oct 6 10:27:27 2014 Martin Duerst + + * tool/unicode_norm_gen.rb: Data generation script imported from + https://github.com/duerst/eprun/blob/master/lib/generate.rb + Mon Oct 6 10:15:15 2014 Martin Duerst * tool/downloader.rb: Adjust example in documentation for diff --git a/tool/unicode_norm_gen.rb b/tool/unicode_norm_gen.rb new file mode 100644 index 0000000000..90eba75f3a --- /dev/null +++ b/tool/unicode_norm_gen.rb @@ -0,0 +1,178 @@ +# coding: utf-8 + +# Copyright 2010-2013 Ayumu Nojima (野島 歩) and Martin J. Dürst (duerst@it.aoyama.ac.jp) +# available under the same licence as Ruby itself +# (see http://www.ruby-lang.org/en/LICENSE.txt) + +class Integer + def to_UTF8() + if self>0xFFFF + "\\u{#{to_s(16).upcase}}" + elsif CombiningClass[self] or self=='\\'.ord or self=='"'.ord + "\\u#{to_s(16).upcase.rjust(4, '0')}" + else + chr Encoding::UTF_8 + end + end +end + +class Array + def line_slice (new_line) # joins items, 16 items per line + each_slice(16).collect(&:join).join new_line + end + + def to_UTF8() collect(&:to_UTF8).join end + + def to_regexp_chars # converts an array of Integers to character ranges + sort.inject([]) do |ranges, value| + if ranges.last and ranges.last[1]+1>=value + ranges.last[1] = value + ranges + else + ranges << [value, value] + end + end.collect do |first, last| + case last-first + when 0 + first.to_UTF8 + when 1 + first.to_UTF8 + last.to_UTF8 + else + first.to_UTF8 + '-' + last.to_UTF8 + end + end.line_slice "\" +\n \"" + end +end + +class Hash + def to_hash_string + collect do |key, value| + "\"#{key.to_UTF8}\"=>\"#{value.to_UTF8}\", " + end.line_slice "\n " + end +end + +# read the file 'CompositionExclusions.txt' +composition_exclusions = IO.readlines("../data/CompositionExclusions.txt") + .select { |line| line =~ /^[A-Z0-9]{4,5}/ } + .collect { |line| line.split(' ').first.hex } + +decomposition_table = {} +kompatible_table = {} +CombiningClass = {} # constant to allow use in Integer#to_UTF8 + +# read the file 'UnicodeData.txt' +IO.foreach("../data/UnicodeData.txt") do |line| + codepoint, name, _2, char_class, _4, decomposition, *_rest = line.split(";") + + case decomposition + when /^[0-9A-F]/ + decomposition_table[codepoint.hex] = decomposition.split(' ').collect(&:hex) + when /^$/ and (char_class!="0" or decomposition!="") + warn "Unexpected: Character range with data relevant to normalization!" + end +end + +# calculate compositions from decompositions +composition_table = decomposition_table.reject do |character, decomposition| + composition_exclusions.member? character or # predefined composition exclusion + decomposition.length<=1 or # Singleton Decomposition + CombiningClass[character] or # character is not a Starter + CombiningClass[decomposition.first] # decomposition begins with a character that is not a Starter +end.invert + +# recalculate composition_exclusions +composition_exclusions = decomposition_table.keys - composition_table.values + +accent_array = CombiningClass.keys + composition_table.keys.collect(&:last) + +composition_starters = composition_table.keys.collect(&:first) + +hangul_no_trailing = 0xAC00.step(0xD7A3, 28).to_a + +# expand decomposition table values +decomposition_table.each do |key, value| + position = 0 + while position < value.length + if decomposition = decomposition_table[value[position]] + decomposition_table[key] = value = value.dup # avoid overwriting composition_table key + value[position, 1] = decomposition + else + position += 1 + end + end +end + +# deal with relationship between canonical and kompatibility decompositions +decomposition_table.each do |key, value| + value = value.dup + expanded = false + position = 0 + while position < value.length + if decomposition = kompatible_table[value[position]] + value[position, 1] = decomposition + expanded = true + else + position += 1 + end + end + kompatible_table[key] = value if expanded +end + +class_table_str = CombiningClass.collect do |key, value| + "\"#{key.to_UTF8}\"=>#{value}, " +end.line_slice "\n " + +# generate normalization tables file +open("normalize_tables.rb", "w").print <