1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
|
#!/usr/bin/ruby
# Usage:
# $ wget http://www.unicode.org/Public/UNIDATA/CaseFolding.txt
# $ ruby CaseFolding.rb CaseFolding.txt > ../enc/unicode/casefold.h
class CaseFolding
module Util
module_function
def hex_seq(v)
v.map {|i| "0x%04x" % i}.join(", ")
end
def print_table_1(dest, data)
for k, v in data.sort
sk = (Array === k and k.length > 1) ? "{#{hex_seq(k)}}" : ("0x%04x" % k)
dest.print(" {#{sk}, {#{v.length}, {#{hex_seq(v)}}}},\n")
end
end
def print_table(dest, type, data)
dest.print("static const #{type}_Type #{type}_Table[] = {\n")
i = 0
data.each do |n, d|
dest.print("#define #{n} (*(#{type}_Type (*)[#{d.size}])(#{type}_Table+#{i}))\n")
i += d.size
print_table_1(dest, d)
end
dest.print("};\n\n")
end
end
include Util
attr_reader :fold, :fold_locale, :unfold, :unfold_locale
def load(filename)
pattern = /([0-9A-F]{4,6}); ([CFT]); ([0-9A-F]{4,6})(?: ([0-9A-F]{4,6}))?(?: ([0-9A-F]{4,6}))?;/
@fold = fold = {}
@unfold = unfold = [{}, {}, {}]
turkic = []
IO.foreach(filename) do |line|
next unless res = pattern.match(line)
ch_from = res[1].to_i(16)
if res[2] == 'T'
# Turkic case folding
turkic << ch_from
next
end
# store folding data
ch_to = res[3..6].inject([]) do |a, i|
break a unless i
a << i.to_i(16)
end
fold[ch_from] = ch_to
# store unfolding data
i = ch_to.length - 1
(unfold[i][ch_to] ||= []) << ch_from
end
# move locale dependent data to (un)fold_locale
@fold_locale = fold_locale = {}
@unfold_locale = unfold_locale = [{}, {}]
for ch_from in turkic
key = fold[ch_from]
i = key.length - 1
unfold_locale[i][i == 0 ? key[0] : key] = unfold[i].delete(key)
fold_locale[ch_from] = fold.delete(ch_from)
end
self
end
def display(dest)
# print the header
dest.print("/* DO NOT EDIT THIS FILE. */\n")
dest.print("/* Generated by enc/unicode/case-folding.rb */\n\n")
# print folding data
# CaseFold + CaseFold_Locale
name = "CaseFold_11"
print_table(dest, name, "CaseFold"=>fold, "CaseFold_Locale"=>fold_locale)
# print unfolding data
# CaseUnfold_11 + CaseUnfold_11_Locale
name = "CaseUnfold_11"
print_table(dest, name, name=>unfold[0], "#{name}_Locale"=>unfold_locale[0])
# CaseUnfold_12 + CaseUnfold_12_Locale
name = "CaseUnfold_12"
print_table(dest, name, name=>unfold[1], "#{name}_Locale"=>unfold_locale[1])
# CaseUnfold_13
name = "CaseUnfold_13"
print_table(dest, name, name=>unfold[2])
# table sizes
fold_table_size = fold.size + fold_locale.size
dest.printf("#define FOLD_TABLE_SIZE\t\t%d\n", (fold_table_size * 1.2))
unfold1_table_size = unfold[0].size + unfold_locale[0].size
dest.printf("#define UNFOLD1_TABLE_SIZE\t%d\n", (unfold1_table_size * 1.2))
unfold2_table_size = unfold[1].size + unfold_locale[1].size
dest.printf("#define UNFOLD2_TABLE_SIZE\t%d\n", (unfold2_table_size * 1.5))
unfold3_table_size = unfold[2].size
dest.printf("#define UNFOLD3_TABLE_SIZE\t%d\n", (unfold3_table_size * 1.7))
end
def self.load(*args)
new.load(*args)
end
end
if $0 == __FILE__
require 'optparse'
dest = nil
fold_1 = false
ARGV.options do |opt|
opt.banner << " [INPUT]"
opt.on("--output-file=FILE", "-o", "output to the FILE instead of STDOUT") {|output|
dest = (output unless output == '-')
}
opt.parse!
abort(opt.to_s) if ARGV.size > 1
end
filename = ARGV[0] || 'CaseFolding.txt'
data = CaseFolding.load(filename)
if dest
open(dest, "wb") do |f|
data.display(f)
end
else
data.display(STDOUT)
end
end
|
