aboutsummaryrefslogtreecommitdiffstats
path: root/enc/unicode/case-folding.rb
blob: cade542385e462921230bc063e162f1a67933c80 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
#!/usr/bin/ruby

# Usage (for case folding only):
#   $ wget http://www.unicode.org/Public/UNIDATA/CaseFolding.txt
#   $ ruby case-folding.rb CaseFolding.txt -o casefold.h
#  or (for case folding and case mapping):
#   $ wget http://www.unicode.org/Public/UNIDATA/CaseFolding.txt
#   $ wget http://www.unicode.org/Public/UNIDATA/UnicodeData.txt
#   $ wget http://www.unicode.org/Public/UNIDATA/SpecialCasing.txt
#   $ ruby case-folding.rb -m . -o casefold.h
# using -d or --debug will include UTF-8 characters in comments for debugging

class CaseFolding
  module Util
    module_function

    def hex_seq(v)
      v.map {|i| "0x%04x" % i}.join(", ")
    end

    def print_table_1(dest, type, mapping_data, data)
      for k, v in data = data.sort
        sk = (Array === k and k.length > 1) ? "{#{hex_seq(k)}}" : ("0x%04x" % k)
        ck = cv = ''
        ck = ' /* ' + Array(k).pack("U*") + ' */' if @debug
        cv = ' /* ' + Array(v).map{|c|[c].pack("U*")}.join(", ") + ' */' if @debug
        dest.print("  {#{sk}#{ck}, {#{v.length}#{mapping_data.flags(k, type, v)}, {#{hex_seq(v)}#{cv}}}},\n")
      end
      data
    end

    def print_table(dest, type, mapping_data, data)
      dest.print("static const #{type}_Type #{type}_Table[] = {\n")
      i = 0
      ret = data.inject([]) do |a, (n, d)|
        dest.print("#define #{n} (*(#{type}_Type (*)[#{d.size}])(#{type}_Table+#{i}))\n")
        i += d.size
        a.concat(print_table_1(dest, type, mapping_data, d))
      end
      dest.print("};\n\n")
      ret
    end
  end

  include Util

  attr_reader :fold, :fold_locale, :unfold, :unfold_locale

  def load(filename)
    pattern = /([0-9A-F]{4,6}); ([CFT]); ([0-9A-F]{4,6})(?: ([0-9A-F]{4,6}))?(?: ([0-9A-F]{4,6}))?;/

    @fold = fold = {}
    @unfold = unfold = [{}, {}, {}]
    @debug = false
    turkic = []

    IO.foreach(filename) do |line|
      next unless res = pattern.match(line)
      ch_from = res[1].to_i(16)

      if res[2] == 'T'
        # Turkic case folding
        turkic << ch_from
        next
      end

      # store folding data
      ch_to = res[3..6].inject([]) do |a, i|
        break a unless i
        a << i.to_i(16)
      end
      fold[ch_from] = ch_to

      # store unfolding data
      i = ch_to.length - 1
      (unfold[i][ch_to] ||= []) << ch_from
    end

    # move locale dependent data to (un)fold_locale
    @fold_locale = fold_locale = {}
    @unfold_locale = unfold_locale = [{}, {}]
    for ch_from in turkic
      key = fold[ch_from]
      i = key.length - 1
      unfold_locale[i][i == 0 ? key[0] : key] = unfold[i].delete(key)
      fold_locale[ch_from] = fold.delete(ch_from)
    end
    self
  end

  def range_check(code)
    "#{code} <= MAX_CODE_VALUE && #{code} >= MIN_CODE_VALUE"
  end

  def lookup_hash(key, type, data)
    hash = "onigenc_unicode_#{key}_hash"
    lookup = "onigenc_unicode_#{key}_lookup"
    arity = Array(data[0][0]).size
    gperf = %W"gperf -7 -k#{[*1..(arity*3)].join(",")} -F,-1 -c -j1 -i1 -t -T -E -C -H #{hash} -N #{lookup} -n"
    argname = arity > 1 ? "codes" : "code"
    argdecl = "const OnigCodePoint #{arity > 1 ? "*": ""}#{argname}"
    n = 7
    m = (1 << n) - 1
    min, max = data.map {|c, *|c}.flatten.minmax
    src = IO.popen(gperf, "r+") {|f|
      f << "short\n%%\n"
      data.each_with_index {|(k, _), i|
        k = Array(k)
        ks = k.map {|j| [(j >> n*2) & m, (j >> n) & m, (j) & m]}.flatten.map {|c| "\\x%.2x" % c}.join("")
        f.printf "\"%s\", ::::/*%s*/ %d\n", ks, k.map {|c| "0x%.4x" % c}.join(","), i
      }
      f << "%%\n"
      f.close_write
      f.read
    }
    src.sub!(/^(#{hash})\s*\(.*?\).*?\n\{\n(.*)^\}/m) {
      name = $1
      body = $2
      body.gsub!(/\(unsigned char\)str\[(\d+)\]/, "bits_#{arity > 1 ? 'at' : 'of'}(#{argname}, \\1)")
      "#{name}(#{argdecl})\n{\n#{body}}"
    }
    src.sub!(/const short *\*\n^(#{lookup})\s*\(.*?\).*?\n\{\n(.*)^\}/m) {
      name = $1
      body = $2
      body.sub!(/\benum\s+\{(\n[ \t]+)/, "\\&MIN_CODE_VALUE = 0x#{min.to_s(16)},\\1""MAX_CODE_VALUE = 0x#{max.to_s(16)},\\1")
      body.gsub!(/(#{hash})\s*\(.*?\)/, "\\1(#{argname})")
      body.gsub!(/\{"",-1}/, "-1")
      body.gsub!(/\{"(?:[^"]|\\")+", *::::(.*)\}/, '\1')
      body.sub!(/(\s+if\s)\(len\b.*\)/) do
        "#$1(" <<
          (arity > 1 ? (0...arity).map {|i| range_check("#{argname}[#{i}]")}.join(" &&\n      ") : range_check(argname)) <<
          ")"
      end
      v = nil
      body.sub!(/(if\s*\(.*MAX_HASH_VALUE.*\)\n([ \t]*))\{(.*?)\n\2\}/m) {
        pre = $1
        indent = $2
        s = $3
        s.sub!(/const char *\* *(\w+)( *= *wordlist\[\w+\]).\w+/, 'short \1 = wordlist[key]')
        v = $1
        s.sub!(/\bif *\(.*\)/, "if (#{v} >= 0 && code#{arity}_equal(#{argname}, #{key}_Table[#{v}].from))")
        "#{pre}{#{s}\n#{indent}}"
      }
      body.sub!(/\b(return\s+&)([^;]+);/, '\1'"#{key}_Table[#{v}].to;")
      "static const #{type} *\n#{name}(#{argdecl})\n{\n#{body}}"
    }
    src
  end

  def display(dest, mapping_data)
    # print the header
    dest.print("/* DO NOT EDIT THIS FILE. */\n")
    dest.print("/* Generated by enc/unicode/case-folding.rb */\n\n")

    # print folding data

    # CaseFold + CaseFold_Locale
    name = "CaseFold_11"
    data = print_table(dest, name, mapping_data, "CaseFold"=>fold, "CaseFold_Locale"=>fold_locale)
    dest.print lookup_hash(name, "CodePointList3", data)

    # print unfolding data

    # CaseUnfold_11 + CaseUnfold_11_Locale
    name = "CaseUnfold_11"
    data = print_table(dest, name, mapping_data, name=>unfold[0], "#{name}_Locale"=>unfold_locale[0])
    dest.print lookup_hash(name, "CodePointList3", data)

    # CaseUnfold_12 + CaseUnfold_12_Locale
    name = "CaseUnfold_12"
    data = print_table(dest, name, mapping_data, name=>unfold[1], "#{name}_Locale"=>unfold_locale[1])
    dest.print lookup_hash(name, "CodePointList2", data)

    # CaseUnfold_13
    name = "CaseUnfold_13"
    data = print_table(dest, name, mapping_data, name=>unfold[2])
    dest.print lookup_hash(name, "CodePointList2", data)

    # TitleCase
    dest.print mapping_data.specials_output
  end

  def debug!
    @debug = true
  end

  def self.load(*args)
    new.load(*args)
  end
end

class MapItem
  attr_accessor :upper, :lower, :title, :code

  def initialize(code, upper, lower, title)
    @code = code
    @upper = upper unless upper == ''
    @lower = lower unless lower == ''
    @title = title unless title == ''
  end
end

class CaseMapping
  def initialize (mapping_directory)
    @mappings = {}
    @specials = []
    @specials_length = 0
    IO.readlines(File.expand_path('UnicodeData.txt', mapping_directory), encoding: Encoding::ASCII_8BIT).each do |line|
      next if line =~ /^</
      code, _1,_2,_3,_4,_5,_6,_7,_8,_9,_10,_11, upper, lower, title = line.chomp.split ';'
      unless upper and lower and title and (upper+lower+title)==''
        @mappings[code] = MapItem.new(code, upper, lower, title)
      end
    end

    IO.readlines(File.expand_path('SpecialCasing.txt', mapping_directory), encoding: Encoding::ASCII_8BIT).each do |line|
      line.chomp!
      line, comment = line.split(/ *#/)
      next if not line or line == ''
      code, lower, title, upper, conditions = line.split(/ *; */)
      unless conditions
        item = @mappings[code]
        item.lower = lower
        item.title = title
        item.upper = upper
      end
    end
  end

  def flags(from, type, to)
    # types: CaseFold_11, CaseUnfold_11, CaseUnfold_12, CaseUnfold_13
    flags = ""
    flags += '|F' if type=='CaseFold_11'
    from = Array(from).map {|i| "%04X" % i}.join(" ")
    to   = Array(to).map {|i| "%04X" % i}.join(" ")
    to = to.split(/ /).first  if type=='CaseUnfold_11'
    item = @mappings[from]
    if item
      flags += '|U'  if to==item.upper
      flags += '|D'  if to==item.lower
      specials_index = nil
      specials = []
      unless item.upper == item.title
        specials << item.title
        flags += '|ST'
        if item.code == item.title
          flags += '|IT'
        end

      end
      unless item.lower.nil? or item.lower==from or item.lower==to
        specials << item.lower
        flags += '|SL'
      end
      unless item.upper.nil? or item.upper==from or item.upper==to
        specials << item.upper
        flags += '|SU'
      end
      if specials.first
        flags += "|I(#{@specials_length})"
        @specials_length += specials.map { |s| s.split(/ /).length }.reduce(:+)
        @specials << specials
      end
    end
    flags
  end

  def debug!
    @debug = true
  end

  def specials_output
    "OnigCodePoint CaseMappingSpecials[] = {\n" +
    @specials.map do |sps|
      '   ' + sps.map do |sp|
        chars = sp.split(/ /)
        ct = ' /* ' + Array(chars).map{|c|[c.to_i(16)].pack("U*")}.join(", ") + ' */' if @debug
        " L(#{chars.length})|#{chars.map {|c| "0x"+c }.join(', ')}#{ct},"
      end.join + "\n"
    end.join + "};\n"
  end

  def self.load(*args)
    new(*args)
  end
end

class CaseMappingDummy
  def flags(from, type, to)
    ""
  end

  def titlecase_output()  ''  end
  def debug!()  end
end

if $0 == __FILE__
  require 'optparse'
  dest = nil
  mapping_directory = nil
  mapping_data = nil
  debug = false
  fold_1 = false
  ARGV.options do |opt|
    opt.banner << " [INPUT]"
    opt.on("--output-file=FILE", "-o", "output to the FILE instead of STDOUT") {|output|
      dest = (output unless output == '-')
    }
    opt.on('--mapping-data-directory=DIRECTORY', '-m', 'data DIRECTORY of mapping files') { |directory|
      mapping_directory = directory
    }
    opt.on('--debug', '-d') {
      debug = true
    }
    opt.parse!
    abort(opt.to_s) if ARGV.size > 1
  end
  if mapping_directory
    if ARGV[0]
      warn "Either specify directory or individual file, but not both."
      exit
    end
    filename = File.expand_path('CaseFolding.txt', mapping_directory)
    mapping_data = CaseMapping.load(mapping_directory)
  end
  filename ||= ARGV[0] || 'CaseFolding.txt'
  mapping_data ||= CaseMappingDummy.new

  data = CaseFolding.load(filename)
  if debug
    data.debug!
    mapping_data.debug!
  end
  if dest
    open(dest, "wb") do |f|
      data.display(f, mapping_data)
    end
  else
    data.display(STDOUT, mapping_data)
  end
end