7 files changed, 215 insertions, 0 deletions
diff --git a/ext/ripper/tools/generate-eventids1.rb b/ext/ripper/tools/generate-eventids1.rb
new file mode 100755
index 0000000000..de0e27b89c
--- /dev/null
+++ b/ext/ripper/tools/generate-eventids1.rb
@@ -0,0 +1,18 @@
+#
+# generate-eventids1.rb
+#
+
+ids = ARGF.map {|s| s.strip }
+
+ids.each do |id|
+  puts "static ID ripper_id_#{id};"
+end
+
+puts
+puts 'static void'
+puts 'ripper_init_eventids1()'
+puts '{'
+ids.each do |id|
+  puts %Q[    ripper_id_#{id} = rb_intern("on__#{id}");]
+end
+puts '}'
diff --git a/ext/ripper/tools/generate-param-macros.rb b/ext/ripper/tools/generate-param-macros.rb
new file mode 100755
index 0000000000..b19f6e8d5c
--- /dev/null
+++ b/ext/ripper/tools/generate-param-macros.rb
@@ -0,0 +1,14 @@
+off = true
+ARGF.each do |line|
+  case line
+  when /RIPPER_PARAMS_DECL_BEGIN/
+    off = false
+  when /RIPPER_PARAMS_DECL_END/
+    exit
+  when /ripper/
+    next if off
+    var = line.scan(/\w+/).last or next
+    base = var.sub(/ripper_/, '')
+    puts %"\#define #{base}\t\t(parser->ripper_#{base})"
+  end
+end
diff --git a/ext/ripper/tools/generate-ripper_rb.rb b/ext/ripper/tools/generate-ripper_rb.rb
new file mode 100755
index 0000000000..c6b2aea13f
--- /dev/null
+++ b/ext/ripper/tools/generate-ripper_rb.rb
@@ -0,0 +1,44 @@
+#
+# generate-ripper_rb.rb
+# Creates ripper.rb, filling in default event handlers, given a basic 
+# template, the list of parser events (ids1), and a list of lexer 
+# events (ids2).
+#
+
+def main
+  template, ids1, ids2 = *ARGV
+  File.foreach(template) do |line|
+    case line
+    when /\A\#include handlers1/
+      File.foreach(ids1) do |line|
+        id, arity = line.split
+        arity = arity.to_i
+        puts
+        puts "  def on__#{id}(#{argdecl(arity)})"
+        puts "    #{arity == 0 ? 'nil' : 'a'}"
+        puts "  end"
+      end
+    when /\A\#include handlers2/
+      File.foreach(ids2) do |line|
+        id, arity = line.split
+        arity = arity.to_i
+        puts
+        puts "  def on__#{id}(token)"
+        puts "    token"
+        puts "  end"
+      end
+    when /\A\#include (.*)/
+      raise "unknown operation: #include #{$1}"
+    else
+      print line
+    end
+  end
+end
+
+# Generate generic arg list depending on arity (n)
+# n:: [Integer] arity of method
+def argdecl( n )
+  %w(a b c d e f g h i j k l m)[0, n].join(', ')
+end
+
+main
diff --git a/ext/ripper/tools/list-parse-event-ids.rb b/ext/ripper/tools/list-parse-event-ids.rb
new file mode 100755
index 0000000000..2936f9b092
--- /dev/null
+++ b/ext/ripper/tools/list-parse-event-ids.rb
@@ -0,0 +1,38 @@
+#
+# list-parse-event-ids.rb
+#
+
+require 'getopts'
+
+def usage( status )
+  (status == 0 ? $stdout : $stderr).print(<<EOS)
+Usage: #{File.basename($0)} [-a] filename
+EOS
+  exit status
+end
+
+def main
+  getopts('a') or usage(1)
+  extract_ids(ARGF).each do |id, arity|
+    if $OPT_a
+    then puts "#{id} #{arity}"
+    else puts id
+    end
+  end
+end
+
+def extract_ids( f )
+  results = []
+  f.each do |line|
+    next if /\A\#\s*define\s+s?dispatch/ === line
+    next if /ripper_dispatch/ === line
+    if a = line.scan(/dispatch(\d)\((\w+)/)
+      a.each do |arity, event|
+        results.push [event, arity.to_i]
+      end
+    end
+  end
+  results.uniq.sort
+end
+
+main
diff --git a/ext/ripper/tools/list-scan-event-ids.rb b/ext/ripper/tools/list-scan-event-ids.rb
new file mode 100755
index 0000000000..6f25362b5d
--- /dev/null
+++ b/ext/ripper/tools/list-scan-event-ids.rb
@@ -0,0 +1,32 @@
+#
+# list-scan-event-ids.rb
+#
+
+require 'getopts'
+
+def usage(status)
+  (status == 0 ? $stdout : $stderr).puts(<<EOS)
+Usage: #{File.basename($0)} eventids2.c
+    -a    print IDs with arity.
+EOS
+  exit status
+end
+
+def main
+  ok = getopts('a', 'help')
+  usage 0 if $OPT_help
+  usage 1 unless ok
+  extract_ids(ARGF).sort.each do |id|
+    if $OPT_a
+      puts "#{id} 1"
+    else
+      puts id
+    end
+  end
+end
+
+def extract_ids(f)
+  (f.read.scan(/ripper_id_(\w+)/).flatten - ['scan']).uniq
+end
+
+main
diff --git a/ext/ripper/tools/preproc.rb b/ext/ripper/tools/preproc.rb
new file mode 100755
index 0000000000..a2dba36e02
--- /dev/null
+++ b/ext/ripper/tools/preproc.rb
@@ -0,0 +1,57 @@
+def main
+  prelude
+  grammar
+  usercode
+end
+
+def prelude
+  while line = ARGF.gets
+    case line
+    when %r</\*%%%\*/>
+      puts '/*'
+    when %r</\*%>
+      puts '*/'
+    when %r<%\*/>
+      puts
+    when /\A%%/
+      puts '%%'
+      return
+    when /\A%token/
+      puts line.sub(/<\w+>/, '<val>')
+    when /\A%type/
+      puts line.sub(/<\w+>/, '<val>')
+    else
+      print line
+    end
+  end
+end
+
+def grammar
+  while line = ARGF.gets
+    case line
+    when %r</\*%%%\*/>
+      puts '#if 0'
+    when %r</\*%c%\*/>
+      puts '/*'
+    when %r</\*%c>
+      puts '*/'
+    when %r</\*%>
+      puts '#endif'
+    when %r<%\*/>
+      puts
+    when /\A%%/
+      puts '%%'
+      return
+    else
+      print line
+    end
+  end
+end
+
+def usercode
+  while line = ARGF.gets
+    print line
+  end
+end
+
+main
diff --git a/ext/ripper/tools/strip.rb b/ext/ripper/tools/strip.rb
new file mode 100755
index 0000000000..99413c361d
--- /dev/null
+++ b/ext/ripper/tools/strip.rb
@@ -0,0 +1,12 @@
+last_is_void = false
+ARGF.each do |line|
+  if line.strip.empty?
+    #puts() unless last_is_void
+    last_is_void = true
+  elsif /\A\#/ === line
+    ;
+  else
+    print line
+    last_is_void = false
+  end
+end