id.def: token_ops

* defs/id.def (token_ops): gather associations between IDs,
  operators, and parser tokens.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@52458 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

5 files changed, 76 insertions, 88 deletions

diff --git a/ChangeLog b/ChangeLog
index 242f21a512..42eb8dac9a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,8 @@
+Thu Nov  5 13:03:58 2015  Nobuyoshi Nakada  <nobu@ruby-lang.org>
+
+	* defs/id.def (token_ops): gather associations between IDs,
+	  operators, and parser tokens.
+
 Thu Nov  5 10:17:17 2015  Nobuyoshi Nakada  <nobu@ruby-lang.org>
 
 	* ext/socket/socket.c (make_addrinfo): use RARRAY_ASET for
diff --git a/defs/id.def b/defs/id.def
index f79e5ef4cf..d871200cf9 100644
--- a/defs/id.def
+++ b/defs/id.def
@@ -65,6 +65,42 @@ firstline, predefined = __LINE__+1, %[\
   -                                                     debug#created_line
 ]
 
+# VM ID         OP      Parser Token
+token_ops = %[\
+  Dot2          ..      DOT2
+  Dot3          ...     DOT3
+  UPlus         +@      UPLUS
+  UMinus        -@      UMINUS
+  Pow           **      POW
+  DSTAR         **
+  Cmp           <=>     CMP
+  PLUS          +
+  MINUS         -
+  MULT          *
+  DIV           /
+  MOD           %
+  LTLT          <<      LSHFT
+  GTGT          >>      RSHFT
+  LT            <
+  LE            <=      LEQ
+  GT            >
+  GE            >=      GEQ
+  Eq            ==      EQ
+  Eqq           ===     EQQ
+  Neq           !=      NEQ
+  Not           !
+  Backquote     `
+  EqTilde       =~      MATCH
+  NeqTilde      !~      NMATCH
+  AREF          []
+  ASET          []=
+  COLON2        ::
+  COLON3        ::
+  ANDOP         &&
+  OROP          ||
+  DOTQ          .?
+]
+
 class KeywordError < RuntimeError
   def self.raise(mesg, line)
     super(self, mesg, ["#{__FILE__}:#{line}", *caller])
@@ -79,6 +115,7 @@ global_ids = []
 const_ids = []
 class_ids = []
 attrset_ids = []
+token_op_ids = []
 names = {}
 predefined.split(/^/).each_with_index do |line, num|
   next if /^#/ =~ line
@@ -117,6 +154,14 @@ predefined.split(/^/).each_with_index do |line, num|
   end << token
   predefined_ids[token] = name
 end
+token_ops.split(/^/).each do |line|
+  next if /^#/ =~ line
+  line.sub!(/\s+#.*/, '')
+  id, op, token = line.split
+  next unless id and op
+  token ||= (id unless /\A\W\z/ =~ op)
+  token_op_ids << [id, op, token]
+end
 {
   "LOCAL" => local_ids,
   "INSTANCE" => instance_ids,
@@ -126,4 +171,5 @@ end
   "ATTRSET" => attrset_ids,
   :preserved => preserved_ids,
   :predefined => predefined_ids,
+  :token_op => token_op_ids,
 }
diff --git a/symbol.c b/symbol.c
index 795c22d836..f46acf22b7 100644
--- a/symbol.c
+++ b/symbol.c
@@ -30,55 +30,6 @@ static ID register_static_symid_str(ID, VALUE);
 
 #define is_identchar(p,e,enc) (rb_enc_isalnum((unsigned char)(*(p)),(enc)) || (*(p)) == '_' || !ISASCII(*(p)))
 
-#define tUPLUS  RUBY_TOKEN(UPLUS)
-#define tUMINUS RUBY_TOKEN(UMINUS)
-#define tPOW    RUBY_TOKEN(POW)
-#define tCMP    RUBY_TOKEN(CMP)
-#define tEQ     RUBY_TOKEN(EQ)
-#define tEQQ    RUBY_TOKEN(EQQ)
-#define tNEQ    RUBY_TOKEN(NEQ)
-#define tGEQ    RUBY_TOKEN(GEQ)
-#define tLEQ    RUBY_TOKEN(LEQ)
-#define tMATCH  RUBY_TOKEN(MATCH)
-#define tNMATCH RUBY_TOKEN(NMATCH)
-#define tDOT2   RUBY_TOKEN(DOT2)
-#define tDOT3   RUBY_TOKEN(DOT3)
-#define tAREF   RUBY_TOKEN(AREF)
-#define tASET   RUBY_TOKEN(ASET)
-#define tLSHFT  RUBY_TOKEN(LSHFT)
-#define tRSHFT  RUBY_TOKEN(RSHFT)
-#define tCOLON2 RUBY_TOKEN(COLON2)
-#define tANDOP  RUBY_TOKEN(ANDOP)
-#define tOROP   RUBY_TOKEN(OROP)
-#define tDOTQ   RUBY_TOKEN(DOTQ)
-
-static const struct {
-    unsigned short token;
-    const char name[3], term;
-} op_tbl[] = {
-    {tDOT2,	".."},
-    {tDOT3,	"..."},
-    {tPOW,	"**"},
-    {tUPLUS,	"+@"},
-    {tUMINUS,	"-@"},
-    {tCMP,	"<=>"},
-    {tGEQ,	">="},
-    {tLEQ,	"<="},
-    {tEQ,	"=="},
-    {tEQQ,	"==="},
-    {tNEQ,	"!="},
-    {tMATCH,	"=~"},
-    {tNMATCH,	"!~"},
-    {tAREF,	"[]"},
-    {tASET,	"[]="},
-    {tLSHFT,	"<<"},
-    {tRSHFT,	">>"},
-    {tCOLON2,   "::"},
-    {tANDOP,    "&&"},
-    {tOROP,     "||"},
-    {tDOTQ,     ".?"},
-};
-
 #define op_tbl_count numberof(op_tbl)
 STATIC_ASSERT(op_tbl_name_size, sizeof(op_tbl[0].name) == 3);
 #define op_tbl_len(i) (!op_tbl[i].name[1] ? 1 : !op_tbl[i].name[2] ? 2 : 3)
diff --git a/template/id.c.tmpl b/template/id.c.tmpl
index cac213a8fb..477a76bc26 100644
--- a/template/id.c.tmpl
+++ b/template/id.c.tmpl
@@ -13,7 +13,23 @@
 <%
 defs = File.join(File.dirname(File.dirname(erb.filename)), "defs/id.def")
 ids = eval(File.read(defs), binding, defs)
+ops = ids[:token_op].uniq {|id, op, token| token && op}
 %>
+% ops.each do |_id, _op, token|
+%   next unless token
+#define t<%=token%> RUBY_TOKEN(<%=token%>)
+% end
+
+static const struct {
+    unsigned short token;
+    const char name[3], term;
+} op_tbl[] = {
+% ops.each do |_id, op, token|
+%   next unless token
+    {t<%=token%>, "<%=op%>"},
+% end
+};
+
 static void
 Init_id(void)
 {
diff --git a/template/id.h.tmpl b/template/id.h.tmpl
index 5e7f117c24..0db35c1802 100644
--- a/template/id.h.tmpl
+++ b/template/id.h.tmpl
@@ -15,12 +15,6 @@ require 'optparse'
 
 op_id_offset = 128
 
-token_op_ids = %w[
-  tDOT2 tDOT3 tUPLUS tUMINUS tPOW tDSTAR tCMP tLSHFT tRSHFT
-  tLEQ tGEQ tEQ tEQQ tNEQ tMATCH tNMATCH tAREF tASET
-  tCOLON2 tCOLON3 tANDOP tOROP tDOTQ
-]
-
 defs = File.join(File.dirname(File.dirname(erb.filename)), "defs/id.def")
 ids = eval(File.read(defs), binding, defs)
 types = ids.keys.grep(/^[A-Z]/)
@@ -57,43 +51,19 @@ enum ruby_id_types {
 #define symIFUNC ID2SYM(idIFUNC)
 #define symCFUNC ID2SYM(idCFUNC)
 
-% token_op_ids.each_with_index do |token, index|
-#define RUBY_TOKEN_<%=token[/\At(.+)\z/, 1]%> <%=op_id_offset + index%>
+% index = op_id_offset
+% ids[:token_op].each do |_id, _op, token|
+%   next unless token
+#define RUBY_TOKEN_<%=token%> <%=index%>
+%   index += 1
 % end
 #define RUBY_TOKEN(t) RUBY_TOKEN_##t
 
 enum ruby_method_ids {
-    idDot2 = RUBY_TOKEN(DOT2),
-    idDot3 = RUBY_TOKEN(DOT3),
-    idUPlus = RUBY_TOKEN(UPLUS),
-    idUMinus = RUBY_TOKEN(UMINUS),
-    idPow = RUBY_TOKEN(POW),
-    idCmp = RUBY_TOKEN(CMP),
-    idPLUS = '+',
-    idMINUS = '-',
-    idMULT = '*',
-    idDIV = '/',
-    idMOD = '%',
-    idLT = '<',
-    idLTLT = RUBY_TOKEN(LSHFT),
-    idLE = RUBY_TOKEN(LEQ),
-    idGT = '>',
-    idGTGT = RUBY_TOKEN(RSHFT),
-    idGE = RUBY_TOKEN(GEQ),
-    idEq = RUBY_TOKEN(EQ),
-    idEqq = RUBY_TOKEN(EQQ),
-    idNeq = RUBY_TOKEN(NEQ),
-    idNot = '!',
-    idBackquote = '`',
-    idEqTilde = RUBY_TOKEN(MATCH),
-    idNeqTilde = RUBY_TOKEN(NMATCH),
-    idAREF = RUBY_TOKEN(AREF),
-    idASET = RUBY_TOKEN(ASET),
-    idCOLON2 = RUBY_TOKEN(COLON2),
-    idANDOP = RUBY_TOKEN(ANDOP),
-    idOROP = RUBY_TOKEN(OROP),
-    idDOTQ = RUBY_TOKEN(DOTQ),
-    tPRESERVED_ID_BEGIN = <%=op_id_offset + token_op_ids.size - 1%>,
+% ids[:token_op].uniq {|_, op| op}.each do |id, op, token|
+    id<%=id%> = <%=token ? "RUBY_TOKEN(#{token})" : "'#{op}'"%>,
+% end
+    tPRESERVED_ID_BEGIN = <%=index-1%>,
 % ids[:preserved].each do |token|
     id<%=token%>,
 % end