delete tool/instruction.rb (2nd try)

Previous commit changed insns.def format. Now is the time for its
generators. In doing so I chose to modernize the system, not just
patch.  My attempt includes

- extensive use of Onigumo regular expressions
- split from one big file (instruction.rb) into separated MVC
- partial view

Also, let me take this opportunity to kill old unused features
such as

- stack caching
- minsns / yasmdata which are never seriously used
- yarvarch document generation (moved to doc/)
- vast majority of unused arguments to insns2vm.rb

This commit generates VM source codes that cleanly compile, and
the generated binary passes tests.  At least for me.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@61784 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

4 files changed, 220 insertions, 0 deletions

diff --git a/tool/ruby_vm/loaders/insns_def.rb b/tool/ruby_vm/loaders/insns_def.rb
new file mode 100644
index 0000000000..58748c3ca6
--- /dev/null
+++ b/tool/ruby_vm/loaders/insns_def.rb
@@ -0,0 +1,92 @@
+#! /your/favourite/path/to/ruby
+# -*- mode: ruby; coding: utf-8; indent-tabs-mode: nil; ruby-indent-level: 2 -*-
+# -*- frozen_string_literal: true; -*-
+# -*- warn_indent: true; -*-
+#
+# Copyright (c) 2017 Urabe, Shyouhei.  All rights reserved.
+#
+# This file is  a part of the programming language  Ruby.  Permission is hereby
+# granted, to  either redistribute and/or  modify this file, provided  that the
+# conditions  mentioned in  the file  COPYING are  met.  Consult  the file  for
+# details.
+
+require_relative '../helpers/scanner'
+
+json    = []
+scanner = RubyVM::Scanner.new '../../../insns.def'
+path    = scanner.__FILE__
+grammar = %r'
+    (?<comment>  /[*] [^*]* [*]+ (?: [^*/] [^*]* [*]+ )* /         ){0}
+    (?<keyword>  typedef | extern | static | auto | register |
+                 struct  | union  | enum                           ){0}
+    (?<C>        (?: \g<block> | [^{}]+ )*                         ){0}
+    (?<block>    \{ \g<ws>* ^ \g<C> $ \g<ws>* \}                   ){0}
+    (?<ws>       \g<comment> | \s                                  ){0}
+    (?<ident>    [_a-zA-Z] [0-9_a-zA-Z]*                           ){0}
+    (?<type>     (?: \g<keyword> \g<ws>+ )* \g<ident>              ){0}
+    (?<arg>      \g<type> \g<ws>+ \g<ident> | \.\.\.               ){0}
+    (?<argv>     (?# empty ) |
+                 void        |
+                 \g<arg> (?: \g<ws>* , \g<ws>* \g<arg> \g<ws>* )*  ){0}
+    (?<pragma>   \g<ws>* // \s* attr \g<ws>+
+                 (?<pragma:type> \g<type>   )              \g<ws>+
+                 (?<pragma:name> \g<ident>  )              \g<ws>*
+                 =                                         \g<ws>*
+                 (?<pragma:expr> .+?;       )              \g<ws>* ){0}
+    (?<insn>     DEFINE_INSN                               \g<ws>+
+                 (?<insn:name>   \g<ident>  )              \g<ws>*
+     [(] \g<ws>* (?<insn:opes>   \g<argv>   ) \g<ws>* [)]  \g<ws>*
+     [(] \g<ws>* (?<insn:pops>   \g<argv>   ) \g<ws>* [)]  \g<ws>*
+     [(] \g<ws>* (?<insn:rets>   \g<argv>   ) \g<ws>* [)]  \g<ws>* ){0}
+'x
+
+until scanner.eos? do
+  next if scanner.scan(/#{grammar}\g<ws>+/o)
+  split = -> (v) {
+    case v when /\Avoid\z/ then
+      []
+    else
+      v.split(/, */)
+    end
+  }
+
+  l1   = scanner.scan!(/#{grammar}\g<insn>/o)
+  name = scanner["insn:name"]
+  ope  = split.(scanner["insn:opes"])
+  pop  = split.(scanner["insn:pops"])
+  ret  = split.(scanner["insn:rets"])
+
+  attrs = []
+  while l2 = scanner.scan(/#{grammar}\g<pragma>/o) do
+    attrs << {
+      location: [path, l2],
+      name: scanner["pragma:name"],
+      type: scanner["pragma:type"],
+      expr: scanner["pragma:expr"],
+    }
+  end
+
+  l3 = scanner.scan!(/#{grammar}\g<block>/o)
+  json << {
+    name: name,
+    location: [path, l1],
+    signature: {
+      name: name,
+      ope: ope,
+      pop: pop,
+      ret: ret,
+    },
+    attributes: attrs,
+    expr: {
+      location: [path, l3],
+      expr: scanner["block"],
+    },
+  }
+end
+
+RubyVM::InsnsDef = json
+
+if __FILE__ == $0 then
+  require 'json'
+  JSON.dump RubyVM::InsnsDef, STDOUT
+end
diff --git a/tool/ruby_vm/loaders/opt_insn_unif_def.rb b/tool/ruby_vm/loaders/opt_insn_unif_def.rb
new file mode 100644
index 0000000000..a5af409e71
--- /dev/null
+++ b/tool/ruby_vm/loaders/opt_insn_unif_def.rb
@@ -0,0 +1,34 @@
+#! /your/favourite/path/to/ruby
+# -*- mode: ruby; coding: utf-8; indent-tabs-mode: nil; ruby-indent-level: 2 -*-
+# -*- frozen_string_literal: true; -*-
+# -*- warn_indent: true; -*-
+#
+# Copyright (c) 2017 Urabe, Shyouhei.  All rights reserved.
+#
+# This file is  a part of the programming language  Ruby.  Permission is hereby
+# granted, to  either redistribute and/or  modify this file, provided  that the
+# conditions  mentioned in  the file  COPYING are  met.  Consult  the file  for
+# details.
+
+require_relative '../helpers/scanner'
+
+json    = []
+scanner = RubyVM::Scanner.new '../../../defs/opt_insn_unif.def'
+path    = scanner.__FILE__
+until scanner.eos? do
+  next  if scanner.scan(/ ^ (?: \#.* )? \n /x)
+  break if scanner.scan(/ ^ __END__ $ /x)
+
+  pos = scanner.scan!(/(?<series>  (?: [\ \t]* \w+ )+ ) \n /mx)
+  json << {
+    location: [path, pos],
+    signature: scanner["series"].strip.split
+  }
+end
+
+RubyVM::OptInsnUnifDef = json
+
+if __FILE__ == $0 then
+  require 'json'
+  JSON.dump RubyVM::OptInsnUnifDef, STDOUT
+end
diff --git a/tool/ruby_vm/loaders/opt_operand_def.rb b/tool/ruby_vm/loaders/opt_operand_def.rb
new file mode 100644
index 0000000000..5c94b4bced
--- /dev/null
+++ b/tool/ruby_vm/loaders/opt_operand_def.rb
@@ -0,0 +1,57 @@
+#! /your/favourite/path/to/ruby
+# -*- mode: ruby; coding: utf-8; indent-tabs-mode: nil; ruby-indent-level: 2 -*-
+# -*- frozen_string_literal: true; -*-
+# -*- warn_indent: true; -*-
+#
+# Copyright (c) 2017 Urabe, Shyouhei.  All rights reserved.
+#
+# This file is  a part of the programming language  Ruby.  Permission is hereby
+# granted, to  either redistribute and/or  modify this file, provided  that the
+# conditions  mentioned in  the file  COPYING are  met.  Consult  the file  for
+# details.
+
+require_relative '../helpers/scanner'
+
+json    = []
+scanner = RubyVM::Scanner.new '../../../defs/opt_operand.def'
+path    = scanner.__FILE__
+grammar = %r/
+    (?<comment> \# .+? \n                      ){0}
+    (?<ws>      \g<comment> | \s               ){0}
+    (?<insn>    \w+                            ){0}
+    (?<paren>   \( (?: \g<paren> | [^()]+)* \) ){0}
+    (?<expr>    (?: \g<paren> | [^(),\ \n] )+  ){0}
+    (?<remain>  \g<expr>                       ){0}
+    (?<arg>     \g<expr>                       ){0}
+    (?<extra>   , \g<ws>* \g<remain>           ){0}
+    (?<args>    \g<arg> \g<extra>*             ){0}
+    (?<decl>    \g<insn> \g<ws>+ \g<args> \n   ){0}
+/mx
+
+until scanner.eos? do
+  break if scanner.scan(/ ^ __END__ $ /x)
+  next  if scanner.scan(/#{grammar} \g<ws>+ /ox)
+
+  line = scanner.scan!(/#{grammar} \g<decl> /mox)
+  insn = scanner["insn"]
+  args = scanner["args"]
+  ary  = []
+  until args.strip.empty? do
+    tmp = StringScanner.new args
+    tmp.scan(/#{grammar} \g<args> /mox)
+    ary << tmp["arg"]
+    args = tmp["remain"]
+    break unless args
+  end
+  json << {
+    location: [path, line],
+    signature: [insn, ary]
+  }
+end
+
+RubyVM::OptOperandDef = json
+
+if __FILE__ == $0 then
+  require 'json'
+  JSON.dump RubyVM::OptOperandDef, STDOUT
+end
diff --git a/tool/ruby_vm/loaders/vm_opts_h.rb b/tool/ruby_vm/loaders/vm_opts_h.rb
new file mode 100644
index 0000000000..f898fb36a4
--- /dev/null
+++ b/tool/ruby_vm/loaders/vm_opts_h.rb
@@ -0,0 +1,37 @@
+#! /your/favourite/path/to/ruby
+# -*- mode: ruby; coding: utf-8; indent-tabs-mode: nil; ruby-indent-level: 2 -*-
+# -*- frozen_string_literal: true; -*-
+# -*- warn_indent: true; -*-
+#
+# Copyright (c) 2017 Urabe, Shyouhei.  All rights reserved.
+#
+# This file is  a part of the programming language  Ruby.  Permission is hereby
+# granted, to  either redistribute and/or  modify this file, provided  that the
+# conditions  mentioned in  the file  COPYING are  met.  Consult  the file  for
+# details.
+
+require_relative '../helpers/scanner'
+
+json    = {}
+scanner = RubyVM::Scanner.new '../../../vm_opts.h'
+grammar = %r/
+    (?<ws>      \u0020 ){0}
+    (?<key>     \w+    ){0}
+    (?<value>   0|1    ){0}
+    (?<define>  \#define \g<ws>+ OPT_\g<key> \g<ws>+ \g<value> \g<ws>*\n )
+/mx
+
+until scanner.eos? do
+  if scanner.scan grammar then
+    json[scanner['key']] = ! scanner['value'].to_i.zero? # not nonzero?
+  else
+    scanner.scan(/.*\n/)
+  end
+end
+
+RubyVM::VmOptsH = json
+
+if __FILE__ == $0 then
+  require 'json'
+  JSON.dump RubyVM::VmOptsH, STDOUT
+end