[Feature #20257] Rearchitect Ripper

Introduce another semantic value stack for Ripper so that
Ripper can manage both Node and Ruby Object separately.
This rearchitectutre of Ripper solves these issues.
Therefore adding test cases for them.

* [Bug 10436] https://bugs.ruby-lang.org/issues/10436
* [Bug 18988] https://bugs.ruby-lang.org/issues/18988
* [Bug 20055] https://bugs.ruby-lang.org/issues/20055

Checked the differences of `Ripper.sexp` for files under `/test/ruby`
are only on test_pattern_matching.rb.
The differences comes from the differences between
`new_hash_pattern_tail` functions between parser and Ripper.
Ripper `new_hash_pattern_tail` didn’t call `assignable` then
`kw_rest_arg` wasn’t marked as local variable.
This is also fixed by this commit.

```
--- a/./tmp/before/test_pattern_matching.rb
+++ b/./tmp/after/test_pattern_matching.rb
@@ -3607,7 +3607,7 @@
                  [:in,
                   [:hshptn, nil, [], [:var_field, [:@ident, “a”, [984, 13]]]],
                   [[:binary,
-                    [:vcall, [:@ident, “a”, [985, 10]]],
+                    [:var_ref, [:@ident, “a”, [985, 10]]],
                     :==,
                     [:hash, nil]]],
                   nil]]],
@@ -3662,7 +3662,7 @@
                  [:in,
                   [:hshptn, nil, [], [:var_field, [:@ident, “a”, [993, 13]]]],
                   [[:binary,
-                    [:vcall, [:@ident, “a”, [994, 10]]],
+                    [:var_ref, [:@ident, “a”, [994, 10]]],
                     :==,
                     [:hash,
                      [:assoclist_from_args,
@@ -3813,7 +3813,7 @@
                    [:command,
                     [:@ident, “raise”, [1022, 10]],
                     [:args_add_block,
-                     [[:vcall, [:@ident, “b”, [1022, 16]]]],
+                     [[:var_ref, [:@ident, “b”, [1022, 16]]]],
                      false]]],
                   [:else, [[:var_ref, [:@kw, “true”, [1024, 10]]]]]]]],
                nil,
@@ -3876,7 +3876,7 @@
                      [:@int, “0”, [1033, 15]]],
                     :“&&“,
                     [:binary,
-                     [:vcall, [:@ident, “b”, [1033, 20]]],
+                     [:var_ref, [:@ident, “b”, [1033, 20]]],
                      :==,
                      [:hash, nil]]]],
                   nil]]],
@@ -3946,7 +3946,7 @@
                      [:@int, “0”, [1042, 15]]],
                     :“&&“,
                     [:binary,
-                     [:vcall, [:@ident, “b”, [1042, 20]]],
+                     [:var_ref, [:@ident, “b”, [1042, 20]]],
                      :==,
                      [:hash,
                       [:assoclist_from_args,
@@ -5206,7 +5206,7 @@
                      [[:assoc_new,
                        [:@label, “c:“, [1352, 22]],
                        [:@int, “0”, [1352, 25]]]]]],
-                   [:vcall, [:@ident, “r”, [1352, 29]]]],
+                   [:var_ref, [:@ident, “r”, [1352, 29]]]],
                   false]]],
                [:binary,
                 [:call,
@@ -5299,7 +5299,7 @@
                       [:assoc_new,
                        [:@label, “c:“, [1367, 34]],
                        [:@int, “0”, [1367, 37]]]]]],
-                   [:vcall, [:@ident, “r”, [1367, 41]]]],
+                   [:var_ref, [:@ident, “r”, [1367, 41]]]],
                   false]]],
                [:binary,
                 [:call,
@@ -5931,7 +5931,7 @@
              [:in,
               [:hshptn, nil, [], [:var_field, [:@ident, “r”, [1533, 11]]]],
               [[:binary,
-                [:vcall, [:@ident, “r”, [1534, 8]]],
+                [:var_ref, [:@ident, “r”, [1534, 8]]],
                 :==,
                 [:hash,
                  [:assoclist_from_args,
```

5 files changed, 20 insertions, 30 deletions

diff --git a/ext/ripper/ripper_init.c.tmpl b/ext/ripper/ripper_init.c.tmpl
index a2a5e4b4cf..1cd701844c 100644
--- a/ext/ripper/ripper_init.c.tmpl
+++ b/ext/ripper/ripper_init.c.tmpl
@@ -54,27 +54,6 @@ static const rb_data_type_t parser_data_type = {
     0, 0, RUBY_TYPED_FREE_IMMEDIATELY
 };
 
-ID
-ripper_get_id(VALUE v)
-{
-    NODE *nd;
-    if (!RB_TYPE_P(v, T_NODE)) return 0;
-    nd = (NODE *)v;
-    if (!nd_type_p(nd, NODE_RIPPER)) return 0;
-    return RNODE_RIPPER(nd)->nd_vid;
-}
-
-VALUE
-ripper_get_value(VALUE v)
-{
-    NODE *nd;
-    if (UNDEF_P(v)) return Qnil;
-    if (!RB_TYPE_P(v, T_NODE)) return v;
-    nd = (NODE *)v;
-    if (!nd_type_p(nd, NODE_RIPPER)) return Qnil;
-    return RNODE_RIPPER(nd)->nd_rval;
-}
-
 static VALUE
 ripper_lex_get_generic(struct parser_params *p, VALUE src)
 {
@@ -607,5 +586,8 @@ InitVM_ripper(void)
      */
     rb_define_global_const("SCRIPT_LINES__", Qnil);
 #endif
+    rb_ripper_none = rb_obj_alloc(rb_cObject);
+    rb_obj_freeze(rb_ripper_none);
+    rb_gc_register_mark_object(rb_ripper_none);
 
 }
diff --git a/ext/ripper/ripper_init.h b/ext/ripper/ripper_init.h
index 82ff13b95f..e9e7bc7e5f 100644
--- a/ext/ripper/ripper_init.h
+++ b/ext/ripper/ripper_init.h
@@ -1,6 +1,7 @@
 #ifndef RIPPER_INIT_H
 #define RIPPER_INIT_H
 
+extern VALUE rb_ripper_none;
 VALUE ripper_get_value(VALUE v);
 ID ripper_get_id(VALUE v);
 PRINTF_ARGS(void ripper_compile_error(struct parser_params*, const char *fmt, ...), 2, 3);
diff --git a/ext/ripper/tools/dsl.rb b/ext/ripper/tools/dsl.rb
index f89e07d65e..8f03233574 100644
--- a/ext/ripper/tools/dsl.rb
+++ b/ext/ripper/tools/dsl.rb
@@ -18,14 +18,15 @@ class DSL
   NAME_PATTERN = /(?>\$|\d+|[a-zA-Z_][a-zA-Z0-9_]*|\[[a-zA-Z_.][-a-zA-Z0-9_.]*\])(?>(?:\.|->)[a-zA-Z_][a-zA-Z0-9_]*)*/.source
   NOT_REF_PATTERN = /(?>\#.*|[^\"$@]*|"(?>\\.|[^\"])*")/.source
 
-  def initialize(code, options)
+  def initialize(code, options, lineno = nil)
+    @lineno = lineno
     @events = {}
     @error = options.include?("error")
     @brace = options.include?("brace")
     if options.include?("final")
       @final = "p->result"
     else
-      @final = (options.grep(/\A\$#{NAME_PATTERN}\z/o)[0] || "$$")
+      @final = (options.grep(/\A\$#{NAME_PATTERN}\z/o)[0] || "p->s_lvalue")
     end
     @vars = 0
 
@@ -33,8 +34,11 @@ class DSL
     p = p = "p"
 
     @code = ""
-    code = code.gsub(%r[\G#{NOT_REF_PATTERN}\K[$@]#{TAG_PATTERN}?#{NAME_PATTERN}]o, '"\&"')
+    code = code.gsub(%r[\G#{NOT_REF_PATTERN}\K(\$|\$:|@)#{TAG_PATTERN}?#{NAME_PATTERN}]o, '"\&"')
     @last_value = eval(code)
+  rescue SyntaxError
+    $stderr.puts "error on line #{@lineno}" if @lineno
+    raise
   end
 
   attr_reader :events
@@ -65,11 +69,15 @@ class DSL
     vars = []
     args.each do |arg|
       vars << v = new_var
-      @code << "#{ v }=#{ arg };"
+      if arg =~ /\A\$:#{NAME_PATTERN}\z/
+        @code << "#{ v }=get_value(#{arg});"
+      else
+        @code << "#{ v }=#{ arg };"
+      end
     end
     v = new_var
     d = "dispatch#{ args.size }(#{ [event, *vars].join(",") })"
-    d = "#{ vars.last }==Qundef ? #{ vars.first } : #{ d }" if qundef_check
+    d = "#{ vars.last }==rb_ripper_none ? #{ vars.first } : #{ d }" if qundef_check
     @code << "#{ v }=#{ d };"
     v
   end
@@ -88,4 +96,3 @@ class DSL
     name
   end
 end
-
diff --git a/ext/ripper/tools/generate.rb b/ext/ripper/tools/generate.rb
index c44b4ba325..27aa53bce0 100644
--- a/ext/ripper/tools/generate.rb
+++ b/ext/ripper/tools/generate.rb
@@ -167,14 +167,14 @@ require_relative "dsl"
 def read_ids1_with_locations(path)
   h = {}
   File.open(path) {|f|
-    f.each do |line|
+    f.each.with_index(1) do |line, i|
       next if /\A\#\s*define\s+dispatch/ =~ line
       next if /ripper_dispatch/ =~ line
       line.scan(/\bdispatch(\d)\((\w+)/) do |arity, event|
         (h[event] ||= []).push [f.lineno, arity.to_i]
       end
       if line =~ %r</\*% *ripper(?:\[(.*?)\])?: *(.*?) *%\*/>
-        gen = DSL.new($2, ($1 || "").split(","))
+        gen = DSL.new($2, ($1 || "").split(","), i)
         gen.generate
         gen.events.each do |event, arity|
           (h[event] ||= []).push [f.lineno, arity.to_i]
diff --git a/ext/ripper/tools/preproc.rb b/ext/ripper/tools/preproc.rb
index a0d5e79d7d..981237a585 100644
--- a/ext/ripper/tools/preproc.rb
+++ b/ext/ripper/tools/preproc.rb
@@ -61,7 +61,7 @@ def prelude(f, out)
       return
     when /\A%token/, /\A%type/, /\A} <node(?>_\w+)?>/
       # types in %union which have corresponding set_yylval_* macro.
-      out << line.sub(/<(?:node(?>_\w+)?|num|id)>/, '<val>')
+      out << line
     when /^enum lex_state_(?:bits|e) \{/
       lex_state_def = true
       out << line