[Universal Parser] Reduce dependence on RArray in parse.y

- Introduce `rb_parser_ary_t` structure to partly eliminate RArray from parse.y
  - In this patch, `parser_params->tokens` and `parser_params->ast->node_buffer->tokens` are now `rb_parser_ary_t *`
  - Instead, `ast_node_all_tokens()` internally creates a Ruby Array object from the `rb_parser_ary_t`
  - Also, delete `rb_ast_tokens()` and `rb_ast_set_tokens()` in node.c

- Implement `rb_parser_str_escape()`
  - This is a port of the `rb_str_escape()` function in string.c
  - `rb_parser_str_escape()` does not depend on `VALUE` (RString)
  - Instead, it uses `rb_parser_stirng_t *`
  - This function works when --dump=y option passed

- Because WIP of the universal parser, similar functions like `rb_parser_tokens_free()` exist in both node.c and parse.y. Refactoring them may be needed in some way in the future

- Although we considered redesigning the structure: `ast->node_buffer->tokens` into `ast->tokens`, we leave it as it is because `rb_ast_t` is an imemo. (We will address it in the future)

1 files changed, 26 insertions, 1 deletions

diff --git a/ast.c b/ast.c
index 66e237b1f2..f716bb995c 100644
--- a/ast.c
+++ b/ast.c
@@ -774,10 +774,35 @@ ast_node_last_column(rb_execution_context_t *ec, VALUE self)
 static VALUE
 ast_node_all_tokens(rb_execution_context_t *ec, VALUE self)
 {
+    long i;
     struct ASTNodeData *data;
+    rb_parser_ary_t *parser_tokens;
+    rb_parser_ast_token_t *parser_token;
+    VALUE str, loc, token, all_tokens;
+
     TypedData_Get_Struct(self, struct ASTNodeData, &rb_node_type, data);
 
-    return rb_ast_tokens(data->ast);
+    parser_tokens = data->ast->node_buffer->tokens;
+    if (parser_tokens == NULL) {
+        return Qnil;
+    }
+
+    all_tokens = rb_ary_new2(parser_tokens->len);
+    for (i = 0; i < parser_tokens->len; i++) {
+        parser_token = parser_tokens->data[i];
+        str = rb_str_new(parser_token->str->ptr, parser_token->str->len);
+        loc = rb_ary_new_from_args(4,
+            INT2FIX(parser_token->loc.beg_pos.lineno),
+            INT2FIX(parser_token->loc.beg_pos.column),
+            INT2FIX(parser_token->loc.end_pos.lineno),
+            INT2FIX(parser_token->loc.end_pos.column)
+        );
+        token = rb_ary_new_from_args(4, INT2FIX(parser_token->id), ID2SYM(rb_intern(parser_token->type_name)), str, loc);
+        rb_ary_push(all_tokens, token);
+    }
+    rb_obj_freeze(all_tokens);
+
+    return all_tokens;
 }
 
 static VALUE