[Feature #19719] Universal Parser

Introduce Universal Parser mode for the parser.
This commit includes these changes:

* Introduce `UNIVERSAL_PARSER` macro. All of CRuby related functions
  are passed via `struct rb_parser_config_struct` when this macro is enabled.
* Add CI task with 'cppflags=-DUNIVERSAL_PARSER' for ubuntu.

1 files changed, 113 insertions, 10 deletions

diff --git a/internal/parse.h b/internal/parse.h
index f242c384ad..9e257cf39d 100644
--- a/internal/parse.h
+++ b/internal/parse.h
@@ -8,18 +8,121 @@
  *             file COPYING are met.  Consult the file for details.
  * @brief      Internal header for the parser.
  */
-#include "ruby/ruby.h"          /* for VALUE */
+#include <limits.h>
+#include "rubyparser.h"
+#include "internal/static_assert.h"
+
+#ifdef UNIVERSAL_PARSER
+#define rb_encoding void
+#endif
+
 struct rb_iseq_struct;          /* in vm_core.h */
 
+#define STRTERM_HEREDOC IMEMO_FL_USER0
+
+/* structs for managing terminator of string literal and heredocment */
+typedef struct rb_strterm_literal_struct {
+    union {
+        VALUE dummy;
+        long nest;
+    } u0;
+    union {
+        VALUE dummy;
+        long func;	    /* STR_FUNC_* (e.g., STR_FUNC_ESCAPE and STR_FUNC_EXPAND) */
+    } u1;
+    union {
+        VALUE dummy;
+        long paren;	    /* '(' of `%q(...)` */
+    } u2;
+    union {
+        VALUE dummy;
+        long term;	    /* ')' of `%q(...)` */
+    } u3;
+} rb_strterm_literal_t;
+
+#define HERETERM_LENGTH_BITS ((SIZEOF_VALUE - 1) * CHAR_BIT - 1)
+
+typedef struct rb_strterm_heredoc_struct {
+    VALUE lastline;	/* the string of line that contains `<<"END"` */
+    long offset;	/* the column of END in `<<"END"` */
+    int sourceline;	/* lineno of the line that contains `<<"END"` */
+    unsigned length	/* the length of END in `<<"END"` */
+#if HERETERM_LENGTH_BITS < SIZEOF_INT * CHAR_BIT
+    : HERETERM_LENGTH_BITS
+# define HERETERM_LENGTH_MAX ((1U << HERETERM_LENGTH_BITS) - 1)
+#else
+# define HERETERM_LENGTH_MAX UINT_MAX
+#endif
+    ;
+#if HERETERM_LENGTH_BITS < SIZEOF_INT * CHAR_BIT
+    unsigned quote: 1;
+    unsigned func: 8;
+#else
+    uint8_t quote;
+    uint8_t func;
+#endif
+} rb_strterm_heredoc_t;
+STATIC_ASSERT(rb_strterm_heredoc_t, sizeof(rb_strterm_heredoc_t) <= 4 * SIZEOF_VALUE);
+
+typedef struct rb_strterm_struct {
+    VALUE flags;
+    union {
+        rb_strterm_literal_t literal;
+        rb_strterm_heredoc_t heredoc;
+    } u;
+} rb_strterm_t;
+
 /* parse.y */
-VALUE rb_parser_set_yydebug(VALUE, VALUE);
-void *rb_parser_load_file(VALUE parser, VALUE name);
-void rb_parser_keep_script_lines(VALUE vparser);
-void rb_parser_error_tolerant(VALUE vparser);
-void rb_parser_keep_tokens(VALUE vparser);
-
-RUBY_SYMBOL_EXPORT_BEGIN
-VALUE rb_parser_set_context(VALUE, const struct rb_iseq_struct *, int);
-RUBY_SYMBOL_EXPORT_END
+void rb_ruby_parser_mark(void *ptr);
+size_t rb_ruby_parser_memsize(const void *ptr);
+
+void rb_ruby_parser_set_options(rb_parser_t *p, int print, int loop, int chomp, int split);
+rb_parser_t *rb_ruby_parser_set_context(rb_parser_t *p, const struct rb_iseq_struct *base, int main);
+void rb_ruby_parser_keep_script_lines(rb_parser_t *p);
+void rb_ruby_parser_error_tolerant(rb_parser_t *p);
+rb_ast_t* rb_ruby_parser_compile_file_path(rb_parser_t *p, VALUE fname, VALUE file, int start);
+void rb_ruby_parser_keep_tokens(rb_parser_t *p);
+rb_ast_t* rb_ruby_parser_compile_generic(rb_parser_t *p, VALUE (*lex_gets)(VALUE, int), VALUE fname, VALUE input, int start);
+rb_ast_t* rb_ruby_parser_compile_string_path(rb_parser_t *p, VALUE f, VALUE s, int line);
+VALUE rb_ruby_parser_encoding(rb_parser_t *p);
+int rb_ruby_parser_end_seen_p(rb_parser_t *p);
+int rb_ruby_parser_set_yydebug(rb_parser_t *p, int flag);
+
+int rb_reg_named_capture_assign_iter_impl(struct parser_params *p, const char *s, long len, rb_encoding *enc, NODE **succ_block, const rb_code_location_t *loc);
+
+#ifdef RIPPER
+void ripper_parser_mark(void *ptr);
+void ripper_parser_free(void *ptr);
+size_t ripper_parser_memsize(const void *ptr);
+void ripper_error(struct parser_params *p);
+VALUE ripper_value(struct parser_params *p);
+int rb_ruby_parser_get_yydebug(rb_parser_t *p);
+void rb_ruby_parser_set_value(rb_parser_t *p, VALUE value);
+int rb_ruby_parser_error_p(rb_parser_t *p);
+VALUE rb_ruby_parser_debug_output(rb_parser_t *p);
+void rb_ruby_parser_set_debug_output(rb_parser_t *p, VALUE output);
+VALUE rb_ruby_parser_parsing_thread(rb_parser_t *p);
+void rb_ruby_parser_set_parsing_thread(rb_parser_t *p, VALUE parsing_thread);
+void rb_ruby_parser_ripper_initialize(rb_parser_t *p, VALUE (*gets)(struct parser_params*,VALUE), VALUE input, VALUE sourcefile_string, const char *sourcefile, int sourceline);
+VALUE rb_ruby_parser_result(rb_parser_t *p);
+rb_encoding *rb_ruby_parser_enc(rb_parser_t *p);
+VALUE rb_ruby_parser_ruby_sourcefile_string(rb_parser_t *p);
+int rb_ruby_parser_ruby_sourceline(rb_parser_t *p);
+int rb_ruby_parser_lex_state(rb_parser_t *p);
+void rb_ruby_ripper_parse0(rb_parser_t *p);
+int rb_ruby_ripper_dedent_string(rb_parser_t *p, VALUE string, int width);
+VALUE rb_ruby_ripper_lex_get_str(rb_parser_t *p, VALUE s);
+int rb_ruby_ripper_initialized_p(rb_parser_t *p);
+void rb_ruby_ripper_parser_initialize(rb_parser_t *p);
+long rb_ruby_ripper_column(rb_parser_t *p);
+long rb_ruby_ripper_token_len(rb_parser_t *p);
+VALUE rb_ruby_ripper_lex_lastline(rb_parser_t *p);
+VALUE rb_ruby_ripper_lex_state_name(struct parser_params *p, int state);
+struct parser_params *rb_ruby_ripper_parser_allocate(void);
+#endif
+
+#ifdef UNIVERSAL_PARSER
+#undef rb_encoding
+#endif
 
 #endif /* INTERNAL_PARSE_H */