Switch the default parser from parse.y to Prism

This commit switches the default parser to Prism. There are a
couple of additional changes related to this that are a part of
this as well to make this happen.

* Switch the default parser in parse.h
* Remove the Prism-specific workflow and add a parse.y-specific
  workflow to CI so that it continues to be tested
* Update a few test exclusions since Prism has the correct
  behavior but parse.y doesn't per
  https://bugs.ruby-lang.org/issues/20504.
* Skips a couple of tests on RBS which are failing because they
  are using RubyVM::AbstractSyntaxTree.of.

Fixes [Feature #20564]

15 files changed, 68 insertions, 31 deletions

diff --git a/.github/workflows/bundled_gems.yml b/.github/workflows/bundled_gems.yml
index feafb9bca8..3a1f6b70f4 100644
--- a/.github/workflows/bundled_gems.yml
+++ b/.github/workflows/bundled_gems.yml
@@ -103,7 +103,7 @@ jobs:
         timeout-minutes: 30
         env:
           RUBY_TESTOPTS: '-q --tty=no'
-          TEST_BUNDLED_GEMS_ALLOW_FAILURES: ''
+          TEST_BUNDLED_GEMS_ALLOW_FAILURES: 'typeprof'
         if: ${{ steps.diff.outputs.gems }}
 
       - name: Commit
diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
index ecd1838123..fee998c70e 100644
--- a/.github/workflows/macos.yml
+++ b/.github/workflows/macos.yml
@@ -138,7 +138,7 @@ jobs:
         timeout-minutes: 60
         env:
           RUBY_TESTOPTS: '-q --tty=no'
-          TEST_BUNDLED_GEMS_ALLOW_FAILURES: ''
+          TEST_BUNDLED_GEMS_ALLOW_FAILURES: 'typeprof'
           PRECHECK_BUNDLED_GEMS: 'no'
 
       - name: make skipped tests
diff --git a/.github/workflows/prism.yml b/.github/workflows/parsey.yml
index de0243cd8f..73e0f4847c 100644
--- a/.github/workflows/prism.yml
+++ b/.github/workflows/parsey.yml
@@ -1,4 +1,4 @@
-name: Prism
+name: parse.y
 on:
   push:
     paths-ignore:
@@ -33,31 +33,31 @@ jobs:
       matrix:
         include:
           - test_task: test
-            run_opts: '--parser=prism'
+            run_opts: '--parser=parse.y'
             testopts: '-v --tty=no'
             timeout: 30
           - test_task: test-all
-            run_opts: '--parser=prism'
-            testopts: '-q --tty=no --excludes-dir="../src/test/.excludes-prism" --exclude="error_highlight/test_error_highlight.rb"'
+            run_opts: '--parser=parse.y'
+            testopts: '-q --tty=no --excludes-dir="../src/test/.excludes-parsey"'
             timeout: 40
           - test_task: test-spec
-            run_opts: '--parser=prism'
-            specopts: '-T --parser=prism'
+            run_opts: '--parser=parse.y'
+            specopts: '-T --parser=parse.y'
             timeout: 10
           - test_task: test-tool
-            run_opts: '--parser=prism'
+            run_opts: '--parser=parse.y'
             testopts: '-v --tty=no'
             timeout: 30
           - test_task: test-bundler-parallel
-            run_opts: '--parser=prism'
+            run_opts: '--parser=parse.y'
             testopts: '-v --tty=no'
             timeout: 50
           - test_task: test-bundled-gems
-            run_opts: '--parser=prism'
+            run_opts: '--parser=parse.y'
             testopts: '-v --tty=no'
             timeout: 30
           - test_task: test-syntax-suggest
-            run_opts: '--parser=prism'
+            run_opts: '--parser=parse.y'
             testopts: '-v --tty=no'
             timeout: 30
       fail-fast: false
@@ -105,6 +105,7 @@ jobs:
           RUBY_TESTOPTS: ${{ matrix.testopts }}
           RUN_OPTS: ${{ matrix.run_opts }}
           SPECOPTS: ${{ matrix.specopts }}
+          TEST_BUNDLED_GEMS_ALLOW_FAILURES: 'typeprof'
 
       - uses: ./.github/actions/slack
         with:
diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
index 85d401dccc..6b58d07110 100644
--- a/.github/workflows/ubuntu.yml
+++ b/.github/workflows/ubuntu.yml
@@ -144,7 +144,7 @@ jobs:
         timeout-minutes: ${{ matrix.timeout || 40 }}
         env:
           RUBY_TESTOPTS: '-q --tty=no'
-          TEST_BUNDLED_GEMS_ALLOW_FAILURES: ''
+          TEST_BUNDLED_GEMS_ALLOW_FAILURES: 'typeprof'
           PRECHECK_BUNDLED_GEMS: 'no'
 
       - name: make skipped tests
diff --git a/.github/workflows/yjit-macos.yml b/.github/workflows/yjit-macos.yml
index 304f3c3513..601e25e9d0 100644
--- a/.github/workflows/yjit-macos.yml
+++ b/.github/workflows/yjit-macos.yml
@@ -134,7 +134,7 @@ jobs:
         timeout-minutes: 60
         env:
           RUBY_TESTOPTS: '-q --tty=no'
-          TEST_BUNDLED_GEMS_ALLOW_FAILURES: ''
+          TEST_BUNDLED_GEMS_ALLOW_FAILURES: 'typeprof'
           SYNTAX_SUGGEST_TIMEOUT: '5'
           PRECHECK_BUNDLED_GEMS: 'no'
         continue-on-error: ${{ matrix.continue-on-test_task || false }}
diff --git a/.github/workflows/yjit-ubuntu.yml b/.github/workflows/yjit-ubuntu.yml
index e585c90ae8..b1f8bcc77a 100644
--- a/.github/workflows/yjit-ubuntu.yml
+++ b/.github/workflows/yjit-ubuntu.yml
@@ -187,7 +187,7 @@ jobs:
         timeout-minutes: 90
         env:
           RUBY_TESTOPTS: '-q --tty=no'
-          TEST_BUNDLED_GEMS_ALLOW_FAILURES: ''
+          TEST_BUNDLED_GEMS_ALLOW_FAILURES: 'typeprof'
           PRECHECK_BUNDLED_GEMS: 'no'
           SYNTAX_SUGGEST_TIMEOUT: '5'
           YJIT_BINDGEN_DIFF_OPTS: '--exit-code'
diff --git a/internal/parse.h b/internal/parse.h
index fdf2a10183..88230fd177 100644
--- a/internal/parse.h
+++ b/internal/parse.h
@@ -16,7 +16,7 @@
 // 0: parse.y
 // 1: Prism
 #ifndef RB_DEFAULT_PARSER
-#define RB_DEFAULT_PARSER 0
+#define RB_DEFAULT_PARSER 1
 #endif
 
 #ifdef UNIVERSAL_PARSER
diff --git a/prism_compile.c b/prism_compile.c
index 0be5c92078..29511e7d39 100644
--- a/prism_compile.c
+++ b/prism_compile.c
@@ -1399,10 +1399,12 @@ pm_compile_hash_elements(rb_iseq_t *iseq, const pm_node_t *node, const pm_node_l
         switch (PM_NODE_TYPE(element)) {
           case PM_ASSOC_NODE: {
             // Pre-allocation check (this branch can be omitted).
-            if (PM_NODE_FLAG_P(element, PM_NODE_FLAG_STATIC_LITERAL) && (
-                (!static_literal && ((index + min_tmp_hash_length) < elements->size)) ||
-                (first_chunk && stack_length == 0)
-              )) {
+            if (
+                PM_NODE_FLAG_P(element, PM_NODE_FLAG_STATIC_LITERAL) && (
+                    (!static_literal && ((index + min_tmp_hash_length) < elements->size)) ||
+                    (first_chunk && stack_length == 0)
+                )
+            ) {
                 // Count the elements that are statically-known.
                 size_t count = 1;
                 while (index + count < elements->size && PM_NODE_FLAG_P(elements->nodes[index + count], PM_NODE_FLAG_STATIC_LITERAL)) count++;
@@ -5641,6 +5643,14 @@ pm_compile_constant_path_operator_write_node(rb_iseq_t *iseq, const pm_constant_
 }
 
 /**
+ * Many nodes in Prism can be marked as a static literal, which means slightly
+ * different things depending on which node it is. Occasionally we need to omit
+ * container nodes from static literal checks, which is where this macro comes
+ * in.
+ */
+#define PM_CONTAINER_P(node) (PM_NODE_TYPE_P(node, PM_ARRAY_NODE) || PM_NODE_TYPE_P(node, PM_HASH_NODE) || PM_NODE_TYPE_P(node, PM_RANGE_NODE))
+
+/**
  * Compiles a prism node into instruction sequences.
  *
  * iseq -            The current instruction sequence object (used for locals)
@@ -5867,12 +5877,21 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret,
                         new_array_size++;
                     }
                 }
-                else if (PM_NODE_FLAG_P(element, PM_NODE_FLAG_STATIC_LITERAL) && !static_literal && ((index + min_tmp_array_size) < elements->size)) {
+                else if (
+                    PM_NODE_FLAG_P(element, PM_NODE_FLAG_STATIC_LITERAL) &&
+                    !PM_CONTAINER_P(element) &&
+                    !static_literal &&
+                    ((index + min_tmp_array_size) < elements->size)
+                ) {
                     // If we have a static literal, then there's the potential
                     // to group a bunch of them together with a literal array
                     // and then concat them together.
                     size_t right_index = index + 1;
-                    while (right_index < elements->size && PM_NODE_FLAG_P(elements->nodes[right_index], PM_NODE_FLAG_STATIC_LITERAL)) right_index++;
+                    while (
+                        right_index < elements->size &&
+                        PM_NODE_FLAG_P(elements->nodes[right_index], PM_NODE_FLAG_STATIC_LITERAL) &&
+                        !PM_CONTAINER_P(elements->nodes[right_index])
+                    ) right_index++;
 
                     size_t tmp_array_size = right_index - index;
                     if (tmp_array_size >= min_tmp_array_size) {
@@ -8975,7 +8994,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret,
                     pm_node_t *value = cast->value;
                     name = cast->name;
 
-                    if (PM_NODE_FLAG_P(value, PM_NODE_FLAG_STATIC_LITERAL) && !(PM_NODE_TYPE_P(value, PM_ARRAY_NODE) || PM_NODE_TYPE_P(value, PM_HASH_NODE) || PM_NODE_TYPE_P(value, PM_RANGE_NODE))) {
+                    if (PM_NODE_FLAG_P(value, PM_NODE_FLAG_STATIC_LITERAL) && !PM_CONTAINER_P(value)) {
                        rb_ary_push(default_values, pm_static_literal_value(iseq, value, scope_node));
                     }
                     else {
@@ -9285,7 +9304,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret,
                     pm_node_t *value = cast->value;
                     name = cast->name;
 
-                    if (!PM_NODE_FLAG_P(value, PM_NODE_FLAG_STATIC_LITERAL) || PM_NODE_TYPE_P(value, PM_ARRAY_NODE) || PM_NODE_TYPE_P(value, PM_HASH_NODE) || PM_NODE_TYPE_P(value, PM_RANGE_NODE)) {
+                    if (!PM_NODE_FLAG_P(value, PM_NODE_FLAG_STATIC_LITERAL) || PM_CONTAINER_P(value)) {
                         LABEL *end_label = NEW_LABEL(location.line);
 
                         pm_local_index_t index = pm_lookup_local_index(iseq, scope_node, name, 0);
@@ -9815,6 +9834,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret,
     }
 }
 
+#undef PM_CONTAINER_P
+
 /** True if the given iseq can have pre execution blocks. */
 static inline bool
 pm_iseq_pre_execution_p(rb_iseq_t *iseq)
diff --git a/test/.excludes-prism/TestM17N.rb b/test/.excludes-parsey/TestM17N.rb
index 7f8c44d02a..7f8c44d02a 100644
--- a/test/.excludes-prism/TestM17N.rb
+++ b/test/.excludes-parsey/TestM17N.rb
diff --git a/test/.excludes-prism/TestMixedUnicodeEscape.rb b/test/.excludes-parsey/TestMixedUnicodeEscape.rb
index 7bf964ebf1..7bf964ebf1 100644
--- a/test/.excludes-prism/TestMixedUnicodeEscape.rb
+++ b/test/.excludes-parsey/TestMixedUnicodeEscape.rb
diff --git a/test/.excludes-prism/TestRubyLiteral.rb b/test/.excludes-parsey/TestRubyLiteral.rb
index 853f23a3b9..853f23a3b9 100644
--- a/test/.excludes-prism/TestRubyLiteral.rb
+++ b/test/.excludes-parsey/TestRubyLiteral.rb
diff --git a/test/ruby/test_literal.rb b/test/ruby/test_literal.rb
index 8732d8f0d0..9b294bc8ea 100644
--- a/test/ruby/test_literal.rb
+++ b/test/ruby/test_literal.rb
@@ -241,8 +241,9 @@ class TestRubyLiteral < Test::Unit::TestCase
   def test_dregexp
     assert_instance_of Regexp, /re#{'ge'}xp/
     assert_equal(/regexp/, /re#{'ge'}xp/)
-    bug3903 = '[ruby-core:32682]'
-    assert_raise(SyntaxError, bug3903) {eval('/[#{"\x80"}]/')}
+
+    # [ruby-core:32682]
+    eval('/[#{"\x80"}]/')
   end
 
   def test_array
diff --git a/test/ruby/test_m17n.rb b/test/ruby/test_m17n.rb
index 630ba269c0..93366ed02a 100644
--- a/test/ruby/test_m17n.rb
+++ b/test/ruby/test_m17n.rb
@@ -1439,20 +1439,20 @@ class TestM17N < Test::Unit::TestCase
     assert_regexp_usascii_literal('//', Encoding::US_ASCII)
     assert_regexp_usascii_literal('/#{ }/', Encoding::US_ASCII)
     assert_regexp_usascii_literal('/#{"a"}/', Encoding::US_ASCII)
-    assert_regexp_usascii_literal('/#{%q"\x80"}/', Encoding::ASCII_8BIT)
-    assert_regexp_usascii_literal('/#{"\x80"}/', nil, SyntaxError)
+    assert_regexp_usascii_literal('/#{%q"\x80"}/', Encoding::US_ASCII)
+    assert_regexp_usascii_literal('/#{"\x80"}/', Encoding::ASCII_8BIT)
 
     assert_regexp_usascii_literal('/a/', Encoding::US_ASCII)
     assert_regexp_usascii_literal('/a#{ }/', Encoding::US_ASCII)
     assert_regexp_usascii_literal('/a#{"a"}/', Encoding::US_ASCII)
     assert_regexp_usascii_literal('/a#{%q"\x80"}/', Encoding::ASCII_8BIT)
-    assert_regexp_usascii_literal('/a#{"\x80"}/', nil, SyntaxError)
+    assert_regexp_usascii_literal('/a#{"\x80"}/', Encoding::ASCII_8BIT)
 
     assert_regexp_usascii_literal('/\x80/', Encoding::ASCII_8BIT)
     assert_regexp_usascii_literal('/\x80#{ }/', Encoding::ASCII_8BIT)
     assert_regexp_usascii_literal('/\x80#{"a"}/', Encoding::ASCII_8BIT)
     assert_regexp_usascii_literal('/\x80#{%q"\x80"}/', Encoding::ASCII_8BIT)
-    assert_regexp_usascii_literal('/\x80#{"\x80"}/', nil, SyntaxError)
+    assert_regexp_usascii_literal('/\x80#{"\x80"}/', Encoding::ASCII_8BIT)
 
     assert_regexp_usascii_literal('/\u1234/', Encoding::UTF_8)
     assert_regexp_usascii_literal('/\u1234#{ }/', Encoding::UTF_8)
diff --git a/test/ruby/test_mixed_unicode_escapes.rb b/test/ruby/test_mixed_unicode_escapes.rb
index f0b4cc691f..a30b5c19f5 100644
--- a/test/ruby/test_mixed_unicode_escapes.rb
+++ b/test/ruby/test_mixed_unicode_escapes.rb
@@ -18,12 +18,12 @@ class TestMixedUnicodeEscape < Test::Unit::TestCase
     assert_raise(SyntaxError) { eval %q("\u{1234}�")}
 
     # also should not work for Regexp
-    assert_raise(SyntaxError) { eval %q(/#{"\u1234"}#{"�"}/)}
     assert_raise(RegexpError) { eval %q(/\u{1234}#{nil}�/)}
     assert_raise(RegexpError) { eval %q(/�#{nil}\u1234/)}
 
     # String interpolation turns into an expression and we get
     # a different kind of error, but we still can't mix these
+    assert_raise(Encoding::CompatibilityError) { eval %q(/#{"\u1234"}#{"�"}/)}
     assert_raise(Encoding::CompatibilityError) { eval %q("\u{1234}#{nil}�")}
     assert_raise(Encoding::CompatibilityError) { eval %q("�#{nil}\u1234")}
   end
diff --git a/tool/rbs_skip_tests b/tool/rbs_skip_tests
index 786e049136..650d1828d5 100644
--- a/tool/rbs_skip_tests
+++ b/tool/rbs_skip_tests
@@ -63,3 +63,17 @@ TestInstanceNetHTTPResponse depending on external resources
 
 test_initialize(TempfileRemoverSingletonTest) Remover class is removed at https://github.com/ruby/tempfile/commit/753ab16642fdc3cf92e1cf3dd1d80e8f75128735
 test_call(TempfileRemoverTest) Remover class is removed at https://github.com/ruby/tempfile/commit/753ab16642fdc3cf92e1cf3dd1d80e8f75128735
+
+# Switching to prism resulted in an unexpected error, `RuntimeError: cannot get AST for ISEQ compiled by prism`, raised by `RubyVM::AbstractSyntax.of(method)`
+test_1(RBS::RuntimePrototypeTest)
+test_decls_for_anonymous_class_or_module(RBS::RuntimePrototypeTest)
+test_decls_structure(RBS::RuntimePrototypeTest)
+test_define_method_alias(RBS::RuntimePrototypeTest)
+test_for_initialize_type(RBS::RuntimePrototypeTest)
+test_for_overwritten_module_name(RBS::RuntimePrototypeTest)
+test_for_yield(RBS::RuntimePrototypeTest)
+test_include_owner(RBS::RuntimePrototypeTest)
+test_merge_types(RBS::RuntimePrototypeTest)
+test_reflection(RBS::RuntimePrototypeTest)
+test_todo(RBS::RuntimePrototypeTest)
+test_of(RubyVM::AbstractSyntaxTreeSingletonTest)