Make any hash values fixable [Bug #17488]

As hnum is an unsigned st_index_t, the result of RSHIFT may not be
in the fixable range.

Co-authored-by: NeoCat <[email protected]>

2 files changed, 29 insertions, 8 deletions

diff --git a/hash.c b/hash.c
index c3a512b8e9..0ed2692f24 100644
--- a/hash.c
+++ b/hash.c
@@ -223,15 +223,10 @@ any_hash(VALUE a, st_index_t (*other_func)(VALUE))
       default:
 	hnum = other_func(a);
     }
-#if SIZEOF_LONG < SIZEOF_ST_INDEX_T
-    if (hnum > 0)
-	hnum &= (unsigned long)-1 >> 2;
+    if ((SIGNED_VALUE)hnum > 0)
+	hnum &= FIXNUM_MAX;
     else
-	hnum |= ~((unsigned long)-1 >> 2);
-#else
-    hnum <<= 1;
-    hnum = RSHIFT(hnum, 1);
-#endif
+	hnum |= FIXNUM_MIN;
     return (long)hnum;
 }
 
diff --git a/test/ruby/test_hash.rb b/test/ruby/test_hash.rb
index c4b93836c6..62d8b3f836 100644
--- a/test/ruby/test_hash.rb
+++ b/test/ruby/test_hash.rb
@@ -1865,4 +1865,30 @@ class TestHash < Test::Unit::TestCase
       {a: 1}.each(&->(k, v) {})
     end
   end
+
+  def test_any_hash_fixable
+    20.times do
+      assert_separately([], "#{<<~"begin;"}\n#{<<~'end;'}")
+      begin;
+        require "delegate"
+        typename = DelegateClass(String)
+
+        hash = {
+          "Int" => true,
+          "Float" => true,
+          "String" => true,
+          "Boolean" => true,
+          "WidgetFilter" => true,
+          "WidgetAggregation" => true,
+          "WidgetEdge" => true,
+          "WidgetSortOrder" => true,
+          "WidgetGrouping" => true,
+        }
+
+        hash.each_key do |key|
+          assert_send([hash, :key?, typename.new(key)])
+        end
+      end;
+    end
+  end
 end