[ruby/openssl] cipher: make output buffer String independent

OpenSSL::Cipher#update accepts a String as the second argument to be
used as the output buffer. The buffer must be directly writable, in
other words, it must not be frozen and not a shared string.

rb_str_resize() does not make the String independent if the String
already has the intended length. Use the rb_str_modify() family instead
to check it.

Fixes: https://bugs.ruby-lang.org/issues/20937

https://github.com/ruby/openssl/commit/1de3b80a46

2 files changed, 28 insertions, 1 deletions

diff --git a/ext/openssl/ossl_cipher.c b/ext/openssl/ossl_cipher.c
index 5a491d89ed..3f07c09e4d 100644
--- a/ext/openssl/ossl_cipher.c
+++ b/ext/openssl/ossl_cipher.c
@@ -408,7 +408,10 @@ ossl_cipher_update(int argc, VALUE *argv, VALUE self)
         str = rb_str_new(0, out_len);
     } else {
         StringValue(str);
-        rb_str_resize(str, out_len);
+        if ((long)rb_str_capacity(str) >= out_len)
+            rb_str_modify(str);
+        else
+            rb_str_modify_expand(str, out_len - RSTRING_LEN(str));
     }
 
     if (!ossl_cipher_update_long(ctx, (unsigned char *)RSTRING_PTR(str), &out_len, in, in_len))
diff --git a/test/openssl/test_cipher.rb b/test/openssl/test_cipher.rb
index 41885fd59b..cd0b3dcb44 100644
--- a/test/openssl/test_cipher.rb
+++ b/test/openssl/test_cipher.rb
@@ -128,6 +128,30 @@ class OpenSSL::TestCipher < OpenSSL::TestCase
     assert_equal pt, cipher.update(ct) << cipher.final
   end
 
+  def test_update_with_buffer
+    cipher = OpenSSL::Cipher.new("aes-128-ecb").encrypt
+    cipher.random_key
+    expected = cipher.update("data") << cipher.final
+    assert_equal 16, expected.bytesize
+
+    # Buffer is supplied
+    cipher.reset
+    buf = String.new
+    assert_same buf, cipher.update("data", buf)
+    assert_equal expected, buf + cipher.final
+
+    # Buffer is frozen
+    cipher.reset
+    assert_raise(FrozenError) { cipher.update("data", String.new.freeze) }
+
+    # Buffer is a shared string [ruby-core:120141] [Bug #20937]
+    cipher.reset
+    buf = "x" * 1024
+    shared = buf[-("data".bytesize + 32)..-1]
+    assert_same shared, cipher.update("data", shared)
+    assert_equal expected, shared + cipher.final
+  end
+
   def test_ciphers
     ciphers = OpenSSL::Cipher.ciphers
     assert_kind_of Array, ciphers