openssl: improve handling of password for encrypted PEM

* ext/openssl/ossl.c (ossl_pem_passwd_value): Added. Convert the
  argument to String with StringValue() and validate the length is in
  4..PEM_BUFSIZE. PEM_BUFSIZE is a macro defined in OpenSSL headers.
  (ossl_pem_passwd_cb): When reading/writing encrypted PEM format, we
  used to pass the password to PEM_def_callback() directly but it was
  problematic. It is not NUL character safe. And surprisingly, it
  silently truncates the password to 1024 bytes.  [GH ruby/openssl#51]

* ext/openssl/ossl.h: Add function prototype declaration of newly
  added ossl_pem_passwd_value().

* ext/openssl/ossl_pkey.c (ossl_pkey_new_from_data): Use
  ossl_pem_passwd_value() to validate the password String.

* ext/openssl/ossl_pkey_dsa.c (ossl_dsa_initialize, ossl_dsa_export):
  ditto.

* ext/openssl/ossl_pkey_ec.c (ossl_ec_key_initialize,
  ossl_ec_key_to_string): ditto.

* ext/openssl/ossl_pkey_rsa.c (ossl_rsa_initialize, ossl_rsa_export):
  ditto.

* test/openssl/test_pkey_{dsa,ec,rsa}.rb: test this.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@55087 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 9 insertions, 19 deletions

diff --git a/ext/openssl/ossl_pkey_ec.c b/ext/openssl/ossl_pkey_ec.c
index 26eb5e8acf..d4153db68c 100644
--- a/ext/openssl/ossl_pkey_ec.c
+++ b/ext/openssl/ossl_pkey_ec.c
@@ -168,7 +168,6 @@ static VALUE ossl_ec_key_initialize(int argc, VALUE *argv, VALUE self)
     EC_KEY *ec = NULL;
     VALUE arg, pass;
     VALUE group = Qnil;
-    char *passwd = NULL;
 
     GetPKey(self, pkey);
     if (pkey->pkey.ec)
@@ -188,15 +187,15 @@ static VALUE ossl_ec_key_initialize(int argc, VALUE *argv, VALUE self)
         	ec = EC_KEY_new();
         	group = arg;
         } else {
-            BIO *in = ossl_obj2bio(arg);
+            BIO *in;
 
-            if (!NIL_P(pass)) {
-		passwd = StringValuePtr(pass);
-	    }
-	    ec = PEM_read_bio_ECPrivateKey(in, NULL, ossl_pem_passwd_cb, passwd);
+	    pass = ossl_pem_passwd_value(pass);
+	    in = ossl_obj2bio(arg);
+
+	    ec = PEM_read_bio_ECPrivateKey(in, NULL, ossl_pem_passwd_cb, (void *)pass);
             if (!ec) {
 		OSSL_BIO_reset(in);
-		ec = PEM_read_bio_EC_PUBKEY(in, NULL, ossl_pem_passwd_cb, passwd);
+		ec = PEM_read_bio_EC_PUBKEY(in, NULL, ossl_pem_passwd_cb, (void *)pass);
             }
             if (!ec) {
 		OSSL_BIO_reset(in);
@@ -473,7 +472,6 @@ static VALUE ossl_ec_key_to_string(VALUE self, VALUE ciph, VALUE pass, int forma
     BIO *out;
     int i = -1;
     int private = 0;
-    char *password = NULL;
     VALUE str;
 
     Require_EC_KEY(self, ec);
@@ -493,20 +491,12 @@ static VALUE ossl_ec_key_to_string(VALUE self, VALUE ciph, VALUE pass, int forma
     switch(format) {
     case EXPORT_PEM:
     	if (private) {
-	    const EVP_CIPHER *cipher;
+	    const EVP_CIPHER *cipher = NULL;
 	    if (!NIL_P(ciph)) {
 		cipher = GetCipherPtr(ciph);
-		if (!NIL_P(pass)) {
-		    StringValue(pass);
-		    if (RSTRING_LENINT(pass) < OSSL_MIN_PWD_LEN)
-			ossl_raise(eOSSLError, "OpenSSL requires passwords to be at least four characters long");
-		    password = RSTRING_PTR(pass);
-		}
-	    }
-	    else {
-		cipher = NULL;
+		pass = ossl_pem_passwd_value(pass);
 	    }
-            i = PEM_write_bio_ECPrivateKey(out, ec, cipher, NULL, 0, NULL, password);
+            i = PEM_write_bio_ECPrivateKey(out, ec, cipher, NULL, 0, ossl_pem_passwd_cb, (void *)pass);
     	} else {
             i = PEM_write_bio_EC_PUBKEY(out, ec);
         }