[ruby/openssl] x509, ssl, pkcs7: try to parse as DER-encoding first

Methods that take both PEM-encoding and DER-encoding have not been
consistent in the order in which encoding to attempt to parse.

A DER-encoding may contain a valid PEM block ("\n-----BEGIN ..-----" to
"-----END ...-----") embedded within it. Also, the PEM-encoding parser
allows arbitrary data around the PEM block and silently skips it. As a
result, attempting to parse data in DER-encoding as PEM-encoding first
can incorrectly finds the embedded PEM block instead.

This commit ensures that DER encoding will always be attempted before
PEM encoding. OpenSSL::X509::Certificate is one of the updated classes.
With this, the following will always be true:

    # obj is an OpenSSL::X509::Certificate
    obj == OpenSSL::X509::Certificate.new(obj.to_der)
    obj == OpenSSL::X509::Certificate.new(obj.to_pem)

https://github.com/ruby/openssl/commit/b280eb1fd0

1 files changed, 10 insertions, 7 deletions

diff --git a/ext/openssl/ossl_x509cert.c b/ext/openssl/ossl_x509cert.c
index 5376bff08d..1385411069 100644
--- a/ext/openssl/ossl_x509cert.c
+++ b/ext/openssl/ossl_x509cert.c
@@ -115,24 +115,27 @@ static VALUE
 ossl_x509_initialize(int argc, VALUE *argv, VALUE self)
 {
     BIO *in;
-    X509 *x509, *x = DATA_PTR(self);
+    X509 *x509, *x509_orig = RTYPEDDATA_DATA(self);
     VALUE arg;
 
+    rb_check_frozen(self);
     if (rb_scan_args(argc, argv, "01", &arg) == 0) {
 	/* create just empty X509Cert */
 	return self;
     }
     arg = ossl_to_der_if_possible(arg);
     in = ossl_obj2bio(&arg);
-    x509 = PEM_read_bio_X509(in, &x, NULL, NULL);
-    DATA_PTR(self) = x;
+    x509 = d2i_X509_bio(in, NULL);
     if (!x509) {
-	OSSL_BIO_reset(in);
-	x509 = d2i_X509_bio(in, &x);
-	DATA_PTR(self) = x;
+        OSSL_BIO_reset(in);
+        x509 = PEM_read_bio_X509(in, NULL, NULL, NULL);
     }
     BIO_free(in);
-    if (!x509) ossl_raise(eX509CertError, NULL);
+    if (!x509)
+        ossl_raise(eX509CertError, "PEM_read_bio_X509");
+
+    RTYPEDDATA_DATA(self) = x509;
+    X509_free(x509_orig);
 
     return self;
 }