[ruby/openssl] Fix test_pkey_dsa.rb in FIPS.

Note that I created the `dsa2048.pem` and signature text
(`signature_encoded.txt`), that is used as a text to create the `signature0` in
the `test_sign_verify` by the following steps with the `openssl` CLI on FIPS
module.

```
$ OPENSSL_DIR="${HOME}/.local/openssl-3.3.0-dev-fips-debug-1f03d33ef5"
$ export OPENSSL_CONF="${OPENSSL_DIR}/ssl/openssl_fips.cnf"

$ "${OPENSSL_DIR}/bin/openssl" dsaparam -out dsaparam2048.pem 2048
$ "${OPENSSL_DIR}/bin/openssl" gendsa -out dsa2048.pem dsaparam2048.pem

$ echo -n "Sign me!" > data.txt
$ "${OPENSSL_DIR}/bin/openssl" dgst -sha256 -sign dsa2048.pem data.txt > signature.txt
$ cat signature.txt | base64 > signature_encoded.txt
```

Skip the `test_DSAPrivateKey_encrypted` on FIPS because AES-128-CBC, the
password based encryption used in the PEM format uses MD5 for deriving the
encryption key from the password, and MD5 is not FIPS-approved.
See also the comment on the `test/openssl/utils.rb#omit_on_fips`.

https://github.com/ruby/openssl/commit/4bdcb419a9

2 files changed, 40 insertions, 15 deletions

diff --git a/test/openssl/fixtures/pkey/dsa2048.pem b/test/openssl/fixtures/pkey/dsa2048.pem
new file mode 100644
index 0000000000..3f22b22b58
--- /dev/null
+++ b/test/openssl/fixtures/pkey/dsa2048.pem
@@ -0,0 +1,15 @@
+-----BEGIN PRIVATE KEY-----
+MIICXgIBADCCAjYGByqGSM44BAEwggIpAoIBAQDXZhJ/dQoWkQELzjzlx8FtIp96
+voCYe5NY0H8j0jz7GyHpXt41+MteqkZK3/Ah+cNR9uG8iEYArAZ71LcWotfee2Gz
+xdxozr9bRt0POYhO2YIsfMpBrEskPsDH2g/2nFV8l4OJgxU2qZUrF4PN5ha+Mu6u
+sVtN8hjvAvnbf4Pxn0b8NN9f4PJncroUa8acv5WsV85E1RW7NYCefggU4LytYIHg
+euRF9eY9gVCX5MkUgW2xODHIYJhwk/+5lJxG7qUsSahD/nPHO/yoWgdVHq2DkdTq
+KYXkAxx2PJcTBOHTglhE6mgCbEKp8vcfElnBWyCT6QykclZiPXXD2JV829J/Ah0A
+vYa+/G/gUZiomyejVje6UsGoCc+vInxmovOL8QKCAQEAhnKEigYPw6u8JY7v5iGo
+Ylz8qiMFYmaJCwevf3KCjWeEXuNO4OrKdfzkQl1tPuGLioYFfP1A2yGosjdUdLEB
+0JqnzlKxUp+G6RfBj+WYzbgc5hr7t0M+reAJh09/hDzqfxjcgiHstq7mpRXBP8Y7
+iu27s7TRYJNSAYRvWcXNSBEUym3mHBBbZn7VszYooSrn60/iZ8I+VY1UF/fgqhbj
+JfaaZNQCDO9K3Vb3rsXoYd8+bOZIen9uHB+pNjMqhpl4waysqrlpGFeeqdxivH6S
+vkrHLs6/eWVMnS08RdcryoCrI3Bm8mMBKQglDwKLnWLfzG565qEhslzyCd/l9k9a
+cwQfAh0Ao8/g72fSFmo04FizM7DZJSIPqDLjfZu9hLvUFA==
+-----END PRIVATE KEY-----
diff --git a/test/openssl/test_pkey_dsa.rb b/test/openssl/test_pkey_dsa.rb
index 3f64a80e32..4c93f2869d 100644
--- a/test/openssl/test_pkey_dsa.rb
+++ b/test/openssl/test_pkey_dsa.rb
@@ -31,11 +31,6 @@ class OpenSSL::TestPKeyDSA < OpenSSL::PKeyTestCase
   def test_generate
     # DSA.generate used to call DSA_generate_parameters_ex(), which adjusts the
     # size of q according to the size of p
-    key1024 = OpenSSL::PKey::DSA.generate(1024)
-    assert_predicate key1024, :private?
-    assert_equal 1024, key1024.p.num_bits
-    assert_equal 160, key1024.q.num_bits
-
     key2048 = OpenSSL::PKey::DSA.generate(2048)
     assert_equal 2048, key2048.p.num_bits
     assert_equal 256, key2048.q.num_bits
@@ -47,28 +42,41 @@ class OpenSSL::TestPKeyDSA < OpenSSL::PKeyTestCase
     end
   end
 
+  def test_generate_on_non_fips
+    # DSA with 1024 bits is invalid on FIPS 186-4.
+    # https://github.com/openssl/openssl/commit/49ed5ba8f62875074f04417189147fd3dda072ab
+    omit_on_fips
+
+    key1024 = OpenSSL::PKey::DSA.generate(1024)
+    assert_predicate key1024, :private?
+    assert_equal 1024, key1024.p.num_bits
+    assert_equal 160, key1024.q.num_bits
+  end
+
   def test_sign_verify
-    dsa512 = Fixtures.pkey("dsa512")
+    # The DSA valid size is 2048 or 3072 on FIPS.
+    # https://github.com/openssl/openssl/blob/7649b5548e5c0352b91d9d3ed695e42a2ac1e99c/providers/common/securitycheck.c#L185-L188
+    dsa = Fixtures.pkey("dsa2048")
     data = "Sign me!"
     if defined?(OpenSSL::Digest::DSS1)
-      signature = dsa512.sign(OpenSSL::Digest.new('DSS1'), data)
-      assert_equal true, dsa512.verify(OpenSSL::Digest.new('DSS1'), signature, data)
+      signature = dsa.sign(OpenSSL::Digest.new('DSS1'), data)
+      assert_equal true, dsa.verify(OpenSSL::Digest.new('DSS1'), signature, data)
     end
 
-    signature = dsa512.sign("SHA256", data)
-    assert_equal true, dsa512.verify("SHA256", signature, data)
+    signature = dsa.sign("SHA256", data)
+    assert_equal true, dsa.verify("SHA256", signature, data)
 
     signature0 = (<<~'end;').unpack1("m")
-      MCwCFH5h40plgU5Fh0Z4wvEEpz0eE9SnAhRPbkRB8ggsN/vsSEYMXvJwjGg/
-      6g==
+      MD4CHQC0zmRkVOAHJTm28fS5PVUv+4LtBeNaKqr/yfmVAh0AsTcLqofWHoW8X5oWu8AOvngOcFVZ
+      cLTvhY3XNw==
     end;
-    assert_equal true, dsa512.verify("SHA256", signature0, data)
+    assert_equal true, dsa.verify("SHA256", signature0, data)
     signature1 = signature0.succ
-    assert_equal false, dsa512.verify("SHA256", signature1, data)
+    assert_equal false, dsa.verify("SHA256", signature1, data)
   end
 
   def test_sign_verify_raw
-    key = Fixtures.pkey("dsa512")
+    key = Fixtures.pkey("dsa2048")
     data = 'Sign me!'
     digest = OpenSSL::Digest.digest('SHA1', data)
 
@@ -127,6 +135,8 @@ class OpenSSL::TestPKeyDSA < OpenSSL::PKeyTestCase
   end
 
   def test_DSAPrivateKey_encrypted
+    omit_on_fips
+
     # key = abcdef
     dsa512 = Fixtures.pkey("dsa512")
     pem = <<~EOF