Fix issues noted in review

Also fix the overhead calculaation for OAEP

Author: timlegge

RSA.xs

@@ -392,14 +392,19 @@ EVP_PKEY*  _load_rsa_key(SV* p_keyStringSv,
 
 void check_max_message_length(rsaData* p_rsa, int from_length) {
     int size;
+    int max_len = -1;
+    const char *pad_name = NULL;
+    EVP_MD *md_obj;
+    int digest_length = 0;
     /* Pre-validate plaintext length before calling OpenSSL.
        Only applies to encryption direction (encrypt, private_encrypt),
        not to decryption (decrypt, public_decrypt) where input is ciphertext. */
     size = EVP_PKEY_get_size(p_rsa->rsa);
-    int max_len = -1;
-    const char *pad_name = NULL;
+
     if (p_rsa->padding == RSA_PKCS1_OAEP_PADDING) {
-        max_len = size - 42;  /* 2 * SHA1_DIGEST_LENGTH + 2 */
+        md_obj = get_md_bynid(p_rsa->hashMode); /* croak()s on unknown NID */
+        digest_length = EVP_MD_size(md_obj);
+        max_len = size - (2 * digest_length) - 2;  /* 2 * SHA1_DIGEST_LENGTH + 2 */
         pad_name = "OAEP";
     } else if (p_rsa->padding == RSA_PKCS1_PADDING) {
         max_len = size - 11;  /* PKCS#1 v1.5 overhead */

t/crypto.t

@@ -15,11 +15,11 @@ Crypt::OpenSSL::RSA->import_random_seed();
 my $rsa  = Crypt::OpenSSL::RSA->generate_key(2048);
 my $rsa2 = Crypt::OpenSSL::RSA->generate_key(2048);
 my $key_size = $rsa->size();  # 256 bytes for 2048-bit key
-
+my $hash_size = 32;           # SHA256 (default is 32 bytes)
 # --- OAEP boundary tests ---
 
 $rsa->use_pkcs1_oaep_padding();
-my $oaep_max = $key_size - 42;  # SHA-1 OAEP overhead
+my $oaep_max = $key_size - (2 * $hash_size) - 2; # SHA256 OAEP overhead
 
 # Max-length plaintext that fits OAEP
 {

t/error.t

@@ -86,7 +86,8 @@ ok($@, "decrypt croaks on empty ciphertext");
 # --- Plaintext too large for padding mode ---
 
 $rsa->use_pkcs1_oaep_padding();
-my $max_oaep = $rsa->size() - 42;
+my $hash_size = 32;           # SHA256 (default is 32 bytes)
+my $max_oaep = $rsa->size() - (2 * $hash_size) - 2;  # SHA-1 OAEP overhead
 my $too_large = "x" x ($max_oaep + 1);
 eval { $rsa->encrypt($too_large) };
 ok($@, "encrypt croaks when plaintext exceeds OAEP max size");

t/padding.t

@@ -117,6 +117,15 @@ my %padding_methods = (
                        #'sslv23'      => {'sign' => 0, 'encrypt' => 0, 'pad' => 11},
                     );
 
+my %digest_lengths = (
+    'md5'       => 16,
+    'sha1'      => 20,
+    'sha224'     => 28,
+    'sha256'     => 32,
+    'sha384'     => 48,
+    'sha512'     => 64,
+    'ripemd160'  => 20,
+);
 
 foreach my $padding (keys %padding_methods) {
     diag $padding;
@@ -157,7 +166,12 @@ foreach my $padding (keys %padding_methods) {
 
         # Valid encryption methods with padding
         if ($encrypt) {
-           _Test_Encrypt_And_Decrypt( $rsa->size() - $pad, $rsa, 0, $padding, $hash );
+           my $size = $rsa->size();
+           my $max_len = $size - $pad;
+           if ( $padding eq 'pkcs1_oaep' && $hash =~ /sha/ ) {
+               $max_len = $size - ($digest_lengths{$hash} * 2) - 2; 
+           }
+           _Test_Encrypt_And_Decrypt( $max_len, $rsa, 0, $padding, $hash );
         }
 
     }

t/rsa.t

@@ -80,7 +80,9 @@ _Test_Encrypt_And_Decrypt( $rsa->size(), $rsa, 1 );
 $rsa->use_pkcs1_oaep_padding();
 
 # private_encrypt does not work with pkcs1_oaep_padding
-_Test_Encrypt_And_Decrypt( $rsa->size() - 42, $rsa, 0 );
+my $hash_size = 32;           # SHA256 (default is 32 bytes)
+my $oaep_max = $rsa->size - (2 * $hash_size) - 2;
+_Test_Encrypt_And_Decrypt( $oaep_max, $rsa, 0 );
 
 #FIXME - use_sslv23_padding seems to fail on decryption.  openssl bug?