Fix [Dev] intermittent SSL alert received: HANDSHAKE_FAILURE - error 552 - phase 2

This JSS code workaround takes steps to make sure ONLY SSL Server Cert private keys are marked
as NOT temporary. We have a case on a certain HSM where NSS makes a copy of the SSL Server private key on the token and
hands it to our JSS SSLEngine implementation. After serveral separate incoming SSL connections, the Java GC
attempts to clean up the private key references periodically. Since the SSL Server private key copy is marked as temporary,
the normal call to clean up a private key reference has been known to delete the key from the token. Since NSS uses this copy
over a long perid of time, any early deletion of this key will make subsequent SSL connections to fail, until the server is reset.

This fix makes sure the private key refernces kept inour java objects are NOT set to temporary. When the GC runs all calls to clean
up the private key reference will not accidentally remove the key from the token.

Fixed to adust to the fact that JSSEngineImpl no longer has a cert and key object, instead some ArrayList.

Author: jmagne

base/src/main/java/org/mozilla/jss/pkcs11/PK11PrivKey.java

@@ -76,6 +76,12 @@ public String getAlgorithm() {
      */
     public native PK11PubKey getPublicKey();
 
+    /**
+     * Sets the private key PRBool pkcs11IsTemp property.
+     * Use with care
+     */
+    public native void setTemporary(boolean isTemporary);
+
     /**
      * Imports a PrivateKeyInfo, storing it as a temporary PrivateKey
      * on the given token.

base/src/main/java/org/mozilla/jss/ssl/javax/JSSEngineReferenceImpl.java

@@ -417,8 +417,17 @@ private void initServer() throws SSLException {
         List<PK11Cert> lstCerts = new ArrayList<>();
         for (Pair<PK11Cert, PK11PrivKey> pairKeys: certs) {
             lstCerts.add(pairKeys.getLeft());
+            PK11PrivKey key = pairKeys.getRight();
+ 
+            // Workaround to account for NSS giving us a copy of the actual SSL Server private key.
+            // This is to keep calls to SECKEY_DestroyPrivateKey from blowing the long lived SSL cert
+            // private key off the token.
+
+            if(key != null) {
+                key.setTemporary(false);
+            }
         }
-        
+ 
         session.setLocalCertificates(lstCerts.toArray(new PK11Cert[0]));
 
         // Create a small server session cache.

lib/jss.map

@@ -529,6 +529,7 @@ JSS_5.9.0 {
     global:
 Java_org_mozilla_jss_pkcs11_PK11KeyPairGenerator_generateMLDSAKeyPair;
 Java_org_mozilla_jss_pkcs11_PK11KeyPairGenerator_generateMLDSAKeyPairWithOpFlags;
+Java_org_mozilla_jss_pkcs11_PK11PrivKey_setTemporary;
     local:
         *;
 };

native/src/main/native/org/mozilla/jss/pkcs11/PK11PrivKey.c

@@ -756,3 +756,28 @@ Java_org_mozilla_jss_pkcs11_PK11PrivKey_getPublicKey
 
     return JSS_PK11_wrapPubKey(env, &pubKey);
 }
+
+/**********************************************************************
+ * PK11PrivKey.setTemporary
+ * Workaround to account for SSL Server Cert private keys created as copies by NSS on a token.
+ * Can either make it temporary of not temporary based on boolean argument.
+ * Not recommended for normal use.
+ */
+JNIEXPORT void JNICALL
+Java_org_mozilla_jss_pkcs11_PK11PrivKey_setTemporary
+    (JNIEnv *env, jobject this, jboolean isTemporary)
+{
+    SECKEYPrivateKey *key = NULL;
+
+    PR_ASSERT(env!=NULL && this!=NULL);
+
+    /***************************************************
+     * Get the private key and slot C structures
+     ***************************************************/
+    if( JSS_PK11_getPrivKeyPtr(env, this, &key) != PR_SUCCESS) {
+        PR_ASSERT( (*env)->ExceptionOccurred(env) != NULL);
+        return;
+    }
+
+    key->pkcs11IsTemp = isTemporary;
+}