Small speedups for AES-GCM:

* Eliminate a copy in the GHASH(length) computation at the end.
* In the tag computation, instead of copying the 128-bit GHASH
  result into a u8 array and doing a bytewise XOR loop, load
  it into a u128 and do a vector XOR.

Author: brian-pane

graviola/src/low/aarch64/ghash.rs

@@ -102,11 +102,18 @@ impl<'a> Ghash<'a> {
         }
     }
 
-    pub(crate) fn into_bytes(self) -> [u8; 16] {
-        to_u128(self.current).to_be_bytes()
+    #[cfg(test)]
+    fn into_bytes(self) -> [u8; 16] {
+        self.into_u128().to_be_bytes()
     }
 
-    fn one_block(&mut self, block: u128) {
+    pub(crate) fn into_u128(self) -> u128 {
+        // SAFETY: u128 and uint64x2_t have the same size and meaning of bits
+        unsafe { mem::transmute(self.current) }
+    }
+
+    // Input the 16 bytes of `block` to the computation.
+    pub(crate) fn one_block(&mut self, block: u128) {
         // SAFETY: this crate requires the `neon` cpu feature
         self.current = unsafe { veorq_u64(self.current, from_u128(block)) };
         self.current = mul(self.current, self.table.powers[0]);
@@ -326,12 +333,6 @@ fn from_u128(u: u128) -> uint64x2_t {
     unsafe { mem::transmute(u) }
 }
 
-#[inline]
-fn to_u128(u: uint64x2_t) -> u128 {
-    // SAFETY: u128 and uint64x2_t have the same size and meaning of bits
-    unsafe { mem::transmute(u) }
-}
-
 #[inline]
 #[target_feature(enable = "neon")]
 // Make a copy of `u` with the bytes reversed, and cast to `uint64x2_t`.

graviola/src/low/x86_64/ghash.rs

@@ -206,7 +206,7 @@ impl<'a> Ghash<'a> {
 
         for chunk in whole_blocks.by_ref() {
             let u = u128::from_be_bytes(chunk.try_into().unwrap());
-            self.one_block(u128_to_m128i(u));
+            self.one_block(u);
         }
 
         let bytes = whole_blocks.remainder();
@@ -215,7 +215,7 @@ impl<'a> Ghash<'a> {
             block[..bytes.len()].copy_from_slice(bytes);
 
             let u = u128::from_be_bytes(block);
-            self.one_block(u128_to_m128i(u));
+            self.one_block(u);
         }
     }
 
@@ -224,6 +224,11 @@ impl<'a> Ghash<'a> {
         unsafe { self._into_bytes() }
     }
 
+    pub(crate) fn into_u128(self) -> u128 {
+        // SAFETY: sizeof(u128) == sizeof(__m128i), all bits have same meaning
+        unsafe { mem::transmute(self.current) }
+    }
+
     #[target_feature(enable = "sse2,ssse3")]
     fn _into_bytes(self) -> [u8; 16] {
         let mut out: i128 = 0;
@@ -233,7 +238,9 @@ impl<'a> Ghash<'a> {
         out.to_le_bytes()
     }
 
-    fn one_block(&mut self, block: __m128i) {
+    // Input the 16 bytes of `block` to the computation.
+    pub(crate) fn one_block(&mut self, block: u128) {
+        let block = u128_to_m128i(block);
         // SAFETY: this crate requires the `avx` and `pclmulqdq` cpu features
         unsafe {
             self.current = _mm_xor_si128(self.current, block);

graviola/src/mid/aes_gcm.rs

@@ -66,16 +66,13 @@ impl AesGcm {
         // computations. see low::generic::aes_gcm for model version.
         aes_gcm::encrypt(&self.key, &mut ghash, &counter, aad, cipher_inout);
 
-        let mut lengths = [0u8; 16];
-        lengths[..8].copy_from_slice(&((aad.len() * 8) as u64).to_be_bytes());
-        lengths[8..].copy_from_slice(&((cipher_inout.len() * 8) as u64).to_be_bytes());
-        ghash.add(&lengths);
+        let lengths: u128 = Self::pack_bit_lengths(aad, cipher_inout);
+        ghash.one_block(lengths);
 
-        let final_xi = ghash.into_bytes();
-
-        for ((out, x), e) in tag_out.iter_mut().zip(final_xi.iter()).zip(e_y0.iter()) {
-            *out = *x ^ *e;
-        }
+        let final_xi = ghash.into_u128();
+        let e = u128::from_be_bytes(e_y0);
+        let tag = final_xi ^ e;
+        tag_out.copy_from_slice(&tag.to_be_bytes());
     }
 
     /// Decrypts and verifies the given message.
@@ -107,17 +104,11 @@ impl AesGcm {
 
         aes_gcm::decrypt(&self.key, &mut ghash, &counter, aad, cipher_inout);
 
-        let mut lengths = [0u8; 16];
-        lengths[..8].copy_from_slice(&((aad.len() * 8) as u64).to_be_bytes());
-        lengths[8..].copy_from_slice(&((cipher_inout.len() * 8) as u64).to_be_bytes());
-        ghash.add(&lengths);
-
-        let mut actual_tag = ghash.into_bytes();
-        for (out, e) in actual_tag.iter_mut().zip(e_y0.iter()) {
-            *out ^= *e;
-        }
+        let lengths: u128 = Self::pack_bit_lengths(aad, cipher_inout);
+        ghash.one_block(lengths);
 
-        if ct_equal(&actual_tag, tag) {
+        let actual_tag = ghash.into_u128() ^ u128::from_be_bytes(e_y0);
+        if ct_equal(&actual_tag.to_be_bytes(), tag) {
             Ok(())
         } else {
             // avoid unauthenticated plaintext leak
@@ -126,6 +117,12 @@ impl AesGcm {
         }
     }
 
+    fn pack_bit_lengths(high: &[u8], low: &[u8]) -> u128 {
+        let lengths = (high.len() * 8) as u128;
+        let lengths = lengths << 64;
+        lengths | ((low.len() * 8) as u128)
+    }
+
     fn nonce_to_y0(&self, nonce: &[u8; 12]) -> [u8; 16] {
         let mut y0 = [0u8; 16];
         y0[..12].copy_from_slice(nonce);