Skip to content

Refactor: Use SIMD4x64 in the x86 SHA-512 compression function #5779

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
randombit merged 1 commit into from
Aug 1, 2026
+55 −50
Merged

Refactor: Use SIMD4x64 in the x86 SHA-512 compression function #5779

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 55 additions & 50 deletions src/lib/hash/sha2_64/sha2_64_x86/sha2_64_x86.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,41 +1,56 @@
/*
* (C) 2025 Jack Lloyd
* (C) 2026 Kagan Can Sit
*
* Botan is released under the Simplified BSD License (see license.txt)
*/

#include <botan/internal/sha2_64.h>

#include <botan/internal/isa_extn.h>
#include <immintrin.h>
#include <botan/internal/simd_4x64.h>

namespace Botan {

namespace {

// NOLINTBEGIN(portability-simd-intrinsics)

BOTAN_FORCE_INLINE BOTAN_FN_ISA_SHA512 void sha512_msg_expand(__m256i& m0, __m256i& m1, __m256i& m2, __m256i& m3) {
m3 = _mm256_sha512msg1_epi64(m3, _mm256_extracti128_si256(m0, 0));
m2 = _mm256_add_epi64(m2, _mm256_permute4x64_epi64(_mm256_blend_epi32(m0, m1, 3), 0b00111001));
m2 = _mm256_sha512msg2_epi64(m2, m1);
BOTAN_FORCE_INLINE BOTAN_FN_ISA_SHA512 SIMD_4x64 sha512_msg1(const SIMD_4x64& x, const SIMD_4x64& w_lo) {
return SIMD_4x64(_mm256_sha512msg1_epi64(x.raw(), _mm256_extracti128_si256(w_lo.raw(), 0)));
}

BOTAN_FORCE_INLINE BOTAN_FN_ISA_SHA512 void sha512_4rounds(__m256i& state0,
__m256i& state1,
const __m256i msg,
const __m256i K) {
const auto tmp = _mm256_add_epi64(msg, K);
state0 = _mm256_sha512rnds2_epi64(state0, state1, _mm256_extracti128_si256(tmp, 0));
state1 = _mm256_sha512rnds2_epi64(state1, state0, _mm256_extracti128_si256(tmp, 1));
BOTAN_FORCE_INLINE BOTAN_FN_ISA_SHA512 SIMD_4x64 sha512_msg2(const SIMD_4x64& x, const SIMD_4x64& y) {
return SIMD_4x64(_mm256_sha512msg2_epi64(x.raw(), y.raw()));
}

BOTAN_FORCE_INLINE BOTAN_FN_ISA_AVX2 void permute_state(__m256i& state0, __m256i& state1) {
state0 = _mm256_shuffle_epi32(state0, 0b01001110);
state1 = _mm256_shuffle_epi32(state1, 0b01001110);
auto statet = state0;
state0 = _mm256_permute2x128_si256(state0, state1, 0x13);
state1 = _mm256_permute2x128_si256(statet, state1, 0x02);
BOTAN_FORCE_INLINE BOTAN_FN_ISA_SHA512 void sha512_msg_expand(SIMD_4x64& m0,
SIMD_4x64& m1,
SIMD_4x64& m2,
SIMD_4x64& m3) {
m3 = sha512_msg1(m3, m0);
m2 += SIMD_4x64::permute_4x64<0b00111001>(SIMD_4x64(_mm256_blend_epi32(m0.raw(), m1.raw(), 0b0011)));
m2 = sha512_msg2(m2, m1);
}

BOTAN_FORCE_INLINE BOTAN_FN_ISA_SHA512 void sha512_4rounds(SIMD_4x64& state0,
SIMD_4x64& state1,
const SIMD_4x64& msg,
const SIMD_4x64& K) {
const auto tmp = msg + K;
state0 = SIMD_4x64(_mm256_sha512rnds2_epi64(state0.raw(), state1.raw(), _mm256_extracti128_si256(tmp.raw(), 0)));
state1 = SIMD_4x64(_mm256_sha512rnds2_epi64(state1.raw(), state0.raw(), _mm256_extracti128_si256(tmp.raw(), 1)));
}

BOTAN_FORCE_INLINE BOTAN_FN_ISA_AVX2 void permute_state(SIMD_4x64& state0, SIMD_4x64& state1) {
__m256i s0 = _mm256_shuffle_epi32(state0.raw(), 0b01001110);
__m256i s1 = _mm256_shuffle_epi32(state1.raw(), 0b01001110);
const __m256i statet = s0;
s0 = _mm256_permute2x128_si256(s0, s1, 0x13);
s1 = _mm256_permute2x128_si256(statet, s1, 0x02);

state0 = SIMD_4x64(s0);
state1 = SIMD_4x64(s1);
}

// NOLINTEND(portability-simd-intrinsics)
Expand Down Expand Up @@ -63,63 +78,53 @@ void SHA_512::compress_digest_x86(digest_type& digest, std::span<const uint8_t>
0x431D67C49C100D4C, 0x4CC5D4BECB3E42B6, 0x597F299CFC657E2A, 0x5FCB6FAB3AD6FAEC, 0x6C44198C4A475817,
};

// NOLINTBEGIN(portability-simd-intrinsics) TODO Use SIMD_4x64 here

const __m256i* K_mm = reinterpret_cast<const __m256i*>(K);

const __m256i bswap_mask =
_mm256_set_epi64x(0x08090a0b0c0d0e0f, 0x0001020304050607, 0x08090a0b0c0d0e0f, 0x0001020304050607);

__m256i* digest_mm = reinterpret_cast<__m256i*>(digest.data());
const __m256i* input_mm = reinterpret_cast<const __m256i*>(input.data());

auto state0 = _mm256_loadu_si256(digest_mm);
auto state1 = _mm256_loadu_si256(digest_mm + 1);
auto state0 = SIMD_4x64::load_le(digest.data());
auto state1 = SIMD_4x64::load_le(digest.data() + 4);

permute_state(state0, state1);

const uint8_t* in = input.data();

for(size_t i = 0; i != blocks; ++i) {
const auto state0_save = state0;
const auto state1_save = state1;

auto m0 = _mm256_shuffle_epi8(_mm256_loadu_si256(input_mm + 0), bswap_mask);
auto m1 = _mm256_shuffle_epi8(_mm256_loadu_si256(input_mm + 1), bswap_mask);
auto m2 = _mm256_shuffle_epi8(_mm256_loadu_si256(input_mm + 2), bswap_mask);
auto m3 = _mm256_shuffle_epi8(_mm256_loadu_si256(input_mm + 3), bswap_mask);
auto m0 = SIMD_4x64::load_be(in + 0 * 32);
auto m1 = SIMD_4x64::load_be(in + 1 * 32);
auto m2 = SIMD_4x64::load_be(in + 2 * 32);
auto m3 = SIMD_4x64::load_be(in + 3 * 32);

sha512_4rounds(state0, state1, m0, _mm256_load_si256(&K_mm[0]));
sha512_4rounds(state0, state1, m1, _mm256_load_si256(&K_mm[1]));
m0 = _mm256_sha512msg1_epi64(m0, _mm256_extracti128_si256(m1, 0));
sha512_4rounds(state0, state1, m0, SIMD_4x64::load_le(&K[4 * 0]));
sha512_4rounds(state0, state1, m1, SIMD_4x64::load_le(&K[4 * 1]));
m0 = sha512_msg1(m0, m1);

for(size_t r = 2; r != 18; r += 4) {
sha512_4rounds(state0, state1, m2, _mm256_load_si256(&K_mm[r + 0]));
sha512_4rounds(state0, state1, m2, SIMD_4x64::load_le(&K[4 * (r + 0)]));
sha512_msg_expand(m2, m3, m0, m1);

sha512_4rounds(state0, state1, m3, _mm256_load_si256(&K_mm[r + 1]));
sha512_4rounds(state0, state1, m3, SIMD_4x64::load_le(&K[4 * (r + 1)]));
sha512_msg_expand(m3, m0, m1, m2);

sha512_4rounds(state0, state1, m0, _mm256_load_si256(&K_mm[r + 2]));
sha512_4rounds(state0, state1, m0, SIMD_4x64::load_le(&K[4 * (r + 2)]));
sha512_msg_expand(m0, m1, m2, m3);

sha512_4rounds(state0, state1, m1, _mm256_load_si256(&K_mm[r + 3]));
sha512_4rounds(state0, state1, m1, SIMD_4x64::load_le(&K[4 * (r + 3)]));
sha512_msg_expand(m1, m2, m3, m0);
}

sha512_4rounds(state0, state1, m2, _mm256_load_si256(&K_mm[18]));
sha512_4rounds(state0, state1, m3, _mm256_load_si256(&K_mm[19]));
sha512_4rounds(state0, state1, m2, SIMD_4x64::load_le(&K[4 * 18]));
sha512_4rounds(state0, state1, m3, SIMD_4x64::load_le(&K[4 * 19]));

state0 = _mm256_add_epi64(state0, state0_save);
state1 = _mm256_add_epi64(state1, state1_save);
state0 += state0_save;
state1 += state1_save;

input_mm += 4;
in += 4 * 32;
}

permute_state(state0, state1);

_mm256_storeu_si256(digest_mm, state0);
_mm256_storeu_si256(digest_mm + 1, state1);

// NOLINTEND(portability-simd-intrinsics)
state0.store_le(digest.data());
state1.store_le(digest.data() + 4);
}

} // namespace Botan
Loading