Add optional CPU cycle measurement to graviola-bench

* This was motivated by the observation that the elapsed
  time for the Graviola benchmarks on macOS can sometimes
  vary by 10% from one run to the next, probably due to
  some combination of varying CPU frequency and random
  background daemons. In past projects, I've found that
  the CPU's cycle counter, while still noisy, is less
  noisy than the wall clock.
* This change adds a feature `__bench_cycles` that
  replaces the standard Criterion elapsed time metric
  with a measurement of elapsed CPU cycles for the
  process on aarch64.
* It should be straightforward to add x86_64 support
  in a later change, either with inline assembly or
  by using the Linux `perf` API. (The latter could
  also provide access to metrics like instructions
  retired and branch mispredictions.)
* Because the counter-timer register is per-core,
  this implementation may return incorrect results if
  a single benchmark runs long enough for the OS to
  reschedule it to a different core or to interrupt
  the execution with some other process. It may be
  beneficial to add CPU pinning in the future.

Author: brian-pane

graviola-bench/Cargo.toml

@@ -36,6 +36,7 @@ openssl = "0.10"
 [features]
 __bench_openssl = []
 __bench_codspeed = ["dep:codspeed-criterion-compat"]
+__bench_cycles = []
 
 [[bench]]
 name = "x25519"

graviola-bench/benches/aes_gcm.rs

@@ -1,5 +1,7 @@
 mod criterion;
-use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
+use criterion::{
+    BenchmarkId, Criterion, CustomMeasurement, Throughput, criterion_group, criterion_main,
+};
 
 fn test_ring_aes_gcm(key: &ring::aead::LessSafeKey, nonce: &[u8; 12], aad: &[u8], plain: &[u8]) {
     let mut ct = plain.to_vec();
@@ -46,7 +48,7 @@ fn test_graviola_aes_gcm(key: &graviola::aead::AesGcm, nonce: &[u8; 12], aad: &[
     key.encrypt(nonce, aad, &mut ct, &mut tag);
 }
 
-fn aes128_gcm(c: &mut Criterion) {
+fn aes128_gcm(c: &mut Criterion<CustomMeasurement>) {
     let key = [0u8; 16];
     let nonce = [0u8; 12];
     let aad = [0u8; 32];
@@ -92,7 +94,7 @@ fn aes128_gcm(c: &mut Criterion) {
     }
 }
 
-fn aes256_gcm(c: &mut Criterion) {
+fn aes256_gcm(c: &mut Criterion<CustomMeasurement>) {
     let key = [0u8; 32];
     let nonce = [0u8; 12];
     let aad = [0u8; 32];
@@ -138,5 +140,5 @@ fn aes256_gcm(c: &mut Criterion) {
     }
 }
 
-criterion_group!(benches, aes128_gcm, aes256_gcm);
+custom_benchmark_group!(benches, aes128_gcm, aes256_gcm);
 criterion_main!(benches);

graviola-bench/benches/chacha20poly1305.rs

@@ -1,4 +1,5 @@
 mod criterion;
+use crate::criterion::CustomMeasurement;
 use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
 
 fn test_ring_chacha(key: &ring::aead::LessSafeKey, nonce: &[u8; 12], aad: &[u8], plain: &[u8]) {
@@ -43,7 +44,7 @@ fn test_graviola_chacha(
     key.encrypt(nonce, aad, &mut ct, &mut tag);
 }
 
-fn bench_chacha20poly1305(c: &mut Criterion) {
+fn bench_chacha20poly1305(c: &mut Criterion<CustomMeasurement>) {
     let key = [0u8; 32];
     let nonce = [0u8; 12];
     let aad = [0u8; 32];
@@ -89,5 +90,5 @@ fn bench_chacha20poly1305(c: &mut Criterion) {
     }
 }
 
-criterion_group!(benches, bench_chacha20poly1305);
+custom_benchmark_group!(benches, bench_chacha20poly1305);
 criterion_main!(benches);

graviola-bench/benches/criterion/aarch64.rs

@@ -0,0 +1,73 @@
+use criterion::Throughput;
+use criterion::measurement::{Measurement, ValueFormatter};
+use std::arch::asm;
+
+// Measurement plug-in for Criterion that measures elapsed CPU cycles
+// by reading the ARM counter-timer hardware register.
+
+pub struct CycleCount;
+
+impl Measurement for CycleCount {
+    type Intermediate = u64;
+    type Value = u64;
+
+    fn start(&self) -> Self::Intermediate {
+        self.get_current()
+    }
+
+    fn end(&self, i: Self::Intermediate) -> Self::Value {
+        self.get_current() - i
+    }
+
+    fn add(&self, v1: &Self::Value, v2: &Self::Value) -> Self::Value {
+        v1 + v2
+    }
+
+    fn zero(&self) -> Self::Value {
+        0
+    }
+
+    fn to_f64(&self, value: &Self::Value) -> f64 {
+        *value as _
+    }
+
+    fn formatter(&self) -> &dyn ValueFormatter {
+        &CycleCountFormatter
+    }
+}
+
+impl CycleCount {
+    fn get_current(&self) -> u64 {
+        let mut value = 0;
+        unsafe {
+            asm!(
+                "isb",
+                "mrs {destination}, CNTVCT_EL0",
+                "isb",
+                destination = inout(reg) value,
+            )
+        }
+        value
+    }
+}
+
+struct CycleCountFormatter;
+
+impl ValueFormatter for CycleCountFormatter {
+    fn scale_values(&self, _typical_value: f64, _values: &mut [f64]) -> &'static str {
+        "cycle"
+    }
+
+    fn scale_throughputs(
+        &self,
+        _typical_value: f64,
+        _throughput: &Throughput,
+        _values: &mut [f64],
+    ) -> &'static str {
+        "elem/cycle"
+    }
+
+    fn scale_for_machines(&self, _values: &mut [f64]) -> &'static str {
+        "cycle"
+    }
+}

graviola-bench/benches/criterion/mod.rs

@@ -1,4 +1,100 @@
+#[cfg(not(all(feature = "__bench_cycles", target_arch = "aarch64")))]
+use std::time::{Duration, Instant};
+
 #[cfg(feature = "__bench_codspeed")]
 pub use codspeed_criterion_compat::*;
 #[cfg(not(feature = "__bench_codspeed"))]
 pub use criterion::*;
+
+#[cfg(all(feature = "__bench_cycles", target_arch = "aarch64"))]
+mod aarch64;
+
+// A measurement type for use with Criterion.
+// Depending on the features enabled, it wraps either Criterion's
+// standard `WallTime` metric or a target-specific measurement.
+pub struct CustomMeasurement(
+    #[cfg(not(all(feature = "__bench_cycles", target_arch = "aarch64")))] measurement::WallTime,
+    #[cfg(all(feature = "__bench_cycles", target_arch = "aarch64"))] aarch64::CycleCount,
+);
+
+impl measurement::Measurement for CustomMeasurement {
+    #[cfg(not(all(feature = "__bench_cycles", target_arch = "aarch64")))]
+    type Intermediate = Instant;
+
+    #[cfg(not(all(feature = "__bench_cycles", target_arch = "aarch64")))]
+    type Value = Duration;
+
+    #[cfg(all(feature = "__bench_cycles", target_arch = "aarch64"))]
+    type Intermediate = u64;
+
+    #[cfg(all(feature = "__bench_cycles", target_arch = "aarch64"))]
+    type Value = u64;
+
+    fn start(&self) -> Self::Intermediate {
+        self.0.start()
+    }
+
+    fn end(&self, i: Self::Intermediate) -> Self::Value {
+        self.0.end(i)
+    }
+
+    fn add(&self, v1: &Self::Value, v2: &Self::Value) -> Self::Value {
+        self.0.add(v1, v2)
+    }
+
+    fn zero(&self) -> Self::Value {
+        self.0.zero()
+    }
+
+    fn to_f64(&self, value: &Self::Value) -> f64 {
+        self.0.to_f64(value)
+    }
+
+    fn formatter(&self) -> &dyn measurement::ValueFormatter {
+        self.0.formatter()
+        //&CustomMeasurementFormatter
+    }
+}
+
+impl CustomMeasurement {
+    #[allow(dead_code)]
+    #[cfg(not(all(feature = "__bench_cycles", target_arch = "aarch64")))]
+    pub fn new() -> Self {
+        CustomMeasurement(measurement::WallTime)
+    }
+
+    #[allow(dead_code)]
+    #[cfg(all(feature = "__bench_cycles", target_arch = "aarch64"))]
+    pub fn new() -> Self {
+        CustomMeasurement(aarch64::CycleCount)
+    }
+}
+
+/*
+struct CustomMeasurementFormatter;
+
+impl measurement::ValueFormatter for CustomMeasurementFormatter {
+    fn scale_values(&self, _typical_value: f64, _values: &mut [f64]) -> &'static str {
+        "cycle"
+    }
+
+    fn scale_throughputs(&self, _typical_value: f64, _throughput: &Throughput, _values: &mut [f64]) -> &'static str {
+        "elem/sycle"
+    }
+
+    fn scale_for_machines(&self, _values: &mut [f64]) -> &'static str {
+        "cycle"
+    }
+}
+*/
+
+#[macro_export]
+macro_rules! custom_benchmark_group {
+    ($name:ident, $( $target:path ),+ $(,)*) => {
+        criterion_group!{
+            name = $name;
+            config = Criterion::default().with_measurement(CustomMeasurement::new());
+            targets = $( $target ),+
+        }
+    }
+}

graviola-bench/benches/ed25519.rs

@@ -1,10 +1,11 @@
 mod criterion;
 use std::hint::black_box;
 
+use crate::criterion::CustomMeasurement;
 use criterion::{Criterion, Throughput, criterion_group, criterion_main};
 
 // Generate a fresh key and make a PKCS8 document of it.
-fn keygen(c: &mut Criterion) {
+fn keygen(c: &mut Criterion<CustomMeasurement>) {
     let mut group = c.benchmark_group("ed25519-keygen");
     group.throughput(Throughput::Elements(1));
 
@@ -53,7 +54,7 @@ fn keygen(c: &mut Criterion) {
     });
 }
 
-fn verify(c: &mut Criterion) {
+fn verify(c: &mut Criterion<CustomMeasurement>) {
     let mut group = c.benchmark_group("ed25519-verify");
     group.throughput(Throughput::Elements(1));
 
@@ -102,7 +103,7 @@ fn verify(c: &mut Criterion) {
     });
 }
 
-fn sign(c: &mut Criterion) {
+fn sign(c: &mut Criterion<CustomMeasurement>) {
     let mut group = c.benchmark_group("ed25519-sign");
     group.throughput(Throughput::Elements(1));
     let message = b"\x31\x32\x33\x34\x30\x30";
@@ -151,5 +152,5 @@ fn sign(c: &mut Criterion) {
     });
 }
 
-criterion_group!(benches, keygen, verify, sign);
+custom_benchmark_group!(benches, keygen, verify, sign);
 criterion_main!(benches);

graviola-bench/benches/hmac.rs

@@ -1,6 +1,7 @@
 mod criterion;
 use std::hint::black_box;
 
+use crate::criterion::CustomMeasurement;
 use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
 use hmac::Mac;
 
@@ -60,7 +61,7 @@ fn test_graviola_sha384(key: &[u8], data: &[u8]) {
     black_box(ctx.finish());
 }
 
-fn sha256(c: &mut Criterion) {
+fn sha256(c: &mut Criterion<CustomMeasurement>) {
     let mut group = c.benchmark_group("hmac-sha256");
     let key = [0xff; 48];
 
@@ -89,7 +90,7 @@ fn sha256(c: &mut Criterion) {
     }
 }
 
-fn sha384(c: &mut Criterion) {
+fn sha384(c: &mut Criterion<CustomMeasurement>) {
     let mut group = c.benchmark_group("hmac-sha384");
     let key = [0xff; 48];
 
@@ -118,5 +119,5 @@ fn sha384(c: &mut Criterion) {
     }
 }
 
-criterion_group!(benches, sha256, sha384);
+custom_benchmark_group!(benches, sha256, sha384);
 criterion_main!(benches);

graviola-bench/benches/mlkem768.rs

@@ -1,11 +1,11 @@
 mod criterion;
 use std::hint::black_box;
 
-use criterion::{Criterion, Throughput, criterion_group, criterion_main};
+use criterion::{Criterion, CustomMeasurement, Throughput, criterion_group, criterion_main};
 use rand_core::RngCore;
 
 // client first operation
-fn mlkem768_keygen(c: &mut Criterion) {
+fn mlkem768_keygen(c: &mut Criterion<CustomMeasurement>) {
     let mut group = c.benchmark_group("mlkem768-keygen");
     group.throughput(Throughput::Elements(1));
 
@@ -46,7 +46,7 @@ fn mlkem768_keygen(c: &mut Criterion) {
 }
 
 // server operation
-fn mlkem768_encaps(c: &mut Criterion) {
+fn mlkem768_encaps(c: &mut Criterion<CustomMeasurement>) {
     let mut group = c.benchmark_group("mlkem768-encaps");
     group.throughput(Throughput::Elements(1));
 
@@ -116,7 +116,7 @@ fn mlkem768_encaps(c: &mut Criterion) {
 }
 
 // client second operation
-fn mlkem768_decaps(c: &mut Criterion) {
+fn mlkem768_decaps(c: &mut Criterion<CustomMeasurement>) {
     let mut group = c.benchmark_group("mlkem768-decaps");
     group.throughput(Throughput::Elements(1));
 
@@ -178,7 +178,7 @@ fn mlkem768_decaps(c: &mut Criterion) {
 }
 
 // combined operation: keygen, encaps-encode, encaps-decode, encaps, decaps
-fn mlkem768_combined(c: &mut Criterion) {
+fn mlkem768_combined(c: &mut Criterion<CustomMeasurement>) {
     let mut group = c.benchmark_group("mlkem768-combined");
     group.throughput(Throughput::Elements(1));
 
@@ -242,7 +242,7 @@ fn mlkem768_combined(c: &mut Criterion) {
     });
 }
 
-criterion_group!(
+custom_benchmark_group!(
     benches,
     mlkem768_keygen,
     mlkem768_encaps,