ml-dsa: deterministic hint-decode canonicity regression tests

Adds unit-level tests exercising Signature::decode directly against the
HintBitUnpack canonicity clauses (FIPS 204 Algorithm 21), the
GHSA-5x2r-hc65-25f9 / CVE-2026-24850 site. They complement the Wycheproof
verification corpus (which covers end-to-end verification of the
repeated-hint vector) with decoder-local checks: strict per-polynomial
index ordering (the bug was <= where < is required), monotonic bounded
cuts, zero padding after the last cut, the empty hint, the cut == omega
upper boundary, and the legal boundary that identical index values may
occur in different polynomials. Run for all three parameter sets.

Verified: pass on current main, and the repeated-index cases fail if the
strict < in hint.rs is reverted to <=.

Author: amarshat

ml-dsa/src/lib.rs

@@ -558,3 +558,170 @@ mod test {
         test_debug::<MlDsa87>();
     }
 }
+
+// Deterministic, unit-level regression tests for the HintBitUnpack canonicity
+// clauses (FIPS 204 Algorithm 21), the GHSA-5x2r-hc65-25f9 / CVE-2026-24850 site.
+// These complement the Wycheproof verification corpus (which covers end-to-end
+// verification of the repeated-hint vector) by exercising `Signature::decode`
+// directly and pinning each decoder-local rule: strict per-polynomial index
+// ordering (the bug was `<=` where `<` is required), monotonic bounded cuts, zero
+// padding after the last cut, and the legal boundary that identical index values
+// may occur in different polynomials. Run for all three parameter sets.
+//
+// The encoded hint is the trailing omega + k bytes of the signature: omega index
+// bytes then k cumulative cut bytes.
+#[cfg(test)]
+mod hint_decode_regression {
+    use super::*;
+    use hybrid_array::typenum::Unsigned;
+
+    const MAX_K: usize = 8; // ML-DSA-87 k
+    const MAX_OMEGA: usize = 80; // ML-DSA-44 omega
+
+    fn omega<P: MlDsaParams>() -> usize {
+        <P::Omega as Unsigned>::USIZE
+    }
+    fn kk<P: MlDsaParams>() -> usize {
+        <P::K as Unsigned>::USIZE
+    }
+
+    fn base_sig<P: MlDsaParams>() -> EncodedSignature<P> {
+        let ssk = SigningKey::<P>::from_seed(&Array::default());
+        let esk = ssk.expanded_key();
+        let rnd = Array([0u8; 32]);
+        let m = b"hint regression";
+        #[allow(deprecated)]
+        let sig = esk.sign_internal(&[m], &rnd);
+        sig.encode()
+    }
+
+    // Overwrite the hint region: `indices` (zero-padded to omega) then the k cuts.
+    fn with_hint<P: MlDsaParams>(
+        mut enc: EncodedSignature<P>,
+        indices: &[u8],
+        cuts: &[u8],
+    ) -> EncodedSignature<P> {
+        let (om, k) = (omega::<P>(), kk::<P>());
+        assert_eq!(cuts.len(), k);
+        let off = enc.len() - (om + k);
+        for x in &mut enc[off..] {
+            *x = 0;
+        }
+        enc[off..off + indices.len()].copy_from_slice(indices);
+        enc[off + om..off + om + k].copy_from_slice(cuts);
+        enc
+    }
+
+    fn decodes<P: MlDsaParams>(enc: &EncodedSignature<P>) -> bool {
+        Signature::<P>::decode(enc).is_some()
+    }
+
+    // k cumulative cut bytes all equal to `v` (so polynomial 0 holds `v` hints).
+    fn cut_buf<P: MlDsaParams>(v: u8) -> [u8; MAX_K] {
+        let mut c = [0u8; MAX_K];
+        for x in c.iter_mut().take(kk::<P>()) {
+            *x = v;
+        }
+        c
+    }
+
+    fn run_cases<P: MlDsaParams>() {
+        let om = omega::<P>();
+        let k = kk::<P>();
+        let omb = u8::try_from(om).expect("omega fits in u8");
+
+        // control: strictly increasing indices [5, 9] in polynomial 0
+        assert!(decodes::<P>(&with_hint::<P>(
+            base_sig::<P>(),
+            &[5, 9],
+            &cut_buf::<P>(2)[..k]
+        )));
+
+        // GHSA-5x2r-hc65-25f9: a repeated index within one polynomial (the `<=` bug)
+        assert!(!decodes::<P>(&with_hint::<P>(
+            base_sig::<P>(),
+            &[5, 5],
+            &cut_buf::<P>(2)[..k]
+        )));
+
+        // decreasing index within one polynomial
+        assert!(!decodes::<P>(&with_hint::<P>(
+            base_sig::<P>(),
+            &[9, 5],
+            &cut_buf::<P>(2)[..k]
+        )));
+
+        // non-monotonic cuts [1, 0, ...]
+        let mut dc = cut_buf::<P>(1);
+        dc[1] = 0;
+        assert!(!decodes::<P>(&with_hint::<P>(
+            base_sig::<P>(),
+            &[5],
+            &dc[..k]
+        )));
+
+        // a cut strictly above omega
+        let mut co = cut_buf::<P>(0);
+        co[k - 1] = omb + 1;
+        assert!(!decodes::<P>(&with_hint::<P>(
+            base_sig::<P>(),
+            &[],
+            &co[..k]
+        )));
+
+        // upper valid boundary: a cut EQUAL to omega, polynomial 0 using all omega
+        // hints as strictly increasing indices 0..omega
+        let mut full = [0u8; MAX_OMEGA];
+        for (i, x) in full.iter_mut().enumerate().take(om) {
+            *x = u8::try_from(i).expect("index fits in u8");
+        }
+        assert!(decodes::<P>(&with_hint::<P>(
+            base_sig::<P>(),
+            &full[..om],
+            &cut_buf::<P>(omb)[..k]
+        )));
+
+        // the empty hint (all cuts zero, all indices zero)
+        assert!(decodes::<P>(&with_hint::<P>(
+            base_sig::<P>(),
+            &[],
+            &cut_buf::<P>(0)[..k]
+        )));
+
+        // nonzero byte immediately after the last used index (start of padding)
+        let mut enc = with_hint::<P>(base_sig::<P>(), &[5], &cut_buf::<P>(1)[..k]);
+        let off = enc.len() - (om + k);
+        enc[off + 1] = 7;
+        assert!(!decodes::<P>(&enc));
+
+        // nonzero byte in the final padding slot
+        let mut enc = with_hint::<P>(base_sig::<P>(), &[5], &cut_buf::<P>(1)[..k]);
+        let last = enc.len() - k - 1;
+        enc[last] = 7;
+        assert!(!decodes::<P>(&enc));
+
+        // legal: the same index value in DIFFERENT polynomials (strictness is per-poly)
+        let mut cx = cut_buf::<P>(2);
+        cx[0] = 1; // poly 0 = [5], poly 1 = [5]
+        assert!(decodes::<P>(&with_hint::<P>(
+            base_sig::<P>(),
+            &[5, 5],
+            &cx[..k]
+        )));
+    }
+
+    #[test]
+    fn mldsa44() {
+        run_cases::<MlDsa44>();
+    }
+
+    #[test]
+    fn mldsa65() {
+        run_cases::<MlDsa65>();
+    }
+
+    #[test]
+    fn mldsa87() {
+        run_cases::<MlDsa87>();
+    }
+}