Skip to content

Commit d564e14

Browse files
范坤范坤
authored andcommitted
harden LA verify/state-update kernels
- la_verify_kvbuffer: re-check V % ilp_rows == 0 AFTER the ilp_rows->8 promotion (the pre-promotion assert could let a partial row-block be silently skipped); zero the sH0 M-padding rows before GEMM1 so the MMA fragment is well-defined instead of consuming stale/NaN SMEM. - assert K == 128 in the verify (MMA + shuffle) and state-update entry points, documenting the hardcoded head-dim assumption.
1 parent a20187d commit d564e14

2 files changed

Lines changed: 12 additions & 1 deletion

File tree

cula/lightning/la_state_update_kvbuffer.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,7 @@ def linear_attention_state_update_kvbuffer(
232232
"""
233233
B, T_k, H, K = k.shape
234234
assert T_k == T, f"k.shape[1]={T_k} doesn't match T={T}"
235+
assert K == 128, f"K={K} != 128: kernel hardcodes K=128 (threads_per_group, lane K-coverage)"
235236
_, _, HV, V = v.shape
236237
pool_size = s.shape[0]
237238

cula/lightning/la_verify_kvbuffer.py

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,12 @@ def la_verify_kvbuffer_kernel(
268268
h_g = cute.local_tile(gH0, (1, vec_size), (v_base + row, lane_id))
269269
h_s = cute.local_tile(sH0_w, (1, vec_size), (row, lane_id))
270270
cute.autovec_copy(h_g, h_s)
271+
# Zero the M-padding rows (ilp_rows..BT-1). GEMM1 reads all BT rows;
272+
# their outputs are unused, but leaving stale/NaN SMEM as MMA inputs
273+
# is unclean — explicitly zero so the fragment is well-defined.
274+
for row in cutlass.range_constexpr(ilp_rows, BT):
275+
for c in cutlass.range_constexpr(vec_size):
276+
sH0_w[(row, lane_id * vec_size + c)] = cutlass.Float32(0.0)
271277
cute.arch.sync_warp() # make sH0 writes visible to this warp's GEMM1
272278

273279
# (b) GEMM1: HQ[row, t] = h0_row . q_t, over the full K.
@@ -453,6 +459,7 @@ def linear_attention_verify_kvbuffer(
453459

454460
B, T_q, H, K = q.shape
455461
assert T_q == T, f"q.shape[1]={T_q} doesn't match T={T}"
462+
assert K == 128, f"K={K} != 128: kernel hardcodes K=128 (threads_per_group, KP=K+4, lane K-coverage)"
456463
_, _, HV, V = v.shape
457464
pool_size = s.shape[0]
458465

@@ -462,12 +469,14 @@ def linear_attention_verify_kvbuffer(
462469

463470
tile_v, vec_size, ilp_rows, _use_smem_v = get_mtp_config(B, T, HV, V, True)
464471
assert T <= 8, f"T={T} > 8: MMA kernel's BT=8 token staging only covers T ≤ 8"
465-
assert V % ilp_rows == 0, f"V={V} % ilp_rows={ilp_rows} ≠ 0: partial row-blocks would be silently skipped"
466472
# The MMA tile has M=8 valid rows, so process 8 V-rows per warp per block:
467473
# this fills the fragment (vs ilp_rows=4 wasting half the MMA) and halves the
468474
# number of row-blocks. Only applies when the V-rows-per-warp is a multiple of 8.
469475
if ilp_rows < 8 and (tile_v // 4) % 8 == 0:
470476
ilp_rows = 8
477+
# Re-check after the promotion above: a partial row-block (V not a multiple of
478+
# the final ilp_rows) would be silently skipped by the kernel's bounds guard.
479+
assert V % ilp_rows == 0, f"V={V} % ilp_rows={ilp_rows} ≠ 0: partial row-blocks would be silently skipped"
471480

472481
cache_key = (
473482
B,
@@ -847,6 +856,7 @@ def linear_attention_verify_kvbuffer_shuffle(
847856
"""
848857
B, T_q, H, K = q.shape
849858
assert T_q == T, f"q.shape[1]={T_q} doesn't match T={T}"
859+
assert K == 128, f"K={K} != 128: kernel hardcodes K=128 (threads_per_group, lane K-coverage)"
850860
_, _, HV, V = v.shape
851861
pool_size = s.shape[0]
852862

0 commit comments

Comments
 (0)