Skip to content

Commit f25641b

Browse files
[AMD] Guard degenerate axis info in atomic RMW lowering (#993)
Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent 5a68d49 commit f25641b

2 files changed

Lines changed: 33 additions & 3 deletions

File tree

test/Conversion/amd/tle_local_pointers_to_llvm.mlir

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,3 +68,25 @@ module attributes {"ttg.num-ctas" = 1 : i32, "ttg.num-warps" = 1 : i32, ttg.targ
6868
tt.return
6969
}
7070
}
71+
72+
// -----
73+
74+
#blocked = #ttg.blocked<{sizePerThread = [1], threadsPerWarp = [32], warpsPerCTA = [1], order = [0]}>
75+
#shared = #ttg.swizzled_shared<{vec = 1, perPhase = 1, maxPhase = 1, order = [0]}>
76+
#smem = #ttg.shared_memory
77+
78+
module attributes {"ttg.num-ctas" = 1 : i32, "ttg.num-warps" = 1 : i32, ttg.target = "hip:gfx1201", "ttg.threads-per-warp" = 32 : i32} {
79+
// tle.local_pointers has no axis-info visitor, so its result carries a
80+
// degenerate rank-0 entry. The atomic lowering must tolerate that instead of
81+
// indexing the contiguity vector, which is only a hint for intra-wave reduce.
82+
// CHECK-LABEL: llvm.func @local_pointers_atomic_rmw
83+
// CHECK: llvm.getelementptr {{.*}}!llvm.ptr<3>
84+
// CHECK: llvm.atomicrmw fadd {{.*}} syncscope("workgroup") monotonic : !llvm.ptr<3>, f32
85+
// CHECK-NOT: tle.local_pointers
86+
tt.func public @local_pointers_atomic_rmw(%idx: tensor<32xi32, #blocked>, %val: tensor<32xf32, #blocked>) {
87+
%buf = ttg.local_alloc : () -> !ttg.memdesc<32xf32, #shared, #smem, mutable>
88+
%ptrs = "tle.local_pointers"(%buf, %idx) : (!ttg.memdesc<32xf32, #shared, #smem, mutable>, tensor<32xi32, #blocked>) -> tensor<32x!tt.ptr<f32, 3>, #blocked>
89+
%res = tt.atomic_rmw fadd, relaxed, cta, %ptrs, %val : (tensor<32x!tt.ptr<f32, 3>, #blocked>, tensor<32xf32, #blocked>) -> tensor<32xf32, #blocked>
90+
tt.return
91+
}
92+
}

third_party/amd/lib/TritonAMDGPUToLLVM/LoadStoreOpToLLVM.cpp

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1887,10 +1887,18 @@ struct AtomicRMWOpConversion
18871887
!enableIntraWaveReduce;
18881888
numElems = tensorTy.getNumElements();
18891889

1890+
// Ops without an axis-info visitor, such as tle.local_pointers, yield a
1891+
// rank-0 entry that cannot be indexed. The contiguity only refines the
1892+
// intra-wave reduce decision, so leave that disabled instead.
18901893
auto threadOrder = getThreadOrder(tensorTy);
1891-
unsigned contigWithinLanes =
1892-
axisAnalysisPass.getAxisInfo(ptr)->getContiguity(threadOrder.front());
1893-
enableIntraWaveReduce &= contigWithinLanes == 1;
1894+
auto *ptrAxisInfo = axisAnalysisPass.getAxisInfo(ptr);
1895+
if (!ptrAxisInfo || ptrAxisInfo->getRank() == 0 || threadOrder.empty()) {
1896+
enableIntraWaveReduce = false;
1897+
} else {
1898+
unsigned contigWithinLanes =
1899+
ptrAxisInfo->getContiguity(threadOrder.front());
1900+
enableIntraWaveReduce &= contigWithinLanes == 1;
1901+
}
18941902
}
18951903

18961904
auto vecTy = vec_ty(valueElemTy, vec);

0 commit comments

Comments
 (0)