@@ -3429,9 +3429,7 @@ struct TTAtomicRMWOpLowering : SharedConversionPattern<triton::AtomicRMWOp> {
34293429 auto mask =
34303430 adaptor.getMask ()
34313431 ? adaptor.getMask ()
3432- : rewriter
3433- .create <arith::ConstantIntOp>(loc, 1 , 1 )
3434- .getResult ();
3432+ : rewriter.create <arith::ConstantIntOp>(loc, 1 , 1 ).getResult ();
34353433 auto zero = rewriter.create <arith::ConstantIndexOp>(loc, 0 );
34363434 auto masterWarpId = getMasterThreadId (op.getOperation ());
34373435 auto isMasterThread =
@@ -3472,8 +3470,7 @@ struct TTAtomicRMWOpLowering : SharedConversionPattern<triton::AtomicRMWOp> {
34723470 auto dimXVal =
34733471 rewriter.create <arith::ConstantIndexOp>(loc, clusterDimX);
34743472 auto flatCtaId = rewriter.create <arith::AddIOp>(
3475- loc, ctaIdX,
3476- rewriter.create <arith::MulIOp>(loc, ctaIdY, dimXVal));
3473+ loc, ctaIdX, rewriter.create <arith::MulIOp>(loc, ctaIdY, dimXVal));
34773474 auto totalCTAsVal =
34783475 rewriter.create <arith::ConstantIndexOp>(loc, totalCTAs);
34793476 auto one = rewriter.create <arith::ConstantIndexOp>(loc, 1 );
@@ -3496,12 +3493,11 @@ struct TTAtomicRMWOpLowering : SharedConversionPattern<triton::AtomicRMWOp> {
34963493 ArrayRef<int64_t >{ShapedType::kDynamic }, elemType);
34973494 auto buffer = innerBuilder.create <gcu::PtrToMemRefOp>(
34983495 loc, dynMemType, ptr);
3499- auto oldVal = emitSoftwareRMW (
3500- innerBuilder, loc, op.getAtomicRmwOp (), buffer, zero,
3501- val, elemType);
3502- innerBuilder.create <memref::StoreOp>(loc, oldVal,
3503- localMem,
3504- ValueRange{zero});
3496+ auto oldVal = emitSoftwareRMW (innerBuilder, loc,
3497+ op.getAtomicRmwOp (), buffer,
3498+ zero, val, elemType);
3499+ innerBuilder.create <memref::StoreOp>(
3500+ loc, oldVal, localMem, ValueRange{zero});
35053501 innerBuilder.create <scf::YieldOp>(loc);
35063502 });
35073503 ctaBuilder.create <scf::YieldOp>(loc);
@@ -3593,8 +3589,7 @@ struct TTAtomicRMWOpLowering : SharedConversionPattern<triton::AtomicRMWOp> {
35933589 auto warpIds = getWarpIds (rewriter, loc, op.getType ());
35943590 for (unsigned i = 0 ; i < resultType.getRank (); ++i) {
35953591 warpOffsets.push_back (rewriter.create <arith::MulIOp>(
3596- loc,
3597- rewriter.create <arith::ConstantIntOp>(loc, numElems[i], 32 ),
3592+ loc, rewriter.create <arith::ConstantIntOp>(loc, numElems[i], 32 ),
35983593 rewriter.create <arith::IndexCastOp>(loc, rewriter.getI32Type (),
35993594 warpIds[i])));
36003595 }
@@ -3628,18 +3623,21 @@ struct TTAtomicRMWOpLowering : SharedConversionPattern<triton::AtomicRMWOp> {
36283623 .getResult ()
36293624 : elemBuilder.create <arith::ConstantIntOp>(loc, 1 , 1 )
36303625 .getResult ();
3631- Value thread_select = elemBuilder.create <arith::CmpIOp>(
3632- loc, arith::CmpIPredicate::eq, mask, true_bool).getResult ();
3626+ Value thread_select =
3627+ elemBuilder
3628+ .create <arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
3629+ mask, true_bool)
3630+ .getResult ();
36333631
36343632 if (hasWarpRedundancy) {
3635- auto threadId = elemBuilder. create <gpu::ThreadIdOp>(
3636- loc, gpu::Dimension::x);
3633+ auto threadId =
3634+ elemBuilder. create <gpu::ThreadIdOp>( loc, gpu::Dimension::x);
36373635 auto tidI32 = elemBuilder.create <arith::IndexCastOp>(
36383636 loc, elemBuilder.getI32Type (), threadId);
36393637 auto bitmaskVal = elemBuilder.create <arith::ConstantIntOp>(
36403638 loc, warpBitmask, 32 );
3641- auto shifted = elemBuilder. create <arith::ShRUIOp>(
3642- loc, bitmaskVal, tidI32);
3639+ auto shifted =
3640+ elemBuilder. create <arith::ShRUIOp>( loc, bitmaskVal, tidI32);
36433641 auto oneI32 =
36443642 elemBuilder.create <arith::ConstantIntOp>(loc, 1 , 32 );
36453643 auto bit =
@@ -3648,8 +3646,11 @@ struct TTAtomicRMWOpLowering : SharedConversionPattern<triton::AtomicRMWOp> {
36483646 elemBuilder.create <arith::ConstantIntOp>(loc, 0 , 32 );
36493647 auto isNonRedundant = elemBuilder.create <arith::CmpIOp>(
36503648 loc, arith::CmpIPredicate::ne, bit, zeroI32);
3651- thread_select = elemBuilder.create <arith::AndIOp>(
3652- loc, thread_select, isNonRedundant.getResult ()).getResult ();
3649+ thread_select =
3650+ elemBuilder
3651+ .create <arith::AndIOp>(loc, thread_select,
3652+ isNonRedundant.getResult ())
3653+ .getResult ();
36533654 }
36543655
36553656 elemBuilder.create <scf::IfOp>(
@@ -3686,28 +3687,31 @@ struct TTAtomicRMWOpLowering : SharedConversionPattern<triton::AtomicRMWOp> {
36863687 .getResult ()
36873688 : builder.create <arith::ConstantIntOp>(loc, 1 , 1 )
36883689 .getResult ();
3689- Value thread_select = builder.create <arith::CmpIOp>(
3690- loc, arith::CmpIPredicate::eq, mask, true_bool).getResult ();
3690+ Value thread_select =
3691+ builder
3692+ .create <arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
3693+ mask, true_bool)
3694+ .getResult ();
36913695
36923696 if (hasWarpRedundancy) {
3693- auto threadId = builder. create <gpu::ThreadIdOp>(
3694- loc, gpu::Dimension::x);
3697+ auto threadId =
3698+ builder. create <gpu::ThreadIdOp>( loc, gpu::Dimension::x);
36953699 auto tidI32 = builder.create <arith::IndexCastOp>(
36963700 loc, builder.getI32Type (), threadId);
3697- auto bitmaskVal = builder.create <arith::ConstantIntOp>(
3698- loc, warpBitmask, 32 );
3699- auto shifted = builder.create <arith::ShRUIOp>(
3700- loc, bitmaskVal, tidI32);
3701- auto oneI32 =
3702- builder.create <arith::ConstantIntOp>(loc, 1 , 32 );
3703- auto bit =
3704- builder.create <arith::AndIOp>(loc, shifted, oneI32);
3705- auto zeroI32 =
3706- builder.create <arith::ConstantIntOp>(loc, 0 , 32 );
3701+ auto bitmaskVal =
3702+ builder.create <arith::ConstantIntOp>(loc, warpBitmask, 32 );
3703+ auto shifted =
3704+ builder.create <arith::ShRUIOp>(loc, bitmaskVal, tidI32);
3705+ auto oneI32 = builder.create <arith::ConstantIntOp>(loc, 1 , 32 );
3706+ auto bit = builder.create <arith::AndIOp>(loc, shifted, oneI32);
3707+ auto zeroI32 = builder.create <arith::ConstantIntOp>(loc, 0 , 32 );
37073708 auto isNonRedundant = builder.create <arith::CmpIOp>(
37083709 loc, arith::CmpIPredicate::ne, bit, zeroI32);
3709- thread_select = builder.create <arith::AndIOp>(
3710- loc, thread_select, isNonRedundant.getResult ()).getResult ();
3710+ thread_select =
3711+ builder
3712+ .create <arith::AndIOp>(loc, thread_select,
3713+ isNonRedundant.getResult ())
3714+ .getResult ();
37113715 }
37123716
37133717 builder.create <scf::IfOp>(
@@ -3745,7 +3749,7 @@ struct TTAtomicRMWOpLowering : SharedConversionPattern<triton::AtomicRMWOp> {
37453749 auto v = builder.create <memref::LoadOp>(
37463750 loc, sharedOutput, relayIdx (builder, loc, iters));
37473751 builder.create <memref::StoreOp>(loc, v, output, iters);
3748- });
3752+ });
37493753 }
37503754 }
37513755 leaveTritionOp (rewriter, op.getOperation ());
0 commit comments