kernel-proxy: per-context QPs for thread-isolated RDMA posting

Author: MC952-arch

flagcx/core/flagcx_hetero.cc

@@ -789,8 +789,8 @@ flagcxResult_t flagcxHeteroFlushRma(flagcxHeteroComm_t comm, int peer,
       return flagcxRemoteError;
     usleep(100);
   }
-  // Check rmaError after wait: kernel proxy direct-post path advances doneSeqs
-  // unconditionally (even on failure) to prevent flush hangs.
+  // Final rmaError check: kernel proxy or network failures set rmaError;
+  // catch errors that occurred after doneSeqs reached the target.
   if (__atomic_load_n(&proxy->rmaError, __ATOMIC_ACQUIRE))
     return flagcxRemoteError;
   return flagcxSuccess;
@@ -831,8 +831,8 @@ flagcxResult_t flagcxHeteroFlushAllRma(flagcxHeteroComm_t comm) {
       usleep(100);
     }
   }
-  // Check rmaError after wait: kernel proxy direct-post path advances doneSeqs
-  // unconditionally (even on failure) to prevent flush hangs.
+  // Final rmaError check: kernel proxy or network failures set rmaError;
+  // catch errors that occurred after doneSeqs reached the target.
   if (__atomic_load_n(&proxy->rmaError, __ATOMIC_ACQUIRE))
     return flagcxRemoteError;
   return flagcxSuccess;

flagcx/core/include/onesided.h

@@ -23,10 +23,20 @@ struct flagcxOneSideHandleInfo {
   void *localMrHandle; // local rank's MR handle for deregMr
   void *localRecvComm; // recvComm used for MR registration (PD match)
   // Full-mesh IB connections (including self loopback, aligned with NCCL GIN)
-  void **fullSendComms; // [nRanks] per-peer sendComm (NULL if not owner)
-  void **fullRecvComms; // [nRanks] per-peer recvComm (NULL if not owner)
+  void **fullSendComms; // [nRanks] per-peer sendComm — alias for
+                        // contextSendComms[0]
+  void **fullRecvComms; // [nRanks] per-peer recvComm — alias for
+                        // contextRecvComms[0]
   int nRanks;           // number of ranks (for cleanup iteration)
 
+  // Per-context QP arrays for thread isolation (NCCL GIN pattern).
+  // Context 0 = RMA proxy; contexts 1..N = kernel proxy threads.
+  // Each context has its own full-mesh of RC QPs so no QP is shared
+  // across threads. All contexts share the same MR handles/rkeys (same PD).
+  void ***contextSendComms; // [nContexts][nRanks]
+  void ***contextRecvComms; // [nContexts][nRanks]
+  int nContexts;            // 1 + nKernelProxies
+
   // Symmetric memory window for intra-node D2D bypass (CE path).
   // NULL if VMM not available or window not registered with
   // FLAGCX_WIN_COLL_SYMMETRIC.