Add TX checksum offload for Windows XDP (#5080)

Author: mtfriesen

src/generated/linux/datapath_raw_xdp_win.c.clog.h

@@ -29,6 +29,25 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
+/*----------------------------------------------------------
+// Decoder Ring for GetTxOffloadConfig
+// [ lib] %u, %u
+// QuicTraceLogInfo(
+        GetTxOffloadConfig,
+        "[ lib] %u, %u",
+        Queue->Interface->ActualIfIndex, *TxChecksumOffload);
+// arg2 = arg2 = Queue->Interface->ActualIfIndex = arg2
+// arg3 = arg3 = *TxChecksumOffload = arg3
+----------------------------------------------------------*/
+#ifndef _clog_4_ARGS_TRACE_GetTxOffloadConfig
+#define _clog_4_ARGS_TRACE_GetTxOffloadConfig(uniqueId, encoded_arg_string, arg2, arg3)\
+tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, GetTxOffloadConfig , arg2, arg3);\
+
+#endif
+
+
+
+
 /*----------------------------------------------------------
 // Decoder Ring for FoundVF
 // [ xdp][%p] Found NetSvc-VF interfaces. NetSvc IfIdx:%lu, VF IfIdx:%lu

src/generated/linux/datapath_raw_xdp_win.c.clog.h.lttng.h

@@ -1,6 +1,28 @@
 
 
 
+/*----------------------------------------------------------
+// Decoder Ring for GetTxOffloadConfig
+// [ lib] %u, %u
+// QuicTraceLogInfo(
+        GetTxOffloadConfig,
+        "[ lib] %u, %u",
+        Queue->Interface->ActualIfIndex, *TxChecksumOffload);
+// arg2 = arg2 = Queue->Interface->ActualIfIndex = arg2
+// arg3 = arg3 = *TxChecksumOffload = arg3
+----------------------------------------------------------*/
+TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, GetTxOffloadConfig,
+    TP_ARGS(
+        unsigned int, arg2,
+        unsigned int, arg3), 
+    TP_FIELDS(
+        ctf_integer(unsigned int, arg2, arg2)
+        ctf_integer(unsigned int, arg3, arg3)
+    )
+)
+
+
+
 /*----------------------------------------------------------
 // Decoder Ring for FoundVF
 // [ xdp][%p] Found NetSvc-VF interfaces. NetSvc IfIdx:%lu, VF IfIdx:%lu

src/inc/quic_datapath.h

@@ -191,15 +191,18 @@ typedef struct CXPLAT_RAW_TCP_STATE {
     uint32_t SequenceNumber;
 } CXPLAT_RAW_TCP_STATE;
 
+//
+// An opaque queue type. Each queue is associated with a single RSS queue on a
+// single interface.
+//
+typedef struct CXPLAT_QUEUE CXPLAT_QUEUE;
+
 //
 // Structure to represent a network route.
 //
 typedef struct CXPLAT_ROUTE {
 
-    //
-    // The (RSS) queue that this route is primarily associated with.
-    //
-    void* Queue;
+    CXPLAT_QUEUE* Queue;
 
     QUIC_ADDR RemoteAddress;
     QUIC_ADDR LocalAddress;

src/manifest/clog.sidecar

@@ -5888,6 +5888,22 @@
       ],
       "macroName": "QuicTraceLogConnWarning"
     },
+    "GetTxOffloadConfig": {
+      "ModuleProperites": {},
+      "TraceString": "[ lib] %u, %u",
+      "UniqueId": "GetTxOffloadConfig",
+      "splitArgs": [
+        {
+          "DefinationEncoding": "u",
+          "MacroVariableName": "arg2"
+        },
+        {
+          "DefinationEncoding": "u",
+          "MacroVariableName": "arg3"
+        }
+      ],
+      "macroName": "QuicTraceLogInfo"
+    },
     "HandshakeConfirmedAck": {
       "ModuleProperites": {},
       "TraceString": "[conn][%p] Handshake confirmed (ack)",
@@ -15624,6 +15640,11 @@
         "TraceID": "GetPacketTypeFailure",
         "EncodingString": "[conn][%p] Failed to get packet type for control frames, 0x%x"
       },
+      {
+        "UniquenessHash": "72895611-1b21-d389-c78b-ab44b14b458f",
+        "TraceID": "GetTxOffloadConfig",
+        "EncodingString": "[ lib] %u, %u"
+      },
       {
         "UniquenessHash": "eeb31991-927b-2f3c-bfb7-12b7dea50ead",
         "TraceID": "HandshakeConfirmedAck",

src/platform/datapath_epoll.c

@@ -1827,7 +1827,7 @@ CxPlatSocketContextRecvComplete(
         QUIC_ADDR* LocalAddr = &IoBlock->Route.LocalAddress;
         QUIC_ADDR* RemoteAddr = &IoBlock->Route.RemoteAddress;
         CxPlatConvertFromMappedV6(RemoteAddr, RemoteAddr);
-        IoBlock->Route.Queue = SocketContext;
+        IoBlock->Route.Queue = (CXPLAT_QUEUE*)SocketContext;
 
         //
         // Process the ancillary control messages to get the local address,
@@ -2133,7 +2133,7 @@ CxPlatSocketReceiveTcpData(
         }
 
         IoBlock->Route.State = RouteResolved;
-        IoBlock->Route.Queue = SocketContext;
+        IoBlock->Route.Queue = (CXPLAT_QUEUE*)SocketContext;
         IoBlock->RefCount = 0;
 
         uint8_t* Buffer = (uint8_t*)IoBlock + DatapathPartition->Datapath->RecvBlockBufferOffset;
@@ -2235,10 +2235,10 @@ SendDataAlloc(
     CXPLAT_DBG_ASSERT(Socket != NULL);
     CXPLAT_DBG_ASSERT(Socket->Type != CXPLAT_SOCKET_UDP || Config->MaxPacketSize <= MAX_UDP_PAYLOAD_LENGTH);
     if (Config->Route->Queue == NULL) {
-        Config->Route->Queue = &Socket->SocketContexts[0];
+        Config->Route->Queue = (CXPLAT_QUEUE*)&Socket->SocketContexts[0];
     }
 
-    CXPLAT_SOCKET_CONTEXT* SocketContext = Config->Route->Queue;
+    CXPLAT_SOCKET_CONTEXT* SocketContext = (CXPLAT_SOCKET_CONTEXT*)Config->Route->Queue;
     CXPLAT_DBG_ASSERT(SocketContext->Binding == Socket);
     CXPLAT_DBG_ASSERT(SocketContext->Binding->Datapath == SocketContext->DatapathPartition->Datapath);
     CXPLAT_SEND_DATA* SendData = CxPlatPoolAlloc(&SocketContext->DatapathPartition->SendBlockPool);

src/platform/datapath_kqueue.c

@@ -1108,7 +1108,7 @@ CxPlatSocketContextRecvComplete(
         CxPlatConvertFromMappedV6(RemoteAddr, RemoteAddr);
     }
 
-    RecvPacket->Route->Queue = SocketContext;
+    RecvPacket->Route->Queue = (CXPLAT_QUEUE*)SocketContext;
     RecvPacket->TypeOfService = 0;
     RecvPacket->HopLimitTTL = 0; // TODO: We are not supporting this on MacOS (yet) unless there's a business need.
 
@@ -1609,10 +1609,10 @@ CxPlatSendDataAlloc(
     CXPLAT_DBG_ASSERT(Socket != NULL);
 
     if (Config->Route->Queue == NULL) {
-        Config->Route->Queue = &Socket->SocketContexts[0];
+        Config->Route->Queue = (CXPLAT_QUEUE*)&Socket->SocketContexts[0];
     }
 
-    CXPLAT_SOCKET_CONTEXT* SocketContext = Config->Route->Queue;
+    CXPLAT_SOCKET_CONTEXT* SocketContext = (CXPLAT_SOCKET_CONTEXT*)Config->Route->Queue;
     CXPLAT_SEND_DATA* SendData = CxPlatPoolAlloc(&SocketContext->DatapathPartition->SendDataPool);
     if (SendData != NULL) {
         CxPlatZeroMemory(SendData, sizeof(*SendData));
@@ -2083,7 +2083,7 @@ CxPlatSocketSend(
     UNREFERENCED_PARAMETER(Socket);
     CXPLAT_DBG_ASSERT(Route->Queue);
     CxPlatSocketSendInternal(
-        Route->Queue,
+        (CXPLAT_SOCKET_CONTEXT*)Route->Queue,
         &Route->LocalAddress,
         &Route->RemoteAddress,
         SendData,

src/platform/datapath_raw.c

@@ -354,12 +354,11 @@ RawSocketSend(
         CASTED_CLOG_BYTEARRAY(sizeof(Route->LocalAddress), &Route->LocalAddress));
     CXPLAT_DBG_ASSERT(Route->State == RouteResolved);
     CXPLAT_DBG_ASSERT(Route->Queue != NULL);
-    const CXPLAT_INTERFACE* Interface = CxPlatDpRawGetInterfaceFromQueue(Route->Queue);
 
     CxPlatFramingWriteHeaders(
-        Socket, Route, &SendData->Buffer, SendData->ECN, SendData->DSCP,
-        Interface->OffloadStatus.Transmit.NetworkLayerXsum,
-        Interface->OffloadStatus.Transmit.TransportLayerXsum,
+        Socket, Route, SendData, &SendData->Buffer, SendData->ECN, SendData->DSCP,
+        CxPlatDpRawIsL3TxXsumOffloadedOnQueue(Route->Queue),
+        CxPlatDpRawIsL4TxXsumOffloadedOnQueue(Route->Queue),
         Route->TcpState.SequenceNumber,
         Route->TcpState.AckNumber,
         TH_ACK);

src/platform/datapath_raw.h

@@ -170,7 +170,27 @@ CxPlatDpRawAssignQueue(
 _IRQL_requires_max_(DISPATCH_LEVEL)
 const CXPLAT_INTERFACE*
 CxPlatDpRawGetInterfaceFromQueue(
-    _In_ const void* Queue
+    _In_ const CXPLAT_QUEUE* Queue
+    );
+
+//
+// Returns whether the L3 (i.e., network) layer transmit checksum offload is
+// enabled on the queue.
+//
+_IRQL_requires_max_(DISPATCH_LEVEL)
+BOOLEAN
+CxPlatDpRawIsL3TxXsumOffloadedOnQueue(
+    _In_ const CXPLAT_QUEUE* Queue
+    );
+
+//
+// Returns whether the L3 (i.e., transport) layer transmit checksum offload is
+// enabled on the queue.
+//
+_IRQL_requires_max_(DISPATCH_LEVEL)
+BOOLEAN
+CxPlatDpRawIsL4TxXsumOffloadedOnQueue(
+    _In_ const CXPLAT_QUEUE* Queue
     );
 
 typedef struct HEADER_BACKFILL {
@@ -250,6 +270,27 @@ CxPlatDpRawTxEnqueue(
     _In_ CXPLAT_SEND_DATA* SendData
     );
 
+//
+// Sets the TX send object to have the specified L3 checksum offload settings.
+//
+_IRQL_requires_max_(DISPATCH_LEVEL)
+void
+CxPlatDpRawTxSetL3ChecksumOffload(
+    _In_ CXPLAT_SEND_DATA* SendData
+    );
+
+//
+// Sets the TX send object to have the specified L4 checksum offload settings.
+//
+_IRQL_requires_max_(DISPATCH_LEVEL)
+void
+CxPlatDpRawTxSetL4ChecksumOffload(
+    _In_ CXPLAT_SEND_DATA* SendData,
+    _In_ BOOLEAN IsIpv6,
+    _In_ BOOLEAN IsTcp,
+    _In_ uint8_t L4HeaderLength
+    );
+
 //
 // Raw Socket Interface
 //
@@ -372,6 +413,7 @@ void
 CxPlatFramingWriteHeaders(
     _In_ CXPLAT_SOCKET_RAW* Socket,
     _In_ const CXPLAT_ROUTE* Route,
+    _Inout_ CXPLAT_SEND_DATA* SendData,
     _Inout_ QUIC_BUFFER* Buffer,
     _In_ CXPLAT_ECN_TYPE ECN,
     _In_ uint8_t DSCP,

src/platform/datapath_raw_dpdk.c

@@ -535,12 +535,31 @@ CxPlatDpRawAssignQueue(
 _IRQL_requires_max_(DISPATCH_LEVEL)
 const CXPLAT_INTERFACE*
 CxPlatDpRawGetInterfaceFromQueue(
-    _In_ const void* Queue
+    _In_ const CXPLAT_QUEUE* Queue
     )
 {
     return (const CXPLAT_INTERFACE*)Queue;
 }
 
+_IRQL_requires_max_(DISPATCH_LEVEL)
+BOOLEAN
+CxPlatDpRawIsL3TxXsumOffloadedOnQueue(
+    _In_ const CXPLAT_QUEUE* Queue
+    )
+{
+    return CxPlatDpRawGetInterfaceFromQueue(Queue)->OffloadStatus.Transmit.NetworkLayerXsum;
+}
+
+
+_IRQL_requires_max_(DISPATCH_LEVEL)
+BOOLEAN
+CxPlatDpRawIsL4TxXsumOffloadedOnQueue(
+    _In_ const CXPLAT_QUEUE* Queue
+    )
+{
+    return CxPlatDpRawGetInterfaceFromQueue(Queue)->OffloadStatus.Transmit.TransportLayerXsum;
+}
+
 static
 void
 CxPlatDpdkRx(
@@ -684,6 +703,30 @@ CxPlatDpRawTxEnqueue(
     CxPlatPoolFree(Packet);
 }
 
+_IRQL_requires_max_(DISPATCH_LEVEL)
+void
+CxPlatDpRawTxSetL3ChecksumOffload(
+    _In_ CXPLAT_SEND_DATA* SendData
+    )
+{
+    UNREFERENCED_PARAMETER(SendData);
+}
+
+_IRQL_requires_max_(DISPATCH_LEVEL)
+void
+CxPlatDpRawTxSetL4ChecksumOffload(
+    _In_ CXPLAT_SEND_DATA* SendData,
+    _In_ BOOLEAN IsIpv6,
+    _In_ BOOLEAN IsTcp,
+    _In_ uint8_t L4HeaderLength
+    )
+{
+    UNREFERENCED_PARAMETER(SendData);
+    UNREFERENCED_PARAMETER(IsIpv6);
+    UNREFERENCED_PARAMETER(IsTcp);
+    UNREFERENCED_PARAMETER(L4HeaderLength);
+}
+
 static
 void
 CxPlatDpdkTx(