Fix data race in connection pool by running QuicConnStart on the connection's worker thread

src/core/api.c

@@ -450,6 +450,7 @@ MsQuicConnectionStart(
     Oper->API_CALL.Context->CONN_START.ServerName = ServerNameCopy;
     Oper->API_CALL.Context->CONN_START.ServerPort = ServerPort;
     Oper->API_CALL.Context->CONN_START.Family = Family;
+    Oper->API_CALL.Context->CONN_START.FailSilently = FALSE;
     ServerNameCopy = NULL;
 
     //

src/core/connection.c

@@ -1988,6 +1988,15 @@ QuicConnStart(
     }
 
     if (QUIC_FAILED(Status)) {
+        if (StartFlags & QUIC_CONN_START_FLAG_FAIL_SILENTLY) {
+            //
+            // This connection was created internally (e.g. by the connection
+            // pool) and was never returned to the application. Mark it
+            // internally owned so that closing it locally below drives shutdown
+            // completion.
+            //
+            Connection->State.ExternalOwner = FALSE;
+        }
         QuicConnCloseLocally(
             Connection,
             StartFlags & QUIC_CONN_START_FLAG_FAIL_SILENTLY ?
@@ -7783,7 +7792,8 @@ QuicConnProcessApiOperation(
                 ApiCtx->CONN_START.Family,
                 ApiCtx->CONN_START.ServerName,
                 ApiCtx->CONN_START.ServerPort,
-                QUIC_CONN_START_FLAG_NONE);
+                ApiCtx->CONN_START.FailSilently ?
+                    QUIC_CONN_START_FLAG_FAIL_SILENTLY : QUIC_CONN_START_FLAG_NONE);
         ApiCtx->CONN_START.ServerName = NULL;
         break;
 

src/core/connection_pool.c

@@ -269,6 +269,51 @@ QuicConnPoolQueueConnectionClose(
     }
 }
 
+//
+// Start the connection on its worker thread and wait for completion.
+//
+static
+_IRQL_requires_max_(PASSIVE_LEVEL)
+QUIC_STATUS
+QuicConnPoolStartConnection(
+    _In_ QUIC_CONNECTION* Connection,
+    _In_ QUIC_CONFIGURATION* Configuration,
+    _In_ QUIC_ADDRESS_FAMILY Family,
+    _In_z_ const char* ServerName,
+    _In_ uint16_t ServerPort
+    )
+{
+    QUIC_STATUS StartStatus = QUIC_STATUS_INTERNAL_ERROR;
+    CXPLAT_EVENT CompletionEvent;
+    QUIC_OPERATION Oper = { 0 };
+    QUIC_API_CONTEXT ApiCtx;
+
+    Oper.Type = QUIC_OPER_TYPE_API_CALL;
+    Oper.FreeAfterProcess = FALSE;          // Stack-allocated; worker must not free it.
+    Oper.API_CALL.Context = &ApiCtx;
+
+    CxPlatEventInitialize(&CompletionEvent, TRUE, FALSE);
+    ApiCtx.Type = QUIC_API_TYPE_CONN_START;
+    ApiCtx.Status = &StartStatus;
+    ApiCtx.Completed = &CompletionEvent;
+    ApiCtx.CONN_START.Configuration = Configuration;
+    ApiCtx.CONN_START.ServerName = ServerName;  // Ownership passes to QuicConnStart.
+    ApiCtx.CONN_START.ServerPort = ServerPort;
+    ApiCtx.CONN_START.Family = Family;
+    ApiCtx.CONN_START.FailSilently = TRUE;
+
+    //
+    // No op-level Configuration ref is needed: we block until the op completes
+    // and the caller keeps the Configuration alive; QuicConnSetConfiguration
+    // takes its own QUIC_CONF_REF_CONNECTION ref.
+    //
+    QuicConnQueueOper(Connection, &Oper);
+    CxPlatEventWaitForever(CompletionEvent);
+    CxPlatEventUninitialize(CompletionEvent);
+
+    return StartStatus;
+}
+
 
 static
 _IRQL_requires_max_(PASSIVE_LEVEL)
@@ -306,6 +351,13 @@ QuicConnPoolTryCreateConnection(
     (*Connection)->ClientCallbackHandler = Handler;
     (*Connection)->ClientContext = Context;
 
+    //
+    // Hold a create-scope reference across the rest of this function. The start
+    // operation is processed on the connection's worker thread. This extra reference
+    // guarantees the connection stays alive until this function is done with it.
+    //
+    QuicConnAddRef(*Connection, QUIC_CONN_REF_HANDLE_OWNER);
+
     //
     // Set the calculated remote address and local address to get the desired
     // RSS CPU.
@@ -372,13 +424,12 @@ QuicConnPoolTryCreateConnection(
         }
     }
 
-    Status = QuicConnStart(
+    Status = QuicConnPoolStartConnection(
         *Connection,
         Configuration,
         Family,
         ServerName,
-        ServerPort,
-        QUIC_CONN_START_FLAG_FAIL_SILENTLY);
+        ServerPort);
 
     ServerName = NULL; // The connection now owns the ServerName.
 
@@ -391,15 +442,12 @@ QuicConnPoolTryCreateConnection(
         CXPLAT_FREE(ServerName, QUIC_POOL_SERVERNAME);
     }
 
-    if (QUIC_FAILED(Status) && *Connection != NULL) {
-        //
-        // This connection has never left MsQuic back to the application.
-        // Mark it as internally owned so no notification is sent to the app,
-        // the closing logic will handle the final deref.
-        //
-        (*Connection)->State.ExternalOwner = FALSE;
-        QuicConnPoolQueueConnectionClose(*Connection, FALSE);
-        *Connection = NULL;
+    if (*Connection != NULL) {
+        QUIC_CONNECTION* PoolConnection = *Connection;
+        if (QUIC_FAILED(Status)) {
+            *Connection = NULL;
+        }
+        QuicConnRelease(PoolConnection, QUIC_CONN_REF_HANDLE_OWNER);
     }
 
     return Status;

src/core/operation.h

@@ -108,6 +108,7 @@ typedef struct QUIC_API_CONTEXT {
             const char* ServerName;
             uint16_t ServerPort;
             QUIC_ADDRESS_FAMILY Family;
+            BOOLEAN FailSilently;       // Don't notify the app on start failure.
         } CONN_START;
         struct {
             QUIC_CONFIGURATION* Configuration;