Merge remote-tracking branch 'origin' into hari/pr_29392_dupe

Author: hariharans29

.github/workflows/linux_cuda_plugin_ci.yml

@@ -22,7 +22,7 @@ jobs:
     name: Build Linux CUDA Plugin EP x64 Release
     uses: ./.github/workflows/reusable_linux_build.yml
     with:
-      pool_name: "onnxruntime-github-Ubuntu2204-AMD-CPU"
+      pool_name: "onnxruntime-github-Ubuntu2204-AMD-CPU" # Build pool
       build_config: Release
       architecture: x64
       dockerfile_path: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda
@@ -39,6 +39,7 @@ jobs:
         --cudnn_home=/usr/local/cuda-13.0
         --enable_cuda_profiling
         --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86
+        --cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=ON
         --cmake_extra_defines onnxruntime_QUICK_BUILD=ON
         --cmake_extra_defines onnxruntime_BUILD_CUDA_EP_AS_PLUGIN=ON
       python_path_prefix: 'PATH=/opt/python/cp312-cp312/bin:$PATH'
@@ -55,6 +56,7 @@ jobs:
     runs-on:
       - self-hosted
       - "1ES.Pool=onnxruntime-github-linux-a10"
+      - "1ES.ImageOverride=onnxruntime-ubuntu2204-CUDA-A10-Test"
       - "JobId=test-linux-cuda-plugin-x64-release-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}"
     permissions:
       contents: read
@@ -109,63 +111,38 @@ jobs:
             "${{ steps.build_docker_image_step.outputs.full-image-name }}" \
             nvidia-smi
 
-      # --- Install the ORT wheel and run CUDA plugin EP tests ---
-      - name: Run CUDA Plugin EP Python Tests
+      - name: Run CUDA Plugin EP C++ Tests
         run: |
           docker run --rm --gpus all \
             -v ${{ github.workspace }}:/onnxruntime_src \
             -v ${{ runner.temp }}/Release:/build/Release \
             -e NVIDIA_VISIBLE_DEVICES=all \
+            -e ORT_UNIT_TEST_MAIN_DYNAMIC_PLUGIN_EP_CONFIG_JSON='{"ep_library_registration_name":"CUDAExecutionProvider","ep_library_path":"libonnxruntime_providers_cuda.so","selected_ep_name":"CUDAExecutionProvider","tests_to_skip":["BeamSearchTest.*","ContribOpAttentionTest.AttentionPastState_dynamic","ContribOpAttentionTest.Attention_Mask1D_Fp32_B2_S64","ContribOpAttentionTest.Attention_Mask2D_Fp32_B2_S32","ConvTest.Conv2D_MatMul_*","CudaPluginArenaTest.DeviceAllocator_ArenaReusesMemory","CudaPluginArenaTest.DeviceAllocator_CudaMemoryIsValid","CudaPluginArenaTest.DeviceAllocator_IsArena","CudaPluginArenaTest.DeviceAllocator_MultipleAllocations","DeformConvTest.Group1OffsetGroup2","GRUTest.*","GreedySearchTest.*","InferenceSessionTests.Test2LayerNestedSubgraph","InferenceSessionTests.Test3LayerNestedSubgraph","InferenceSessionTests.TestArenaShrinkageAfterRun","LSTMTest.*","LogSoftmaxOperator.*","LongformerAttentionTest.*","LoraAdapterTest.VerifyDeviceCopy","PackedAttentionTest.*","PlannerTest.MultiStream","PlannerTest.MultiStream1StreamWaitFor2Streams","PlannerTest.MultiStream2NodesSameStreamConsumedBy1NodeInDifferentStream","PlannerTest.MultiStreamCudaEPNodeCPUOutput","PlannerTest.MultiStreamMultiOutput","RNNTest.*","ReductionOpTest.ArgMax2D_select_last","ReductionOpTest.ArgMin_do_not_keepdims_2_select_last","SamplingTest.*","Scan.MixedExecutionProvidersUnknownDimInSubgraphOutput","Scan8.*","SessionStateTest.TestLayeringPartitioning","SessionStateTest.TestResourceAwarePartitioning_LargeLimit","SessionStateTest.TestResourceAwarePartitioning_NoLimit","SkipGroupNormTest.*","TopKOperator.NthElementBFloat16_NegativeVals"]}' \
             ${{ steps.build_docker_image_step.outputs.full-image-name }} \
             bash -c "
               set -ex
-              export PATH=/opt/python/cp312-cp312/bin:\$PATH
-              # Ensure libcudart.so.13 is findable regardless of host-runner NVIDIA Container Toolkit configuration.
-              # The CUDA runtime library lives in the container image at /usr/local/cuda-13.0/lib64, but the
-              # LD_LIBRARY_PATH may not include this path when the runner's NVIDIA toolkit only mounts driver
-              # libraries at /usr/local/nvidia/lib64.
-              export LD_LIBRARY_PATH=/usr/local/cuda-13.0/lib64:\${LD_LIBRARY_PATH:-}
-
-              # Install the ORT wheel
-              python -m pip install /build/Release/Release/dist/onnxruntime*.whl
-
-              # Install test dependencies
-              python -m pip install numpy onnx
-              python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
-
-              # Set plugin path and run tests
-              export ORT_CUDA_PLUGIN_PATH=/build/Release/Release/libonnxruntime_providers_cuda_plugin.so
-              echo \"ORT_CUDA_PLUGIN_PATH=\$ORT_CUDA_PLUGIN_PATH\"
-              ls -la \$ORT_CUDA_PLUGIN_PATH
-
-              cd /onnxruntime_src/onnxruntime/test/python/transformers
-              python test_cuda_plugin_ep.py
+              export LD_LIBRARY_PATH=/build/Release/Release:/usr/local/cuda-13.0/lib64:\${LD_LIBRARY_PATH:-}
+              cd /build/Release/Release
+              ./onnxruntime_test_all
+              ./onnxruntime_provider_test
             "
 
-      # --- Run the CUDA plugin EP C++ GoogleTest binary ---
-      # onnxruntime_provider_test is built into the artifact and links the plugin tests
-      # (gated by ORT_UNIT_TEST_HAS_CUDA_PLUGIN_EP). These tests register the plugin .so via
-      # GetSharedLibraryFileName("onnxruntime_providers_cuda_plugin"), which returns the
-      # platform-specific filename without a directory component. Run from /build/Release/Release
-      # so that filename resolves to the plugin .so built there.
-      # The filter covers every CUDA plugin EP suite linked into this binary:
-      #   CudaPlugin*              -> CudaPluginUserStreamGraphTest, CudaPluginArenaTest,
-      #                              CudaPluginPartitioningTest, CudaPluginProfilingTest
-      #   CudaResourcePartitioning* -> CudaResourcePartitioningTest
-      - name: Run CUDA Plugin EP C++ Tests
+      - name: Run CUDA Plugin EP Python Tests
         run: |
           docker run --rm --gpus all \
             -v ${{ github.workspace }}:/onnxruntime_src \
             -v ${{ runner.temp }}/Release:/build/Release \
             -e NVIDIA_VISIBLE_DEVICES=all \
+            -e ORT_TEST_CUDA_PLUGIN_EP=1 \
             ${{ steps.build_docker_image_step.outputs.full-image-name }} \
             bash -c "
               set -ex
               export PATH=/opt/python/cp312-cp312/bin:\$PATH
-              # Make libcudart.so.13 (and the plugin's CUDA deps) findable; see note above.
               export LD_LIBRARY_PATH=/build/Release/Release:/usr/local/cuda-13.0/lib64:\${LD_LIBRARY_PATH:-}
-
-              cd /build/Release/Release
-              ls -la onnxruntime_provider_test libonnxruntime_providers_cuda_plugin.so
-              ./onnxruntime_provider_test --gtest_filter='CudaPlugin*:CudaResourcePartitioning*'
+              python -m pip install /build/Release/Release/dist/onnxruntime*.whl
+              python -m pip install numpy onnx
+              python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
+              cd /onnxruntime_src/onnxruntime/test/python/transformers
+              python test_cuda_plugin_ep.py
             "
+

.github/workflows/windows_cuda_no_cudnn.yml

@@ -192,7 +192,7 @@ jobs:
         shell: cmd
 
       - name: Install torch for CPU only
-        run: python -m pip install torch
+        run: python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
         working-directory: ${{ github.workspace }}
         shell: cmd
 
@@ -229,7 +229,7 @@ jobs:
       - name: Verify CUDA plugin has no direct cuDNN dependency
         shell: pwsh
         run: |
-          $pluginPath = "${{ runner.temp }}\build\Release\Release\onnxruntime_providers_cuda_plugin.dll"
+          $pluginPath = "${{ runner.temp }}\build\Release\Release\onnxruntime_providers_cuda.dll"
           if (-not (Test-Path $pluginPath)) {
             Write-Error "CUDA plugin EP library not found at $pluginPath"
             exit 1
@@ -245,10 +245,8 @@ jobs:
         working-directory: ${{ github.workspace }}\onnxruntime\test\python\transformers
         shell: pwsh
         run: |
-          $env:ORT_CUDA_PLUGIN_PATH = "${{ runner.temp }}\build\Release\Release\onnxruntime_providers_cuda_plugin.dll"
           $env:ORT_TEST_CUDA_PLUGIN_EP = "1"
           $env:ORT_TEST_CUDA_PLUGIN_NO_CUDNN = "1"
-          Write-Host "ORT_CUDA_PLUGIN_PATH=$env:ORT_CUDA_PLUGIN_PATH"
           python test_cuda_plugin_ep.py
           if ($lastExitCode -ne 0) {
             exit $lastExitCode

.github/workflows/windows_cuda_plugin.yml

@@ -157,7 +157,7 @@ jobs:
         shell: cmd
 
       - name: Install torch for CPU only
-        run: python -m pip install torch
+        run: python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
         working-directory: ${{ github.workspace }}
         shell: cmd
 
@@ -194,20 +194,32 @@ jobs:
         shell: pwsh
         run: nvidia-smi
 
+      - name: Run Tests
+        working-directory: ${{ runner.temp }}\build\Release\Release
+        shell: pwsh
+        run: |
+          $env:ORT_UNIT_TEST_MAIN_DYNAMIC_PLUGIN_EP_CONFIG_JSON = '{"ep_library_registration_name":"CUDAExecutionProvider","ep_library_path":"onnxruntime_providers_cuda.dll","selected_ep_name":"CUDAExecutionProvider","tests_to_skip":["BeamSearchTest.*","ContribOpAttentionTest.AttentionPastState_dynamic","ContribOpAttentionTest.Attention_Mask1D_Fp32_B2_S64","ContribOpAttentionTest.Attention_Mask2D_Fp32_B2_S32","ConvTest.Conv2D_MatMul_*","ConvTest.Conv3D_2","CudaPluginArenaTest.DeviceAllocator_ArenaReusesMemory","CudaPluginArenaTest.DeviceAllocator_CudaMemoryIsValid","CudaPluginArenaTest.DeviceAllocator_IsArena","CudaPluginArenaTest.DeviceAllocator_MultipleAllocations","DeformConvTest.Group1OffsetGroup2","GRUTest.*","GreedySearchTest.*","InferenceSessionTests.Test2LayerNestedSubgraph","InferenceSessionTests.Test3LayerNestedSubgraph","InferenceSessionTests.TestArenaShrinkageAfterRun","LSTMTest.*","LogSoftmaxOperator.*","LongformerAttentionTest.*","LoraAdapterTest.VerifyDeviceCopy","PackedAttentionTest.*","PlannerTest.MultiStream","PlannerTest.MultiStream1StreamWaitFor2Streams","PlannerTest.MultiStream2NodesSameStreamConsumedBy1NodeInDifferentStream","PlannerTest.MultiStreamCudaEPNodeCPUOutput","PlannerTest.MultiStreamMultiOutput","RNNTest.*","ReductionOpTest.ArgMax2D_select_last","ReductionOpTest.ArgMin_do_not_keepdims_2_select_last","SamplingTest.*","Scan.MixedExecutionProvidersUnknownDimInSubgraphOutput","Scan8.*","SessionStateTest.TestLayeringPartitioning","SessionStateTest.TestResourceAwarePartitioning_LargeLimit","SessionStateTest.TestResourceAwarePartitioning_NoLimit","SkipGroupNormTest.*","TopKOperator.NthElementBFloat16_NegativeVals"]}'
+
+          .\onnxruntime_test_all.exe
+          if ($lastExitCode -ne 0) {
+            exit $lastExitCode
+          }
+
+          .\onnxruntime_provider_test.exe
+          if ($lastExitCode -ne 0) {
+            exit $lastExitCode
+          }
+
       - name: Run CUDA Plugin EP Python Tests
         working-directory: ${{ github.workspace }}\onnxruntime\test\python\transformers
         shell: pwsh
         run: |
-          $env:ORT_CUDA_PLUGIN_PATH = "${{ runner.temp }}\build\Release\Release\onnxruntime_providers_cuda_plugin.dll"
-          Write-Host "ORT_CUDA_PLUGIN_PATH=$env:ORT_CUDA_PLUGIN_PATH"
-          if (-not (Test-Path $env:ORT_CUDA_PLUGIN_PATH)) {
-            Write-Error "CUDA plugin EP library not found at $env:ORT_CUDA_PLUGIN_PATH"
-            exit 1
-          }
+          $env:ORT_TEST_CUDA_PLUGIN_EP = "1"
           python test_cuda_plugin_ep.py
           if ($lastExitCode -ne 0) {
             exit $lastExitCode
           }
+
     env:
       DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
       setVcvars: true

VERSION_NUMBER

@@ -1 +1 @@
-1.28.0
+1.29.0

cmake/CMakeLists.txt

@@ -76,7 +76,7 @@ option(onnxruntime_USE_CUDA "Build with CUDA support" OFF)
 cmake_dependent_option(onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS "Build with CUDA unit tests" OFF "onnxruntime_USE_CUDA;onnxruntime_BUILD_UNIT_TESTS" OFF)
 
 cmake_dependent_option(onnxruntime_USE_CUDA_NHWC_OPS "Build CUDA with NHWC op support" ON "onnxruntime_USE_CUDA" OFF)
-cmake_dependent_option(onnxruntime_BUILD_CUDA_EP_AS_PLUGIN "Build CUDA EP as a separate plugin shared library" OFF "onnxruntime_USE_CUDA" OFF)
+cmake_dependent_option(onnxruntime_BUILD_CUDA_EP_AS_PLUGIN "Build CUDA EP as a separate plugin shared library instead of the legacy in-tree provider" OFF "onnxruntime_USE_CUDA" OFF)
 option(onnxruntime_CUDA_MINIMAL "Build CUDA without any operations apart from memcpy ops. Usefuel for a very minial TRT build" OFF)
 option(onnxruntime_ENABLE_CUDA_LINE_NUMBER_INFO "When building with CUDA support, generate device code line number information." OFF)
 option(onnxruntime_USE_OPENVINO "Build with OpenVINO support" OFF)
@@ -1800,10 +1800,6 @@ foreach(onnxruntime_cmake_file ${ONNXRUNTIME_CMAKE_FILES})
   include(${onnxruntime_cmake_file}.cmake)
 endforeach()
 
-# CUDA EP Plugin build (independent shared library)
-if (onnxruntime_USE_CUDA AND onnxruntime_BUILD_CUDA_EP_AS_PLUGIN)
-  include(onnxruntime_providers_cuda_plugin.cmake)
-endif()
 if (UNIX)
   option(BUILD_PKGCONFIG_FILES "Build and install pkg-config files" ON)
 else()

cmake/onnxruntime_java.cmake

@@ -162,9 +162,12 @@ if (WIN32)
     if (TARGET onnxruntime_providers_shared)
       add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:onnxruntime_providers_shared> ${JAVA_PACKAGE_LIB_DIR}/$<TARGET_FILE_NAME:onnxruntime_providers_shared>)
     endif()
-    if (onnxruntime_USE_CUDA)
+    if (onnxruntime_USE_CUDA AND NOT onnxruntime_BUILD_CUDA_EP_AS_PLUGIN)
       add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:onnxruntime_providers_cuda> ${JAVA_PACKAGE_LIB_DIR}/$<TARGET_FILE_NAME:onnxruntime_providers_cuda>)
     endif()
+    if (onnxruntime_USE_CUDA AND onnxruntime_BUILD_CUDA_EP_AS_PLUGIN)
+      add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:onnxruntime_providers_cuda_plugin> ${JAVA_PACKAGE_LIB_DIR}/$<TARGET_FILE_NAME:onnxruntime_providers_cuda_plugin>)
+    endif()
     if (onnxruntime_USE_DNNL)
       add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:onnxruntime_providers_dnnl> ${JAVA_PACKAGE_LIB_DIR}/$<TARGET_FILE_NAME:onnxruntime_providers_dnnl>)
     endif()
@@ -210,9 +213,12 @@ else()
   if (TARGET onnxruntime_providers_shared)
     add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:onnxruntime_providers_shared> ${JAVA_PACKAGE_LIB_DIR}/$<TARGET_LINKER_FILE_NAME:onnxruntime_providers_shared>)
   endif()
-  if (onnxruntime_USE_CUDA)
+  if (onnxruntime_USE_CUDA AND NOT onnxruntime_BUILD_CUDA_EP_AS_PLUGIN)
     add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:onnxruntime_providers_cuda> ${JAVA_PACKAGE_LIB_DIR}/$<TARGET_LINKER_FILE_NAME:onnxruntime_providers_cuda>)
   endif()
+  if (onnxruntime_USE_CUDA AND onnxruntime_BUILD_CUDA_EP_AS_PLUGIN)
+    add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:onnxruntime_providers_cuda_plugin> ${JAVA_PACKAGE_LIB_DIR}/$<TARGET_LINKER_FILE_NAME:onnxruntime_providers_cuda_plugin>)
+  endif()
   if (onnxruntime_USE_DNNL)
     add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:onnxruntime_providers_dnnl> ${JAVA_PACKAGE_LIB_DIR}/$<TARGET_LINKER_FILE_NAME:onnxruntime_providers_dnnl>)
   endif()

cmake/onnxruntime_providers.cmake

@@ -62,7 +62,7 @@ endfunction()
 if(onnxruntime_USE_VITISAI)
   set(PROVIDERS_VITISAI onnxruntime_providers_vitisai)
 endif()
-if(onnxruntime_USE_CUDA)
+if(onnxruntime_USE_CUDA AND NOT onnxruntime_BUILD_CUDA_EP_AS_PLUGIN)
   set(PROVIDERS_CUDA onnxruntime_providers_cuda)
 endif()
 if(onnxruntime_USE_COREML)
@@ -114,9 +114,12 @@ if(onnxruntime_USE_SNPE)
 endif()
 
 include(onnxruntime_providers_cpu.cmake)
-if (onnxruntime_USE_CUDA)
+if (onnxruntime_USE_CUDA AND NOT onnxruntime_BUILD_CUDA_EP_AS_PLUGIN)
   include(onnxruntime_providers_cuda.cmake)
 endif()
+if (onnxruntime_USE_CUDA AND onnxruntime_BUILD_CUDA_EP_AS_PLUGIN)
+  include(onnxruntime_providers_cuda_plugin.cmake)
+endif()
 
 if (onnxruntime_USE_DNNL)
   include(onnxruntime_providers_dnnl.cmake)

cmake/onnxruntime_providers_cuda_plugin.cmake

@@ -2,7 +2,7 @@
 # Licensed under the MIT License.
 
 # Build the CUDA Execution Provider as a plugin shared library.
-# This file is included from the main CMakeLists.txt when onnxruntime_BUILD_CUDA_EP_AS_PLUGIN=ON.
+# This file is included from onnxruntime_providers.cmake when onnxruntime_BUILD_CUDA_EP_AS_PLUGIN=ON.
 
 message(STATUS "Building CUDA EP as plugin shared library")
 
@@ -458,7 +458,7 @@ endif()
 
 # Set output name and solution folder
 set_target_properties(onnxruntime_providers_cuda_plugin PROPERTIES
-    OUTPUT_NAME "onnxruntime_providers_cuda_plugin"
+  OUTPUT_NAME "onnxruntime_providers_cuda"
     FOLDER "ONNXRuntime"
 )
 

cmake/onnxruntime_python.cmake

@@ -1085,7 +1085,7 @@ if (DEFINED ENV{OPENVINO_MANYLINUX})
     )
 endif()
 
-if (onnxruntime_USE_CUDA)
+if (onnxruntime_USE_CUDA AND NOT onnxruntime_BUILD_CUDA_EP_AS_PLUGIN)
     add_custom_command(
       TARGET onnxruntime_pybind11_state POST_BUILD
       COMMAND ${CMAKE_COMMAND} -E copy
@@ -1095,6 +1095,15 @@ if (onnxruntime_USE_CUDA)
     )
 endif()
 
+if (onnxruntime_USE_CUDA AND onnxruntime_BUILD_CUDA_EP_AS_PLUGIN)
+  add_custom_command(
+    TARGET onnxruntime_pybind11_state POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy
+      $<TARGET_FILE:onnxruntime_providers_cuda_plugin>
+      $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/capi/
+  )
+endif()
+
 if (onnxruntime_USE_CANN)
     add_custom_command(
       TARGET onnxruntime_pybind11_state POST_BUILD