WinML - Source Changes

Author: chrisdMSFT

src/config.cpp

@@ -706,6 +706,12 @@ struct Model_Element : JSON::Element {
       v_.decoder_start_token_id = static_cast<int>(JSON::Get<double>(value));
     } else if (name == "sep_token_id") {
       v_.sep_token_id = static_cast<int>(JSON::Get<double>(value));
+    } else if (name == "hardware_device_type") {
+      v_.hardware_device_type = JSON::Get<std::string_view>(value);
+    } else if (name == "hardware_device_id") {
+      v_.hardware_device_id = static_cast<uint32_t>(JSON::Get<double>(value));
+    } else if (name == "hardware_vendor_id") {
+      v_.hardware_vendor_id = static_cast<uint32_t>(JSON::Get<double>(value));
     } else {
       throw JSON::unknown_value_error{};
     }

src/config.h

@@ -247,6 +247,11 @@ struct Config {
 
     } decoder;
 
+    // EP device filters
+    std::optional<std::string> hardware_device_type;  // CPU, GPU, NPU
+    std::optional<uint32_t> hardware_device_id;
+    std::optional<uint32_t> hardware_vendor_id;
+
   } model;
 
   struct Search {

src/models/model.cpp

@@ -589,12 +589,141 @@ DeviceInterface* SetProviderSessionOptions(OrtSessionOptions& session_options,
         p_device = GetDeviceInterface(DeviceType::NvTensorRtRtx);
       }
 
+#if USE_WINML
+      // Get model device config
+      std::optional<uint32_t> config_device_id = config.model.hardware_device_id;
+      std::optional<uint32_t> config_vendor_id = config.model.hardware_vendor_id;
+      std::optional<std::string> config_device_type = config.model.hardware_device_type;
+      // for OpenVINO, use "device_type" in provider_options exclusively if it's provided
+      std::optional<std::string> config_ov_device_type = std::nullopt;
+      if (provider_options.name == "OpenVINO") {
+        for (auto& option : provider_options.options) {
+          if (option.first == "device_type") {
+            config_ov_device_type = option.second;
+          }
+        }
+        if (config_ov_device_type.has_value()) {
+          config_device_id = std::nullopt;
+          config_vendor_id = std::nullopt;
+          config_device_type = std::nullopt;
+        } else if (!(config_device_id.has_value() || config_vendor_id.has_value() || config_device_type.has_value())) {
+          config_ov_device_type = "CPU";
+        }
+      }
+      std::optional<OrtHardwareDeviceType> config_device_type_enum;
+      if (config_device_type.has_value()) {
+        if (*config_device_type == "CPU") {
+          config_device_type_enum = OrtHardwareDeviceType_CPU;
+        } else if (*config_device_type == "GPU") {
+          config_device_type_enum = OrtHardwareDeviceType_GPU;
+        } else if (*config_device_type == "NPU") {
+          config_device_type_enum = OrtHardwareDeviceType_NPU;
+        } else {
+          throw std::runtime_error("Unsupported hardware device type: " + *config_device_type);
+        }
+      }
+
+      // Match EP device with EP name in provider options and model device config
+      // include\onnxruntime\core\graph\constants.h
+      const static std::unordered_map<std::string, std::string> s_providerNameToExecutionProvider{
+          {"QNN", "QNNExecutionProvider"},
+          {"WebGPU", "WebGpuExecutionProvider"},
+          {"OpenVINO", "OpenVINOExecutionProvider"},
+          {"VitisAI", "VitisAIExecutionProvider"},
+          {"NvTensorRtRtx", "NvTensorRTRTXExecutionProvider"},
+      };
+      std::string epName{};
+      if (auto search = s_providerNameToExecutionProvider.find(provider_options.name); search != s_providerNameToExecutionProvider.end()) {
+        epName = search->second;
+      }
+
+      size_t num_devices;
+      const OrtEpDevice* const* device_ptrs;
+      Ort::api->GetEpDevices(&GetOrtEnv(), &device_ptrs, &num_devices);
+      std::vector<const OrtEpDevice*> ep_devices_ptrs;
+      ep_devices_ptrs.reserve(num_devices);
+      for (size_t i = 0; i < num_devices; ++i) {
+        const OrtHardwareDevice* hardware_device = Ort::api->EpDevice_Device(device_ptrs[i]);
+        const uint32_t hardware_device_id = Ort::api->HardwareDevice_DeviceId(hardware_device);
+        const uint32_t hardware_vendor_id = Ort::api->HardwareDevice_VendorId(hardware_device);
+        const OrtHardwareDeviceType hardware_device_type = Ort::api->HardwareDevice_Type(hardware_device);
+
+        auto check_ov_device_type = [&config_ov_device_type, &provider_options](const OrtEpDevice* device_ptr) -> bool {
+          if (provider_options.name != "OpenVINO") {
+            return true;
+          } else if (!config_ov_device_type.has_value()) {
+            return true;
+          } else {
+            const OrtKeyValuePairs* keyvals = Ort::api->EpDevice_EpMetadata(device_ptr);
+            size_t num_entries;
+            const char* const* keys = nullptr;
+            const char* const* values = nullptr;
+            Ort::api->GetKeyValuePairs(keyvals, &keys, &values, &num_entries);
+            for (int kvi = 0; kvi < num_entries; kvi++) {
+              const std::string key = keys[kvi];
+              const std::string val = values[kvi];
+              if (key == "ov_device" && val == config_ov_device_type) {
+                return true;
+              }
+            }
+            return false;
+          }
+        };
+        bool hardware_device_id_matched = (!config_device_id.has_value()) || config_device_id.value() == hardware_device_id;
+        bool hardware_vendor_id_matched = (!config_vendor_id.has_value()) || config_vendor_id.value() == hardware_vendor_id;
+        bool hardware_device_type_matched = (!config_device_type_enum.has_value()) ||
+                                            config_device_type_enum.value() == hardware_device_type;
+        bool hardware_ov_device_type_matched = check_ov_device_type(device_ptrs[i]);
+
+        // Append matched EP device
+        if (Ort::api->EpDevice_EpName(device_ptrs[i]) == epName &&
+            hardware_device_id_matched &&
+            hardware_vendor_id_matched &&
+            hardware_device_type_matched &&
+            hardware_ov_device_type_matched) {
+          ep_devices_ptrs.push_back(device_ptrs[i]);
+          // WinML Hotfix: DML and WebGPU EP factories currently only support one device at a time
+          // OpenVINO also supports only one device at a time
+          if (provider_options.name == "DML" || provider_options.name == "WebGPU" || provider_options.name == "OpenVINO") {
+            break;
+          }
+        }
+      }
+
+      // No need to append if we can't find a device.
+      if (!ep_devices_ptrs.empty()) {
+        std::vector<const char*> keys, values;
+        for (auto& option : provider_options.options) {
+          // WinML Hotfix: remove backend_type and backend_path from QNN provider options
+          static const std::set<std::string> qnn_options_to_remove{"backend_type", "backend_path"};
+          if (provider_options.name == "QNN" &&
+              qnn_options_to_remove.find(option.first) != qnn_options_to_remove.end()) {
+            continue;
+          }
+
+          // 'device_type' is not a supported option for OpenVINO when SessionOptionsAppendExecutionProvider_V2 is used.
+          if (provider_options.name == "OpenVINO" && option.first == "device_type") {
+            continue;
+          }
+          keys.emplace_back(option.first.c_str());
+          values.emplace_back(option.second.c_str());
+        }
+
+        Ort::api->SessionOptionsAppendExecutionProvider_V2(
+            &session_options,
+            &GetOrtEnv(),
+            ep_devices_ptrs.data(), ep_devices_ptrs.size(),
+            keys.data(), values.data(), keys.size());
+      }
+#else
       std::vector<const char*> keys, values;
       for (auto& option : provider_options.options) {
         keys.emplace_back(option.first.c_str());
         values.emplace_back(option.second.c_str());
       }
       session_options.AppendExecutionProvider(provider_options.name.c_str(), keys.data(), values.data(), keys.size());
+
+#endif
     }
   }
   return p_device;

src/python/CMakeLists.txt

@@ -26,7 +26,9 @@ source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR} FILES ${python_srcs})
 if(BUILD_WHEEL)
   set(WHEEL_FILES_DIR "${CMAKE_BINARY_DIR}/wheel")
   message("Setting up wheel files in : ${WHEEL_FILES_DIR}")
-  if(USE_CUDA)
+  if(USE_WINML)
+    set(TARGET_NAME "onnxruntime-genai-winml")
+  elseif(USE_CUDA)
     set(TARGET_NAME "onnxruntime-genai-cuda")
   elseif(USE_ROCM)
     set(TARGET_NAME "onnxruntime-genai-rocm")
@@ -45,6 +47,7 @@ if(BUILD_WHEEL)
   file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/package_description.md" DESTINATION ${WHEEL_FILES_DIR}/)
   file(COPY "${CMAKE_SOURCE_DIR}/ThirdPartyNotices.txt" DESTINATION ${WHEEL_TARGET_NAME}/)
   file(COPY "${CMAKE_SOURCE_DIR}/LICENSE" DESTINATION ${WHEEL_TARGET_NAME}/)
+
   add_custom_command(TARGET python POST_BUILD
     COMMAND ${CMAKE_COMMAND} -E copy
     ${ortgenai_embed_libs} $<TARGET_FILE:python>

src/python/py/_dll_directory.py

@@ -40,6 +40,14 @@ def add_onnxruntime_dependency(package_id: str):
             import ctypes
             _ = ctypes.CDLL(dml_path)
 
+        # temporary workaround for onnxruntime.dll loading
+        ort_path = os.path.join(ort_package_path, "capi", "onnxruntime.dll")
+        # The dependent onnxruntime package may have multiple execution providers.
+        # Check to see if onnxruntime.dll exists before trying to load it.
+        if os.path.exists(ort_path):
+            import ctypes
+            _ = ctypes.CDLL(ort_path)
+
     elif _is_linux() or _is_macos():
         import importlib.util
         import ctypes

src/python/setup.py.in

@@ -19,7 +19,7 @@ if not path.exists(long_description_file):
 with open(long_description_file, encoding="utf-8") as fdesc:
     long_description = fdesc.read()
 
-extras = ["ThirdPartyNotices.txt", "LICENSE"]
+extras = ["ThirdPartyNotices.txt", "LICENSE", "MSIX/**/*.msix"]
 package_name = '@TARGET_NAME@'
 
 
@@ -46,6 +46,13 @@ def _onnxruntime_dependency() -> str:
 
     return dependency if not ort_version else dependency + ">=" + ort_version
 
+def _get_install_requires():
+    install_requires = [
+        'numpy>=1.21.6',
+    ]
+    if package_name != "onnxruntime-genai-winml":
+        install_requires.append(_onnxruntime_dependency())
+    return install_requires
 
 setup(
     name=package_name,
@@ -56,10 +63,7 @@ setup(
     packages=['onnxruntime_genai', 'onnxruntime_genai.models'],
     include_package_data=True,
     package_data={'': ['*.pyd', '*.dll', '*.so*', '*.dylib'] + extras},
-    install_requires=[
-        'numpy>=1.21.6',
-        _onnxruntime_dependency(),
-    ],
+    install_requires=_get_install_requires(),
     distclass=BinaryDistribution,
     author="Microsoft Corporation",
     author_email="onnxruntime-genai@microsoft.com",