Enable re-init after OgaShutdown

CMakeLists.txt

@@ -94,6 +94,19 @@ if(MSVC)
     "$<$<COMPILE_LANGUAGE:C>:/WX>"
     "$<$<COMPILE_LANGUAGE:CXX>:/WX>"
   )
+
+  # MSVC 19.50+ (VS 2026) introduced C4875: non-string literal argument to [[gsl::suppress]] is
+  # deprecated. The GSL version used by onnxruntime_extensions (and other deps) hasn't adopted
+  # string-literal syntax yet. Suppress globally so third-party headers build cleanly.
+  # Also silence the STL1011 static assertion for <experimental/coroutine> being deprecated
+  # (ocos_operators in onnxruntime_extensions still uses it; the fix is upstream).
+  if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "19.50")
+    add_compile_options(
+      "$<$<COMPILE_LANGUAGE:C>:/wd4875>"
+      "$<$<COMPILE_LANGUAGE:CXX>:/wd4875>"
+    )
+    add_compile_definitions(_SILENCE_EXPERIMENTAL_COROUTINE_DEPRECATION_WARNINGS)
+  endif()
 endif()
 
 include(cmake/ortlib.cmake)
@@ -155,23 +168,39 @@ endif()
 
 find_package(Threads REQUIRED)
 
+# genai is split into an OBJECT library (all sources + their compile/link usage requirements) and
+# the SHARED library that ships it. White-box test binaries (e.g. reinit_tests) link the object
+# library directly to reach internal symbols; the shipped DLL and every other consumer keep linking
+# the SHARED onnxruntime-genai target. Usage requirements are PUBLIC on the object library so both
+# the shared library and such tests inherit them. The shared library links the object library
+# PRIVATE, so DLL consumers (python / java / benchmark / unit_tests) do not inherit genai's private
+# dependencies -- they just link the shared onnxruntime-genai library directly.
+add_library(onnxruntime-genai-obj OBJECT ${generator_srcs})
+target_include_directories(onnxruntime-genai-obj PUBLIC ${ORT_HEADER_DIR})
+target_include_directories(onnxruntime-genai-obj PUBLIC ${onnxruntime_extensions_SOURCE_DIR}/shared/api)
+target_link_libraries(onnxruntime-genai-obj PUBLIC onnxruntime_extensions)
+target_link_directories(onnxruntime-genai-obj PUBLIC ${ORT_LIB_DIR})
+target_link_libraries(onnxruntime-genai-obj PUBLIC Threads::Threads)
+
 if(WIN32)
-  add_library(onnxruntime-genai SHARED ${generator_srcs} "${GENERATORS_ROOT}/dll/onnxruntime-genai.rc")
+  add_library(onnxruntime-genai SHARED "${GENERATORS_ROOT}/dll/onnxruntime-genai.rc")
   target_compile_definitions(onnxruntime-genai PRIVATE VERSION_INFO=\"${VERSION_INFO}\")
   target_compile_definitions(onnxruntime-genai PRIVATE VERSION_MAJOR=${VERSION_MAJOR})
   target_compile_definitions(onnxruntime-genai PRIVATE VERSION_MINOR=${VERSION_MINOR})
   target_compile_definitions(onnxruntime-genai PRIVATE VERSION_PATCH=${VERSION_PATCH})
   target_compile_definitions(onnxruntime-genai PRIVATE VERSION_SUFFIX=${VERSION_SUFFIX})
   target_compile_definitions(onnxruntime-genai PRIVATE FILE_NAME=\"onnxruntime-genai.dll\")
 else()
-  add_library(onnxruntime-genai SHARED ${generator_srcs})
+  add_library(onnxruntime-genai SHARED)
 endif()
+# Bring in the object library's compiled objects and its (PUBLIC) usage requirements. PRIVATE so
+# the requirements are not re-exported to consumers of the shared library.
+target_link_libraries(onnxruntime-genai PRIVATE onnxruntime-genai-obj)
 
-target_include_directories(onnxruntime-genai PRIVATE ${ORT_HEADER_DIR})
-target_include_directories(onnxruntime-genai PRIVATE ${onnxruntime_extensions_SOURCE_DIR}/shared/api)
-target_link_libraries(onnxruntime-genai PRIVATE onnxruntime_extensions)
-target_link_directories(onnxruntime-genai PRIVATE ${ORT_LIB_DIR})
-target_link_libraries(onnxruntime-genai PRIVATE Threads::Threads)
+# Group the genai library targets under one Visual Studio solution folder so the object library and
+# the shared library it produces appear together rather than as loose entries at the solution root.
+# Cosmetic only (IDE generators); no effect on the build.
+set_target_properties(onnxruntime-genai-obj onnxruntime-genai PROPERTIES FOLDER "GenAI")
 
 # The genai library itself is always embedded in the shared library
 list(APPEND ortgenai_embed_libs "$<TARGET_FILE:onnxruntime-genai>")
@@ -181,14 +210,14 @@ list(APPEND ortgenai_embed_libs "$<TARGET_FILE:onnxruntime-genai>")
 if(CMAKE_SYSTEM_NAME STREQUAL "Android" OR CMAKE_SYSTEM_NAME STREQUAL "Linux" OR (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND (NOT BUILD_APPLE_FRAMEWORK) AND (NOT MAC_CATALYST)))
   add_compile_definitions(_ORT_GENAI_USE_DLOPEN)
 else()
-  target_link_libraries(onnxruntime-genai PRIVATE ${ONNXRUNTIME_LIB})
+  target_link_libraries(onnxruntime-genai-obj PUBLIC ${ONNXRUNTIME_LIB})
   if(USE_WINML)
     target_link_options(onnxruntime-genai PRIVATE "/DELAYLOAD:${ONNXRUNTIME_LIB}")
   endif()
 endif()
 
 if(APPLE)
-target_link_libraries(onnxruntime-genai PRIVATE "-framework Foundation" "-framework CoreML")
+target_link_libraries(onnxruntime-genai-obj PUBLIC "-framework Foundation" "-framework CoreML")
 endif()
 
 
@@ -227,6 +256,7 @@ if((USE_CUDA OR USE_TRT_RTX) AND CMAKE_CUDA_COMPILER)
   target_include_directories(onnxruntime-genai-cuda PRIVATE ${GENERATORS_ROOT})
   target_link_libraries(onnxruntime-genai-cuda PRIVATE cublasLt cublas curand cufft cudart)
   set_target_properties(onnxruntime-genai-cuda PROPERTIES LINKER_LANGUAGE CUDA)
+  set_target_properties(onnxruntime-genai-cuda PROPERTIES FOLDER "GenAI")
   add_dependencies(onnxruntime-genai onnxruntime-genai-cuda)
   source_group(TREE ${GENERATORS_ROOT}/cuda FILES ${generator_cudalib_srcs})
   list(APPEND ortgenai_embed_libs "$<TARGET_FILE:onnxruntime-genai-cuda>")
@@ -243,11 +273,11 @@ endif()
 
 
 if(USE_GUIDANCE)
-  target_include_directories(onnxruntime-genai PUBLIC ${llguidance_SOURCE_DIR}/parser/)
-  target_link_libraries(onnxruntime-genai PRIVATE llguidance)
+  target_include_directories(onnxruntime-genai-obj PUBLIC ${llguidance_SOURCE_DIR}/parser/)
+  target_link_libraries(onnxruntime-genai-obj PUBLIC llguidance)
   if (WIN32)
     # bcrypt is needed for the rust std lib
-    target_link_libraries(onnxruntime-genai PRIVATE bcrypt)
+    target_link_libraries(onnxruntime-genai-obj PUBLIC bcrypt)
   endif()
   if(MSVC)
     # The Rust llguidance static library is always compiled against the release MSVC CRT
@@ -260,12 +290,14 @@ if(USE_GUIDANCE)
     # Fix: explicitly add the debug CRT import libraries in Debug builds. Explicitly specified
     # libraries are not affected by /NODEFAULTLIB directives (those only suppress /DEFAULTLIB
     # auto-linking). Also suppress the conflicting release CRT to avoid LNK4098 warnings.
-    target_link_libraries(onnxruntime-genai PRIVATE
+    # PUBLIC/INTERFACE so any binary linking the object library (the shared lib and white-box tests)
+    # gets the same fixups.
+    target_link_libraries(onnxruntime-genai-obj PUBLIC
       $<$<CONFIG:Debug>:msvcrtd.lib>
       $<$<CONFIG:Debug>:ucrtd.lib>
       $<$<CONFIG:Debug>:vcruntimed.lib>
     )
-    target_link_options(onnxruntime-genai PRIVATE
+    target_link_options(onnxruntime-genai-obj INTERFACE
       $<$<CONFIG:Debug>:/NODEFAULTLIB:msvcrt.lib>
       $<$<CONFIG:Debug>:/NODEFAULTLIB:ucrt.lib>
       $<$<CONFIG:Debug>:/NODEFAULTLIB:vcruntime.lib>
@@ -277,24 +309,24 @@ if(USE_GUIDANCE)
 endif()
 
 if(CMAKE_GENERATOR_TOOLSET MATCHES "Visual Studio")
-  target_compile_options(onnxruntime-genai PRIVATE "/sdl")
+  target_compile_options(onnxruntime-genai-obj PRIVATE "/sdl")
 endif()
 
 if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
-  set_target_properties(onnxruntime-genai PROPERTIES POSITION_INDEPENDENT_CODE ON)
-  target_link_libraries(onnxruntime-genai PUBLIC dl)  # For dlopen & co
+  set_target_properties(onnxruntime-genai-obj PROPERTIES POSITION_INDEPENDENT_CODE ON)
+  target_link_libraries(onnxruntime-genai-obj PUBLIC dl)  # For dlopen & co
 endif()
 
 if(USE_DML)
   list(APPEND ortgenai_embed_libs "${D3D12_LIB_DIR}/D3D12Core.dll")
-  target_include_directories(onnxruntime-genai PRIVATE $<TARGET_PROPERTY:${WIL_TARGET},INTERFACE_INCLUDE_DIRECTORIES>)
-  target_include_directories(onnxruntime-genai PRIVATE $<TARGET_PROPERTY:${DIRECTX_HEADERS_TARGET},INTERFACE_INCLUDE_DIRECTORIES>/directx)
-  target_include_directories(onnxruntime-genai PRIVATE $<TARGET_PROPERTY:${DIRECTX_HEADERS_TARGET},INTERFACE_INCLUDE_DIRECTORIES>)
-  target_include_directories(onnxruntime-genai PRIVATE ${DML_HEADER_DIR})
-  target_include_directories(onnxruntime-genai PRIVATE ${D3D12_HEADER_DIR})
-  target_link_directories(onnxruntime-genai PRIVATE ${DML_LIB_DIR})
-  target_link_directories(onnxruntime-genai PRIVATE ${D3D12_LIB_DIR})
-  target_link_libraries(onnxruntime-genai PRIVATE d3d12.lib dxcore.lib dxguid.lib dxgi.lib)
+  target_include_directories(onnxruntime-genai-obj PRIVATE $<TARGET_PROPERTY:${WIL_TARGET},INTERFACE_INCLUDE_DIRECTORIES>)
+  target_include_directories(onnxruntime-genai-obj PRIVATE $<TARGET_PROPERTY:${DIRECTX_HEADERS_TARGET},INTERFACE_INCLUDE_DIRECTORIES>/directx)
+  target_include_directories(onnxruntime-genai-obj PRIVATE $<TARGET_PROPERTY:${DIRECTX_HEADERS_TARGET},INTERFACE_INCLUDE_DIRECTORIES>)
+  target_include_directories(onnxruntime-genai-obj PRIVATE ${DML_HEADER_DIR})
+  target_include_directories(onnxruntime-genai-obj PRIVATE ${D3D12_HEADER_DIR})
+  target_link_directories(onnxruntime-genai-obj PUBLIC ${DML_LIB_DIR})
+  target_link_directories(onnxruntime-genai-obj PUBLIC ${D3D12_LIB_DIR})
+  target_link_libraries(onnxruntime-genai-obj PUBLIC d3d12.lib dxcore.lib dxguid.lib dxgi.lib)
 
   get_filename_component(PACKAGES_DIR ${CMAKE_CURRENT_BINARY_DIR}/_deps ABSOLUTE)
   set(DXC_PACKAGE_DIR ${PACKAGES_DIR}/Microsoft.Direct3D.DXC.1.7.2308.12)
@@ -318,7 +350,7 @@ if(USE_DML)
   )
 
   add_dependencies(RESTORE_PACKAGES nuget)
-  add_dependencies(onnxruntime-genai RESTORE_PACKAGES)
+  add_dependencies(onnxruntime-genai-obj RESTORE_PACKAGES)
 endif()
 
 if(ANDROID)

docs/EnableReinitAfterOgaShutdown.md

@@ -0,0 +1,172 @@
+# Plan: Clean `OgaShutdown` and re-initialization
+
+`OgaShutdown` returns genai to a *just-loaded* state. It tears down all of
+genai's ONNX Runtime-derived global state -- the `OrtEnv`, the device
+interfaces, the genai add-on libraries, and the trivial-session allocators --
+and a subsequent genai call transparently re-initializes with a fresh `OrtEnv`.
+This lets a host that recreates its wrapper in-process (for example Foundry
+Local's `Manager`) get a clean environment -- fresh logging and env
+configuration -- without restarting the process.
+
+## Background
+
+`OrtEnv` is a process-wide, refcounted singleton. genai holds one reference to
+it inside its global state (`OrtGlobals`). Previously that state was created
+once via a function-static and never rebuilt, so after `OgaShutdown` reset it
+genai could not be used again in the same process. The changes here move all
+env-scoped state into a single owner (`OrtGlobals`) that is destroyed on
+shutdown and rebuilt on demand.
+
+## `OrtGlobals` owns all env-scoped state
+
+`OrtGlobals` (`src/generators.cpp`, `src/generators.h`) is the single owner of
+everything whose lifetime is tied to the `OrtEnv`:
+
+- `env_` -- the `OrtEnv`.
+- `owned_interfaces_` -- the in-process `DeviceInterface` instances (CPU,
+  WebGPU, QNN, OpenVINO, RyzenAI), owned directly.
+- `cuda_library_` -- the genai CUDA add-on library handle (see below).
+- `device_interfaces_` -- a non-owning `DeviceType -> DeviceInterface*` lookup
+  that indexes the owners above (and the module-owned DML interface).
+- `device_allocators_` -- the trivial-session allocators (`{session, allocator}`
+  per device type) that bootstrap the non-CPU device allocators.
+- `graph_session_cache_` -- cached dynamically-built sessions (Cast, TopK, ...).
+
+Because all of this lives in one object, tearing genai down is a single
+`OrtGlobals` destruction and re-initialization is a single `OrtGlobals`
+construction.
+
+## Re-creatable globals
+
+`g_ort_globals` is a `std::unique_ptr<OrtGlobals>` that `GetOrtGlobals()` builds
+on demand:
+
+- `GetOrtGlobals()` lazily (re)creates the globals whenever they are null, so
+  the first call after `OgaShutdown` rebuilds them with a fresh `OrtEnv`.
+- Creation is guarded by a mutex rather than `std::call_once` (which is
+  process-once and would prevent re-init), so concurrent first-use -- or
+  first-use after a shutdown -- cannot double-construct the globals.
+- A process-exit guard, `g_process_exiting`, is set by the `EnsureShutdown`
+  static destructor so `GetOrtGlobals()` cannot resurrect the globals during
+  static destruction.
+- `Shutdown()` (behind `OgaShutdown`) simply resets `g_ort_globals`;
+  `~OrtGlobals` performs all of the teardown.
+
+The `OrtGlobals` constructor bootstraps the CPU interface through a member call
+(`this->GetDeviceInterface(DeviceType::CPU)`) rather than the free
+`GetDeviceInterface()` -> `GetOrtGlobals()` path, so it does not re-enter
+`GetOrtGlobals()` while still being constructed.
+
+## Device interface ownership
+
+Each in-process EP exposes a `CreateXInterface()` factory that returns a
+`std::unique_ptr<DeviceInterface>`. `OrtGlobals::GetDeviceInterface()` creates
+the interface on first use (under a lock), stores it in `owned_interfaces_`, and
+caches a raw pointer in `device_interfaces_`. This replaces the previous per-EP
+`GetXInterface()` accessors that held the interface in a function/namespace
+static (and, for RyzenAI, a `std::call_once`), none of which could be rebuilt
+after a shutdown. Because the interfaces are now owned by `OrtGlobals`,
+`~OrtGlobals` drops them all, and the hard-coded `RyzenAIInterface::Shutdown()`
+that `Shutdown()` used to call is gone.
+
+## CUDA add-on library
+
+The genai CUDA add-on library (`onnxruntime-genai-cuda`) provides its
+`DeviceInterface` from inside the DLL (`g_cuda_device`), along with the CUDA
+stream and a file-static allocator pointer. `OrtGlobals` owns the loaded library
+handle (`cuda_library_`) and holds only a non-owning pointer to the interface in
+`device_interfaces_`. Unloading the library at teardown runs the add-on's static
+destructors, so the interface, the stream, and the file-static allocator all
+cease to exist between cycles -- genai returns to a just-loaded state with no
+cached add-on state.
+
+## Teardown order
+
+`~OrtGlobals` destroys state in the reverse of construction order, so everything
+that uses env-owned resources is gone before the env itself:
+
+1. `graph_session_cache_` -- genai-side session caches of dynamically created models for pre/post processing.
+2. `device_interfaces_` (the non-owning index), then `owned_interfaces_`, then
+   `cuda_library_` -- the device interfaces and the CUDA add-on library.
+3. `device_allocators_` -- the trivial-session allocators. Each entry bundles its
+   own dedicated dummy `OrtSession`; the allocator wraps that bundled session (not
+   any session in `graph_session_cache_`), so the relative order of steps 1 and 3
+   is not a correctness constraint. Within each entry the `OrtSession` is declared
+   before the `Ort::Allocator`, so the allocator is destroyed before its session.
+4. `env_` -- the `OrtEnv`. If genai held the last reference, ORT destroys the
+   environment here.
+
+Ending all allocator use before the env is destroyed keeps teardown correct even
+if a future interface or buffer were to dereference a cached allocator in its own
+destructor.
+
+## CPU allocator
+
+The CPU interface allocates through ONNX Runtime's process-global default
+allocator (`Ort::Allocator::GetWithDefaultOptions()`), not an env-scoped one, so
+nothing genai caches for CPU dangles when the env is destroyed. The only change
+CPU needs for re-init is that its `InitOrt` is idempotent: `OrtGlobals` re-runs
+`CreateAndRegisterAllocator` on the new env and calls `InitOrt` again with the
+same process-global allocator each cycle, so the previous
+`assert(!ort_allocator_)` is removed.
+
+## DML
+
+DML is a deliberate exception to the ownership model above, for two reasons:
+
+1. Its interface is built from the model's `luid` / `device_index` provider
+   options (parsed in `src/dml/session_options.cpp` and passed to
+   `InitDmlInterface`), which the generic `GetDeviceInterface(DeviceType::DML)`
+   cannot supply.
+2. DML objects launch background threads that must be released promptly, so
+   `Model::~Model` tears the DML interface down per-`Model` via
+   `CloseDmlInterface()`.
+
+Because DML is destroyed per-`Model`, `OrtGlobals::GetDeviceInterface` must
+**not** cache the DML interface -- a cached pointer would dangle after the first
+DML model is freed and be handed to a later one. The `#if USE_DML` arm therefore
+returns the live `GetDmlInterface()` instead of memoizing it (all other EPs,
+which live for the env cycle, are cached). DML is already re-init-safe through
+this per-`Model` teardown: after the last DML model is destroyed, no DML
+interface or `Dml::` static survives into the next env cycle.
+
+## RyzenAI
+
+RyzenAI is owned by `OrtGlobals` like the other in-process EPs (created via
+`CreateRyzenAIInterface(env)`), and the hard-coded `RyzenAIInterface::Shutdown()`
+is removed -- teardown now happens when `~OrtGlobals` drops the interface. EP-library
+registration is per-`OrtEnv` (ORT keys it by registration name), so on each env cycle
+the interface registers the RyzenAI EP library on the fresh env even when the EP module
+is already resident in the process from an earlier cycle. Because an `OrtEnv` can outlive
+an `OgaShutdown` when the host holds its own reference, the registration call tolerates
+ORT's "library is already registered" status for that env rather than skipping
+registration up front, so RyzenAI is re-init-safe in both cases.
+
+Teardown of the EP module itself (the `RyzenAI_Shutdown` call in the interface destructor)
+is gated on genai having loaded it: each interface records at construction, before it
+registers/loads the EP, whether the module was already resident. genai calls
+`RyzenAI_Shutdown` only when it was not -- i.e. genai's own registration loaded it. If an
+external owner had already loaded it, genai never shuts it down, so it cannot tear down EP
+state another owner still relies on. ORT unloads the EP library when the interface's env is
+torn down, so the next re-init cycle re-evaluates this against a freshly unloaded module.
+
+## Lifetime contract
+
+No `Model`, `Generator`, `Tokenizer`, `OgaTensor`, `Engine`, `Request`, or any
+object that holds device memory may outlive `OgaShutdown`. The caller must
+destroy every such object before calling `OgaShutdown`; doing otherwise is
+undefined behavior (typically a crash when the buffer is freed through a
+now-invalid allocator). Debug builds report any leaked genai objects at shutdown.
+
+## Testing
+
+Re-initialization is covered by a dedicated `reinit_tests` executable, kept
+separate from `unit_tests` because it calls `OgaShutdown`, which resets
+process-global state and so must not share a process with the public-API suite.
+`reinit_tests` links the genai object library (`onnxruntime-genai-obj`) rather
+than the shared library, giving white-box access to `GetDeviceInterface` so it
+can force each available device interface to be created and torn down directly
+across repeated shutdown -> re-init cycles (no model load required). The
+object-library split -- the shipped shared library and the white-box tests both
+consume `onnxruntime-genai-obj` -- lets the tests reach internal symbols without
+exporting them from the shipped DLL.

src/cpu/interface.cpp

@@ -49,7 +49,9 @@ struct CpuInterface : DeviceInterface {
   DeviceType GetType() const override { return DeviceType::CPU; }
 
   void InitOrt(const OrtApi& /*api*/, Ort::Allocator& allocator) override {
-    assert(!ort_allocator_);
+    // Idempotent: on re-initialization this is called again with the same process-global default
+    // allocator (CpuMemory allocates through Ort::Allocator::GetWithDefaultOptions(), which is not
+    // env-scoped), so re-assigning the same pointer is harmless.
     ort_allocator_ = &allocator;
   }
 
@@ -166,9 +168,8 @@ struct CpuInterface : DeviceInterface {
   void Synchronize() override {}  // Nothing to do as CPU is always in sync with itself
 };
 
-DeviceInterface* GetCpuInterface() {
-  static std::unique_ptr<CpuInterface> g_cpu = std::make_unique<CpuInterface>();
-  return g_cpu.get();
+std::unique_ptr<DeviceInterface> CreateCpuInterface() {
+  return std::make_unique<CpuInterface>();
 }
 
 }  // namespace Generators