Stage python sources per file instead of copying the tree (NVIDIA#4875)

CopyPythonFiles copied all of python/ into the build directory with a
single always-run command. That copy raced the MLIR machinery's symlink
rules for cudaq/mlir (intermittent "failed to create symbolic link ...:
File exists" in parallel builds), declared no per-file outputs so ninja
could neither order it nor reject duplicate producers, re-copied every
file on every build, and spilled extension/ sources into binary
directories.

Stage the cudaq package and tests with one declared rule per file
(cudaq_stage_python_sources, modeled on add_mlir_python_sources_target
without its install and export machinery), and generate _metadata.py
from a command with a declared output. cudaq/mlir stays owned solely by
the MLIR staging. Content with no build-tree consumers (metapackages,
README, extension/ sources) is no longer copied; wheels take pure python
sources from the source tree via wheel.packages and non-wheel installs
use install(DIRECTORY), both unchanged.

See the failure at:
https://github.com/NVIDIA/cuda-quantum/actions/runs/28880392782/job/85669399820

---------

Signed-off-by: mitchdz <mitch_dz@hotmail.com>
Signed-off-by: mdzurick <mitch_dz@hotmail.com>

Author: mitchdz

cmake/modules/BuildHelpers.cmake

@@ -92,3 +92,31 @@ function(add_target_libs_to_wheel nvqir_backend_lib_or_config)
         message(WARNING "Unknown file extension of ${nvqir_backend_lib_or_config} file. It will be ignored.")
     endif()
 endfunction()
+
+# Stage python sources into the build tree, one symlink rule per file.
+# Modeled on MLIR's add_mlir_python_sources_target minus its install and
+# export machinery; these sources are installed through other mechanisms.
+function(cudaq_stage_python_sources name)
+    cmake_parse_arguments(ARG "" "ROOT_DIR;OUTPUT_DIRECTORY" "SOURCES" ${ARGN})
+    if(ARG_UNPARSED_ARGUMENTS)
+        message(FATAL_ERROR "Unhandled arguments to cudaq_stage_python_sources(${name}): ${ARG_UNPARSED_ARGUMENTS}")
+    endif()
+
+    set(_dest_paths "")
+    foreach(_rel_path ${ARG_SOURCES})
+        set(_src_path "${ARG_ROOT_DIR}/${_rel_path}")
+        set(_dest_path "${ARG_OUTPUT_DIRECTORY}/${_rel_path}")
+        get_filename_component(_dest_dir "${_dest_path}" DIRECTORY)
+        file(MAKE_DIRECTORY "${_dest_dir}")
+        add_custom_command(
+            OUTPUT "${_dest_path}"
+            COMMENT "Staging python source ${_rel_path}"
+            DEPENDS "${_src_path}"
+            COMMAND "${CMAKE_COMMAND}" -E create_symlink
+                "${_src_path}" "${_dest_path}"
+        )
+        list(APPEND _dest_paths "${_dest_path}")
+    endforeach()
+
+    add_custom_target(${name} DEPENDS ${_dest_paths})
+endfunction()

python/CMakeLists.txt

@@ -24,8 +24,6 @@ set(CMAKE_PLATFORM_NO_VERSIONED_SONAME 1)
 
 add_subdirectory(extension)
 
-file(GLOB_RECURSE PYTHON_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/*.py")
-
 if (CUDA_FOUND)
   enable_language(CUDA)
   find_package(CUDAToolkit REQUIRED)
@@ -38,21 +36,46 @@ else()
 endif()
 
 set(METADATA_FILE "${CMAKE_BINARY_DIR}/python/cudaq/_metadata.py" )
-add_custom_target(
-  CopyPythonFiles ALL
-  COMMAND ${CMAKE_COMMAND} -E copy_directory
-  ${CMAKE_CURRENT_SOURCE_DIR}
-  ${CMAKE_BINARY_DIR}/python
+add_custom_command(
+  OUTPUT "${METADATA_FILE}"
   COMMAND ${CMAKE_COMMAND}
   -DMETADATA_FILE="${METADATA_FILE}"
   -DCUDA_VERSION_MAJOR=${CUDAToolkit_VERSION_MAJOR}
   -DASSERTIONS_ENABLED=${CUDAQ_ASSERTIONS_ENABLED}
   -P ${CMAKE_CURRENT_SOURCE_DIR}/metadata.cmake
-  DEPENDS ${PYTHON_SOURCES}
-  BYPRODUCTS "${METADATA_FILE}"
+  DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/metadata.cmake
 )
 
-add_dependencies(CUDAQuantumPythonModules CopyPythonFiles)
+# cudaq/mlir is excluded: the MLIR machinery in extension/ stages it, and
+# a second producer for the same outputs races it. No CONFIGURE_DEPENDS:
+# a mid-ctest reconfigure relinks libraries under running tests. Adding a
+# python file requires a reconfigure.
+file(GLOB_RECURSE _cudaq_package_sources
+  RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/cudaq"
+  "${CMAKE_CURRENT_SOURCE_DIR}/cudaq/*.py")
+list(FILTER _cudaq_package_sources EXCLUDE REGEX "^mlir/")
+cudaq_stage_python_sources(CUDAQPythonPackageStaging
+  ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/cudaq"
+  OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/python/cudaq"
+  SOURCES ${_cudaq_package_sources})
+
+# The pycudaq-mlir lit suite and the interop pytest run from the build tree.
+# mlir/generated holds artifacts lit writes into the source tree at config
+# load; they are discovered from there, not staged.
+file(GLOB_RECURSE _cudaq_test_sources
+  RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/tests"
+  "${CMAKE_CURRENT_SOURCE_DIR}/tests/*")
+list(FILTER _cudaq_test_sources EXCLUDE REGEX
+  "^mlir/generated/|__pycache__|\\.pyc$|\\.DS_Store$")
+cudaq_stage_python_sources(CUDAQPythonTestStaging
+  ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/tests"
+  OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/python/tests"
+  SOURCES ${_cudaq_test_sources})
+
+add_custom_target(CUDAQPythonStaging ALL DEPENDS "${METADATA_FILE}")
+add_dependencies(CUDAQPythonStaging
+  CUDAQPythonPackageStaging CUDAQPythonTestStaging)
+add_dependencies(CUDAQuantumPythonModules CUDAQPythonStaging)
 
 add_subdirectory(runtime/cudaq/domains/plugins)