Skip to content

Commit 9ed343e

Browse files
Shiva Kumarshivakunv
Shiva Kumar
authored and
shivakunv
committed
gpu-manager: enable kernel module configuration via KernelModuleConfig
Signed-off-by: Shiva Kumar (SW-CLOUD) <shivaku@nvidia.com>
1 parent f9750fb commit 9ed343e

4 files changed

Lines changed: 160 additions & 4 deletions

File tree

‎vgpu-manager/rhel8/nvidia-driver‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ DRIVER_VERSION=${DRIVER_VERSION:?"Missing driver version"}
77
DRIVER_RESET_RETRIES=10
88
DELAY_BEFORE_VF_CREATION=${DELAY_BEFORE_VF_CREATION:-15}
99
RUN_DIR=/run/nvidia
10+
NVIDIA_MODULE_PARAMS=()
1011

1112
# Mount the driver rootfs into the run directory with the exception of sysfs.
1213
_mount_rootfs() {
@@ -52,14 +53,52 @@ _set_fw_search_path() {
5253
echo -n "$nv_fw_search_path" > $fw_path_config_file
5354
}
5455

56+
# For each kernel module configuration file mounted into the container,
57+
# parse the file contents and extract the custom module parameters that
58+
# are to be passed as input to 'modprobe'.
59+
#
60+
# Assumptions:
61+
# - Configuration file is named nvidia.conf
62+
# - Configuration file is mounted inside the container at /drivers.
63+
# - Each line in the file contains at least one parameter, where parameters on the same line
64+
# are space delimited. It is up to the user to properly format the file to ensure
65+
# the correct set of parameters are passed to 'modprobe'.
66+
_get_module_params() {
67+
local base_path="/drivers"
68+
# nvidia
69+
if [ -f "${base_path}/nvidia.conf" ]; then
70+
while IFS="" read -r param || [ -n "$param" ]; do
71+
NVIDIA_MODULE_PARAMS+=("$param")
72+
done <"${base_path}/nvidia.conf"
73+
echo "Module parameters provided for nvidia: ${NVIDIA_MODULE_PARAMS[@]}"
74+
fi
75+
}
76+
5577
_install_driver() {
5678
local tmp_dir=$(mktemp -d)
5779

5880
sh NVIDIA-Linux-${DRIVER_ARCH}-${DRIVER_VERSION}-vgpu-kvm.run --ui=none --no-questions --tmpdir ${tmp_dir} --no-systemd
5981
}
6082

61-
# Currently _install_driver() takes care of loading nvidia modules. Just need to start necessary vgpu daemons
83+
# Load NVIDIA driver kernel modules with custom parameters and start vGPU daemons
6284
_load_driver() {
85+
# Unload modules if they're already loaded so we can reload with custom parameters
86+
if [ -f /sys/module/nvidia_vgpu_vfio/refcnt ] || [ -f /sys/module/nvidia/refcnt ]; then
87+
echo "NVIDIA modules already loaded by installer, unloading to apply custom parameters..."
88+
rmmod nvidia_vgpu_vfio 2>/dev/null || true
89+
rmmod nvidia 2>/dev/null || true
90+
fi
91+
92+
echo "Parsing kernel module parameters..."
93+
_get_module_params
94+
95+
echo "Loading NVIDIA driver kernel modules..."
96+
set -o xtrace +o nounset
97+
modprobe nvidia "${NVIDIA_MODULE_PARAMS[@]}"
98+
modprobe nvidia_vgpu_vfio
99+
set +o xtrace -o nounset
100+
101+
# Start vGPU daemons
63102
/usr/bin/nvidia-vgpud
64103
/usr/bin/nvidia-vgpu-mgr &
65104

‎vgpu-manager/rhel9/nvidia-driver‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ DRIVER_VERSION=${DRIVER_VERSION:?"Missing driver version"}
1919
DRIVER_RESET_RETRIES=10
2020
DELAY_BEFORE_VF_CREATION=${DELAY_BEFORE_VF_CREATION:-15}
2121
RUN_DIR=/run/nvidia
22+
NVIDIA_MODULE_PARAMS=()
2223

2324
# Mount the driver rootfs into the run directory with the exception of sysfs.
2425
_mount_rootfs() {
@@ -64,14 +65,52 @@ _set_fw_search_path() {
6465
echo -n "$nv_fw_search_path" > $fw_path_config_file
6566
}
6667

68+
# For each kernel module configuration file mounted into the container,
69+
# parse the file contents and extract the custom module parameters that
70+
# are to be passed as input to 'modprobe'.
71+
#
72+
# Assumptions:
73+
# - Configuration file is named nvidia.conf
74+
# - Configuration file is mounted inside the container at /drivers.
75+
# - Each line in the file contains at least one parameter, where parameters on the same line
76+
# are space delimited. It is up to the user to properly format the file to ensure
77+
# the correct set of parameters are passed to 'modprobe'.
78+
_get_module_params() {
79+
local base_path="/drivers"
80+
# nvidia
81+
if [ -f "${base_path}/nvidia.conf" ]; then
82+
while IFS="" read -r param || [ -n "$param" ]; do
83+
NVIDIA_MODULE_PARAMS+=("$param")
84+
done <"${base_path}/nvidia.conf"
85+
echo "Module parameters provided for nvidia: ${NVIDIA_MODULE_PARAMS[@]}"
86+
fi
87+
}
88+
6789
_install_driver() {
6890
local tmp_dir=$(mktemp -d)
6991

7092
sh NVIDIA-Linux-${DRIVER_ARCH}-${DRIVER_VERSION}-vgpu-kvm.run --ui=none --no-questions --tmpdir ${tmp_dir} --no-systemd
7193
}
7294

73-
# Currently _install_driver() takes care of loading nvidia modules. Just need to start necessary vgpu daemons
95+
# Load NVIDIA driver kernel modules with custom parameters and start vGPU daemons
7496
_load_driver() {
97+
# Unload modules if they're already loaded so we can reload with custom parameters
98+
if [ -f /sys/module/nvidia_vgpu_vfio/refcnt ] || [ -f /sys/module/nvidia/refcnt ]; then
99+
echo "NVIDIA modules already loaded by installer, unloading to apply custom parameters..."
100+
rmmod nvidia_vgpu_vfio 2>/dev/null || true
101+
rmmod nvidia 2>/dev/null || true
102+
fi
103+
104+
echo "Parsing kernel module parameters..."
105+
_get_module_params
106+
107+
echo "Loading NVIDIA driver kernel modules..."
108+
set -o xtrace +o nounset
109+
modprobe nvidia "${NVIDIA_MODULE_PARAMS[@]}"
110+
modprobe nvidia_vgpu_vfio
111+
set +o xtrace -o nounset
112+
113+
# Start vGPU daemons
75114
/usr/bin/nvidia-vgpud
76115
/usr/bin/nvidia-vgpu-mgr &
77116

‎vgpu-manager/ubuntu22.04/nvidia-driver‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ DRIVER_RESET_RETRIES=10
88
DELAY_BEFORE_VF_CREATION=${DELAY_BEFORE_VF_CREATION:-15}
99
KERNEL_VERSION=$(uname -r)
1010
RUN_DIR=/run/nvidia
11+
NVIDIA_MODULE_PARAMS=()
1112

1213
export DEBIAN_FRONTEND=noninteractive
1314

@@ -133,8 +134,46 @@ _set_fw_search_path() {
133134
echo -n "$nv_fw_search_path" > $fw_path_config_file
134135
}
135136

136-
# Currently _install_driver() takes care of loading nvidia modules. Just need to start necessary vgpu daemons
137+
# For each kernel module configuration file mounted into the container,
138+
# parse the file contents and extract the custom module parameters that
139+
# are to be passed as input to 'modprobe'.
140+
#
141+
# Assumptions:
142+
# - Configuration file is named nvidia.conf
143+
# - Configuration file is mounted inside the container at /drivers.
144+
# - Each line in the file contains at least one parameter, where parameters on the same line
145+
# are space delimited. It is up to the user to properly format the file to ensure
146+
# the correct set of parameters are passed to 'modprobe'.
147+
_get_module_params() {
148+
local base_path="/drivers"
149+
# nvidia
150+
if [ -f "${base_path}/nvidia.conf" ]; then
151+
while IFS="" read -r param || [ -n "$param" ]; do
152+
NVIDIA_MODULE_PARAMS+=("$param")
153+
done <"${base_path}/nvidia.conf"
154+
echo "Module parameters provided for nvidia: ${NVIDIA_MODULE_PARAMS[@]}"
155+
fi
156+
}
157+
158+
# Load NVIDIA driver kernel modules with custom parameters and start vGPU daemons
137159
_load_driver() {
160+
# Unload modules if they're already loaded so we can reload with custom parameters
161+
if [ -f /sys/module/nvidia_vgpu_vfio/refcnt ] || [ -f /sys/module/nvidia/refcnt ]; then
162+
echo "NVIDIA modules already loaded by installer, unloading to apply custom parameters..."
163+
rmmod nvidia_vgpu_vfio 2>/dev/null || true
164+
rmmod nvidia 2>/dev/null || true
165+
fi
166+
167+
echo "Parsing kernel module parameters..."
168+
_get_module_params
169+
170+
echo "Loading NVIDIA driver kernel modules..."
171+
set -o xtrace +o nounset
172+
modprobe nvidia "${NVIDIA_MODULE_PARAMS[@]}"
173+
modprobe nvidia_vgpu_vfio
174+
set +o xtrace -o nounset
175+
176+
# Start vGPU daemons
138177
/usr/bin/nvidia-vgpud
139178
/usr/bin/nvidia-vgpu-mgr &
140179

‎vgpu-manager/ubuntu24.04/nvidia-driver‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ DRIVER_RESET_RETRIES=10
88
DELAY_BEFORE_VF_CREATION=${DELAY_BEFORE_VF_CREATION:-15}
99
KERNEL_VERSION=$(uname -r)
1010
RUN_DIR=/run/nvidia
11+
NVIDIA_MODULE_PARAMS=()
1112

1213
export DEBIAN_FRONTEND=noninteractive
1314

@@ -133,8 +134,46 @@ _install_driver() {
133134
sh NVIDIA-Linux-${DRIVER_ARCH}-${DRIVER_VERSION}-vgpu-kvm.run --ui=none --no-questions --tmpdir ${tmp_dir} --no-systemd
134135
}
135136

136-
# Currently _install_driver() takes care of loading nvidia modules. Just need to start necessary vgpu daemons
137+
# For each kernel module configuration file mounted into the container,
138+
# parse the file contents and extract the custom module parameters that
139+
# are to be passed as input to 'modprobe'.
140+
#
141+
# Assumptions:
142+
# - Configuration file is named nvidia.conf
143+
# - Configuration file is mounted inside the container at /drivers.
144+
# - Each line in the file contains at least one parameter, where parameters on the same line
145+
# are space delimited. It is up to the user to properly format the file to ensure
146+
# the correct set of parameters are passed to 'modprobe'.
147+
_get_module_params() {
148+
local base_path="/drivers"
149+
# nvidia
150+
if [ -f "${base_path}/nvidia.conf" ]; then
151+
while IFS="" read -r param || [ -n "$param" ]; do
152+
NVIDIA_MODULE_PARAMS+=("$param")
153+
done <"${base_path}/nvidia.conf"
154+
echo "Module parameters provided for nvidia: ${NVIDIA_MODULE_PARAMS[@]}"
155+
fi
156+
}
157+
158+
# Load NVIDIA driver kernel modules with custom parameters and start vGPU daemons
137159
_load_driver() {
160+
# Unload modules if they're already loaded so we can reload with custom parameters
161+
if [ -f /sys/module/nvidia_vgpu_vfio/refcnt ] || [ -f /sys/module/nvidia/refcnt ]; then
162+
echo "NVIDIA modules already loaded by installer, unloading to apply custom parameters..."
163+
rmmod nvidia_vgpu_vfio 2>/dev/null || true
164+
rmmod nvidia 2>/dev/null || true
165+
fi
166+
167+
echo "Parsing kernel module parameters..."
168+
_get_module_params
169+
170+
echo "Loading NVIDIA driver kernel modules..."
171+
set -o xtrace +o nounset
172+
modprobe nvidia "${NVIDIA_MODULE_PARAMS[@]}"
173+
modprobe nvidia_vgpu_vfio
174+
set +o xtrace -o nounset
175+
176+
# Start vGPU daemons
138177
/usr/bin/nvidia-vgpud
139178
/usr/bin/nvidia-vgpu-mgr &
140179

0 commit comments

Comments
 (0)