Skip to content

Commit 6ba105b

Browse files
Shiva Kumarshivakunv
Shiva Kumar
authored and
shivakunv
committed
gpu-manager: enable kernel module configuration via KernelModuleConfig
Signed-off-by: Shiva Kumar (SW-CLOUD) <shivaku@nvidia.com>
1 parent f9750fb commit 6ba105b

4 files changed

Lines changed: 196 additions & 4 deletions

File tree

‎vgpu-manager/rhel8/nvidia-driver‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ DRIVER_VERSION=${DRIVER_VERSION:?"Missing driver version"}
77
DRIVER_RESET_RETRIES=10
88
DELAY_BEFORE_VF_CREATION=${DELAY_BEFORE_VF_CREATION:-15}
99
RUN_DIR=/run/nvidia
10+
NVIDIA_MODULE_PARAMS=()
11+
MODPROBE_CONFIG_DIR="/etc/modprobe.d"
1012

1113
# Mount the driver rootfs into the run directory with the exception of sysfs.
1214
_mount_rootfs() {
@@ -52,14 +54,59 @@ _set_fw_search_path() {
5254
echo -n "$nv_fw_search_path" > $fw_path_config_file
5355
}
5456

57+
# For each kernel module configuration file mounted into the container,
58+
# parse the file contents and extract the custom module parameters that
59+
# are to be passed as input to 'modprobe'.
60+
#
61+
# Assumptions:
62+
# - Configuration file is named nvidia.conf
63+
# - Configuration file is mounted inside the container at /drivers.
64+
# - Each line in the file contains at least one parameter, where parameters on the same line
65+
# are space delimited. It is up to the user to properly format the file to ensure
66+
# the correct set of parameters are passed to 'modprobe'.
67+
_get_module_params() {
68+
local base_path="/drivers"
69+
# nvidia
70+
if [ -f "${base_path}/nvidia.conf" ]; then
71+
while IFS="" read -r param || [ -n "$param" ]; do
72+
NVIDIA_MODULE_PARAMS+=("$param")
73+
done <"${base_path}/nvidia.conf"
74+
echo "Module parameters provided for nvidia: ${NVIDIA_MODULE_PARAMS[@]}"
75+
fi
76+
}
77+
5578
_install_driver() {
5679
local tmp_dir=$(mktemp -d)
5780

5881
sh NVIDIA-Linux-${DRIVER_ARCH}-${DRIVER_VERSION}-vgpu-kvm.run --ui=none --no-questions --tmpdir ${tmp_dir} --no-systemd
5982
}
6083

61-
# Currently _install_driver() takes care of loading nvidia modules. Just need to start necessary vgpu daemons
84+
_create_module_params_conf() {
85+
echo "Parsing kernel module parameters..."
86+
_get_module_params
87+
88+
if [ ${#NVIDIA_MODULE_PARAMS[@]} -gt 0 ]; then
89+
echo "Configuring nvidia module parameters in ${MODPROBE_CONFIG_DIR}/nvidia.conf"
90+
echo "options nvidia ${NVIDIA_MODULE_PARAMS[@]}" > ${MODPROBE_CONFIG_DIR}/nvidia.conf
91+
fi
92+
}
93+
94+
# Load NVIDIA driver kernel modules with custom parameters and start vGPU daemons
6295
_load_driver() {
96+
# Unload modules if they're already loaded so we can reload with custom parameters
97+
if [ -f /sys/module/nvidia_vgpu_vfio/refcnt ] || [ -f /sys/module/nvidia/refcnt ]; then
98+
echo "NVIDIA modules already loaded by installer, unloading to apply custom parameters..."
99+
rmmod nvidia_vgpu_vfio 2>/dev/null || true
100+
rmmod nvidia 2>/dev/null || true
101+
fi
102+
103+
echo "Loading NVIDIA driver kernel modules..."
104+
set -o xtrace +o nounset
105+
modprobe nvidia
106+
modprobe nvidia_vgpu_vfio
107+
set +o xtrace -o nounset
108+
109+
# Start vGPU daemons
63110
/usr/bin/nvidia-vgpud
64111
/usr/bin/nvidia-vgpu-mgr &
65112

@@ -181,6 +228,7 @@ init() {
181228
_unmount_rootfs
182229
_create_dev_char_directory
183230
_set_fw_search_path
231+
_create_module_params_conf
184232
_install_driver
185233
_load_driver || exit 1
186234
_mount_rootfs

‎vgpu-manager/rhel9/nvidia-driver‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ DRIVER_VERSION=${DRIVER_VERSION:?"Missing driver version"}
1919
DRIVER_RESET_RETRIES=10
2020
DELAY_BEFORE_VF_CREATION=${DELAY_BEFORE_VF_CREATION:-15}
2121
RUN_DIR=/run/nvidia
22+
NVIDIA_MODULE_PARAMS=()
23+
MODPROBE_CONFIG_DIR="/etc/modprobe.d"
2224

2325
# Mount the driver rootfs into the run directory with the exception of sysfs.
2426
_mount_rootfs() {
@@ -64,14 +66,59 @@ _set_fw_search_path() {
6466
echo -n "$nv_fw_search_path" > $fw_path_config_file
6567
}
6668

69+
# For each kernel module configuration file mounted into the container,
70+
# parse the file contents and extract the custom module parameters that
71+
# are to be passed as input to 'modprobe'.
72+
#
73+
# Assumptions:
74+
# - Configuration file is named nvidia.conf
75+
# - Configuration file is mounted inside the container at /drivers.
76+
# - Each line in the file contains at least one parameter, where parameters on the same line
77+
# are space delimited. It is up to the user to properly format the file to ensure
78+
# the correct set of parameters are passed to 'modprobe'.
79+
_get_module_params() {
80+
local base_path="/drivers"
81+
# nvidia
82+
if [ -f "${base_path}/nvidia.conf" ]; then
83+
while IFS="" read -r param || [ -n "$param" ]; do
84+
NVIDIA_MODULE_PARAMS+=("$param")
85+
done <"${base_path}/nvidia.conf"
86+
echo "Module parameters provided for nvidia: ${NVIDIA_MODULE_PARAMS[@]}"
87+
fi
88+
}
89+
6790
_install_driver() {
6891
local tmp_dir=$(mktemp -d)
6992

7093
sh NVIDIA-Linux-${DRIVER_ARCH}-${DRIVER_VERSION}-vgpu-kvm.run --ui=none --no-questions --tmpdir ${tmp_dir} --no-systemd
7194
}
7295

73-
# Currently _install_driver() takes care of loading nvidia modules. Just need to start necessary vgpu daemons
96+
_create_module_params_conf() {
97+
echo "Parsing kernel module parameters..."
98+
_get_module_params
99+
100+
if [ ${#NVIDIA_MODULE_PARAMS[@]} -gt 0 ]; then
101+
echo "Configuring nvidia module parameters in ${MODPROBE_CONFIG_DIR}/nvidia.conf"
102+
echo "options nvidia ${NVIDIA_MODULE_PARAMS[@]}" > ${MODPROBE_CONFIG_DIR}/nvidia.conf
103+
fi
104+
}
105+
106+
# Load NVIDIA driver kernel modules with custom parameters and start vGPU daemons
74107
_load_driver() {
108+
# Unload modules if they're already loaded so we can reload with custom parameters
109+
if [ -f /sys/module/nvidia_vgpu_vfio/refcnt ] || [ -f /sys/module/nvidia/refcnt ]; then
110+
echo "NVIDIA modules already loaded by installer, unloading to apply custom parameters..."
111+
rmmod nvidia_vgpu_vfio 2>/dev/null || true
112+
rmmod nvidia 2>/dev/null || true
113+
fi
114+
115+
echo "Loading NVIDIA driver kernel modules..."
116+
set -o xtrace +o nounset
117+
modprobe nvidia
118+
modprobe nvidia_vgpu_vfio
119+
set +o xtrace -o nounset
120+
121+
# Start vGPU daemons
75122
/usr/bin/nvidia-vgpud
76123
/usr/bin/nvidia-vgpu-mgr &
77124

@@ -193,6 +240,7 @@ init() {
193240
_unmount_rootfs
194241
_create_dev_char_directory
195242
_set_fw_search_path
243+
_create_module_params_conf
196244
_install_driver
197245
_load_driver || exit 1
198246
_mount_rootfs

‎vgpu-manager/ubuntu22.04/nvidia-driver‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ DRIVER_RESET_RETRIES=10
88
DELAY_BEFORE_VF_CREATION=${DELAY_BEFORE_VF_CREATION:-15}
99
KERNEL_VERSION=$(uname -r)
1010
RUN_DIR=/run/nvidia
11+
NVIDIA_MODULE_PARAMS=()
12+
MODPROBE_CONFIG_DIR="/etc/modprobe.d"
1113

1214
export DEBIAN_FRONTEND=noninteractive
1315

@@ -133,8 +135,53 @@ _set_fw_search_path() {
133135
echo -n "$nv_fw_search_path" > $fw_path_config_file
134136
}
135137

136-
# Currently _install_driver() takes care of loading nvidia modules. Just need to start necessary vgpu daemons
138+
# For each kernel module configuration file mounted into the container,
139+
# parse the file contents and extract the custom module parameters that
140+
# are to be passed as input to 'modprobe'.
141+
#
142+
# Assumptions:
143+
# - Configuration file is named nvidia.conf
144+
# - Configuration file is mounted inside the container at /drivers.
145+
# - Each line in the file contains at least one parameter, where parameters on the same line
146+
# are space delimited. It is up to the user to properly format the file to ensure
147+
# the correct set of parameters are passed to 'modprobe'.
148+
_get_module_params() {
149+
local base_path="/drivers"
150+
# nvidia
151+
if [ -f "${base_path}/nvidia.conf" ]; then
152+
while IFS="" read -r param || [ -n "$param" ]; do
153+
NVIDIA_MODULE_PARAMS+=("$param")
154+
done <"${base_path}/nvidia.conf"
155+
echo "Module parameters provided for nvidia: ${NVIDIA_MODULE_PARAMS[@]}"
156+
fi
157+
}
158+
159+
_create_module_params_conf() {
160+
echo "Parsing kernel module parameters..."
161+
_get_module_params
162+
163+
if [ ${#NVIDIA_MODULE_PARAMS[@]} -gt 0 ]; then
164+
echo "Configuring nvidia module parameters in ${MODPROBE_CONFIG_DIR}/nvidia.conf"
165+
echo "options nvidia ${NVIDIA_MODULE_PARAMS[@]}" > ${MODPROBE_CONFIG_DIR}/nvidia.conf
166+
fi
167+
}
168+
169+
# Load NVIDIA driver kernel modules with custom parameters and start vGPU daemons
137170
_load_driver() {
171+
# Unload modules if they're already loaded so we can reload with custom parameters
172+
if [ -f /sys/module/nvidia_vgpu_vfio/refcnt ] || [ -f /sys/module/nvidia/refcnt ]; then
173+
echo "NVIDIA modules already loaded by installer, unloading to apply custom parameters..."
174+
rmmod nvidia_vgpu_vfio 2>/dev/null || true
175+
rmmod nvidia 2>/dev/null || true
176+
fi
177+
178+
echo "Loading NVIDIA driver kernel modules..."
179+
set -o xtrace +o nounset
180+
modprobe nvidia
181+
modprobe nvidia_vgpu_vfio
182+
set +o xtrace -o nounset
183+
184+
# Start vGPU daemons
138185
/usr/bin/nvidia-vgpud
139186
/usr/bin/nvidia-vgpu-mgr &
140187

@@ -260,6 +307,7 @@ init() {
260307
_install_prerequisites
261308
_create_dev_char_directory
262309
_set_fw_search_path
310+
_create_module_params_conf
263311
_install_driver
264312
_load_driver || exit 1
265313
_mount_rootfs

‎vgpu-manager/ubuntu24.04/nvidia-driver‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ DRIVER_RESET_RETRIES=10
88
DELAY_BEFORE_VF_CREATION=${DELAY_BEFORE_VF_CREATION:-15}
99
KERNEL_VERSION=$(uname -r)
1010
RUN_DIR=/run/nvidia
11+
NVIDIA_MODULE_PARAMS=()
12+
MODPROBE_CONFIG_DIR="/etc/modprobe.d"
1113

1214
export DEBIAN_FRONTEND=noninteractive
1315

@@ -133,8 +135,53 @@ _install_driver() {
133135
sh NVIDIA-Linux-${DRIVER_ARCH}-${DRIVER_VERSION}-vgpu-kvm.run --ui=none --no-questions --tmpdir ${tmp_dir} --no-systemd
134136
}
135137

136-
# Currently _install_driver() takes care of loading nvidia modules. Just need to start necessary vgpu daemons
138+
# For each kernel module configuration file mounted into the container,
139+
# parse the file contents and extract the custom module parameters that
140+
# are to be passed as input to 'modprobe'.
141+
#
142+
# Assumptions:
143+
# - Configuration file is named nvidia.conf
144+
# - Configuration file is mounted inside the container at /drivers.
145+
# - Each line in the file contains at least one parameter, where parameters on the same line
146+
# are space delimited. It is up to the user to properly format the file to ensure
147+
# the correct set of parameters are passed to 'modprobe'.
148+
_get_module_params() {
149+
local base_path="/drivers"
150+
# nvidia
151+
if [ -f "${base_path}/nvidia.conf" ]; then
152+
while IFS="" read -r param || [ -n "$param" ]; do
153+
NVIDIA_MODULE_PARAMS+=("$param")
154+
done <"${base_path}/nvidia.conf"
155+
echo "Module parameters provided for nvidia: ${NVIDIA_MODULE_PARAMS[@]}"
156+
fi
157+
}
158+
159+
_create_module_params_conf() {
160+
echo "Parsing kernel module parameters..."
161+
_get_module_params
162+
163+
if [ ${#NVIDIA_MODULE_PARAMS[@]} -gt 0 ]; then
164+
echo "Configuring nvidia module parameters in ${MODPROBE_CONFIG_DIR}/nvidia.conf"
165+
echo "options nvidia ${NVIDIA_MODULE_PARAMS[@]}" > ${MODPROBE_CONFIG_DIR}/nvidia.conf
166+
fi
167+
}
168+
169+
# Load NVIDIA driver kernel modules with custom parameters and start vGPU daemons
137170
_load_driver() {
171+
# Unload modules if they're already loaded so we can reload with custom parameters
172+
if [ -f /sys/module/nvidia_vgpu_vfio/refcnt ] || [ -f /sys/module/nvidia/refcnt ]; then
173+
echo "NVIDIA modules already loaded by installer, unloading to apply custom parameters..."
174+
rmmod nvidia_vgpu_vfio 2>/dev/null || true
175+
rmmod nvidia 2>/dev/null || true
176+
fi
177+
178+
echo "Loading NVIDIA driver kernel modules..."
179+
set -o xtrace +o nounset
180+
modprobe nvidia
181+
modprobe nvidia_vgpu_vfio
182+
set +o xtrace -o nounset
183+
184+
# Start vGPU daemons
138185
/usr/bin/nvidia-vgpud
139186
/usr/bin/nvidia-vgpu-mgr &
140187

@@ -260,6 +307,7 @@ init() {
260307
_install_prerequisites
261308
_create_dev_char_directory
262309
_set_fw_search_path
310+
_create_module_params_conf
263311
_install_driver
264312
_load_driver || exit 1
265313
_mount_rootfs

0 commit comments

Comments
 (0)