Skip to content

Commit c9d2e19

Browse files
tariq1890tariq1890
committed
[B200+][NVLink5+] reset GPUs after FABRIC_MODE=1 is set
Signed-off-by: Tariq Ibrahim <tibrahim@nvidia.com>
1 parent 92d2dd6 commit c9d2e19

6 files changed

Lines changed: 36 additions & 0 deletions

File tree

‎rhel10/nvidia-driver‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -774,6 +774,12 @@ _start_daemons() {
774774
--fm-pid-file $fm_pid_file \
775775
--nvlsm-config-file $nvlsm_config_file \
776776
--nvlsm-pid-file $nvlsm_pid_file
777+
if [ ! -z "${NVFM_CONFIG_FABRIC_MODE:-}" ] && [ "${NVFM_CONFIG_FABRIC_MODE}" == "1" ]; then
778+
# We reset the GPU devices after Fabric Manager startup to clear stale FM
779+
# partition data after setting FABRIC_MODE=1
780+
echo "Resetting GPUs to clear stale fabric partition states..."
781+
nvidia-smi -r
782+
fi
777783

778784
# If not a NVLink5+ switch, check for the presence of NVLink4 (or below) switches
779785
elif _assert_nvswitch_system; then

‎rhel8/nvidia-driver‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -751,6 +751,12 @@ _start_daemons() {
751751
--fm-pid-file $fm_pid_file \
752752
--nvlsm-config-file $nvlsm_config_file \
753753
--nvlsm-pid-file $nvlsm_pid_file
754+
if [ ! -z "${NVFM_CONFIG_FABRIC_MODE:-}" ] && [ "${NVFM_CONFIG_FABRIC_MODE}" == "1" ]; then
755+
# We reset the GPU devices after Fabric Manager startup to clear stale FM
756+
# partition data after setting FABRIC_MODE=1
757+
echo "Resetting GPUs to clear stale fabric partition states..."
758+
nvidia-smi -r
759+
fi
754760

755761
# If not a NVLink5+ switch, check for the presence of NVLink4 (or below) switches
756762
elif _assert_nvswitch_system; then

‎rhel9/nvidia-driver‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -770,6 +770,12 @@ _start_daemons() {
770770
--fm-pid-file $fm_pid_file \
771771
--nvlsm-config-file $nvlsm_config_file \
772772
--nvlsm-pid-file $nvlsm_pid_file
773+
if [ ! -z "${NVFM_CONFIG_FABRIC_MODE:-}" ] && [ "${NVFM_CONFIG_FABRIC_MODE}" == "1" ]; then
774+
# We reset the GPU devices after Fabric Manager startup to clear stale FM
775+
# partition data after setting FABRIC_MODE=1
776+
echo "Resetting GPUs to clear stale fabric partition states..."
777+
nvidia-smi -r
778+
fi
773779

774780
# If not a NVLink5+ switch, check for the presence of NVLink4 (or below) switches
775781
elif _assert_nvswitch_system; then

‎ubuntu22.04/nvidia-driver‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -715,6 +715,12 @@ _start_daemons() {
715715
--fm-pid-file $fm_pid_file \
716716
--nvlsm-config-file $nvlsm_config_file \
717717
--nvlsm-pid-file $nvlsm_pid_file
718+
if [ ! -z "${NVFM_CONFIG_FABRIC_MODE:-}" ] && [ "${NVFM_CONFIG_FABRIC_MODE}" == "1" ]; then
719+
# We reset the GPU devices after Fabric Manager startup to clear stale FM
720+
# partition data after setting FABRIC_MODE=1
721+
echo "Resetting GPUs to clear stale fabric partition states..."
722+
nvidia-smi -r
723+
fi
718724

719725
# If not a NVLink5+ switch, check for the presence of NVLink4 (or below) switches
720726
elif _assert_nvswitch_system; then

‎ubuntu24.04/nvidia-driver‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -643,6 +643,12 @@ _start_daemons() {
643643
--fm-pid-file $fm_pid_file \
644644
--nvlsm-config-file $nvlsm_config_file \
645645
--nvlsm-pid-file $nvlsm_pid_file
646+
if [ ! -z "${NVFM_CONFIG_FABRIC_MODE:-}" ] && [ "${NVFM_CONFIG_FABRIC_MODE}" == "1" ]; then
647+
# We reset the GPU devices after Fabric Manager startup to clear stale FM
648+
# partition data after setting FABRIC_MODE=1
649+
echo "Resetting GPUs to clear stale fabric partition states..."
650+
nvidia-smi -r
651+
fi
646652

647653
# If not a NVLink5+ switch, check for the presence of NVLink4 (or below) switches
648654
elif _assert_nvswitch_system; then

‎ubuntu26.04/nvidia-driver‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -703,6 +703,12 @@ _start_daemons() {
703703
--fm-pid-file $fm_pid_file \
704704
--nvlsm-config-file $nvlsm_config_file \
705705
--nvlsm-pid-file $nvlsm_pid_file
706+
if [ ! -z "${NVFM_CONFIG_FABRIC_MODE:-}" ] && [ "${NVFM_CONFIG_FABRIC_MODE}" == "1" ]; then
707+
# We reset the GPU devices after Fabric Manager startup to clear stale FM
708+
# partition data after setting FABRIC_MODE=1
709+
echo "Resetting GPUs to clear stale fabric partition states..."
710+
nvidia-smi -r
711+
fi
706712

707713
# If not a NVLink5+ switch, check for the presence of NVLink4 (or below) switches
708714
elif _assert_nvswitch_system; then

0 commit comments

Comments
 (0)