fix: NFS error 2370 retry and mutex to prevent race condition on concurrent PVC provisioning

pkg/driver/nodeserver.go

@@ -23,6 +23,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/cenkalti/backoff/v4"
@@ -48,6 +49,13 @@ type nodeServer struct {
 	Initiator  *initiatorDriver
 	Client     clientset.Interface
 	tools      tools
+
+	// nfsPrivMus serialises ShareNfsPrivilegeSave calls per DSM IP within
+	// this node pod.  Under concurrent PVC creation (e.g. Kasten K10 restore)
+	// multiple NodeStageVolume goroutines would otherwise hammer the same DSM
+	// NFS subsystem simultaneously, causing transient error 2370.
+	nfsPrivMusMu sync.Mutex
+	nfsPrivMus   map[string]*sync.Mutex
 }
 
 func waitForDevicePathToExist(path string) error {
@@ -304,6 +312,15 @@ func getNodeAddress(ctx context.Context, client clientset.Interface) ([]string,
 	return ips, nil
 }
 
+func (ns *nodeServer) getNfsPrivMutex(dsmIp string) *sync.Mutex {
+	ns.nfsPrivMusMu.Lock()
+	defer ns.nfsPrivMusMu.Unlock()
+	if ns.nfsPrivMus[dsmIp] == nil {
+		ns.nfsPrivMus[dsmIp] = &sync.Mutex{}
+	}
+	return ns.nfsPrivMus[dsmIp]
+}
+
 func (ns *nodeServer) setNFSVolumePrivilege(sourcePath string, hostnames []string, authType utils.AuthType) error {
 	// NFSTODO: fix the parsing rule
 	s := strings.Split(strings.TrimPrefix(sourcePath, "//"), "/")
@@ -338,6 +355,14 @@ func (ns *nodeServer) setNFSVolumePrivilege(sourcePath string, hostnames []strin
 		})
 	}
 
+	// Serialise NFS privilege saves per DSM to avoid concurrent calls that
+	// trigger DSM error 2370 (NFS subsystem temporarily busy).
+	// ShareNfsPrivilegeSave already retries on 2370, but serialising here
+	// prevents the thundering herd in the first place.
+	mu := ns.getNfsPrivMutex(dsmIp)
+	mu.Lock()
+	defer mu.Unlock()
+
 	err = dsm.ShareNfsPrivilegeSave(priv)
 	if err != nil {
 		log.Printf("Failed to save share NFS privilege. Priv:%v. %v", priv, err)

pkg/driver/utils.go

@@ -22,6 +22,7 @@ import (
 	"context"
 	"fmt"
 	"strings"
+	"sync"
 
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"github.com/kubernetes-csi/csi-lib-utils/protosanitizer"
@@ -76,8 +77,9 @@ func NewNodeServer(d *Driver) *nodeServer {
 			chapPassword: "",
 			tools:        d.tools,
 		},
-		Client: getK8sClient(),
-		tools:  d.tools,
+		Client:     getK8sClient(),
+		tools:      d.tools,
+		nfsPrivMus: make(map[string]*sync.Mutex),
 	}
 }
 

pkg/dsm/webapi/share.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"net/url"
 	"strconv"
+	"time"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/SynologyOpenSource/synology-csi/pkg/utils"
@@ -413,12 +414,38 @@ func (dsm *DSM) ShareNfsPrivilegeSave(privilege SharePrivilege) error {
 	}
 	params.Add("rule", string(js))
 
-	_, err = dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
-	if err != nil {
-		return err
-	}
-
-	return nil
+	// DSM error 2370 means the NFS subsystem is temporarily busy.
+	// Under concurrent PVC creation (e.g. Kasten K10 restore) the NFS
+	// subsystem serialises updates and can transiently reject calls.
+	// Retry with exponential back-off (capped at 10s) before giving up.
+	// NodeStageVolume has a multi-minute Kubernetes timeout so we have
+	// enough headroom to wait ~45s total across 8 attempts.
+	const maxRetries = 11
+	const maxDelay = 10 * time.Second
+	delay := 500 * time.Millisecond
+	for attempt := 1; attempt <= maxRetries; attempt++ {
+		resp, reqErr := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
+		if reqErr == nil {
+			return nil
+		}
+		if resp.ErrorCode == 2370 {
+			if attempt == maxRetries {
+				break
+			}
+			log.Warnf("NFS subsystem busy for share [%s] (attempt %d/%d), retrying in %v",
+				privilege.ShareName, attempt, maxRetries, delay)
+			time.Sleep(delay)
+			delay *= 2
+			if delay > maxDelay {
+				delay = maxDelay
+			}
+			continue
+		}
+		return reqErr
+	}
+	return utils.NfsSystemBusyError(
+		fmt.Sprintf("share [%s]: NFS subsystem still busy after %d attempts", privilege.ShareName, maxRetries),
+	)
 }
 
 func (dsm *DSM) ShareNfsPrivilegeLoad(shareName string) (SharePrivilege, error) {

pkg/utils/error.go

@@ -80,3 +80,10 @@ func (_ ShareSystemBusyError) Error() string {
 func (e ShareDefaultError) Error() string {
 	return fmt.Sprintf("Share API error. Error code: %d", e.ErrCode)
 }
+
+// NFS errors
+type NfsSystemBusyError string
+
+func (_ NfsSystemBusyError) Error() string {
+	return "NFS system is temporarily busy (error 2370)"
+}