Skip to content

util: improve static PV handling #6328

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
satoru-takeuchi wants to merge 1 commit into
base: devel
Choose a base branch
from
Open

util: improve static PV handling #6328

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/static-pvc.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,9 @@ spec:
volumeMode: Filesystem
```

> [!note]
> `controllerPublishSecretRef` must not be specified for static RBD PVs.
Comment thread
Madhu-1 marked this conversation as resolved.

### RBD Volume Attributes in PV

Below table explains the list of volume attributes can be set when creating a
Expand Down Expand Up @@ -278,6 +281,9 @@ spec:
volumeMode: Filesystem
```

> [!note]
> `controllerPublishSecretRef` must not be specified for static CephFS PVs.

### Node stage secret ref in CephFS PV

For static CephFS PV to work, userID and userKey needs to be specified in the
Expand Down
42 changes: 12 additions & 30 deletions internal/cephfs/controllerserver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1155,21 +1155,12 @@ func (cs *ControllerServer) getServiceAccountRestriction(
req *csi.ControllerPublishVolumeRequest,
) (string, error) {
volumeID := req.GetVolumeId()
secrets := req.GetSecrets()

if secrets == nil {
secretName, secretNamespace, err := util.GetControllerPublishSecretRef(volumeID, util.CephFsType)
if err != nil {
log.WarningLog(ctx, "controller publish secret not found: %v", err)

return "", nil
}

secrets, err = k8s.GetSecret(secretName, secretNamespace)
if err != nil {
return "", status.Errorf(codes.Internal,
"failed to get controller publish secret from k8s: %v", err)
}
secrets, skip, err := util.GetControllerPublishSecrets(ctx, req.GetSecrets(), volumeID, util.CephFsType)
if skip {
return "", nil
}
if err != nil {
return "", status.Errorf(codes.Internal, "%v", err)
}

volOptions, _, err := store.NewVolumeOptionsFromVolID(ctx, volumeID, nil, secrets, cs.ClusterName)
Expand Down Expand Up @@ -1228,21 +1219,12 @@ func (cs *ControllerServer) ControllerUnpublishVolume(
}
defer cs.VolumeLocks.Release(volumeId)

secrets := req.GetSecrets()
if secrets == nil {
secretName, secretNamespace, err := util.GetControllerPublishSecretRef(volumeId, util.CephFsType)
if err != nil {
log.WarningLog(ctx, "controller publish secret not found: %v", err)

// If the secret is not found, return success to not break for older PVs
// without controller-publish secrets.
return &csi.ControllerUnpublishVolumeResponse{}, nil
}

secrets, err = k8s.GetSecret(secretName, secretNamespace)
if err != nil {
return nil, fmt.Errorf("failed to get controller publish secret from k8s: %w", err)
}
secrets, skip, err := util.GetControllerPublishSecrets(ctx, req.GetSecrets(), volumeId, util.CephFsType)
if skip {
return &csi.ControllerUnpublishVolumeResponse{}, nil
}
if err != nil {
return nil, status.Errorf(codes.Internal, "%v", err)
}

volOptions, _, err := store.NewVolumeOptionsFromVolID(ctx, volumeId, nil, secrets, cs.ClusterName)
Expand Down
42 changes: 12 additions & 30 deletions internal/rbd/controllerserver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1755,21 +1755,12 @@ func (cs *ControllerServer) getServiceAccountRestriction(
req *csi.ControllerPublishVolumeRequest,
) (string, error) {
volumeID := req.GetVolumeId()
secrets := req.GetSecrets()

if secrets == nil {
secretName, secretNamespace, err := util.GetControllerPublishSecretRef(volumeID, util.RBDType)
if err != nil {
log.WarningLog(ctx, "controller publish secret not found: %v", err)

return "", nil
}

secrets, err = k8s.GetSecret(secretName, secretNamespace)
if err != nil {
return "", status.Errorf(codes.Internal,
"failed to get controller publish secret from k8s: %v", err)
}
secrets, skip, err := util.GetControllerPublishSecrets(ctx, req.GetSecrets(), volumeID, util.RBDType)
if skip {
return "", nil
}
if err != nil {
return "", status.Errorf(codes.Internal, "%v", err)
}

cr, err := util.NewUserCredentials(secrets)
Expand Down Expand Up @@ -1827,21 +1818,12 @@ func (cs *ControllerServer) ControllerUnpublishVolume(
}
defer cs.VolumeLocks.Release(volumeId)

secrets := req.GetSecrets()
if secrets == nil {
secretName, secretNamespace, err := util.GetControllerPublishSecretRef(volumeId, util.RBDType)
if err != nil {
log.WarningLog(ctx, "controller publish secret not found: %v", err)

// If the secret is not found, return success to not break for older PVs
// without controller-publish secrets.
return &csi.ControllerUnpublishVolumeResponse{}, nil
}

secrets, err = k8s.GetSecret(secretName, secretNamespace)
if err != nil {
return nil, fmt.Errorf("failed to get controller publish secret from k8s: %w", err)
}
secrets, skip, err := util.GetControllerPublishSecrets(ctx, req.GetSecrets(), volumeId, util.RBDType)
if skip {
return &csi.ControllerUnpublishVolumeResponse{}, nil
}
if err != nil {
return nil, status.Errorf(codes.Internal, "%v", err)
}

credentials, err := util.NewAdminCredentials(secrets)
Expand Down
2 changes: 2 additions & 0 deletions internal/util/errors.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,6 @@ var (
ErrMissingConfigForMonitor = errors.New("missing configuration of cluster ID for monitor")
// ErrConfigNotFound is returned when no configuration is found for a cluster ID.
ErrConfigNotFound = errors.New("missing configuration for cluster ID")
// ErrInvalidVolID is returned when the volume ID cannot be decomposed into a CSI identifier.
ErrInvalidVolID = errors.New("invalid volume ID")
)
35 changes: 34 additions & 1 deletion internal/util/util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import (
mount "k8s.io/mount-utils"

"github.com/ceph/ceph-csi/internal/util/k8s"
"github.com/ceph/ceph-csi/internal/util/log"
)

// Driver types to identify type of driver running.
Expand Down Expand Up @@ -349,7 +350,8 @@ func GetControllerPublishSecretRef(volumeId, driverType string) (string, string,
)
err := vi.DecomposeCSIID(volumeId)
if err != nil {
return secretName, secretNamespace, fmt.Errorf("failed to decode volume ID (%s): %w", volumeId, err)
return secretName, secretNamespace, fmt.Errorf("failed to decode volume ID (%s): %w",
volumeId, errors.Join(ErrInvalidVolID, err))
}

secretName, secretNamespace, err = getControllerPublishSecretRef(vi.ClusterID, driverType)
Expand Down Expand Up @@ -395,6 +397,37 @@ func GetControllerPublishSecretRef(volumeId, driverType string) (string, string,
return secretName, secretNamespace, nil
}

// GetControllerPublishSecrets resolves secrets for a controller publish/unpublish operation.
// If reqSecrets is non-nil, return it. Otherwise secrets are fetched from the CSI config.
// When the second return value is true the caller should return early with no error.
func GetControllerPublishSecrets(
ctx context.Context,
reqSecrets map[string]string,
volumeID, driverType string,
) (map[string]string, bool, error) {
if reqSecrets != nil {
return reqSecrets, false, nil
}
secretName, secretNamespace, err := GetControllerPublishSecretRef(volumeID, driverType)
if errors.Is(err, ErrInvalidVolID) || errors.Is(err, ErrConfigNotFound) {
// Possibly the volume is a static/older volume. In this case, we have nothing to do.
// Even if it's not a static/older volume, we should skip handling it because
// we have no way to process handling anyway.
log.WarningLog(ctx, "should skip handling this volume: %v", err)

return nil, true, nil
}
if err != nil {
return nil, false, fmt.Errorf("failed to get controller publish secret ref: %w", err)
}
secrets, err := k8s.GetSecret(secretName, secretNamespace)
if err != nil {
return nil, false, fmt.Errorf("failed to get controller publish secret from k8s: %w", err)
}

return secrets, false, nil
}

func getControllerPublishSecretRef(clusterId, driverType string) (string, string, error) {
var (
err error
Expand Down