diff --git a/charts/cinder-csi-plugin/ci/hostmount-values.yaml b/charts/cinder-csi-plugin/ci/hostmount-values.yaml new file mode 100644 index 0000000000..edcdee2de2 --- /dev/null +++ b/charts/cinder-csi-plugin/ci/hostmount-values.yaml @@ -0,0 +1,3 @@ +secret: + enabled: false + hostMount: true diff --git a/charts/cinder-csi-plugin/ci/secret-enabled-values.yaml b/charts/cinder-csi-plugin/ci/secret-enabled-values.yaml new file mode 100644 index 0000000000..e01b43a244 --- /dev/null +++ b/charts/cinder-csi-plugin/ci/secret-enabled-values.yaml @@ -0,0 +1,8 @@ +secret: + enabled: true + create: true + name: cinder-csi-cloud-config + data: + cloud.conf: | + [Global] + auth-url=http://openstack-control-plane diff --git a/charts/cinder-csi-plugin/templates/cinder-csi-driver.yaml b/charts/cinder-csi-plugin/templates/cinder-csi-driver.yaml index 5b681e4c4d..52af1ad5b8 100644 --- a/charts/cinder-csi-plugin/templates/cinder-csi-driver.yaml +++ b/charts/cinder-csi-plugin/templates/cinder-csi-driver.yaml @@ -1,3 +1,4 @@ +{{- if or (not .Values.splitMode.enabled) (eq .Values.splitMode.type "workload") }} apiVersion: storage.k8s.io/v1 kind: CSIDriver metadata: @@ -8,3 +9,4 @@ spec: volumeLifecycleModes: - Persistent - Ephemeral +{{- end }} diff --git a/charts/cinder-csi-plugin/templates/controllerplugin-deployment.yaml b/charts/cinder-csi-plugin/templates/controllerplugin-deployment.yaml index d1e9426620..5aeaf5cf85 100644 --- a/charts/cinder-csi-plugin/templates/controllerplugin-deployment.yaml +++ b/charts/cinder-csi-plugin/templates/controllerplugin-deployment.yaml @@ -1,3 +1,4 @@ +{{- if or (not .Values.splitMode.enabled) (eq .Values.splitMode.type "management") }} kind: Deployment apiVersion: apps/v1 metadata: @@ -10,7 +11,7 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - replicas: {{ .Values.csi.plugin.controllerPlugin.replicas }} + replicas: {{ and .Values.splitMode.enabled (eq .Values.splitMode.type "management") | ternary .Values.splitMode.replicas .Values.csi.plugin.controllerPlugin.replicas }} strategy: type: {{ .Values.csi.plugin.controllerPlugin.strategy.type }} {{- if eq .Values.csi.plugin.controllerPlugin.strategy.type "RollingUpdate" }} @@ -28,7 +29,11 @@ spec: annotations: {{- include "cinder-csi.controllerplugin.podAnnotations" . | nindent 8 }} spec: - serviceAccount: csi-cinder-controller-sa + {{- if .Values.splitMode.enabled }} + automountServiceAccountToken: false + {{- else }} + serviceAccountName: csi-cinder-controller-sa + {{- end }} securityContext: {{- toYaml .Values.csi.plugin.controllerPlugin.podSecurityContext | nindent 8 }} containers: @@ -43,6 +48,9 @@ spec: - "--timeout={{ .Values.timeout }}" - "--leader-election=true" - "--default-fstype=ext4" + {{- if and .Values.splitMode.enabled (eq .Values.splitMode.type "management") }} + - "--kubeconfig=/etc/kubernetes/kubeconfig" + {{- end }} {{- if .Values.csi.attacher.extraArgs }} {{- with .Values.csi.attacher.extraArgs }} {{- tpl . $ | trim | nindent 12 }} @@ -57,6 +65,11 @@ spec: volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- if and .Values.splitMode.enabled (eq .Values.splitMode.type "management") }} + - name: kubeconfig + mountPath: /etc/kubernetes + readOnly: true + {{- end }} resources: {{ toYaml .Values.csi.attacher.resources | nindent 12 }} - name: csi-provisioner securityContext: @@ -70,6 +83,9 @@ spec: - "--leader-election=true" - "--default-fstype=ext4" - "--extra-create-metadata" + {{- if and .Values.splitMode.enabled (eq .Values.splitMode.type "management") }} + - "--kubeconfig=/etc/kubernetes/kubeconfig" + {{- end }} {{- if .Values.csi.provisioner.extraArgs }} {{- with .Values.csi.provisioner.extraArgs }} {{- tpl . $ | trim | nindent 12 }} @@ -84,6 +100,11 @@ spec: volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- if and .Values.splitMode.enabled (eq .Values.splitMode.type "management") }} + - name: kubeconfig + mountPath: /etc/kubernetes + readOnly: true + {{- end }} resources: {{ toYaml .Values.csi.provisioner.resources | nindent 12 }} {{- if dig "enabled" true .Values.csi.snapshotter }} - name: csi-snapshotter @@ -96,6 +117,9 @@ spec: - "--csi-address=$(ADDRESS)" - "--timeout={{ .Values.timeout }}" - "--leader-election=true" + {{- if and .Values.splitMode.enabled (eq .Values.splitMode.type "management") }} + - "--kubeconfig=/etc/kubernetes/kubeconfig" + {{- end }} {{- if .Values.csi.snapshotter.extraArgs }} {{- with .Values.csi.snapshotter.extraArgs }} {{- tpl . $ | trim | nindent 12 }} @@ -110,6 +134,11 @@ spec: volumeMounts: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir + {{- if and .Values.splitMode.enabled (eq .Values.splitMode.type "management") }} + - name: kubeconfig + mountPath: /etc/kubernetes + readOnly: true + {{- end }} resources: {{ toYaml .Values.csi.snapshotter.resources | nindent 12 }} {{- end }} - name: csi-resizer @@ -123,6 +152,9 @@ spec: - "--timeout={{ .Values.timeout }}" - "--handle-volume-inuse-error=false" - "--leader-election=true" + {{- if and .Values.splitMode.enabled (eq .Values.splitMode.type "management") }} + - "--kubeconfig=/etc/kubernetes/kubeconfig" + {{- end }} {{- if .Values.csi.resizer.extraArgs }} {{- with .Values.csi.resizer.extraArgs }} {{- tpl . $ | trim | nindent 12 }} @@ -137,6 +169,11 @@ spec: volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- if and .Values.splitMode.enabled (eq .Values.splitMode.type "management") }} + - name: kubeconfig + mountPath: /etc/kubernetes + readOnly: true + {{- end }} resources: {{ toYaml .Values.csi.resizer.resources | nindent 12 }} - name: liveness-probe securityContext: @@ -194,7 +231,7 @@ spec: - name: CSI_ENDPOINT value: unix://csi/csi.sock - name: CLOUD_CONFIG - value: /etc/config/{{ .Values.secret.filename }} + value: {{ .Values.secret.path }}/{{ .Values.secret.filename }} {{- if .Values.clusterID }} - name: CLUSTER_NAME value: "{{ .Values.clusterID }}" @@ -223,7 +260,12 @@ spec: volumeMounts: - name: socket-dir mountPath: /csi - {{- with .Values.csi.plugin.volumeMounts }} + {{- if or .Values.secret.enabled .Values.secret.hostMount }} + - name: cloud-config + mountPath: {{ .Values.secret.path }} + readOnly: true + {{- end }} + {{- with .Values.splitMode.enabled | ternary .Values.splitMode.volumeMounts .Values.csi.plugin.volumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} resources: {{ toYaml .Values.csi.plugin.resources | nindent 12 }} @@ -239,11 +281,19 @@ spec: hostPath: path: /etc/config {{- end }} - {{- with .Values.csi.plugin.volumes }} + {{- with .Values.splitMode.enabled | ternary .Values.splitMode.volumes .Values.csi.plugin.volumes }} {{- toYaml . | nindent 8 }} {{- end }} + {{- if and .Values.splitMode.enabled (eq .Values.splitMode.type "management") }} + - name: kubeconfig + secret: + secretName: {{ .Values.splitMode.kubeconfig.secretName }} + items: + - key: {{ .Values.splitMode.kubeconfig.secretKey }} + path: kubeconfig + {{- end }} affinity: {{ toYaml .Values.csi.plugin.controllerPlugin.affinity | nindent 8 }} - nodeSelector: {{ toYaml .Values.csi.plugin.controllerPlugin.nodeSelector | nindent 8 }} + nodeSelector: {{ toYaml (and .Values.splitMode.enabled (eq .Values.splitMode.type "management") | ternary .Values.splitMode.nodeSelector .Values.csi.plugin.controllerPlugin.nodeSelector) | nindent 8 }} tolerations: {{ toYaml .Values.csi.plugin.controllerPlugin.tolerations | nindent 8 }} {{- with .Values.csi.plugin.controllerPlugin.hostAliases }} hostAliases: @@ -256,3 +306,4 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} +{{- end }} diff --git a/charts/cinder-csi-plugin/templates/controllerplugin-podmonitor.yaml b/charts/cinder-csi-plugin/templates/controllerplugin-podmonitor.yaml index 2dd8ee6e4a..7ea8697565 100644 --- a/charts/cinder-csi-plugin/templates/controllerplugin-podmonitor.yaml +++ b/charts/cinder-csi-plugin/templates/controllerplugin-podmonitor.yaml @@ -1,4 +1,4 @@ -{{- if .Values.csi.plugin.podMonitor.enabled }} +{{- if and (or (not .Values.splitMode.enabled) (eq .Values.splitMode.type "management")) .Values.csi.plugin.podMonitor.enabled }} apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: diff --git a/charts/cinder-csi-plugin/templates/controllerplugin-rbac.yaml b/charts/cinder-csi-plugin/templates/controllerplugin-rbac.yaml index 27f05c54e3..3ed67d591f 100644 --- a/charts/cinder-csi-plugin/templates/controllerplugin-rbac.yaml +++ b/charts/cinder-csi-plugin/templates/controllerplugin-rbac.yaml @@ -1,12 +1,15 @@ +{{- if or (not .Values.splitMode.enabled) (eq .Values.splitMode.type "workload") }} # This YAML file contains RBAC API objects, # which are necessary to run csi controller plugin +{{- if not .Values.splitMode.enabled }} apiVersion: v1 kind: ServiceAccount metadata: name: csi-cinder-controller-sa namespace: {{ .Release.Namespace }} --- +{{- end }} kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -33,9 +36,15 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-attacher-binding subjects: + {{- if and .Values.splitMode.enabled (eq .Values.splitMode.type "workload") }} + - kind: User + name: {{ .Values.splitMode.subject.name }} + apiGroup: rbac.authorization.k8s.io + {{- else }} - kind: ServiceAccount name: csi-cinder-controller-sa namespace: {{ .Release.Namespace }} + {{- end }} roleRef: kind: ClusterRole name: csi-attacher-role @@ -80,9 +89,15 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-provisioner-binding subjects: + {{- if and .Values.splitMode.enabled (eq .Values.splitMode.type "workload") }} + - kind: User + name: {{ .Values.splitMode.subject.name }} + apiGroup: rbac.authorization.k8s.io + {{- else }} - kind: ServiceAccount name: csi-cinder-controller-sa namespace: {{ .Release.Namespace }} + {{- end }} roleRef: kind: ClusterRole name: csi-provisioner-role @@ -119,9 +134,15 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-snapshotter-binding subjects: + {{- if and .Values.splitMode.enabled (eq .Values.splitMode.type "workload") }} + - kind: User + name: {{ .Values.splitMode.subject.name }} + apiGroup: rbac.authorization.k8s.io + {{- else }} - kind: ServiceAccount name: csi-cinder-controller-sa namespace: {{ .Release.Namespace }} + {{- end }} roleRef: kind: ClusterRole name: csi-snapshotter-role @@ -160,11 +181,17 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-resizer-binding subjects: + {{- if and .Values.splitMode.enabled (eq .Values.splitMode.type "workload") }} + - kind: User + name: {{ .Values.splitMode.subject.name }} + apiGroup: rbac.authorization.k8s.io + {{- else }} - kind: ServiceAccount name: csi-cinder-controller-sa namespace: {{ .Release.Namespace }} + {{- end }} roleRef: kind: ClusterRole name: csi-resizer-role apiGroup: rbac.authorization.k8s.io ---- +{{- end }} diff --git a/charts/cinder-csi-plugin/templates/custom_storageclass.yaml b/charts/cinder-csi-plugin/templates/custom_storageclass.yaml index 267c731a27..12484d762c 100644 --- a/charts/cinder-csi-plugin/templates/custom_storageclass.yaml +++ b/charts/cinder-csi-plugin/templates/custom_storageclass.yaml @@ -1,3 +1,3 @@ -{{- if .Values.storageClass.custom -}} +{{- if and .Values.storageClass.custom (or (not .Values.splitMode.enabled) (eq .Values.splitMode.type "workload")) -}} {{ .Values.storageClass.custom }} {{- end }} diff --git a/charts/cinder-csi-plugin/templates/nodeplugin-daemonset.yaml b/charts/cinder-csi-plugin/templates/nodeplugin-daemonset.yaml index 29e016dd89..3de3ed3195 100644 --- a/charts/cinder-csi-plugin/templates/nodeplugin-daemonset.yaml +++ b/charts/cinder-csi-plugin/templates/nodeplugin-daemonset.yaml @@ -1,3 +1,4 @@ +{{- if or (not .Values.splitMode.enabled) (eq .Values.splitMode.type "workload") }} kind: DaemonSet apiVersion: apps/v1 metadata: @@ -20,7 +21,7 @@ spec: annotations: {{- include "cinder-csi.nodeplugin.podAnnotations" . | nindent 8 }} spec: - serviceAccount: csi-cinder-node-sa + serviceAccountName: csi-cinder-node-sa # Host network access is usually necessary if using metadata service, # since not all CNIs support link-local addresses, or if the OpenStack # control plane is on a management network that is not advertised to pods @@ -109,7 +110,7 @@ spec: - name: CSI_ENDPOINT value: unix://csi/csi.sock - name: CLOUD_CONFIG - value: /etc/config/{{ .Values.secret.filename }} + value: {{ .Values.secret.path }}/{{ .Values.secret.filename }} {{- if .Values.csi.plugin.extraEnv }} {{- toYaml .Values.csi.plugin.extraEnv | nindent 12 }} {{- end }} @@ -135,7 +136,12 @@ spec: - name: pods-probe-dir mountPath: /dev mountPropagation: "HostToContainer" - {{- with .Values.csi.plugin.volumeMounts }} + {{- if or .Values.secret.enabled .Values.secret.hostMount }} + - name: cloud-config + mountPath: {{ .Values.secret.path }} + readOnly: true + {{- end }} + {{- with .Values.splitMode.enabled | ternary .Values.splitMode.volumeMounts .Values.csi.plugin.volumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} resources: {{ toYaml .Values.csi.plugin.resources | nindent 12 }} @@ -169,7 +175,7 @@ spec: hostPath: path: /etc/config {{- end }} - {{- with .Values.csi.plugin.volumes }} + {{- with .Values.splitMode.enabled | ternary .Values.splitMode.volumes .Values.csi.plugin.volumes }} {{- toYaml . | nindent 8 }} {{- end }} affinity: {{ toYaml .Values.csi.plugin.nodePlugin.affinity | nindent 8 }} @@ -186,3 +192,4 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} +{{- end }} diff --git a/charts/cinder-csi-plugin/templates/nodeplugin-rbac.yaml b/charts/cinder-csi-plugin/templates/nodeplugin-rbac.yaml index 894a8df219..0028c1c44f 100644 --- a/charts/cinder-csi-plugin/templates/nodeplugin-rbac.yaml +++ b/charts/cinder-csi-plugin/templates/nodeplugin-rbac.yaml @@ -1,3 +1,4 @@ +{{- if or (not .Values.splitMode.enabled) (eq .Values.splitMode.type "workload") }} apiVersion: v1 kind: ServiceAccount metadata: @@ -26,3 +27,4 @@ roleRef: kind: ClusterRole name: csi-nodeplugin-role apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/cinder-csi-plugin/templates/storageclass.yaml b/charts/cinder-csi-plugin/templates/storageclass.yaml index ea1fd24cb4..b082bb39b6 100644 --- a/charts/cinder-csi-plugin/templates/storageclass.yaml +++ b/charts/cinder-csi-plugin/templates/storageclass.yaml @@ -1,4 +1,4 @@ -{{- if .Values.storageClass.enabled }} +{{- if and .Values.storageClass.enabled (or (not .Values.splitMode.enabled) (eq .Values.splitMode.type "workload")) }} apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: diff --git a/charts/cinder-csi-plugin/values.yaml b/charts/cinder-csi-plugin/values.yaml index f2aad2c134..79e04cc1d9 100644 --- a/charts/cinder-csi-plugin/values.yaml +++ b/charts/cinder-csi-plugin/values.yaml @@ -86,9 +86,6 @@ csi: - name: cacert mountPath: /etc/cacert readOnly: true - - name: cloud-config - mountPath: /etc/config - readOnly: true nodePlugin: dnsPolicy: ClusterFirstWithHostNet priorityClassName: "system-node-critical" @@ -181,6 +178,7 @@ secret: hostMount: true create: false filename: cloud.conf + path: /etc/config # name: cinder-csi-cloud-config # data: # cloud.conf: |- @@ -236,3 +234,24 @@ imagePullSecrets: [] # add annotations to all pods, deployment, daemonset and podmonitor commonAnnotations: {} + +# External mode for split-cluster deployments (e.g., Cluster API) +# When disabled, deploys both controller and node plugin with standard RBAC +splitMode: + enabled: false + # type: "management" - Deploy controller with external kubeconfig (management cluster) + # type: "workload" - Deploy node plugin with RBAC for external controller (workload cluster) + type: management + # Management cluster settings (type: management) + kubeconfig: + secretName: "" + secretKey: value + replicas: 1 + nodeSelector: {} + # Workload cluster settings (type: workload) + # The User subject name for controller RBAC (identity of external controller) + subject: + name: cinder-csi-controller + # Volumes and volumeMounts for splitMode (overrides csi.plugin.volumes/volumeMounts) + volumes: [] + volumeMounts: [] diff --git a/docs/cinder-csi-plugin/using-cinder-csi-plugin.md b/docs/cinder-csi-plugin/using-cinder-csi-plugin.md index ad1a0f75d7..686d84a221 100644 --- a/docs/cinder-csi-plugin/using-cinder-csi-plugin.md +++ b/docs/cinder-csi-plugin/using-cinder-csi-plugin.md @@ -261,6 +261,37 @@ To install the chart, use the following command: helm install --namespace kube-system --name cinder-csi ./charts/cinder-csi-plugin ``` +#### Split-cluster deployments (splitMode) + +The controller plugin needs OpenStack credentials (via `cloud-config`) to talk to Cinder, while the node plugin only formats and mounts already-attached volumes. In managed setups, e.g. a management cluster provisioning volumes for a workload cluster with Cluster API, you don't want that OpenStack credential present in the workload cluster where end users have access. `splitMode` lets you install the controller plugin into a separate (management) cluster and only the node plugin into the workload cluster, so the workload cluster's end users never have access to OpenStack credentials. + +Set `splitMode.type` to `management` on the cluster that should run the controller plugin: + +```yaml +splitMode: + enabled: true + type: management + kubeconfig: + secretName: workload-cluster-kubeconfig + secretKey: value +``` + +The controller plugin authenticates against the workload cluster's API server using the kubeconfig from `splitMode.kubeconfig.secretName`/`secretKey` instead of its local in-cluster service account, and no node plugin, storage class, or RBAC for the local cluster is installed. + +Set `splitMode.type` to `workload` on the cluster that runs the node plugin: + +```yaml +splitMode: + enabled: true + type: workload + subject: + name: cinder-csi-controller +``` + +Only the node plugin, `CSIDriver` object, storage classes, and RBAC are installed; the RBAC grants access to `splitMode.subject.name`, which must match the identity the external controller plugin authenticates as. No controller deployment is installed on this cluster. + +`splitMode.volumes`/`splitMode.volumeMounts` override `csi.plugin.volumes`/`csi.plugin.volumeMounts` for both the controller and node plugin containers while `splitMode.enabled` is `true`. The plugin binary always requires a `--cloud-config` file to start, so on the workload cluster this is used to supply a `cloud-config` that carries no OpenStack credentials, instead of `secret.enabled`/`secret.hostMount`, keeping real credentials confined to the management cluster. + ## Supported Features * [Dynamic Provisioning](./features.md#dynamic-provisioning)