dsk-dev kubespray 이동

ansible/kubespray/roles/kubernetes-apps/csi_driver/vsphere/defaults/main.yml

@@ -0,0 +1,37 @@
+---
+external_vsphere_vcenter_port: "443"
+external_vsphere_insecure: "true"
+external_vsphere_kubernetes_cluster_id: "kubernetes-cluster-id"
+external_vsphere_version: "7.0u1"
+
+vsphere_syncer_image_tag: "v2.5.1"
+vsphere_csi_attacher_image_tag: "v3.4.0"
+vsphere_csi_controller: "v2.5.1"
+vsphere_csi_liveness_probe_image_tag: "v2.6.0"
+vsphere_csi_provisioner_image_tag: "v3.1.0"
+vsphere_csi_snapshotter_image_tag: "v5.0.1"
+vsphere_csi_node_driver_registrar_image_tag: "v2.5.0"
+vsphere_csi_driver_image_tag: "v2.5.1"
+vsphere_csi_resizer_tag: "v1.4.0"
+
+# Set to kube-system for backward compatibility, should be change to vmware-system-csi on the long run
+vsphere_csi_namespace: "kube-system"
+
+vsphere_csi_controller_replicas: 1
+
+csi_endpoint: '{% if external_vsphere_version >= "7.0u1" %}/csi{% else %}/var/lib/csi/sockets/pluginproxy{% endif %}'
+
+vsphere_csi_aggressive_node_drain: False
+vsphere_csi_aggressive_node_unreachable_timeout: 300
+vsphere_csi_aggressive_node_not_ready_timeout: 300
+
+vsphere_csi_node_affinity: {}
+
+# If this is true, debug information will be displayed but
+# may contain some private data, so it is recommended to set it to false
+# in the production environment.
+unsafe_show_logs: false
+
+# https://github.com/kubernetes-sigs/vsphere-csi-driver/blob/master/docs/book/features/volume_snapshot.md#how-to-enable-volume-snapshot--restore-feature-in-vsphere-csi-
+# according to the above link , we can controler the block-volume-snapshot parameter
+vsphere_csi_block_volume_snapshot: false

ansible/kubespray/roles/kubernetes-apps/csi_driver/vsphere/tasks/main.yml

@@ -0,0 +1,54 @@
+---
+- include_tasks: vsphere-credentials-check.yml
+
+- name: vSphere CSI Driver | Generate CSI cloud-config
+  template:
+    src: "{{ item }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item }}"
+    mode: 0640
+  with_items:
+    - vsphere-csi-cloud-config
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: vSphere CSI Driver | Generate Manifests
+  template:
+    src: "{{ item }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item }}"
+    mode: 0644
+  with_items:
+    - vsphere-csi-namespace.yml
+    - vsphere-csi-driver.yml
+    - vsphere-csi-controller-rbac.yml
+    - vsphere-csi-node-rbac.yml
+    - vsphere-csi-controller-config.yml
+    - vsphere-csi-controller-deployment.yml
+    - vsphere-csi-controller-service.yml
+    - vsphere-csi-node.yml
+  register: vsphere_csi_manifests
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: vSphere CSI Driver | Generate a CSI secret manifest
+  command: "{{ kubectl }} create secret generic vsphere-config-secret --from-file=csi-vsphere.conf={{ kube_config_dir }}/vsphere-csi-cloud-config -n {{ vsphere_csi_namespace }} --dry-run --save-config -o yaml"
+  register: vsphere_csi_secret_manifest
+  when: inventory_hostname == groups['kube_control_plane'][0]
+  no_log: "{{ not (unsafe_show_logs|bool) }}"
+
+- name: vSphere CSI Driver | Apply a CSI secret manifest
+  command:
+    cmd: "{{ kubectl }} apply -f -"
+    stdin: "{{ vsphere_csi_secret_manifest.stdout }}"
+  when: inventory_hostname == groups['kube_control_plane'][0]
+  no_log: "{{ not (unsafe_show_logs|bool) }}"
+
+- name: vSphere CSI Driver | Apply Manifests
+  kube:
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/{{ item.item }}"
+    state: "latest"
+  with_items:
+    - "{{ vsphere_csi_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - not item is skipped
+  loop_control:
+    label: "{{ item.item }}"

ansible/kubespray/roles/kubernetes-apps/csi_driver/vsphere/tasks/vsphere-credentials-check.yml

@@ -0,0 +1,38 @@
+---
+- name: External vSphere Cloud Provider | check external_vsphere_vcenter_ip value
+  fail:
+    msg: "external_vsphere_vcenter_ip is missing"
+  when: external_vsphere_vcenter_ip is not defined or not external_vsphere_vcenter_ip
+
+- name: External vSphere Cloud Provider | check external_vsphere_vcenter_port value
+  fail:
+    msg: "external_vsphere_vcenter_port is missing"
+  when: external_vsphere_vcenter_port is not defined or not external_vsphere_vcenter_port
+
+- name: External vSphere Cloud Provider | check external_vsphere_insecure value
+  fail:
+    msg: "external_vsphere_insecure is missing"
+  when: external_vsphere_insecure is not defined or not external_vsphere_insecure
+
+- name: External vSphere Cloud Provider | check external_vsphere_user value
+  fail:
+    msg: "external_vsphere_user is missing"
+  when: external_vsphere_user is not defined or not external_vsphere_user
+
+- name: External vSphere Cloud Provider | check external_vsphere_password value
+  fail:
+    msg: "external_vsphere_password is missing"
+  when:
+    - external_vsphere_password is not defined or not external_vsphere_password
+
+- name: External vSphere Cloud Provider | check external_vsphere_datacenter value
+  fail:
+    msg: "external_vsphere_datacenter is missing"
+  when:
+    - external_vsphere_datacenter is not defined or not external_vsphere_datacenter
+
+- name: External vSphere Cloud Provider | check external_vsphere_kubernetes_cluster_id value
+  fail:
+    msg: "external_vsphere_kubernetes_cluster_id is missing"
+  when:
+    - external_vsphere_kubernetes_cluster_id is not defined or not external_vsphere_kubernetes_cluster_id

ansible/kubespray/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-cloud-config.j2

@@ -0,0 +1,9 @@
+[Global]
+cluster-id = "{{ external_vsphere_kubernetes_cluster_id }}"
+
+[VirtualCenter "{{ external_vsphere_vcenter_ip }}"]
+insecure-flag = "{{ external_vsphere_insecure }}"
+user = "{{ external_vsphere_user }}"
+password = "{{ external_vsphere_password }}"
+port = "{{ external_vsphere_vcenter_port }}"
+datacenters = "{{ external_vsphere_datacenter }}"

ansible/kubespray/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-config.yml.j2

@@ -0,0 +1,24 @@
+apiVersion: v1
+data:
+  "csi-migration": "false"
+{% if external_vsphere_version >= "7.0" %}
+  "csi-auth-check": "true"
+{% else %}
+  "csi-auth-check": "false"
+{% endif %}
+  "online-volume-extend": "true"
+  "trigger-csi-fullsync": "false"
+  "async-query-volume": "true"
+  "improved-csi-idempotency": "true"
+  "improved-volume-topology": "true"
+  "block-volume-snapshot": "{{ vsphere_csi_block_volume_snapshot }}"
+  "csi-windows-support": "false"
+{% if vsphere_csi_controller is version('v2.5.0', '>=') %}
+  "use-csinode-id": "true"
+  "pv-to-backingdiskobjectid-mapping": "false"
+  "cnsmgr-suspend-create-volume": "false"
+{% endif %}
+kind: ConfigMap
+metadata:
+  name: internal-feature-states.csi.vsphere.vmware.com
+  namespace: "{{ vsphere_csi_namespace }}"

ansible/kubespray/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-deployment.yml.j2

@@ -0,0 +1,220 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: vsphere-csi-controller
+  namespace: "{{ vsphere_csi_namespace }}"
+spec:
+  replicas: {{ vsphere_csi_controller_replicas }}
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+      maxSurge: 0
+  selector:
+    matchLabels:
+      app: vsphere-csi-controller
+  template:
+    metadata:
+      labels:
+        app: vsphere-csi-controller
+        role: vsphere-csi
+    spec:
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: "app"
+                    operator: In
+                    values:
+                      - vsphere-csi-controller
+              topologyKey: "kubernetes.io/hostname"
+      serviceAccountName: vsphere-csi-controller
+      nodeSelector:
+        node-role.kubernetes.io/control-plane: ""
+      tolerations:
+        - operator: "Exists"
+          key: node-role.kubernetes.io/master
+          effect: NoSchedule
+        - operator: "Exists"
+          key: node-role.kubernetes.io/control-plane
+          effect: NoSchedule
+{% if vsphere_csi_aggressive_node_drain %}
+        # set below toleration if you need an aggressive pod eviction in case when
+        # node becomes not-ready or unreachable. Default is 300 seconds if not specified.
+        - key: node.kubernetes.io/not-ready
+          operator: Exists
+          effect: NoExecute
+          tolerationSeconds: {{ vsphere_csi_aggressive_node_not_ready_timeout }}
+        - key: node.kubernetes.io/unreachable
+          operator: Exists
+          effect: NoExecute
+          tolerationSeconds: {{ vsphere_csi_aggressive_node_unreachable_timeout }}
+{% endif %}
+      dnsPolicy: "Default"
+      containers:
+        - name: csi-attacher
+          image: {{ kube_image_repo }}/sig-storage/csi-attacher:{{ vsphere_csi_attacher_image_tag }}
+          args:
+            - "--v=4"
+            - "--timeout=300s"
+            - "--csi-address=$(ADDRESS)"
+            - "--leader-election"
+            - "--kube-api-qps=100"
+            - "--kube-api-burst=100"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+{% if external_vsphere_version >= "7.0" %}
+        - name: csi-resizer
+          image: {{ kube_image_repo }}/sig-storage/csi-resizer:{{ vsphere_csi_resizer_tag }}
+          args:
+            - "--v=4"
+            - "--timeout=300s"
+            - "--csi-address=$(ADDRESS)"
+            - "--handle-volume-inuse-error=false"
+            - "--kube-api-qps=100"
+            - "--kube-api-burst=100"
+            - "--leader-election"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+{% endif %}
+        - name: vsphere-csi-controller
+          image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/driver:{{ vsphere_csi_controller }}
+          args:
+            - "--fss-name=internal-feature-states.csi.vsphere.vmware.com"
+            - "--fss-namespace={{ vsphere_csi_namespace }}"
+            - "--supervisor-fss-namespace={{ vsphere_csi_namespace }}"
+            - "--use-gocsi=false"
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          env:
+            - name: CSI_ENDPOINT
+              value: unix://{{ csi_endpoint }}/csi.sock
+            - name: X_CSI_MODE
+              value: "controller"
+            - name: X_CSI_SPEC_DISABLE_LEN_CHECK
+              value: "true"
+            - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT
+              value: 3m
+            - name: VSPHERE_CSI_CONFIG
+              value: "/etc/cloud/csi-vsphere.conf"
+            - name: LOGGER_LEVEL
+              value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION
+{% if external_vsphere_version >= "7.0u1" %}
+            - name: INCLUSTER_CLIENT_QPS
+              value: "100"
+            - name: INCLUSTER_CLIENT_BURST
+              value: "100"
+{% endif %}
+          volumeMounts:
+            - mountPath: /etc/cloud
+              name: vsphere-config-volume
+              readOnly: true
+            - mountPath: {{ csi_endpoint }}
+              name: socket-dir
+          ports:
+            - name: healthz
+              containerPort: 9808
+              protocol: TCP
+            - name: prometheus
+              containerPort: 2112
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 5
+            failureThreshold: 3
+        - name: liveness-probe
+          image: {{ kube_image_repo }}/sig-storage/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }}
+          args:
+            - "--v=4"
+            - "--csi-address=$(ADDRESS)"
+          env:
+            - name: ADDRESS
+              value: {{ csi_endpoint }}/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: {{ csi_endpoint }}
+        - name: vsphere-syncer
+          image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/syncer:{{ vsphere_syncer_image_tag }}
+          args:
+            - "--leader-election"
+            - "--fss-name=internal-feature-states.csi.vsphere.vmware.com"
+            - "--fss-namespace={{ vsphere_csi_namespace }}"
+            - "--supervisor-fss-namespace={{ vsphere_csi_namespace }}"
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          ports:
+            - containerPort: 2113
+              name: prometheus
+              protocol: TCP
+          env:
+            - name: FULL_SYNC_INTERVAL_MINUTES
+              value: "30"
+            - name: VSPHERE_CSI_CONFIG
+              value: "/etc/cloud/csi-vsphere.conf"
+            - name: LOGGER_LEVEL
+              value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION
+{% if external_vsphere_version >= "7.0u1" %}
+            - name: INCLUSTER_CLIENT_QPS
+              value: "100"
+            - name: INCLUSTER_CLIENT_BURST
+              value: "100"
+{% endif %}
+          volumeMounts:
+            - mountPath: /etc/cloud
+              name: vsphere-config-volume
+              readOnly: true
+        - name: csi-provisioner
+          image: {{ kube_image_repo }}/sig-storage/csi-provisioner:{{ vsphere_csi_provisioner_image_tag }}
+          args:
+            - "--v=4"
+            - "--timeout=300s"
+            - "--csi-address=$(ADDRESS)"
+{% if vsphere_csi_controller is version('v2.2.0', '>=') %}
+            - "--kube-api-qps=100"
+            - "--kube-api-burst=100"
+{% endif %}
+            - "--leader-election"
+            - "--default-fstype=ext4"
+            # needed only for topology aware setup
+            #- "--feature-gates=Topology=true"
+            #- "--strict-topology"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+{% if vsphere_csi_controller is version('v2.5.0', '>=') %}
+        - name: csi-snapshotter
+          image: {{ kube_image_repo }}/sig-storage/csi-snapshotter:{{ vsphere_csi_snapshotter_image_tag }}
+          args:
+            - "--v=4"
+            - "--kube-api-qps=100"
+            - "--kube-api-burst=100"
+            - "--timeout=300s"
+            - "--csi-address=$(ADDRESS)"
+            - "--leader-election"
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+{% endif %}
+      volumes:
+      - name: vsphere-config-volume
+        secret:
+          secretName: vsphere-config-secret
+      - name: socket-dir
+        emptyDir: {}

ansible/kubespray/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-rbac.yml.j2

@@ -0,0 +1,86 @@
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: vsphere-csi-controller
+  namespace: "{{ vsphere_csi_namespace }}"
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: vsphere-csi-controller-role
+rules:
+  - apiGroups: [""]
+    resources: ["nodes", "pods", "configmaps"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+{% if external_vsphere_version >= "7.0" %}
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+{% if external_vsphere_version >= "7.0u1" %}
+    verbs: ["patch"]
+{% else %}
+    verbs: ["update", "patch"]
+{% endif %}
+{% endif %}
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "update", "delete", "patch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+{% if vsphere_csi_controller is version('v2.0.0', '>=') %}
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
+{% endif %}
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses","csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "patch", "update"]
+  - apiGroups: ["cns.vmware.com"]
+    resources: ["triggercsifullsyncs"]
+    verbs: ["create", "get", "update", "watch", "list"]
+  - apiGroups: ["cns.vmware.com"]
+    resources: ["cnsvspherevolumemigrations"]
+    verbs: ["create", "get", "list", "watch", "update", "delete"]
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources: ["customresourcedefinitions"]
+    verbs: ["get", "create", "update"]
+  - apiGroups: ["cns.vmware.com"]
+    resources: ["cnsvolumeoperationrequests"]
+    verbs: ["create", "get", "list", "update", "delete"]
+  - apiGroups: [ "cns.vmware.com" ]
+    resources: [ "csinodetopologies" ]
+    verbs: ["get", "update", "watch", "list"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments/status"]
+    verbs: ["patch"]
+  - apiGroups: [ "snapshot.storage.k8s.io" ]
+    resources: [ "volumesnapshots" ]
+    verbs: [ "get", "list" ]
+  - apiGroups: [ "snapshot.storage.k8s.io" ]
+    resources: [ "volumesnapshotclasses" ]
+    verbs: [ "watch", "get", "list" ]
+  - apiGroups: [ "snapshot.storage.k8s.io" ]
+    resources: [ "volumesnapshotcontents" ]
+    verbs: [ "create", "get", "list", "watch", "update", "delete", "patch" ]
+  - apiGroups: [ "snapshot.storage.k8s.io" ]
+    resources: [ "volumesnapshotcontents/status" ]
+    verbs: [ "update", "patch" ]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: vsphere-csi-controller-binding
+subjects:
+  - kind: ServiceAccount
+    name: vsphere-csi-controller
+    namespace: "{{ vsphere_csi_namespace }}"
+roleRef:
+  kind: ClusterRole
+  name: vsphere-csi-controller-role
+  apiGroup: rbac.authorization.k8s.io

ansible/kubespray/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-service.yml.j2

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: vsphere-csi-controller
+  namespace: "{{ vsphere_csi_namespace }}"
+  labels:
+    app: vsphere-csi-controller
+spec:
+  ports:
+    - name: ctlr
+      port: 2112
+      targetPort: 2112
+      protocol: TCP
+    - name: syncer
+      port: 2113
+      targetPort: 2113
+      protocol: TCP
+  selector:
+    app: vsphere-csi-controller

ansible/kubespray/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-driver.yml.j2

@@ -0,0 +1,7 @@
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  name: csi.vsphere.vmware.com
+spec:
+  attachRequired: true
+  podInfoOnMount: false

ansible/kubespray/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-namespace.yml.j2

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: "{{ vsphere_csi_namespace }}"

ansible/kubespray/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node-rbac.yml.j2

@@ -0,0 +1,55 @@
+---
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: vsphere-csi-node
+  namespace: "{{ vsphere_csi_namespace }}"
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: vsphere-csi-node-cluster-role
+rules:
+  - apiGroups: ["cns.vmware.com"]
+    resources: ["csinodetopologies"]
+    verbs: ["create", "watch", "get", "patch" ]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: vsphere-csi-node-cluster-role-binding
+subjects:
+  - kind: ServiceAccount
+    name: vsphere-csi-node
+    namespace: "{{ vsphere_csi_namespace }}"
+roleRef:
+  kind: ClusterRole
+  name: vsphere-csi-node-cluster-role
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: vsphere-csi-node-role
+  namespace: "{{ vsphere_csi_namespace }}"
+rules:
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "list", "watch"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: vsphere-csi-node-binding
+  namespace: "{{ vsphere_csi_namespace }}"
+subjects:
+  - kind: ServiceAccount
+    name: vsphere-csi-node
+    namespace: "{{ vsphere_csi_namespace }}"
+roleRef:
+  kind: Role
+  name: vsphere-csi-node-role
+  apiGroup: rbac.authorization.k8s.io

ansible/kubespray/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node.yml.j2

@@ -0,0 +1,157 @@
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: vsphere-csi-node
+  namespace: "{{ vsphere_csi_namespace }}"
+spec:
+  selector:
+    matchLabels:
+      app: vsphere-csi-node
+  updateStrategy:
+    type: "RollingUpdate"
+    rollingUpdate:
+      maxUnavailable: 1
+  template:
+    metadata:
+      labels:
+        app: vsphere-csi-node
+        role: vsphere-csi
+    spec:
+      nodeSelector:
+        kubernetes.io/os: linux
+{% if vsphere_csi_node_affinity %}
+      affinity:
+        {{ vsphere_csi_node_affinity | to_nice_yaml | indent(width=8) }}
+{% endif %}
+      serviceAccountName: vsphere-csi-node
+      hostNetwork: true
+      dnsPolicy: "ClusterFirstWithHostNet"
+      containers:
+      - name: node-driver-registrar
+        image: {{ kube_image_repo }}/sig-storage/csi-node-driver-registrar:{{ vsphere_csi_node_driver_registrar_image_tag }}
+{% if external_vsphere_version < "7.0u1" %}
+        lifecycle:
+          preStop:
+            exec:
+              command: ["/bin/sh", "-c", "rm -rf /registration/csi.vsphere.vmware.com-reg.sock /csi/csi.sock"]
+{% endif %}
+        args:
+        - "--v=5"
+        - "--csi-address=$(ADDRESS)"
+        - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
+        env:
+        - name: ADDRESS
+          value: /csi/csi.sock
+        - name: DRIVER_REG_SOCK_PATH
+          value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock
+        volumeMounts:
+        - name: plugin-dir
+          mountPath: /csi
+        - name: registration-dir
+          mountPath: /registration
+        livenessProbe:
+          exec:
+            command:
+            - /csi-node-driver-registrar
+            - --kubelet-registration-path=/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock
+            - --mode=kubelet-registration-probe
+          initialDelaySeconds: 3
+      - name: vsphere-csi-node
+        image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/driver:{{ vsphere_csi_driver_image_tag }}
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        args:
+          - "--fss-name=internal-feature-states.csi.vsphere.vmware.com"
+          - "--fss-namespace={{ vsphere_csi_namespace }}"
+          - "--supervisor-fss-namespace={{ vsphere_csi_namespace }}"
+          - "--use-gocsi=false"
+        imagePullPolicy: "Always"
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: CSI_ENDPOINT
+          value: unix:///csi/csi.sock
+        - name: MAX_VOLUMES_PER_NODE
+          value: "59" # Maximum number of volumes that controller can publish to the node. If value is not set or zero Kubernetes decide how many volumes can be published by the controller to the node.
+        - name: X_CSI_MODE
+          value: "node"
+        - name: X_CSI_SPEC_REQ_VALIDATION
+          value: "false"
+        - name: X_CSI_DEBUG
+          value: "true"
+        - name: X_CSI_SPEC_DISABLE_LEN_CHECK
+          value: "true"
+        - name: LOGGER_LEVEL
+          value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION
+        - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES
+          value: "1"
+        securityContext:
+          privileged: true
+          capabilities:
+            add: ["SYS_ADMIN"]
+          allowPrivilegeEscalation: true
+        volumeMounts:
+        - name: plugin-dir
+          mountPath: /csi
+        - name: pods-mount-dir
+          mountPath: /var/lib/kubelet
+          # needed so that any mounts setup inside this container are
+          # propagated back to the host machine.
+          mountPropagation: "Bidirectional"
+        - name: device-dir
+          mountPath: /dev
+        - name: blocks-dir
+          mountPath: /sys/block
+        - name: sys-devices-dir
+          mountPath: /sys/devices
+        ports:
+          - containerPort: 9808
+            name: healthz
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: healthz
+          initialDelaySeconds: 10
+          timeoutSeconds: 5
+          periodSeconds: 5
+          failureThreshold: 3
+      - name: liveness-probe
+        image: {{ kube_image_repo }}/sig-storage/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }}
+        args:
+{% if external_vsphere_version >= "7.0u1" %}
+          - "--v=4"
+{% endif %}
+          - "--csi-address=/csi/csi.sock"
+        volumeMounts:
+        - name: plugin-dir
+          mountPath: /csi
+      volumes:
+      - name: registration-dir
+        hostPath:
+          path: /var/lib/kubelet/plugins_registry
+          type: Directory
+      - name: plugin-dir
+        hostPath:
+          path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com
+          type: DirectoryOrCreate
+      - name: pods-mount-dir
+        hostPath:
+          path: /var/lib/kubelet
+          type: Directory
+      - name: device-dir
+        hostPath:
+          path: /dev
+      - name: blocks-dir
+        hostPath:
+          path: /sys/block
+          type: Directory
+      - name: sys-devices-dir
+        hostPath:
+          path: /sys/devices
+          type: Directory
+      tolerations:
+        - effect: NoExecute
+          operator: Exists
+        - effect: NoSchedule
+          operator: Exists