update

kubespray/roles/remove-node/post-remove/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+delete_node_retries: 10
+delete_node_delay_seconds: 3

kubespray/roles/remove-node/post-remove/tasks/main.yml

@@ -0,0 +1,11 @@
+---
+- name: remove-node | Delete node
+  command: "{{ kubectl }} delete node {{ kube_override_hostname|default(inventory_hostname) }}"
+  delegate_to: "{{ groups['kube_control_plane']|first }}"
+  # ignore servers that are not nodes
+  when: inventory_hostname in groups['k8s_cluster'] and kube_override_hostname|default(inventory_hostname) in nodes.stdout_lines
+  retries: "{{ delete_node_retries }}"
+  # Sometimes the api-server can have a short window of indisponibility when we delete a master node
+  delay: "{{ delete_node_delay_seconds }}"
+  register: result
+  until: result is not failed

kubespray/roles/remove-node/pre-remove/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+allow_ungraceful_removal: false
+drain_grace_period: 300
+drain_timeout: 360s
+drain_retries: 3
+drain_retry_delay_seconds: 10

kubespray/roles/remove-node/pre-remove/tasks/main.yml

@@ -0,0 +1,38 @@
+---
+- name: remove-node | List nodes
+  command: >-
+    {{ kubectl }} get nodes -o go-template={% raw %}'{{ range .items }}{{ .metadata.name }}{{ "\n" }}{{ end }}'{% endraw %}
+  register: nodes
+  delegate_to: "{{ groups['kube_control_plane']|first }}"
+  changed_when: false
+  run_once: true
+
+- name: remove-node | Drain node except daemonsets resource  # noqa 301
+  command: >-
+    {{ kubectl }} drain
+      --force
+      --ignore-daemonsets
+      --grace-period {{ drain_grace_period }}
+      --timeout {{ drain_timeout }}
+      --delete-emptydir-data {{ kube_override_hostname|default(inventory_hostname) }}
+  # ignore servers that are not nodes
+  when: kube_override_hostname|default(inventory_hostname) in nodes.stdout_lines
+  register: result
+  failed_when: result.rc != 0 and not allow_ungraceful_removal
+  delegate_to: "{{ groups['kube_control_plane']|first }}"
+  until: result.rc == 0 or allow_ungraceful_removal
+  retries: "{{ drain_retries }}"
+  delay: "{{ drain_retry_delay_seconds }}"
+
+- name: remove-node | Wait until Volumes will be detached from the node
+  command: >-
+    {{ kubectl }} get volumeattachments -o go-template={% raw %}'{{ range .items }}{{ .spec.nodeName }}{{ "\n" }}{{ end }}'{% endraw %}
+  register: nodes_with_volumes
+  delegate_to: "{{ groups['kube_control_plane']|first }}"
+  changed_when: false
+  until: not (kube_override_hostname|default(inventory_hostname) in nodes_with_volumes.stdout_lines)
+  retries: 3
+  delay: "{{ drain_grace_period }}"
+  when:
+    - not allow_ungraceful_removal
+    - kube_override_hostname|default(inventory_hostname) in nodes.stdout_lines

kubespray/roles/remove-node/remove-etcd-node/tasks/main.yml

@@ -0,0 +1,55 @@
+---
+- name: Lookup node IP in kubernetes
+  command: >
+    {{ kubectl }} get nodes {{ node }}
+    -o jsonpath='{range .status.addresses[?(@.type=="InternalIP")]}{@.address}{"\n"}{end}'
+  register: remove_node_ip
+  when:
+    - inventory_hostname in groups['etcd']
+    - ip is not defined
+    - access_ip is not defined
+  delegate_to: "{{ groups['etcd']|first }}"
+  failed_when: false
+
+- name: Set node IP
+  set_fact:
+    node_ip: "{{ ip | default(access_ip | default(remove_node_ip.stdout)) | trim }}"
+  when:
+    - inventory_hostname in groups['etcd']
+
+- name: Make sure node_ip is set
+  assert:
+    that: node_ip is defined and node_ip | length > 0
+    msg: "Etcd node ip is not set !"
+  when:
+    - inventory_hostname in groups['etcd']
+
+- name: Lookup etcd member id
+  shell: "{{ bin_dir }}/etcdctl member list | grep {{ node_ip }} | cut -d, -f1"
+  register: etcd_member_id
+  ignore_errors: true  # noqa ignore-errors
+  changed_when: false
+  check_mode: no
+  tags:
+    - facts
+  environment:
+    ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ kube_cert_dir + '/etcd/server.crt' if etcd_deployment_type == 'kubeadm' else etcd_cert_dir + '/admin-' + groups['etcd']|first + '.pem' }}"
+    ETCDCTL_KEY: "{{ kube_cert_dir + '/etcd/server.key' if etcd_deployment_type == 'kubeadm' else etcd_cert_dir + '/admin-' + groups['etcd']|first + '-key.pem' }}"
+    ETCDCTL_CACERT: "{{ kube_cert_dir + '/etcd/ca.crt' if etcd_deployment_type == 'kubeadm' else etcd_cert_dir + '/ca.pem' }}"
+    ETCDCTL_ENDPOINTS: "https://127.0.0.1:2379"
+  delegate_to: "{{ groups['etcd']|first }}"
+  when: inventory_hostname in groups['etcd']
+
+- name: Remove etcd member from cluster
+  command: "{{ bin_dir }}/etcdctl member remove {{ etcd_member_id.stdout }}"
+  environment:
+    ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ kube_cert_dir + '/etcd/server.crt' if etcd_deployment_type == 'kubeadm' else etcd_cert_dir + '/admin-' + groups['etcd']|first + '.pem' }}"
+    ETCDCTL_KEY: "{{ kube_cert_dir + '/etcd/server.key' if etcd_deployment_type == 'kubeadm' else etcd_cert_dir + '/admin-' + groups['etcd']|first + '-key.pem' }}"
+    ETCDCTL_CACERT: "{{ kube_cert_dir + '/etcd/ca.crt' if etcd_deployment_type == 'kubeadm' else etcd_cert_dir + '/ca.pem' }}"
+    ETCDCTL_ENDPOINTS: "https://127.0.0.1:2379"
+  delegate_to: "{{ groups['etcd']|first }}"
+  when:
+    - inventory_hostname in groups['etcd']
+    - etcd_member_id.stdout | length > 0