상태 체크 추가

roles/etcd-restore/tasks/redhat/2-etcd-restore.yml

@@ -2,6 +2,8 @@
 - name: set time
   set_fact:
     time: "{{ lookup('pipe','date +%Y%m%d%H%M') }}"
+  environment:
+    TZ: Asia/Seoul
 
 - name: "[{{ target }} / 2-etcd-restore.yml] etcd original backup directory create"
   file:
@@ -14,9 +16,22 @@
     cp -r /var/lib/etcd/* /tmp/etcd-backup-{{ time }}
   failed_when: false
 
+#- name: "[{{ target }} / 2-etcd-restore.yml] etcd original file delete"
+#  shell: |
+#    rm -rf /var/lib/etcd/*
+
+- name: "[{{ target }} / 2-etcd-restore.yml] find old etcd file and directory"
+  find:
+    path: /var/lib/etcd
+    recurse: yes
+    file_type: any
+  register: find_etcd_file
+
 - name: "[{{ target }} / 2-etcd-restore.yml] etcd original file delete"
-  shell: |
+  file:
-    rm -rf /var/lib/etcd/*
+    path: "{{ item.path }}"
+    state: absent
+  loop: "{{ find_etcd_file.files }}"
 
 - name: "[{{ target }} / 2-etcd-restore.yml] etcd restore directory create"
   file:

roles/etcd-restore/tasks/redhat/3-etcd-restart.yml

@@ -9,16 +9,19 @@
   loop: "{{ etcd_container_ids.stdout_lines }}"
   when: etcd_container_ids.stdout_lines | length > 0
 
-#- name: "[{{ target }} / 3-etcd-restart.yml] Wait for etcd container to be ready (60s)"
+- name: "[{{ target }} / 3-etcd-restart.yml] etcd running check"
-#  pause:
+  shell: crictl ps -a | awk '$7 == "etcd" {print $1}'
-#    seconds: 60
+  register: etcd_running_check
+  until: etcd_running_check.stdout | trim != ""
+  retries: 300
+  delay: 10
 
-#- name: "[{{ target }} / 3-etcd-restart.yml] etcd status"
+- name: "[{{ target }} / 3-etcd-restart.yml] etcd status"
-#  shell: |
+  shell: |
-#    crictl ps -a |grep etcd
+    crictl ps -a |grep etcd
-#  register: etcd_status
+  register: etcd_status
 
-#- name: "[{{ target }} / 3-etcd-restart.yml] etcd status"
+- name: "[{{ target }} / 3-etcd-restart.yml] etcd status"
-#  debug:
+  debug:
-#    msg:
+    msg:
-#      "{{ etcd_status.stdout }}"
+      "{{ etcd_status.stdout }}"

roles/etcd-restore/tasks/redhat/5-kube-apiserver-restart.yml

@@ -0,0 +1,25 @@
+---
+- name: "[{{ target }} / 5-kube-apiserver-restart.yml] get kube-apiserver container id"
+  shell: crictl ps -a |grep kube-apiserver-{{ ansible_hostname }} | awk '{print $1}'
+  register: kube_apiserver_ids
+
+- name: "[{{ target }} / 5-kube-apiserver-restart.yml] kube-apiserver restart"
+  shell: crictl stop {{ item }} && crictl rm {{ item }}
+  loop: "{{ kube_apiserver_ids.stdout_lines }}"
+  when: kube_apiserver_ids.stdout_lines | length > 0
+
+- name: "[{{ target }} / 5-kube-apiserver-restart.yml] kube-apiserver running check"
+  shell: crictl ps -a |grep kube-apiserver-{{ ansible_hostname }}
+  register: apiserver_running_check
+  until: apiserver_running_check.stdout | trim != ""
+  retries: 300
+  delay: 10
+
+- name: "[{{ target }} / 5-kube-apiserver-restart.yml] kube-apiserver status"
+  shell: crictl ps -a |grep kube-apiserver-{{ ansible_hostname }}
+  register: apiserver_status
+
+- name: "[{{ target }} / 5-kube-apiserver-restart.yml] kube-apiserver status"
+  debug:
+    msg:
+      "{{ apiserver_status }}"

roles/etcd-restore/tasks/redhat/6-calico-restart.yml

@@ -0,0 +1,26 @@
+---
+- name: "[{{ host }} / 6-calico-restart.yml] calico restart"
+  shell: kubectl rollout restart ds -n kube-system calico-node
+
+- name: "[{{ host }} / 6-calico-restart.yml] calico status check"
+  shell: kubectl get pods -n kube-system | grep calico-node | wc -l
+  register: calico_node_count
+  changed_when: false
+
+- name: "[{{ host }} / 6-calico-restart.yml] calico status check"
+  shell: kubectl get pods -n kube-system | grep calico-node | grep Running | wc -l
+  register: running_count
+  until: calico_node_count | int == running_count | int
+  retries: 50
+  delay: 10
+  changed_when: false
+
+- name: "[{{ host }} / 6-calico-restart.yml] calico status check"
+  shell: kubectl get pods -n kube-system | grep calico-node
+  register: calico_status
+  changed_when: false
+
+- name: "[{{ host }} / 6-calico-restart.yml] calico status check"
+  debug:
+    msg:
+      "{{ calico_status.stdout }}"

roles/etcd-restore/tasks/redhat/main.yml

@@ -23,3 +23,8 @@
   when:
     - ("host-cluster" in group_names) and target == "host" or
       ("member-cluster" in group_names) and target == "member"
+
+- include: "5-kube-apiserver-restart.yml"
+  when:
+    - ("host-master" in group_names or "host-master-cluster" in group_names) and target == "host" or
+      ("member-master" in group_names or "member-master-cluster" in group_names) and target == "member"