상태 체크 추가

roles/etcd-restore/tasks/redhat/2-etcd-restore.yml

@@ -2,29 +2,44 @@
 - name: set time
   set_fact:
     time: "{{ lookup('pipe','date +%Y%m%d%H%M') }}"
+  environment:
+    TZ: Asia/Seoul
 
-- name: "[{{ target }}  / 2-etcd-restore.yml] etcd original backup directory create"
+- name: "[{{ target }} / 2-etcd-restore.yml] etcd original backup directory create"
   file:
     path: /tmp/etcd-backup-{{ time }}
     state: directory
     mode: '0755'
 
-- name: "[{{ target }}  / 2-etcd-restore.yml] etcd original backup"
+- name: "[{{ target }} / 2-etcd-restore.yml] etcd original backup"
   shell: |
     cp -r /var/lib/etcd/* /tmp/etcd-backup-{{ time }}
   failed_when: false
 
-- name: "[{{ target }}  / 2-etcd-restore.yml] etcd original file delete"
+#- name: "[{{ target }} / 2-etcd-restore.yml] etcd original file delete"
-  shell: |
+#  shell: |
-    rm -rf /var/lib/etcd/*
+#    rm -rf /var/lib/etcd/*
 
-- name: "[{{ target }}  / 2-etcd-restore.yml] etcd restore directory create"
+- name: "[{{ target }} / 2-etcd-restore.yml] find old etcd file and directory"
+  find:
+    path: /var/lib/etcd
+    recurse: yes
+    file_type: any
+  register: find_etcd_file
+
+- name: "[{{ target }} / 2-etcd-restore.yml] etcd original file delete"
+  file:
+    path: "{{ item.path }}"
+    state: absent
+  loop: "{{ find_etcd_file.files }}"
+
+- name: "[{{ target }} / 2-etcd-restore.yml] etcd restore directory create"
   file:
     path: /var/lib/etcd/recover
     state: directory
     mode: '0755'
 
-- name: "[{{ target }}  / 2-etcd-restore.yml] etcd restore"
+- name: "[{{ target }} / 2-etcd-restore.yml] etcd restore"
   shell: |
     etcdctl snapshot restore /tmp/{{ etcd_filename }} \
     --name {{ ansible_hostname }} \
@@ -35,7 +50,7 @@
   when:
     - master_mode == "yes"
 
-- name: "[{{ target }}  / 2-etcd-restore.yml] etcd restore"
+- name: "[{{ target }} / 2-etcd-restore.yml] etcd restore"
   shell: |
     etcdctl snapshot restore /tmp/{{ etcd_filename }} \
     --name {{ ansible_hostname }} \
@@ -46,7 +61,7 @@
   when:
     - master_mode == "no"
 
-- name: "[{{ target }}  / 2-etcd-restore.yml] etcd restore result"
+- name: "[{{ target }} / 2-etcd-restore.yml] etcd restore result"
   debug:
     msg: |
       {% if master_mode_etcdctl_command_result.rc == 0 or single_etcdctl_command_result.rc == 0 %}
@@ -55,16 +70,16 @@
       etcd restore failed!
       {% endif %}
 
-- name: "[{{ target }}  / 2-etcd-restore.yml] delete old etcd member directory"
+- name: "[{{ target }} / 2-etcd-restore.yml] delete old etcd member directory"
   file:
     path: /var/lib/etcd/member
     state: absent
 
-- name: "[{{ target }}  / 2-etcd-restore.yml] change name etcd restore directory"
+- name: "[{{ target }} / 2-etcd-restore.yml] change name etcd restore directory"
   shell: |
     mv /var/lib/etcd/recover/member /var/lib/etcd/member
 
-- name: "[{{ target }}  / 2-etcd-restore.yml] delete recover directory"
+- name: "[{{ target }} / 2-etcd-restore.yml] delete recover directory"
   file:
     path: /var/lib/etcd/recover
     state: absent

roles/etcd-restore/tasks/redhat/3-etcd-restart.yml

@@ -9,16 +9,19 @@
   loop: "{{ etcd_container_ids.stdout_lines }}"
   when: etcd_container_ids.stdout_lines | length > 0
 
-#- name: "[{{ target }} / 3-etcd-restart.yml] Wait for etcd container to be ready (60s)"
+- name: "[{{ target }} / 3-etcd-restart.yml] etcd running check"
-#  pause:
+  shell: crictl ps -a | awk '$7 == "etcd" {print $1}'
-#    seconds: 60
+  register: etcd_running_check
+  until: etcd_running_check.stdout | trim != ""
+  retries: 300
+  delay: 10
 
-#- name: "[{{ target }} / 3-etcd-restart.yml] etcd status"
+- name: "[{{ target }} / 3-etcd-restart.yml] etcd status"
-#  shell: |
+  shell: |
-#    crictl ps -a |grep etcd
+    crictl ps -a |grep etcd
-#  register: etcd_status
+  register: etcd_status
 
-#- name: "[{{ target }} / 3-etcd-restart.yml] etcd status"
+- name: "[{{ target }} / 3-etcd-restart.yml] etcd status"
-#  debug:
+  debug:
-#    msg:
+    msg:
-#      "{{ etcd_status.stdout }}"
+      "{{ etcd_status.stdout }}"

roles/etcd-restore/tasks/redhat/5-kube-apiserver-restart.yml

@@ -0,0 +1,25 @@
+---
+- name: "[{{ target }} / 5-kube-apiserver-restart.yml] get kube-apiserver container id"
+  shell: crictl ps -a |grep kube-apiserver-{{ ansible_hostname }} | awk '{print $1}'
+  register: kube_apiserver_ids
+
+- name: "[{{ target }} / 5-kube-apiserver-restart.yml] kube-apiserver restart"
+  shell: crictl stop {{ item }} && crictl rm {{ item }}
+  loop: "{{ kube_apiserver_ids.stdout_lines }}"
+  when: kube_apiserver_ids.stdout_lines | length > 0
+
+- name: "[{{ target }} / 5-kube-apiserver-restart.yml] kube-apiserver running check"
+  shell: crictl ps -a |grep kube-apiserver-{{ ansible_hostname }}
+  register: apiserver_running_check
+  until: apiserver_running_check.stdout | trim != ""
+  retries: 300
+  delay: 10
+
+- name: "[{{ target }} / 5-kube-apiserver-restart.yml] kube-apiserver status"
+  shell: crictl ps -a |grep kube-apiserver-{{ ansible_hostname }}
+  register: apiserver_status
+
+- name: "[{{ target }} / 5-kube-apiserver-restart.yml] kube-apiserver status"
+  debug:
+    msg:
+      "{{ apiserver_status }}"

roles/etcd-restore/tasks/redhat/6-calico-restart.yml

@@ -0,0 +1,26 @@
+---
+- name: "[{{ host }} / 6-calico-restart.yml] calico restart"
+  shell: kubectl rollout restart ds -n kube-system calico-node
+
+- name: "[{{ host }} / 6-calico-restart.yml] calico status check"
+  shell: kubectl get pods -n kube-system | grep calico-node | wc -l
+  register: calico_node_count
+  changed_when: false
+
+- name: "[{{ host }} / 6-calico-restart.yml] calico status check"
+  shell: kubectl get pods -n kube-system | grep calico-node | grep Running | wc -l
+  register: running_count
+  until: calico_node_count | int == running_count | int
+  retries: 50
+  delay: 10
+  changed_when: false
+
+- name: "[{{ host }} / 6-calico-restart.yml] calico status check"
+  shell: kubectl get pods -n kube-system | grep calico-node
+  register: calico_status
+  changed_when: false
+
+- name: "[{{ host }} / 6-calico-restart.yml] calico status check"
+  debug:
+    msg:
+      "{{ calico_status.stdout }}"

roles/etcd-restore/tasks/redhat/main.yml

@@ -23,3 +23,8 @@
   when:
     - ("host-cluster" in group_names) and target == "host" or
       ("member-cluster" in group_names) and target == "member"
+
+- include: "5-kube-apiserver-restart.yml"
+  when:
+    - ("host-master" in group_names or "host-master-cluster" in group_names) and target == "host" or
+      ("member-master" in group_names or "member-master-cluster" in group_names) and target == "member"