diff --git a/roles/etcd-restore/tasks/redhat/2-etcd-restore.yml b/roles/etcd-restore/tasks/redhat/2-etcd-restore.yml index 49d9851..ddd7179 100644 --- a/roles/etcd-restore/tasks/redhat/2-etcd-restore.yml +++ b/roles/etcd-restore/tasks/redhat/2-etcd-restore.yml @@ -2,29 +2,44 @@ - name: set time set_fact: time: "{{ lookup('pipe','date +%Y%m%d%H%M') }}" + environment: + TZ: Asia/Seoul -- name: "[{{ target }} / 2-etcd-restore.yml] etcd original backup directory create" +- name: "[{{ target }} / 2-etcd-restore.yml] etcd original backup directory create" file: path: /tmp/etcd-backup-{{ time }} state: directory mode: '0755' -- name: "[{{ target }} / 2-etcd-restore.yml] etcd original backup" +- name: "[{{ target }} / 2-etcd-restore.yml] etcd original backup" shell: | cp -r /var/lib/etcd/* /tmp/etcd-backup-{{ time }} failed_when: false -- name: "[{{ target }} / 2-etcd-restore.yml] etcd original file delete" - shell: | - rm -rf /var/lib/etcd/* +#- name: "[{{ target }} / 2-etcd-restore.yml] etcd original file delete" +# shell: | +# rm -rf /var/lib/etcd/* -- name: "[{{ target }} / 2-etcd-restore.yml] etcd restore directory create" +- name: "[{{ target }} / 2-etcd-restore.yml] find old etcd file and directory" + find: + path: /var/lib/etcd + recurse: yes + file_type: any + register: find_etcd_file + +- name: "[{{ target }} / 2-etcd-restore.yml] etcd original file delete" + file: + path: "{{ item.path }}" + state: absent + loop: "{{ find_etcd_file.files }}" + +- name: "[{{ target }} / 2-etcd-restore.yml] etcd restore directory create" file: path: /var/lib/etcd/recover state: directory mode: '0755' -- name: "[{{ target }} / 2-etcd-restore.yml] etcd restore" +- name: "[{{ target }} / 2-etcd-restore.yml] etcd restore" shell: | etcdctl snapshot restore /tmp/{{ etcd_filename }} \ --name {{ ansible_hostname }} \ @@ -35,7 +50,7 @@ when: - master_mode == "yes" -- name: "[{{ target }} / 2-etcd-restore.yml] etcd restore" +- name: "[{{ target }} / 2-etcd-restore.yml] etcd restore" shell: | etcdctl snapshot restore /tmp/{{ etcd_filename }} \ --name {{ ansible_hostname }} \ @@ -46,7 +61,7 @@ when: - master_mode == "no" -- name: "[{{ target }} / 2-etcd-restore.yml] etcd restore result" +- name: "[{{ target }} / 2-etcd-restore.yml] etcd restore result" debug: msg: | {% if master_mode_etcdctl_command_result.rc == 0 or single_etcdctl_command_result.rc == 0 %} @@ -55,16 +70,16 @@ etcd restore failed! {% endif %} -- name: "[{{ target }} / 2-etcd-restore.yml] delete old etcd member directory" +- name: "[{{ target }} / 2-etcd-restore.yml] delete old etcd member directory" file: path: /var/lib/etcd/member state: absent -- name: "[{{ target }} / 2-etcd-restore.yml] change name etcd restore directory" +- name: "[{{ target }} / 2-etcd-restore.yml] change name etcd restore directory" shell: | mv /var/lib/etcd/recover/member /var/lib/etcd/member -- name: "[{{ target }} / 2-etcd-restore.yml] delete recover directory" +- name: "[{{ target }} / 2-etcd-restore.yml] delete recover directory" file: path: /var/lib/etcd/recover state: absent \ No newline at end of file diff --git a/roles/etcd-restore/tasks/redhat/3-etcd-restart.yml b/roles/etcd-restore/tasks/redhat/3-etcd-restart.yml index 7e69bf1..620f1b4 100644 --- a/roles/etcd-restore/tasks/redhat/3-etcd-restart.yml +++ b/roles/etcd-restore/tasks/redhat/3-etcd-restart.yml @@ -9,16 +9,19 @@ loop: "{{ etcd_container_ids.stdout_lines }}" when: etcd_container_ids.stdout_lines | length > 0 -#- name: "[{{ target }} / 3-etcd-restart.yml] Wait for etcd container to be ready (60s)" -# pause: -# seconds: 60 +- name: "[{{ target }} / 3-etcd-restart.yml] etcd running check" + shell: crictl ps -a | awk '$7 == "etcd" {print $1}' + register: etcd_running_check + until: etcd_running_check.stdout | trim != "" + retries: 300 + delay: 10 -#- name: "[{{ target }} / 3-etcd-restart.yml] etcd status" -# shell: | -# crictl ps -a |grep etcd -# register: etcd_status +- name: "[{{ target }} / 3-etcd-restart.yml] etcd status" + shell: | + crictl ps -a |grep etcd + register: etcd_status -#- name: "[{{ target }} / 3-etcd-restart.yml] etcd status" -# debug: -# msg: -# "{{ etcd_status.stdout }}" \ No newline at end of file +- name: "[{{ target }} / 3-etcd-restart.yml] etcd status" + debug: + msg: + "{{ etcd_status.stdout }}" \ No newline at end of file diff --git a/roles/etcd-restore/tasks/redhat/5-kube-apiserver-restart.yml b/roles/etcd-restore/tasks/redhat/5-kube-apiserver-restart.yml new file mode 100644 index 0000000..145ba45 --- /dev/null +++ b/roles/etcd-restore/tasks/redhat/5-kube-apiserver-restart.yml @@ -0,0 +1,25 @@ +--- +- name: "[{{ target }} / 5-kube-apiserver-restart.yml] get kube-apiserver container id" + shell: crictl ps -a |grep kube-apiserver-{{ ansible_hostname }} | awk '{print $1}' + register: kube_apiserver_ids + +- name: "[{{ target }} / 5-kube-apiserver-restart.yml] kube-apiserver restart" + shell: crictl stop {{ item }} && crictl rm {{ item }} + loop: "{{ kube_apiserver_ids.stdout_lines }}" + when: kube_apiserver_ids.stdout_lines | length > 0 + +- name: "[{{ target }} / 5-kube-apiserver-restart.yml] kube-apiserver running check" + shell: crictl ps -a |grep kube-apiserver-{{ ansible_hostname }} + register: apiserver_running_check + until: apiserver_running_check.stdout | trim != "" + retries: 300 + delay: 10 + +- name: "[{{ target }} / 5-kube-apiserver-restart.yml] kube-apiserver status" + shell: crictl ps -a |grep kube-apiserver-{{ ansible_hostname }} + register: apiserver_status + +- name: "[{{ target }} / 5-kube-apiserver-restart.yml] kube-apiserver status" + debug: + msg: + "{{ apiserver_status }}" \ No newline at end of file diff --git a/roles/etcd-restore/tasks/redhat/6-calico-restart.yml b/roles/etcd-restore/tasks/redhat/6-calico-restart.yml new file mode 100644 index 0000000..41d756f --- /dev/null +++ b/roles/etcd-restore/tasks/redhat/6-calico-restart.yml @@ -0,0 +1,26 @@ +--- +- name: "[{{ host }} / 6-calico-restart.yml] calico restart" + shell: kubectl rollout restart ds -n kube-system calico-node + +- name: "[{{ host }} / 6-calico-restart.yml] calico status check" + shell: kubectl get pods -n kube-system | grep calico-node | wc -l + register: calico_node_count + changed_when: false + +- name: "[{{ host }} / 6-calico-restart.yml] calico status check" + shell: kubectl get pods -n kube-system | grep calico-node | grep Running | wc -l + register: running_count + until: calico_node_count | int == running_count | int + retries: 50 + delay: 10 + changed_when: false + +- name: "[{{ host }} / 6-calico-restart.yml] calico status check" + shell: kubectl get pods -n kube-system | grep calico-node + register: calico_status + changed_when: false + +- name: "[{{ host }} / 6-calico-restart.yml] calico status check" + debug: + msg: + "{{ calico_status.stdout }}" \ No newline at end of file diff --git a/roles/etcd-restore/tasks/redhat/main.yml b/roles/etcd-restore/tasks/redhat/main.yml index f4047a7..6b66912 100644 --- a/roles/etcd-restore/tasks/redhat/main.yml +++ b/roles/etcd-restore/tasks/redhat/main.yml @@ -23,3 +23,8 @@ when: - ("host-cluster" in group_names) and target == "host" or ("member-cluster" in group_names) and target == "member" + +- include: "5-kube-apiserver-restart.yml" + when: + - ("host-master" in group_names or "host-master-cluster" in group_names) and target == "host" or + ("member-master" in group_names or "member-master-cluster" in group_names) and target == "member" \ No newline at end of file