update

kubespray/extra_playbooks/roles/etcd/defaults/main.yml

@@ -0,0 +1,122 @@
+---
+# Set etcd user
+etcd_owner: etcd
+
+# Set to false to only do certificate management
+etcd_cluster_setup: true
+etcd_events_cluster_setup: false
+
+# Set to true to separate k8s events to a different etcd cluster
+etcd_events_cluster_enabled: false
+
+etcd_backup_prefix: "/var/backups"
+etcd_data_dir: "/var/lib/etcd"
+
+# Number of etcd backups to retain. Set to a value < 0 to retain all backups
+etcd_backup_retention_count: -1
+
+force_etcd_cert_refresh: true
+etcd_config_dir: /etc/ssl/etcd
+etcd_cert_dir: "{{ etcd_config_dir }}/ssl"
+etcd_cert_dir_mode: "0700"
+etcd_cert_group: root
+# Note: This does not set up DNS entries. It simply adds the following DNS
+# entries to the certificate
+etcd_cert_alt_names:
+  - "etcd.kube-system.svc.{{ dns_domain }}"
+  - "etcd.kube-system.svc"
+  - "etcd.kube-system"
+  - "etcd"
+etcd_cert_alt_ips: []
+
+etcd_script_dir: "{{ bin_dir }}/etcd-scripts"
+
+etcd_heartbeat_interval: "250"
+etcd_election_timeout: "5000"
+
+# etcd_snapshot_count: "10000"
+
+etcd_metrics: "basic"
+
+# Define in inventory to set a separate port for etcd to expose metrics on
+# etcd_metrics_port: 2381
+
+## A dictionary of extra environment variables to add to etcd.env, formatted like:
+##  etcd_extra_vars:
+##    ETCD_VAR1: "value1"
+##    ETCD_VAR2: "value2"
+etcd_extra_vars: {}
+
+# Limits
+# Limit memory only if <4GB memory on host. 0=unlimited
+# This value is only relevant when deploying etcd with `etcd_deployment_type: docker`
+etcd_memory_limit: "{% if ansible_memtotal_mb < 4096 %}512M{% else %}0{% endif %}"
+
+# The default storage size limit is 2G.
+# 8G is a suggested maximum size for normal environments and etcd warns at startup if the configured value exceeds it.
+# etcd_quota_backend_bytes: "2147483648"
+
+# Maximum client request size in bytes the server will accept.
+# etcd is designed to handle small key value pairs typical for metadata.
+# Larger requests will work, but may increase the latency of other requests
+# etcd_max_request_bytes: "1572864"
+
+# Uncomment to set CPU share for etcd
+# etcd_cpu_limit: 300m
+
+etcd_blkio_weight: 1000
+
+etcd_node_cert_hosts: "{{ groups['k8s_cluster'] }}"
+
+etcd_compaction_retention: "8"
+
+# Force clients like etcdctl to use TLS certs (different than peer security)
+etcd_secure_client: true
+
+# Enable peer client cert authentication
+etcd_peer_client_auth: true
+
+# Maximum number of snapshot files to retain (0 is unlimited)
+# etcd_max_snapshots: 5
+
+# Maximum number of wal files to retain (0 is unlimited)
+# etcd_max_wals: 5
+
+# Number of loop retries
+etcd_retries: 4
+
+## Support tls cipher suites.
+# etcd_tls_cipher_suites: {}
+#   - TLS_RSA_WITH_RC4_128_SHA
+#   - TLS_RSA_WITH_3DES_EDE_CBC_SHA
+#   - TLS_RSA_WITH_AES_128_CBC_SHA
+#   - TLS_RSA_WITH_AES_256_CBC_SHA
+#   - TLS_RSA_WITH_AES_128_CBC_SHA256
+#   - TLS_RSA_WITH_AES_128_GCM_SHA256
+#   - TLS_RSA_WITH_AES_256_GCM_SHA384
+#   - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA
+#   - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA
+#   - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA
+#   - TLS_ECDHE_RSA_WITH_RC4_128_SHA
+#   - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA
+#   - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA
+#   - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA
+#   - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
+#   - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256
+#   - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
+#   - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
+#   - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+#   - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+#   - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305
+#   - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256
+#   - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
+#   - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256
+
+# ETCD 3.5.x issue
+# https://groups.google.com/a/kubernetes.io/g/dev/c/B7gJs88XtQc/m/rSgNOzV2BwAJ?utm_medium=email&utm_source=footer
+etcd_experimental_initial_corrupt_check: true
+
+# If this is true, debug information will be displayed but
+# may contain some private data, so it is recommended to set it to false
+# in the production environment.
+unsafe_show_logs: false

kubespray/extra_playbooks/roles/etcd/handlers/backup.yml

@@ -0,0 +1,62 @@
+---
+- name: Backup etcd data
+  command: /bin/true
+  notify:
+    - Refresh Time Fact
+    - Set Backup Directory
+    - Create Backup Directory
+    - Stat etcd v2 data directory
+    - Backup etcd v2 data
+    - Backup etcd v3 data
+  when: etcd_cluster_is_healthy.rc == 0
+
+- name: Refresh Time Fact
+  setup: filter=ansible_date_time
+
+- name: Set Backup Directory
+  set_fact:
+    etcd_backup_directory: "{{ etcd_backup_prefix }}/etcd-{{ ansible_date_time.date }}_{{ ansible_date_time.time }}"
+
+- name: Create Backup Directory
+  file:
+    path: "{{ etcd_backup_directory }}"
+    state: directory
+    owner: root
+    group: root
+    mode: 0600
+
+- name: Stat etcd v2 data directory
+  stat:
+    path: "{{ etcd_data_dir }}/member"
+    get_attributes: no
+    get_checksum: no
+    get_mime: no
+  register: etcd_data_dir_member
+
+- name: Backup etcd v2 data
+  when: etcd_data_dir_member.stat.exists
+  command: >-
+    {{ bin_dir }}/etcdctl backup
+      --data-dir {{ etcd_data_dir }}
+      --backup-dir {{ etcd_backup_directory }}
+  environment:
+    ETCDCTL_API: 2
+  retries: 3
+  register: backup_v2_command
+  until: backup_v2_command.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+
+- name: Backup etcd v3 data
+  command: >-
+    {{ bin_dir }}/etcdctl
+      snapshot save {{ etcd_backup_directory }}/snapshot.db
+  environment:
+    ETCDCTL_API: 3
+    ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses.split(',') | first }}"
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
+  retries: 3
+  register: etcd_backup_v3_command
+  until: etcd_backup_v3_command.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"

kubespray/extra_playbooks/roles/etcd/handlers/backup_cleanup.yml

@@ -0,0 +1,11 @@
+---
+- name: Cleanup etcd backups
+  command: /bin/true
+  notify:
+    - Remove old etcd backups
+
+- name: Remove old etcd backups
+  shell:
+    chdir: "{{ etcd_backup_prefix }}"
+    cmd: "find . -name 'etcd-*' -type d | sort -n | head -n -{{ etcd_backup_retention_count }} | xargs rm -rf"
+  when: etcd_backup_retention_count >= 0

kubespray/extra_playbooks/roles/etcd/handlers/main.yml

@@ -0,0 +1,62 @@
+---
+- name: restart etcd
+  command: /bin/true
+  notify:
+    - Backup etcd data
+    - etcd | reload systemd
+    - reload etcd
+    - wait for etcd up
+    - Cleanup etcd backups
+
+- name: restart etcd-events
+  command: /bin/true
+  notify:
+    - etcd | reload systemd
+    - reload etcd-events
+    - wait for etcd-events up
+
+- import_tasks: backup.yml
+
+- name: etcd | reload systemd
+  systemd:
+    daemon_reload: true
+
+- name: reload etcd
+  service:
+    name: etcd
+    state: restarted
+  when: is_etcd_master
+
+- name: reload etcd-events
+  service:
+    name: etcd-events
+    state: restarted
+  when: is_etcd_master
+
+- name: wait for etcd up
+  uri:
+    url: "https://{% if is_etcd_master %}{{ etcd_address }}{% else %}127.0.0.1{% endif %}:2379/health"
+    validate_certs: no
+    client_cert: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem"
+    client_key: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem"
+  register: result
+  until: result.status is defined and result.status == 200
+  retries: 60
+  delay: 1
+
+- import_tasks: backup_cleanup.yml
+
+- name: wait for etcd-events up
+  uri:
+    url: "https://{% if is_etcd_master %}{{ etcd_address }}{% else %}127.0.0.1{% endif %}:2383/health"
+    validate_certs: no
+    client_cert: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem"
+    client_key: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem"
+  register: result
+  until: result.status is defined and result.status == 200
+  retries: 60
+  delay: 1
+
+- name: set etcd_secret_changed
+  set_fact:
+    etcd_secret_changed: true

kubespray/extra_playbooks/roles/etcd/meta/main.yml

@@ -0,0 +1,8 @@
+---
+dependencies:
+  - role: adduser
+    user: "{{ addusers.etcd }}"
+    when: not (ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_fedora_coreos)
+  - role: adduser
+    user: "{{ addusers.kube }}"
+    when: not (ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_fedora_coreos)

kubespray/extra_playbooks/roles/etcd/tasks/check_certs.yml

@@ -0,0 +1,169 @@
+---
+- name: "Check_certs | Register certs that have already been generated on first etcd node"
+  find:
+    paths: "{{ etcd_cert_dir }}"
+    patterns: "ca.pem,node*.pem,member*.pem,admin*.pem"
+    get_checksum: true
+  delegate_to: "{{ groups['etcd'][0] }}"
+  register: etcdcert_master
+  run_once: true
+
+- name: "Check_certs | Set default value for 'sync_certs', 'gen_certs' and 'etcd_secret_changed' to false"
+  set_fact:
+    sync_certs: false
+    gen_certs: false
+    etcd_secret_changed: false
+
+- name: "Check certs | Register ca and etcd admin/member certs on etcd hosts"
+  stat:
+    path: "{{ etcd_cert_dir }}/{{ item }}"
+    get_attributes: no
+    get_checksum: yes
+    get_mime: no
+  register: etcd_member_certs
+  when: inventory_hostname in groups['etcd']
+  with_items:
+    - ca.pem
+    - member-{{ inventory_hostname }}.pem
+    - member-{{ inventory_hostname }}-key.pem
+    - admin-{{ inventory_hostname }}.pem
+    - admin-{{ inventory_hostname }}-key.pem
+
+- name: "Check certs | Register ca and etcd node certs on kubernetes hosts"
+  stat:
+    path: "{{ etcd_cert_dir }}/{{ item }}"
+  register: etcd_node_certs
+  when: inventory_hostname in groups['k8s_cluster']
+  with_items:
+    - ca.pem
+    - node-{{ inventory_hostname }}.pem
+    - node-{{ inventory_hostname }}-key.pem
+
+- name: "Check_certs | Set 'gen_certs' to true if expected certificates are not on the first etcd node(1/2)"
+  set_fact:
+    gen_certs: true
+  when: force_etcd_cert_refresh or not item in etcdcert_master.files|map(attribute='path') | list
+  run_once: true
+  with_items: "{{ expected_files }}"
+  vars:
+    expected_files: >-
+      ['{{ etcd_cert_dir }}/ca.pem',
+      {% set etcd_members = groups['etcd'] %}
+      {% for host in etcd_members %}
+        '{{ etcd_cert_dir }}/admin-{{ host }}.pem',
+        '{{ etcd_cert_dir }}/admin-{{ host }}-key.pem',
+        '{{ etcd_cert_dir }}/member-{{ host }}.pem',
+        '{{ etcd_cert_dir }}/member-{{ host }}-key.pem',
+      {% endfor %}
+      {% set k8s_nodes = groups['kube_control_plane'] %}
+      {% for host in k8s_nodes %}
+        '{{ etcd_cert_dir }}/node-{{ host }}.pem',
+        '{{ etcd_cert_dir }}/node-{{ host }}-key.pem'
+        {% if not loop.last %}{{','}}{% endif %}
+      {% endfor %}]
+
+- name: "Check_certs | Set 'gen_certs' to true if expected certificates are not on the first etcd node(2/2)"
+  set_fact:
+    gen_certs: true
+  run_once: true
+  with_items: "{{ expected_files }}"
+  vars:
+    expected_files: >-
+      ['{{ etcd_cert_dir }}/ca.pem',
+      {% set etcd_members = groups['etcd'] %}
+      {% for host in etcd_members %}
+        '{{ etcd_cert_dir }}/admin-{{ host }}.pem',
+        '{{ etcd_cert_dir }}/admin-{{ host }}-key.pem',
+        '{{ etcd_cert_dir }}/member-{{ host }}.pem',
+        '{{ etcd_cert_dir }}/member-{{ host }}-key.pem',
+      {% endfor %}
+      {% set k8s_nodes = groups['k8s_cluster']|unique|sort %}
+      {% for host in k8s_nodes %}
+        '{{ etcd_cert_dir }}/node-{{ host }}.pem',
+        '{{ etcd_cert_dir }}/node-{{ host }}-key.pem'
+        {% if not loop.last %}{{','}}{% endif %}
+      {% endfor %}]
+  when:
+    - kube_network_plugin in ["calico", "flannel", "canal", "cilium"] or cilium_deploy_additionally | default(false) | bool
+    - kube_network_plugin != "calico" or calico_datastore == "etcd"
+    - force_etcd_cert_refresh or not item in etcdcert_master.files|map(attribute='path') | list
+
+- name: "Check_certs | Set 'gen_master_certs' object to track whether member and admin certs exist on first etcd node"
+  set_fact:
+    gen_master_certs: |-
+      {
+      {% set etcd_members = groups['etcd'] -%}
+      {% set existing_certs = etcdcert_master.files|map(attribute='path')|list|sort %}
+      {% for host in etcd_members -%}
+        {% set member_cert = "%s/member-%s.pem"|format(etcd_cert_dir, host) %}
+        {% set member_key = "%s/member-%s-key.pem"|format(etcd_cert_dir, host) %}
+        {% set admin_cert = "%s/admin-%s.pem"|format(etcd_cert_dir, host) %}
+        {% set admin_key = "%s/admin-%s-key.pem"|format(etcd_cert_dir, host) %}
+        {% if force_etcd_cert_refresh -%}
+        "{{ host }}": True,
+        {% elif member_cert in existing_certs and member_key in existing_certs and admin_cert in existing_certs and admin_key in existing_certs  -%}
+        "{{ host }}": False,
+        {% else -%}
+        "{{ host }}": True,
+        {% endif -%}
+      {% endfor %}
+      }
+  run_once: true
+
+- name: "Check_certs | Set 'gen_node_certs' object to track whether node certs exist on first etcd node"
+  set_fact:
+    gen_node_certs: |-
+      {
+      {% set k8s_nodes = groups['k8s_cluster'] -%}
+      {% set existing_certs = etcdcert_master.files|map(attribute='path')|list|sort %}
+      {% for host in k8s_nodes -%}
+        {% set host_cert = "%s/node-%s.pem"|format(etcd_cert_dir, host) %}
+        {% set host_key = "%s/node-%s-key.pem"|format(etcd_cert_dir, host) %}
+        {% if force_etcd_cert_refresh -%}
+        "{{ host }}": True,
+        {% elif host_cert in existing_certs and host_key in existing_certs -%}
+        "{{ host }}": False,
+        {% else -%}
+        "{{ host }}": True,
+        {% endif -%}
+      {% endfor %}
+      }
+  run_once: true
+
+- name: "Check_certs | Set 'etcd_member_requires_sync' to true if ca or member/admin cert and key don't exist on etcd member or checksum doesn't match"
+  set_fact:
+    etcd_member_requires_sync: true
+  when:
+    - inventory_hostname in groups['etcd']
+    - (not etcd_member_certs.results[0].stat.exists|default(false)) or
+      (not etcd_member_certs.results[1].stat.exists|default(false)) or
+      (not etcd_member_certs.results[2].stat.exists|default(false)) or
+      (not etcd_member_certs.results[3].stat.exists|default(false)) or
+      (not etcd_member_certs.results[4].stat.exists|default(false)) or
+      (etcd_member_certs.results[0].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[0].stat.path)|map(attribute="checksum")|first|default('')) or
+      (etcd_member_certs.results[1].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[1].stat.path)|map(attribute="checksum")|first|default('')) or
+      (etcd_member_certs.results[2].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[2].stat.path)|map(attribute="checksum")|first|default('')) or
+      (etcd_member_certs.results[3].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[3].stat.path)|map(attribute="checksum")|first|default('')) or
+      (etcd_member_certs.results[4].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[4].stat.path)|map(attribute="checksum")|first|default(''))
+
+- name: "Check_certs | Set 'kubernetes_host_requires_sync' to true if ca or node cert and key don't exist on kubernetes host or checksum doesn't match"
+  set_fact:
+    kubernetes_host_requires_sync: true
+  when:
+    - inventory_hostname in groups['k8s_cluster'] and
+      inventory_hostname not in groups['etcd']
+    - (not etcd_node_certs.results[0].stat.exists|default(false)) or
+      (not etcd_node_certs.results[1].stat.exists|default(false)) or
+      (not etcd_node_certs.results[2].stat.exists|default(false)) or
+      (etcd_node_certs.results[0].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_node_certs.results[0].stat.path)|map(attribute="checksum")|first|default('')) or
+      (etcd_node_certs.results[1].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_node_certs.results[1].stat.path)|map(attribute="checksum")|first|default('')) or
+      (etcd_node_certs.results[2].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_node_certs.results[2].stat.path)|map(attribute="checksum")|first|default(''))
+
+- name: "Check_certs | Set 'sync_certs' to true"
+  set_fact:
+    sync_certs: true
+  when:
+    - etcd_member_requires_sync|default(false) or
+      kubernetes_host_requires_sync|default(false) or
+      (inventory_hostname in gen_master_certs and gen_master_certs[inventory_hostname]) or
+      (inventory_hostname in gen_node_certs and gen_node_certs[inventory_hostname])

kubespray/extra_playbooks/roles/etcd/tasks/configure.yml

@@ -0,0 +1,168 @@
+---
+- name: Configure | Check if etcd cluster is healthy
+  shell: "set -o pipefail && {{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health  2>&1 | grep -v 'Error: unhealthy cluster' >/dev/null"
+  args:
+    executable: /bin/bash
+  register: etcd_cluster_is_healthy
+  failed_when: false
+  changed_when: false
+  check_mode: no
+  run_once: yes
+  when: is_etcd_master and etcd_cluster_setup
+  tags:
+    - facts
+  environment:
+    ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
+    ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
+
+- name: Configure | Check if etcd-events cluster is healthy
+  shell: "set -o pipefail && {{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health  2>&1 | grep -v 'Error: unhealthy cluster' >/dev/null"
+  args:
+    executable: /bin/bash
+  register: etcd_events_cluster_is_healthy
+  failed_when: false
+  changed_when: false
+  check_mode: no
+  run_once: yes
+  when: is_etcd_master and etcd_events_cluster_setup
+  tags:
+    - facts
+  environment:
+    ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
+    ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
+
+- include_tasks: refresh_config.yml
+  when: is_etcd_master
+
+- name: Configure | Copy etcd.service systemd file
+  template:
+    src: "etcd-{{ etcd_deployment_type }}.service.j2"
+    dest: /etc/systemd/system/etcd.service
+    backup: yes
+    mode: 0644
+  when: is_etcd_master and etcd_cluster_setup
+
+- name: Configure | Copy etcd-events.service systemd file
+  template:
+    src: "etcd-events-{{ etcd_deployment_type }}.service.j2"
+    dest: /etc/systemd/system/etcd-events.service
+    backup: yes
+    mode: 0644
+  when: is_etcd_master and etcd_events_cluster_setup
+
+- name: Configure | reload systemd
+  systemd:
+    daemon_reload: true
+  when: is_etcd_master
+
+# when scaling new etcd will fail to start
+- name: Configure | Ensure etcd is running
+  service:
+    name: etcd
+    state: started
+    enabled: yes
+  ignore_errors: "{{ etcd_cluster_is_healthy.rc == 0 }}"  # noqa ignore-errors
+  when: is_etcd_master and etcd_cluster_setup
+
+# when scaling new etcd will fail to start
+- name: Configure | Ensure etcd-events is running
+  service:
+    name: etcd-events
+    state: started
+    enabled: yes
+  ignore_errors: "{{ etcd_events_cluster_is_healthy.rc != 0 }}"  # noqa ignore-errors
+  when: is_etcd_master and etcd_events_cluster_setup
+
+- name: Configure | Wait for etcd cluster to be healthy
+  shell: "set -o pipefail && {{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -v 'Error: unhealthy cluster' >/dev/null"
+  args:
+    executable: /bin/bash
+  register: etcd_cluster_is_healthy
+  until: etcd_cluster_is_healthy.rc == 0
+  retries: "{{ etcd_retries }}"
+  delay: "{{ retry_stagger | random + 3 }}"
+  changed_when: false
+  check_mode: no
+  run_once: yes
+  when:
+    - is_etcd_master
+    - etcd_cluster_setup
+  tags:
+    - facts
+  environment:
+    ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
+    ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
+
+- name: Configure | Wait for etcd-events cluster to be healthy
+  shell: "set -o pipefail && {{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -v 'Error: unhealthy cluster' >/dev/null"
+  args:
+    executable: /bin/bash
+  register: etcd_events_cluster_is_healthy
+  until: etcd_events_cluster_is_healthy.rc == 0
+  retries: "{{ etcd_retries }}"
+  delay: "{{ retry_stagger | random + 3 }}"
+  changed_when: false
+  check_mode: no
+  run_once: yes
+  when:
+    - is_etcd_master
+    - etcd_events_cluster_setup
+  tags:
+    - facts
+  environment:
+    ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
+    ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
+
+- name: Configure | Check if member is in etcd cluster
+  shell: "{{ bin_dir }}/etcdctl member list | grep -q {{ etcd_access_address }}"
+  register: etcd_member_in_cluster
+  ignore_errors: true  # noqa ignore-errors
+  changed_when: false
+  check_mode: no
+  when: is_etcd_master and etcd_cluster_setup
+  tags:
+    - facts
+  environment:
+    ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
+    ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
+
+- name: Configure | Check if member is in etcd-events cluster
+  shell: "{{ bin_dir }}/etcdctl member list | grep -q {{ etcd_access_address }}"
+  register: etcd_events_member_in_cluster
+  ignore_errors: true  # noqa ignore-errors
+  changed_when: false
+  check_mode: no
+  when: is_etcd_master and etcd_events_cluster_setup
+  tags:
+    - facts
+  environment:
+    ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
+    ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
+
+- name: Configure | Join member(s) to etcd cluster one at a time
+  include_tasks: join_etcd_member.yml
+  with_items: "{{ groups['etcd'] }}"
+  when: inventory_hostname == item and etcd_cluster_setup and etcd_member_in_cluster.rc != 0 and etcd_cluster_is_healthy.rc == 0
+
+- name: Configure | Join member(s) to etcd-events cluster one at a time
+  include_tasks: join_etcd-events_member.yml
+  with_items: "{{ groups['etcd'] }}"
+  when: inventory_hostname == item and etcd_events_cluster_setup and etcd_events_member_in_cluster.rc != 0 and etcd_events_cluster_is_healthy.rc == 0

kubespray/extra_playbooks/roles/etcd/tasks/gen_certs_script.yml

@@ -0,0 +1,166 @@
+---
+- name: Gen_certs | create etcd cert dir
+  file:
+    path: "{{ etcd_cert_dir }}"
+    group: "{{ etcd_cert_group }}"
+    state: directory
+    owner: "{{ etcd_owner }}"
+    mode: "{{ etcd_cert_dir_mode }}"
+    recurse: yes
+
+- name: "Gen_certs | create etcd script dir (on {{ groups['etcd'][0] }})"
+  file:
+    path: "{{ etcd_script_dir }}"
+    state: directory
+    owner: root
+    mode: 0700
+  run_once: yes
+  when: inventory_hostname == groups['etcd'][0]
+
+- name: Gen_certs | write openssl config
+  template:
+    src: "openssl.conf.j2"
+    dest: "{{ etcd_config_dir }}/openssl.conf"
+    mode: 0640
+  run_once: yes
+  delegate_to: "{{ groups['etcd'][0] }}"
+  when:
+    - gen_certs|default(false)
+    - inventory_hostname == groups['etcd'][0]
+
+- name: Gen_certs | copy certs generation script
+  template:
+    src: "make-ssl-etcd.sh.j2"
+    dest: "{{ etcd_script_dir }}/make-ssl-etcd.sh"
+    mode: 0700
+  run_once: yes
+  when:
+    - gen_certs|default(false)
+    - inventory_hostname == groups['etcd'][0]
+
+- name: Gen_certs | run cert generation script for etcd and kube control plane nodes
+  command: "bash -x {{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}"
+  environment:
+    - MASTERS: "{% for m in groups['etcd'] %}
+                  {% if gen_master_certs[m] %}
+                    {{ m }}
+                  {% endif %}
+                {% endfor %}"
+    - HOSTS: "{% for h in groups['kube_control_plane'] %}
+                {% if gen_node_certs[h] %}
+                    {{ h }}
+                {% endif %}
+              {% endfor %}"
+  run_once: yes
+  delegate_to: "{{ groups['etcd'][0] }}"
+  when: gen_certs|default(false)
+  notify: set etcd_secret_changed
+
+- name: Gen_certs | run cert generation script for all clients
+  command: "bash -x {{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}"
+  environment:
+    - HOSTS: "{% for h in groups['k8s_cluster'] %}
+                {% if gen_node_certs[h] %}
+                    {{ h }}
+                {% endif %}
+              {% endfor %}"
+  run_once: yes
+  delegate_to: "{{ groups['etcd'][0] }}"
+  when:
+    - kube_network_plugin in ["calico", "flannel", "canal", "cilium"] or cilium_deploy_additionally | default(false) | bool
+    - kube_network_plugin != "calico" or calico_datastore == "etcd"
+    - gen_certs|default(false)
+  notify: set etcd_secret_changed
+
+- name: Gen_certs | Gather etcd member/admin and kube_control_plane clinet certs from first etcd node
+  slurp:
+    src: "{{ item }}"
+  register: etcd_master_certs
+  with_items:
+    - "{{ etcd_cert_dir }}/ca.pem"
+    - "{{ etcd_cert_dir }}/ca-key.pem"
+    - "[{% for node in groups['etcd'] %}
+        '{{ etcd_cert_dir }}/admin-{{ node }}.pem',
+        '{{ etcd_cert_dir }}/admin-{{ node }}-key.pem',
+        '{{ etcd_cert_dir }}/member-{{ node }}.pem',
+        '{{ etcd_cert_dir }}/member-{{ node }}-key.pem',
+        {% endfor %}]"
+    - "[{% for node in (groups['kube_control_plane']) %}
+        '{{ etcd_cert_dir }}/node-{{ node }}.pem',
+        '{{ etcd_cert_dir }}/node-{{ node }}-key.pem',
+        {% endfor %}]"
+  delegate_to: "{{ groups['etcd'][0] }}"
+  when:
+    - inventory_hostname in groups['etcd']
+    - sync_certs|default(false)
+    - inventory_hostname != groups['etcd'][0]
+  notify: set etcd_secret_changed
+
+- name: Gen_certs | Write etcd member/admin and kube_control_plane clinet certs to other etcd nodes
+  copy:
+    dest: "{{ item.item }}"
+    content: "{{ item.content | b64decode }}"
+    group: "{{ etcd_cert_group }}"
+    owner: "{{ etcd_owner }}"
+    mode: 0640
+  with_items: "{{ etcd_master_certs.results }}"
+  when:
+    - inventory_hostname in groups['etcd']
+    - sync_certs|default(false)
+    - inventory_hostname != groups['etcd'][0]
+  loop_control:
+    label: "{{ item.item }}"
+
+- name: Gen_certs | Gather node certs from first etcd node
+  slurp:
+    src: "{{ item }}"
+  register: etcd_master_node_certs
+  with_items:
+    - "[{% for node in groups['k8s_cluster'] %}
+        '{{ etcd_cert_dir }}/node-{{ node }}.pem',
+        '{{ etcd_cert_dir }}/node-{{ node }}-key.pem',
+        {% endfor %}]"
+  delegate_to: "{{ groups['etcd'][0] }}"
+  when:
+    - inventory_hostname in groups['etcd']
+    - inventory_hostname != groups['etcd'][0]
+    - kube_network_plugin in ["calico", "flannel", "canal", "cilium"] or cilium_deploy_additionally | default(false) | bool
+    - kube_network_plugin != "calico" or calico_datastore == "etcd"
+  notify: set etcd_secret_changed
+
+- name: Gen_certs | Write node certs to other etcd nodes
+  copy:
+    dest: "{{ item.item }}"
+    content: "{{ item.content | b64decode }}"
+    group: "{{ etcd_cert_group }}"
+    owner: "{{ etcd_owner }}"
+    mode: 0640
+  with_items: "{{ etcd_master_node_certs.results }}"
+  when:
+    - inventory_hostname in groups['etcd']
+    - inventory_hostname != groups['etcd'][0]
+    - kube_network_plugin in ["calico", "flannel", "canal", "cilium"] or cilium_deploy_additionally | default(false) | bool
+    - kube_network_plugin != "calico" or calico_datastore == "etcd"
+  loop_control:
+    label: "{{ item.item }}"
+
+- include_tasks: gen_nodes_certs_script.yml
+  when:
+    - inventory_hostname in groups['kube_control_plane'] and
+        sync_certs|default(false) and inventory_hostname not in groups['etcd']
+
+- include_tasks: gen_nodes_certs_script.yml
+  when:
+    - kube_network_plugin in ["calico", "flannel", "canal", "cilium"] or cilium_deploy_additionally | default(false) | bool
+    - kube_network_plugin != "calico" or calico_datastore == "etcd"
+    - inventory_hostname in groups['k8s_cluster'] and
+        sync_certs|default(false) and inventory_hostname not in groups['etcd']
+
+- name: Gen_certs | check certificate permissions
+  file:
+    path: "{{ etcd_cert_dir }}"
+    group: "{{ etcd_cert_group }}"
+    state: directory
+    owner: "{{ etcd_owner }}"
+    mode: "{{ etcd_cert_dir_mode }}"
+    recurse: yes

kubespray/extra_playbooks/roles/etcd/tasks/gen_nodes_certs_script.yml

@@ -0,0 +1,32 @@
+---
+- name: Gen_certs | Set cert names per node
+  set_fact:
+    my_etcd_node_certs: [ 'ca.pem',
+                          'node-{{ inventory_hostname }}.pem',
+                          'node-{{ inventory_hostname }}-key.pem']
+  tags:
+    - facts
+
+- name: "Check_certs | Set 'sync_certs' to true on nodes"
+  set_fact:
+    sync_certs: true
+  with_items:
+    - "{{ my_etcd_node_certs }}"
+
+- name: Gen_certs | Gather node certs
+  shell: "set -o pipefail && tar cfz - -C {{ etcd_cert_dir }} {{ my_etcd_node_certs|join(' ') }} | base64 --wrap=0"
+  args:
+    executable: /bin/bash
+    warn: false
+  no_log: "{{ not (unsafe_show_logs|bool) }}"
+  register: etcd_node_certs
+  check_mode: no
+  delegate_to: "{{ groups['etcd'][0] }}"
+  changed_when: false
+
+- name: Gen_certs | Copy certs on nodes
+  shell: "set -o pipefail && base64 -d <<< '{{ etcd_node_certs.stdout|quote }}' | tar xz -C {{ etcd_cert_dir }}"
+  args:
+    executable: /bin/bash
+  no_log: "{{ not (unsafe_show_logs|bool) }}"
+  changed_when: false

kubespray/extra_playbooks/roles/etcd/tasks/install_docker.yml

@@ -0,0 +1,45 @@
+---
+- import_tasks: install_etcdctl_docker.yml
+  when: etcd_cluster_setup
+
+- name: Get currently-deployed etcd version
+  shell: "{{ docker_bin_dir }}/docker ps --filter='name={{ etcd_member_name }}' --format='{{ '{{ .Image }}' }}'"
+  register: etcd_current_docker_image
+  when: etcd_cluster_setup
+
+- name: Get currently-deployed etcd-events version
+  shell: "{{ docker_bin_dir }}/docker ps --filter='name={{ etcd_member_name }}-events' --format='{{ '{{ .Image }}' }}'"
+  register: etcd_events_current_docker_image
+  when: etcd_events_cluster_setup
+
+- name: Restart etcd if necessary
+  command: /bin/true
+  notify: restart etcd
+  when:
+    - etcd_cluster_setup
+    - etcd_image_tag not in etcd_current_docker_image.stdout|default('')
+
+- name: Restart etcd-events if necessary
+  command: /bin/true
+  notify: restart etcd-events
+  when:
+    - etcd_events_cluster_setup
+    - etcd_image_tag not in etcd_events_current_docker_image.stdout|default('')
+
+- name: Install etcd launch script
+  template:
+    src: etcd.j2
+    dest: "{{ bin_dir }}/etcd"
+    owner: 'root'
+    mode: 0750
+    backup: yes
+  when: etcd_cluster_setup
+
+- name: Install etcd-events launch script
+  template:
+    src: etcd-events.j2
+    dest: "{{ bin_dir }}/etcd-events"
+    owner: 'root'
+    mode: 0750
+    backup: yes
+  when: etcd_events_cluster_setup

kubespray/extra_playbooks/roles/etcd/tasks/install_etcdctl_docker.yml

@@ -0,0 +1,11 @@
+---
+- name: Install | Copy etcdctl binary from docker container
+  command: sh -c "{{ docker_bin_dir }}/docker rm -f etcdctl-binarycopy;
+            {{ docker_bin_dir }}/docker create --name etcdctl-binarycopy {{ etcd_image_repo }}:{{ etcd_image_tag }} &&
+            {{ docker_bin_dir }}/docker cp etcdctl-binarycopy:/usr/local/bin/etcdctl {{ bin_dir }}/etcdctl &&
+            {{ docker_bin_dir }}/docker rm -f etcdctl-binarycopy"
+  register: etcdctl_install_result
+  until: etcdctl_install_result.rc == 0
+  retries: "{{ etcd_retries }}"
+  delay: "{{ retry_stagger | random + 3 }}"
+  changed_when: false

kubespray/extra_playbooks/roles/etcd/tasks/install_host.yml

@@ -0,0 +1,41 @@
+---
+- name: Get currently-deployed etcd version
+  command: "{{ bin_dir }}/etcd --version"
+  register: etcd_current_host_version
+  # There's a chance this play could run before etcd is installed at all
+  ignore_errors: true
+  when: etcd_cluster_setup
+
+- name: Restart etcd if necessary
+  command: /bin/true
+  notify: restart etcd
+  when:
+    - etcd_cluster_setup
+    - etcd_version.lstrip('v') not in etcd_current_host_version.stdout|default('')
+
+- name: Restart etcd-events if necessary
+  command: /bin/true
+  notify: restart etcd-events
+  when:
+    - etcd_events_cluster_setup
+    - etcd_version.lstrip('v') not in etcd_current_host_version.stdout|default('')
+
+- name: install | Download etcd and etcdctl
+  include_tasks: "../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.etcd) }}"
+  when: etcd_cluster_setup
+  tags:
+    - never
+    - etcd
+
+- name: install | Copy etcd and etcdctl binary from download dir
+  copy:
+    src: "{{ local_release_dir }}/etcd-{{ etcd_version }}-linux-{{ host_architecture }}/{{ item }}"
+    dest: "{{ bin_dir }}/{{ item }}"
+    mode: 0755
+    remote_src: yes
+  with_items:
+    - etcd
+    - etcdctl
+  when: etcd_cluster_setup

kubespray/extra_playbooks/roles/etcd/tasks/join_etcd-events_member.yml

@@ -0,0 +1,47 @@
+---
+- name: Join Member | Add member to etcd-events cluster  # noqa 301 305
+  shell: "{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} --peer-urls={{ etcd_events_peer_url }}"
+  register: member_add_result
+  until: member_add_result.rc == 0
+  retries: "{{ etcd_retries }}"
+  delay: "{{ retry_stagger | random + 3 }}"
+  environment:
+    ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
+    ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
+
+- include_tasks: refresh_config.yml
+  vars:
+    etcd_events_peer_addresses: >-
+      {% for host in groups['etcd'] -%}
+        {%- if hostvars[host]['etcd_events_member_in_cluster'].rc == 0 -%}
+          {{ "etcd"+loop.index|string }}=https://{{ hostvars[host].etcd_events_access_address | default(hostvars[host].ip | default(fallback_ips[host])) }}:2382,
+        {%- endif -%}
+        {%- if loop.last -%}
+          {{ etcd_member_name }}={{ etcd_events_peer_url }}
+        {%- endif -%}
+      {%- endfor -%}
+
+- name: Join Member | Ensure member is in etcd-events cluster
+  shell: "set -o pipefail && {{ bin_dir }}/etcdctl member list | grep {{ etcd_events_access_address }} >/dev/null"
+  args:
+    executable: /bin/bash
+  register: etcd_events_member_in_cluster
+  changed_when: false
+  check_mode: no
+  tags:
+    - facts
+  environment:
+    ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
+    ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
+
+- name: Configure | Ensure etcd-events is running
+  service:
+    name: etcd-events
+    state: started
+    enabled: yes

kubespray/extra_playbooks/roles/etcd/tasks/join_etcd_member.yml

@@ -0,0 +1,51 @@
+---
+- name: Join Member | Add member to etcd cluster  # noqa 301 305
+  shell: "{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} --peer-urls={{ etcd_peer_url }}"
+  register: member_add_result
+  until: member_add_result.rc == 0 or 'Peer URLs already exists' in member_add_result.stderr
+  failed_when: member_add_result.rc != 0 and 'Peer URLs already exists' not in member_add_result.stderr
+  retries: "{{ etcd_retries }}"
+  delay: "{{ retry_stagger | random + 3 }}"
+  environment:
+    ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
+    ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
+
+- include_tasks: refresh_config.yml
+  vars:
+    etcd_peer_addresses: >-
+      {% for host in groups['etcd'] -%}
+        {%- if hostvars[host]['etcd_member_in_cluster'].rc == 0 -%}
+          {{ "etcd"+loop.index|string }}=https://{{ hostvars[host].etcd_access_address | default(hostvars[host].ip | default(fallback_ips[host])) }}:2380,
+        {%- endif -%}
+        {%- if loop.last -%}
+          {{ etcd_member_name }}={{ etcd_peer_url }}
+        {%- endif -%}
+      {%- endfor -%}
+
+- name: Join Member | Ensure member is in etcd cluster
+  shell: "set -o pipefail && {{ bin_dir }}/etcdctl member list | grep {{ etcd_access_address }} >/dev/null"
+  args:
+    executable: /bin/bash
+  register: etcd_member_in_cluster
+  changed_when: false
+  check_mode: no
+  retries: "{{ etcd_retries }}"
+  delay: "{{ retry_stagger | random + 3 }}"
+  until: etcd_member_in_cluster.rc == 0
+  tags:
+    - facts
+  environment:
+    ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
+    ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
+
+- name: Configure | Ensure etcd is running
+  service:
+    name: etcd
+    state: started
+    enabled: yes

kubespray/extra_playbooks/roles/etcd/tasks/main.yml

@@ -0,0 +1,77 @@
+---
+- include_tasks: check_certs.yml
+  when: cert_management == "script"
+  tags:
+    - etcd-secrets
+    - facts
+
+- include_tasks: "gen_certs_script.yml"
+  when:
+    - cert_management |d('script') == "script"
+  tags:
+    - etcd-secrets
+
+- include_tasks: upd_ca_trust.yml
+  when:
+    - inventory_hostname in groups['etcd']|union(groups['kube_control_plane'])|unique|sort
+  tags:
+    - etcd-secrets
+
+- include_tasks: upd_ca_trust.yml
+  when:
+    - kube_network_plugin in ["calico", "flannel", "canal", "cilium"] or cilium_deploy_additionally | default(false) | bool
+    - kube_network_plugin != "calico" or calico_datastore == "etcd"
+    - inventory_hostname in groups['k8s_cluster']
+  tags:
+    - etcd-secrets
+
+- name: "Gen_certs | Get etcd certificate serials"
+  command: "openssl x509 -in {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem -noout -serial"
+  register: "etcd_client_cert_serial_result"
+  changed_when: false
+  check_mode: no
+  when:
+    - kube_network_plugin in ["calico", "flannel", "canal", "cilium"] or cilium_deploy_additionally | default(false) | bool
+    - kube_network_plugin != "calico" or calico_datastore == "etcd"
+    - inventory_hostname in groups['k8s_cluster']
+  tags:
+    - master
+    - network
+
+- name: Set etcd_client_cert_serial
+  set_fact:
+    etcd_client_cert_serial: "{{ etcd_client_cert_serial_result.stdout.split('=')[1] }}"
+  when:
+    - kube_network_plugin in ["calico", "flannel", "canal", "cilium"] or cilium_deploy_additionally | default(false) | bool
+    - kube_network_plugin != "calico" or calico_datastore == "etcd"
+    - inventory_hostname in groups['k8s_cluster']
+  tags:
+    - master
+    - network
+
+- include_tasks: "install_{{ etcd_deployment_type }}.yml"
+  when: is_etcd_master
+  tags:
+    - upgrade
+
+- include_tasks: configure.yml
+  when: is_etcd_master
+
+- include_tasks: refresh_config.yml
+  when: is_etcd_master
+
+- name: Restart etcd if certs changed
+  command: /bin/true
+  notify: restart etcd
+  when: is_etcd_master and etcd_cluster_setup and etcd_secret_changed|default(false)
+
+- name: Restart etcd-events if certs changed
+  command: /bin/true
+  notify: restart etcd
+  when: is_etcd_master and etcd_events_cluster_setup and etcd_secret_changed|default(false)
+
+# After etcd cluster is assembled, make sure that
+# initial state of the cluster is in `existing`
+# state instead of `new`.
+- include_tasks: refresh_config.yml
+  when: is_etcd_master

kubespray/extra_playbooks/roles/etcd/tasks/refresh_config.yml

@@ -0,0 +1,16 @@
+---
+- name: Refresh config | Create etcd config file
+  template:
+    src: etcd.env.j2
+    dest: /etc/etcd.env
+    mode: 0640
+  notify: restart etcd
+  when: is_etcd_master and etcd_cluster_setup
+
+- name: Refresh config | Create etcd-events config file
+  template:
+    src: etcd-events.env.j2
+    dest: /etc/etcd-events.env
+    mode: 0640
+  notify: restart etcd-events
+  when: is_etcd_master and etcd_events_cluster_setup

kubespray/extra_playbooks/roles/etcd/tasks/upd_ca_trust.yml

@@ -0,0 +1,37 @@
+---
+- name: Gen_certs | target ca-certificate store file
+  set_fact:
+    ca_cert_path: |-
+      {% if ansible_os_family == "Debian" -%}
+      /usr/local/share/ca-certificates/etcd-ca.crt
+      {%- elif ansible_os_family == "RedHat" -%}
+      /etc/pki/ca-trust/source/anchors/etcd-ca.crt
+      {%- elif ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] -%}
+      /etc/ssl/certs/etcd-ca.pem
+      {%- elif ansible_os_family == "Suse" -%}
+      /etc/pki/trust/anchors/etcd-ca.pem
+      {%- elif ansible_os_family == "ClearLinux" -%}
+      /usr/share/ca-certs/etcd-ca.pem
+      {%- endif %}
+  tags:
+    - facts
+
+- name: Gen_certs | add CA to trusted CA dir
+  copy:
+    src: "{{ etcd_cert_dir }}/ca.pem"
+    dest: "{{ ca_cert_path }}"
+    remote_src: true
+    mode: 0640
+  register: etcd_ca_cert
+
+- name: Gen_certs | update ca-certificates (Debian/Ubuntu/SUSE/Flatcar)  # noqa 503
+  command: update-ca-certificates
+  when: etcd_ca_cert.changed and ansible_os_family in ["Debian", "Flatcar", "Flatcar Container Linux by Kinvolk", "Suse"]
+
+- name: Gen_certs | update ca-certificates (RedHat)  # noqa 503
+  command: update-ca-trust extract
+  when: etcd_ca_cert.changed and ansible_os_family == "RedHat"
+
+- name: Gen_certs | update ca-certificates (ClearLinux)  # noqa 503
+  command: clrtrust add "{{ ca_cert_path }}"
+  when: etcd_ca_cert.changed and ansible_os_family == "ClearLinux"

kubespray/extra_playbooks/roles/etcd/templates/etcd-docker.service.j2

@@ -0,0 +1,18 @@
+[Unit]
+Description=etcd docker wrapper
+Wants=docker.socket
+After=docker.service
+
+[Service]
+User=root
+PermissionsStartOnly=true
+EnvironmentFile=-/etc/etcd.env
+ExecStart={{ bin_dir }}/etcd
+ExecStartPre=-{{ docker_bin_dir }}/docker rm -f {{ etcd_member_name | default("etcd") }}
+ExecStop={{ docker_bin_dir }}/docker stop {{ etcd_member_name | default("etcd") }}
+Restart=always
+RestartSec=15s
+TimeoutStartSec=30s
+
+[Install]
+WantedBy=multi-user.target

kubespray/extra_playbooks/roles/etcd/templates/etcd-events-docker.service.j2

@@ -0,0 +1,18 @@
+[Unit]
+Description=etcd docker wrapper
+Wants=docker.socket
+After=docker.service
+
+[Service]
+User=root
+PermissionsStartOnly=true
+EnvironmentFile=-/etc/etcd-events.env
+ExecStart={{ bin_dir }}/etcd-events
+ExecStartPre=-{{ docker_bin_dir }}/docker rm -f {{ etcd_member_name }}-events
+ExecStop={{ docker_bin_dir }}/docker stop {{ etcd_member_name }}-events
+Restart=always
+RestartSec=15s
+TimeoutStartSec=30s
+
+[Install]
+WantedBy=multi-user.target

kubespray/extra_playbooks/roles/etcd/templates/etcd-events-host.service.j2

@@ -0,0 +1,16 @@
+[Unit]
+Description=etcd
+After=network.target
+
+[Service]
+Type=notify
+User=root
+EnvironmentFile=/etc/etcd-events.env
+ExecStart={{ bin_dir }}/etcd
+NotifyAccess=all
+Restart=always
+RestartSec=10s
+LimitNOFILE=40000
+
+[Install]
+WantedBy=multi-user.target

kubespray/extra_playbooks/roles/etcd/templates/etcd-events.env.j2

@@ -0,0 +1,43 @@
+ETCD_DATA_DIR={{ etcd_events_data_dir }}
+ETCD_ADVERTISE_CLIENT_URLS={{ etcd_events_client_url }}
+ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_events_peer_url }}
+ETCD_INITIAL_CLUSTER_STATE={% if etcd_events_cluster_is_healthy.rc == 0 | bool %}existing{% else %}new{% endif %}
+
+ETCD_METRICS={{ etcd_metrics }}
+ETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address }}:2383,https://127.0.0.1:2383
+ETCD_ELECTION_TIMEOUT={{ etcd_election_timeout }}
+ETCD_HEARTBEAT_INTERVAL={{ etcd_heartbeat_interval }}
+ETCD_INITIAL_CLUSTER_TOKEN=k8s_events_etcd
+ETCD_LISTEN_PEER_URLS=https://{{ etcd_address }}:2382
+ETCD_NAME={{ etcd_member_name }}-events
+ETCD_PROXY=off
+ETCD_INITIAL_CLUSTER={{ etcd_events_peer_addresses }}
+ETCD_AUTO_COMPACTION_RETENTION={{ etcd_compaction_retention }}
+{% if etcd_snapshot_count is defined %}
+ETCD_SNAPSHOT_COUNT={{ etcd_snapshot_count }}
+{% endif %}
+{% if etcd_quota_backend_bytes is defined %}
+ETCD_QUOTA_BACKEND_BYTES={{ etcd_quota_backend_bytes }}
+{% endif %}
+{% if etcd_max_request_bytes is defined %}
+ETCD_MAX_REQUEST_BYTES={{ etcd_max_request_bytes }}
+{% endif %}
+
+# TLS settings
+ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
+ETCD_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem
+ETCD_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem
+ETCD_CLIENT_CERT_AUTH={{ etcd_secure_client | lower}}
+
+ETCD_PEER_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
+ETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem
+ETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem
+ETCD_PEER_CLIENT_CERT_AUTH={{ etcd_peer_client_auth }}
+
+{% if etcd_tls_cipher_suites is defined %}
+ETCD_CIPHER_SUITES={% for tls in etcd_tls_cipher_suites %}{{ tls }}{{ "," if not loop.last else "" }}{% endfor %}
+{% endif %}
+
+{% for key, value in etcd_extra_vars.items() %}
+{{ key }}={{ value }}
+{% endfor %}

kubespray/extra_playbooks/roles/etcd/templates/etcd-events.j2

@@ -0,0 +1,21 @@
+#!/bin/bash
+{{ docker_bin_dir }}/docker run \
+  --restart=on-failure:5 \
+  --env-file=/etc/etcd-events.env \
+  --net=host \
+  -v /etc/ssl/certs:/etc/ssl/certs:ro \
+  -v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \
+  -v {{ etcd_events_data_dir }}:{{ etcd_events_data_dir }}:rw \
+  {% if etcd_memory_limit is defined %}
+  --memory={{ etcd_memory_limit|regex_replace('Mi', 'M') }} \
+  {% endif %}
+  {% if etcd_cpu_limit is defined %}
+  --cpu-shares={{ etcd_cpu_limit|regex_replace('m', '') }} \
+  {% endif %}
+  {% if etcd_blkio_weight is defined %}
+  --blkio-weight={{ etcd_blkio_weight }} \
+  {% endif %}
+  --name={{ etcd_member_name }}-events \
+  {{ etcd_image_repo }}:{{ etcd_image_tag }} \
+  /usr/local/bin/etcd \
+  "$@"

kubespray/extra_playbooks/roles/etcd/templates/etcd-host.service.j2

@@ -0,0 +1,16 @@
+[Unit]
+Description=etcd
+After=network.target
+
+[Service]
+Type=notify
+User=root
+EnvironmentFile=/etc/etcd.env
+ExecStart={{ bin_dir }}/etcd
+NotifyAccess=all
+Restart=always
+RestartSec=10s
+LimitNOFILE=40000
+
+[Install]
+WantedBy=multi-user.target

kubespray/extra_playbooks/roles/etcd/templates/etcd.env.j2

@@ -0,0 +1,68 @@
+# Environment file for etcd {{ etcd_version }}
+ETCD_DATA_DIR={{ etcd_data_dir }}
+ETCD_ADVERTISE_CLIENT_URLS={{ etcd_client_url }}
+ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_peer_url }}
+ETCD_INITIAL_CLUSTER_STATE={% if etcd_cluster_is_healthy.rc == 0 | bool %}existing{% else %}new{% endif %}
+
+ETCD_METRICS={{ etcd_metrics }}
+{% if etcd_metrics_port is defined %}
+ETCD_LISTEN_METRICS_URLS=http://{{ etcd_address }}:{{ etcd_metrics_port }},http://127.0.0.1:{{ etcd_metrics_port }}
+{% endif %}
+ETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address }}:2379,https://127.0.0.1:2379
+ETCD_ELECTION_TIMEOUT={{ etcd_election_timeout }}
+ETCD_HEARTBEAT_INTERVAL={{ etcd_heartbeat_interval }}
+ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd
+ETCD_LISTEN_PEER_URLS=https://{{ etcd_address }}:2380
+ETCD_NAME={{ etcd_member_name }}
+ETCD_PROXY=off
+ETCD_INITIAL_CLUSTER={{ etcd_peer_addresses }}
+ETCD_AUTO_COMPACTION_RETENTION={{ etcd_compaction_retention }}
+{% if etcd_snapshot_count is defined %}
+ETCD_SNAPSHOT_COUNT={{ etcd_snapshot_count }}
+{% endif %}
+{% if etcd_quota_backend_bytes is defined %}
+ETCD_QUOTA_BACKEND_BYTES={{ etcd_quota_backend_bytes }}
+{% endif %}
+{% if etcd_max_request_bytes is defined %}
+ETCD_MAX_REQUEST_BYTES={{ etcd_max_request_bytes }}
+{% endif %}
+{% if etcd_log_level is defined %}
+ETCD_LOG_LEVEL={{ etcd_log_level }}
+{% endif %}
+{% if etcd_max_snapshots is defined %}
+ETCD_MAX_SNAPSHOTS={{ etcd_max_snapshots }}
+{% endif %}
+{% if etcd_max_wals is defined %}
+ETCD_MAX_WALS={{ etcd_max_wals }}
+{% endif %}
+# Flannel need etcd v2 API
+ETCD_ENABLE_V2=true
+
+# TLS settings
+ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
+ETCD_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem
+ETCD_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem
+ETCD_CLIENT_CERT_AUTH={{ etcd_secure_client | lower}}
+
+ETCD_PEER_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
+ETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem
+ETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem
+ETCD_PEER_CLIENT_CERT_AUTH={{ etcd_peer_client_auth }}
+
+{% if etcd_tls_cipher_suites is defined %}
+ETCD_CIPHER_SUITES={% for tls in etcd_tls_cipher_suites %}{{ tls }}{{ "," if not loop.last else "" }}{% endfor %}
+{% endif %}
+
+{% for key, value in etcd_extra_vars.items() %}
+{{ key }}={{ value }}
+{% endfor %}
+
+# CLI settings
+ETCDCTL_ENDPOINTS=https://127.0.0.1:2379
+ETCDCTL_CACERT={{ etcd_cert_dir }}/ca.pem
+ETCDCTL_KEY={{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem
+ETCDCTL_CERT={{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem
+
+# ETCD 3.5.x issue
+# https://groups.google.com/a/kubernetes.io/g/dev/c/B7gJs88XtQc/m/rSgNOzV2BwAJ?utm_medium=email&utm_source=footer
+ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK={{ etcd_experimental_initial_corrupt_check }}

kubespray/extra_playbooks/roles/etcd/templates/etcd.j2

@@ -0,0 +1,21 @@
+#!/bin/bash
+{{ docker_bin_dir }}/docker run \
+  --restart=on-failure:5 \
+  --env-file=/etc/etcd.env \
+  --net=host \
+  -v /etc/ssl/certs:/etc/ssl/certs:ro \
+  -v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \
+  -v {{ etcd_data_dir }}:{{ etcd_data_dir }}:rw \
+{% if etcd_memory_limit is defined %}
+  --memory={{ etcd_memory_limit|regex_replace('Mi', 'M') }} \
+{% endif %}
+{% if etcd_cpu_limit is defined %}
+  --cpu-shares={{ etcd_cpu_limit|regex_replace('m', '') }} \
+{% endif %}
+{% if etcd_blkio_weight is defined %}
+  --blkio-weight={{ etcd_blkio_weight }} \
+{% endif %}
+  --name={{ etcd_member_name | default("etcd") }} \
+  {{ etcd_image_repo }}:{{ etcd_image_tag }} \
+  /usr/local/bin/etcd \
+  "$@"

kubespray/extra_playbooks/roles/etcd/templates/make-ssl-etcd.sh.j2

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+# Author: Smana smainklh@gmail.com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o pipefail
+usage()
+{
+    cat << EOF
+Create self signed certificates
+
+Usage : $(basename $0) -f <config> [-d <ssldir>]
+      -h | --help         : Show this message
+      -f | --config       : Openssl configuration file
+      -d | --ssldir       : Directory where the certificates will be installed
+
+               ex :
+               $(basename $0) -f openssl.conf -d /srv/ssl
+EOF
+}
+
+# Options parsing
+while (($#)); do
+    case "$1" in
+        -h | --help)   usage;   exit 0;;
+        -f | --config) CONFIG=${2}; shift 2;;
+        -d | --ssldir) SSLDIR="${2}"; shift 2;;
+        *)
+            usage
+            echo "ERROR : Unknown option"
+            exit 3
+        ;;
+    esac
+done
+
+if [ -z ${CONFIG} ]; then
+    echo "ERROR: the openssl configuration file is missing. option -f"
+    exit 1
+fi
+if [ -z ${SSLDIR} ]; then
+    SSLDIR="/etc/ssl/etcd"
+fi
+
+tmpdir=$(mktemp -d /tmp/etcd_cacert.XXXXXX)
+trap 'rm -rf "${tmpdir}"' EXIT
+cd "${tmpdir}"
+
+mkdir -p "${SSLDIR}"
+
+# Root CA
+if [ -e "$SSLDIR/ca-key.pem" ]; then
+    # Reuse existing CA
+    cp $SSLDIR/{ca.pem,ca-key.pem} .
+else
+    openssl genrsa -out ca-key.pem {{certificates_key_size}} > /dev/null 2>&1
+    openssl req -x509 -new -nodes -key ca-key.pem -days {{certificates_duration}} -out ca.pem -subj "/CN=etcd-ca" > /dev/null 2>&1
+fi
+
+# ETCD member
+if [ -n "$MASTERS" ]; then
+    for host in $MASTERS; do
+        cn="${host%%.*}"
+        # Member key
+        openssl genrsa -out member-${host}-key.pem {{certificates_key_size}} > /dev/null 2>&1
+        openssl req -new -key member-${host}-key.pem -out member-${host}.csr -subj "/CN=etcd-member-${cn}" -config ${CONFIG} > /dev/null 2>&1
+        openssl x509 -req -in member-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member-${host}.pem -days {{certificates_duration}} -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
+
+        # Admin key
+        openssl genrsa -out admin-${host}-key.pem {{certificates_key_size}} > /dev/null 2>&1
+        openssl req -new -key admin-${host}-key.pem -out admin-${host}.csr -subj "/CN=etcd-admin-${cn}" > /dev/null 2>&1
+        openssl x509 -req -in admin-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out admin-${host}.pem -days {{certificates_duration}} -extensions ssl_client  -extfile ${CONFIG} > /dev/null 2>&1
+    done
+fi
+
+# Node keys
+if [ -n "$HOSTS" ]; then
+    for host in $HOSTS; do
+        cn="${host%%.*}"
+        openssl genrsa -out node-${host}-key.pem {{certificates_key_size}} > /dev/null 2>&1
+        openssl req -new -key node-${host}-key.pem -out node-${host}.csr -subj "/CN=etcd-node-${cn}" > /dev/null 2>&1
+        openssl x509 -req -in node-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out node-${host}.pem -days {{certificates_duration}} -extensions ssl_client  -extfile ${CONFIG} > /dev/null 2>&1
+    done
+fi
+
+# Install certs
+if [ -e "$SSLDIR/ca-key.pem" ]; then
+    # No pass existing CA
+    rm -f ca.pem ca-key.pem
+fi
+
+mv *.pem ${SSLDIR}/

kubespray/extra_playbooks/roles/etcd/templates/openssl.conf.j2

@@ -0,0 +1,45 @@
+{% set counter = {'dns': 2,'ip': 1,} %}{% macro increment(dct, key, inc=1)%}{% if dct.update({key: dct[key] + inc}) %} {% endif %}{% endmacro %}[req]
+req_extensions = v3_req
+distinguished_name = req_distinguished_name
+
+[req_distinguished_name]
+
+[ v3_req ]
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+subjectAltName = @alt_names
+
+[ ssl_client ]
+extendedKeyUsage = clientAuth, serverAuth
+basicConstraints = CA:FALSE
+subjectKeyIdentifier=hash
+authorityKeyIdentifier=keyid,issuer
+subjectAltName = @alt_names
+
+[ v3_ca ]
+basicConstraints = CA:TRUE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+subjectAltName = @alt_names
+authorityKeyIdentifier=keyid:always,issuer
+
+[alt_names]
+DNS.1 = localhost
+{% for host in groups['etcd'] %}
+DNS.{{ counter["dns"] }} = {{ host }}{{ increment(counter, 'dns') }}
+{% endfor %}
+{% if apiserver_loadbalancer_domain_name is defined %}
+DNS.{{ counter["dns"] }} = {{ apiserver_loadbalancer_domain_name }}{{ increment(counter, 'dns') }}
+{% endif %}
+{% for etcd_alt_name in etcd_cert_alt_names %}
+DNS.{{ counter["dns"] }} = {{ etcd_alt_name }}{{ increment(counter, 'dns') }}
+{% endfor %}
+{% for host in groups['etcd'] %}
+{% if hostvars[host]['access_ip'] is defined  %}
+IP.{{ counter["ip"] }} = {{ hostvars[host]['access_ip'] }}{{ increment(counter, 'ip') }}
+{% endif %}
+IP.{{ counter["ip"] }} = {{ hostvars[host]['ip'] | default(fallback_ips[host]) }}{{ increment(counter, 'ip') }}
+{% endfor %}
+{% for cert_alt_ip in etcd_cert_alt_ips %}
+IP.{{ counter["ip"] }} = {{ cert_alt_ip }}{{ increment(counter, 'ip') }}
+{% endfor %}
+IP.{{ counter["ip"] }} = 127.0.0.1