update

kubespray/extra_playbooks/roles/kubernetes/node/tasks/cloud-credentials/azure-credential-check.yml

@@ -0,0 +1,82 @@
+---
+- name: check azure_tenant_id value
+  fail:
+    msg: "azure_tenant_id is missing"
+  when: azure_tenant_id is not defined or not azure_tenant_id
+
+- name: check azure_subscription_id value
+  fail:
+    msg: "azure_subscription_id is missing"
+  when: azure_subscription_id is not defined or not azure_subscription_id
+
+- name: check azure_aad_client_id value
+  fail:
+    msg: "azure_aad_client_id is missing"
+  when: azure_aad_client_id is not defined or not azure_aad_client_id
+
+- name: check azure_aad_client_secret value
+  fail:
+    msg: "azure_aad_client_secret is missing"
+  when: azure_aad_client_secret is not defined or not azure_aad_client_secret
+
+- name: check azure_resource_group value
+  fail:
+    msg: "azure_resource_group is missing"
+  when: azure_resource_group is not defined or not azure_resource_group
+
+- name: check azure_location value
+  fail:
+    msg: "azure_location is missing"
+  when: azure_location is not defined or not azure_location
+
+- name: check azure_subnet_name value
+  fail:
+    msg: "azure_subnet_name is missing"
+  when: azure_subnet_name is not defined or not azure_subnet_name
+
+- name: check azure_security_group_name value
+  fail:
+    msg: "azure_security_group_name is missing"
+  when: azure_security_group_name is not defined or not azure_security_group_name
+
+- name: check azure_vnet_name value
+  fail:
+    msg: "azure_vnet_name is missing"
+  when: azure_vnet_name is not defined or not azure_vnet_name
+
+- name: check azure_vnet_resource_group value
+  fail:
+    msg: "azure_vnet_resource_group is missing"
+  when: azure_vnet_resource_group is not defined or not azure_vnet_resource_group
+
+- name: check azure_route_table_name value
+  fail:
+    msg: "azure_route_table_name is missing"
+  when: azure_route_table_name is not defined or not azure_route_table_name
+
+- name: check azure_loadbalancer_sku value
+  fail:
+    msg: "azure_loadbalancer_sku has an invalid value '{{ azure_loadbalancer_sku }}'. Supported values are 'basic', 'standard'"
+  when: azure_loadbalancer_sku not in ["basic", "standard"]
+
+- name: "check azure_exclude_master_from_standard_lb is a bool"
+  assert:
+    that: azure_exclude_master_from_standard_lb |type_debug == 'bool'
+
+- name: "check azure_disable_outbound_snat is a bool"
+  assert:
+    that: azure_disable_outbound_snat |type_debug == 'bool'
+
+- name: "check azure_use_instance_metadata is a bool"
+  assert:
+    that: azure_use_instance_metadata |type_debug == 'bool'
+
+- name: check azure_vmtype value
+  fail:
+    msg: "azure_vmtype is missing. Supported values are 'standard' or 'vmss'"
+  when: azure_vmtype is not defined or not azure_vmtype
+
+- name: check azure_cloud value
+  fail:
+    msg: "azure_cloud has an invalid value '{{ azure_cloud }}'. Supported values are 'AzureChinaCloud', 'AzureGermanCloud', 'AzurePublicCloud', 'AzureUSGovernmentCloud'."
+  when: azure_cloud not in ["AzureChinaCloud", "AzureGermanCloud", "AzurePublicCloud", "AzureUSGovernmentCloud"]

kubespray/extra_playbooks/roles/kubernetes/node/tasks/cloud-credentials/openstack-credential-check.yml

@@ -0,0 +1,34 @@
+---
+- name: check openstack_auth_url value
+  fail:
+    msg: "openstack_auth_url is missing"
+  when: openstack_auth_url is not defined or not openstack_auth_url
+
+- name: check openstack_username value
+  fail:
+    msg: "openstack_username is missing"
+  when: openstack_username is not defined or not openstack_username
+
+- name: check openstack_password value
+  fail:
+    msg: "openstack_password is missing"
+  when: openstack_password is not defined or not openstack_password
+
+- name: check openstack_region value
+  fail:
+    msg: "openstack_region is missing"
+  when: openstack_region is not defined or not openstack_region
+
+- name: check openstack_tenant_id value
+  fail:
+    msg: "one of openstack_tenant_id or openstack_trust_id must be specified"
+  when:
+    - openstack_tenant_id is not defined or not openstack_tenant_id
+    - openstack_trust_id is not defined
+
+- name: check openstack_trust_id value
+  fail:
+    msg: "one of openstack_tenant_id or openstack_trust_id must be specified"
+  when:
+    - openstack_trust_id is not defined or not openstack_trust_id
+    - openstack_tenant_id is not defined

kubespray/extra_playbooks/roles/kubernetes/node/tasks/cloud-credentials/vsphere-credential-check.yml

@@ -0,0 +1,22 @@
+---
+- name: check vsphere environment variables
+  fail:
+    msg: "{{ item.name }} is missing"
+  when: item.value is not defined or not item.value
+  with_items:
+    - name: vsphere_vcenter_ip
+      value: "{{ vsphere_vcenter_ip }}"
+    - name: vsphere_vcenter_port
+      value: "{{ vsphere_vcenter_port }}"
+    - name: vsphere_user
+      value: "{{ vsphere_user }}"
+    - name: vsphere_password
+      value: "{{ vsphere_password }}"
+    - name: vsphere_datacenter
+      value: "{{ vsphere_datacenter }}"
+    - name: vsphere_datastore
+      value: "{{ vsphere_datastore }}"
+    - name: vsphere_working_dir
+      value: "{{ vsphere_working_dir }}"
+    - name: vsphere_insecure
+      value: "{{ vsphere_insecure }}"

kubespray/extra_playbooks/roles/kubernetes/node/tasks/facts.yml

@@ -0,0 +1,57 @@
+---
+- block:
+  - name: look up docker cgroup driver
+    shell: "docker info | grep 'Cgroup Driver' | awk -F': ' '{ print $2; }'"
+    register: docker_cgroup_driver_result
+    changed_when: false
+    check_mode: no
+
+  - name: set kubelet_cgroup_driver_detected fact for docker
+    set_fact:
+      kubelet_cgroup_driver_detected: "{{ docker_cgroup_driver_result.stdout }}"
+  when: container_manager == 'docker'
+
+- block:
+  - name: look up crio cgroup driver
+    shell: "{{ bin_dir }}/crio-status info | grep 'cgroup driver' | awk -F': ' '{ print $2; }'"
+    register: crio_cgroup_driver_result
+    changed_when: false
+
+  - name: set kubelet_cgroup_driver_detected fact for crio
+    set_fact:
+      kubelet_cgroup_driver_detected: "{{ crio_cgroup_driver_result.stdout }}"
+  when: container_manager == 'crio'
+
+- name: set kubelet_cgroup_driver_detected fact for containerd
+  set_fact:
+    kubelet_cgroup_driver_detected: >-
+      {%- if containerd_use_systemd_cgroup -%}systemd{%- else -%}cgroupfs{%- endif -%}
+  when: container_manager == 'containerd'
+
+- name: set kubelet_cgroup_driver
+  set_fact:
+    kubelet_cgroup_driver: "{{ kubelet_cgroup_driver_detected }}"
+  when: kubelet_cgroup_driver is undefined
+
+- name: set kubelet_cgroups options when cgroupfs is used
+  set_fact:
+    kubelet_runtime_cgroups: "{{ kubelet_runtime_cgroups_cgroupfs }}"
+    kubelet_kubelet_cgroups: "{{ kubelet_kubelet_cgroups_cgroupfs }}"
+  when: kubelet_cgroup_driver == 'cgroupfs'
+
+- name: set kubelet_config_extra_args options when cgroupfs is used
+  vars:
+  set_fact:
+    kubelet_config_extra_args: "{{ kubelet_config_extra_args | combine(kubelet_config_extra_args_cgroupfs) }}"
+  when: kubelet_cgroup_driver == 'cgroupfs'
+
+- name: os specific vars
+  include_vars: "{{ item }}"
+  with_first_found:
+  - files:
+    - "{{ ansible_distribution|lower }}-{{ ansible_distribution_version|lower|replace('/', '_') }}.yml"
+    - "{{ ansible_distribution|lower }}-{{ ansible_distribution_release }}.yml"
+    - "{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version|lower|replace('/', '_') }}.yml"
+    - "{{ ansible_distribution|lower }}.yml"
+    - "{{ ansible_os_family|lower }}.yml"
+    skip: true

kubespray/extra_playbooks/roles/kubernetes/node/tasks/install.yml

@@ -0,0 +1,22 @@
+---
+- name: install | Copy kubeadm binary from download dir
+  copy:
+    src: "{{ local_release_dir }}/kubeadm-{{ kubeadm_version }}-{{ image_arch }}"
+    dest: "{{ bin_dir }}/kubeadm"
+    mode: 0755
+    remote_src: true
+  tags:
+    - kubeadm
+  when:
+    - not inventory_hostname in groups['kube_control_plane']
+
+- name: install | Copy kubelet binary from download dir
+  copy:
+    src: "{{ local_release_dir }}/kubelet-{{ kube_version }}-{{ image_arch }}"
+    dest: "{{ bin_dir }}/kubelet"
+    mode: 0755
+    remote_src: true
+  tags:
+    - kubelet
+    - upgrade
+  notify: Node | restart kubelet

kubespray/extra_playbooks/roles/kubernetes/node/tasks/kubelet.yml

@@ -0,0 +1,52 @@
+---
+- name: Set kubelet api version to v1beta1
+  set_fact:
+    kubeletConfig_api_version: v1beta1
+  tags:
+    - kubelet
+    - kubeadm
+
+- name: Write kubelet environment config file (kubeadm)
+  template:
+    src: "kubelet.env.{{ kubeletConfig_api_version }}.j2"
+    dest: "{{ kube_config_dir }}/kubelet.env"
+    setype: "{{ (preinstall_selinux_state != 'disabled') | ternary('etc_t', omit) }}"
+    backup: yes
+    mode: 0640
+  notify: Node | restart kubelet
+  tags:
+    - kubelet
+    - kubeadm
+
+- name: Write kubelet config file
+  template:
+    src: "kubelet-config.{{ kubeletConfig_api_version }}.yaml.j2"
+    dest: "{{ kube_config_dir }}/kubelet-config.yaml"
+    mode: 0640
+  notify: Kubelet | restart kubelet
+  tags:
+    - kubelet
+    - kubeadm
+
+- name: Write kubelet systemd init file
+  template:
+    src: "kubelet.service.j2"
+    dest: "/etc/systemd/system/kubelet.service"
+    backup: "yes"
+    mode: 0644
+  notify: Node | restart kubelet
+  tags:
+    - kubelet
+    - kubeadm
+
+- name: flush_handlers and reload-systemd
+  meta: flush_handlers
+
+- name: Enable kubelet
+  service:
+    name: kubelet
+    enabled: yes
+    state: started
+  tags:
+    - kubelet
+  notify: Kubelet | restart kubelet

kubespray/extra_playbooks/roles/kubernetes/node/tasks/loadbalancer/haproxy.yml

@@ -0,0 +1,34 @@
+---
+- name: haproxy | Cleanup potentially deployed nginx-proxy
+  file:
+    path: "{{ kube_manifest_dir }}/nginx-proxy.yml"
+    state: absent
+
+- name: haproxy | Make haproxy directory
+  file:
+    path: "{{ haproxy_config_dir }}"
+    state: directory
+    mode: 0755
+    owner: root
+
+- name: haproxy | Write haproxy configuration
+  template:
+    src: "loadbalancer/haproxy.cfg.j2"
+    dest: "{{ haproxy_config_dir }}/haproxy.cfg"
+    owner: root
+    mode: 0755
+    backup: yes
+
+- name: haproxy | Get checksum from config
+  stat:
+    path: "{{ haproxy_config_dir }}/haproxy.cfg"
+    get_attributes: no
+    get_checksum: yes
+    get_mime: no
+  register: haproxy_stat
+
+- name: haproxy | Write static pod
+  template:
+    src: manifests/haproxy.manifest.j2
+    dest: "{{ kube_manifest_dir }}/haproxy.yml"
+    mode: 0640

kubespray/extra_playbooks/roles/kubernetes/node/tasks/loadbalancer/kube-vip.yml

@@ -0,0 +1,13 @@
+---
+- name: kube-vip  | Check cluster settings for kube-vip
+  fail:
+    msg: "kube-vip require kube_proxy_strict_arp = true, see https://github.com/kube-vip/kube-vip/blob/main/docs/kubernetes/arp/index.md"
+  when:
+    - kube_proxy_mode == 'ipvs' and not kube_proxy_strict_arp
+    - kube_vip_arp_enabled
+
+- name: kube-vip | Write static pod
+  template:
+    src: manifests/kube-vip.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-vip.yml"
+    mode: 0640

kubespray/extra_playbooks/roles/kubernetes/node/tasks/loadbalancer/nginx-proxy.yml

@@ -0,0 +1,34 @@
+---
+- name: haproxy | Cleanup potentially deployed haproxy
+  file:
+    path: "{{ kube_manifest_dir }}/haproxy.yml"
+    state: absent
+
+- name: nginx-proxy | Make nginx directory
+  file:
+    path: "{{ nginx_config_dir }}"
+    state: directory
+    mode: 0700
+    owner: root
+
+- name: nginx-proxy | Write nginx-proxy configuration
+  template:
+    src: "loadbalancer/nginx.conf.j2"
+    dest: "{{ nginx_config_dir }}/nginx.conf"
+    owner: root
+    mode: 0755
+    backup: yes
+
+- name: nginx-proxy | Get checksum from config
+  stat:
+    path: "{{ nginx_config_dir }}/nginx.conf"
+    get_attributes: no
+    get_checksum: yes
+    get_mime: no
+  register: nginx_stat
+
+- name: nginx-proxy | Write static pod
+  template:
+    src: manifests/nginx-proxy.manifest.j2
+    dest: "{{ kube_manifest_dir }}/nginx-proxy.yml"
+    mode: 0640

kubespray/extra_playbooks/roles/kubernetes/node/tasks/main.yml

@@ -0,0 +1,193 @@
+---
+- import_tasks: facts.yml
+  tags:
+    - facts
+
+- import_tasks: pre_upgrade.yml
+  tags:
+    - kubelet
+
+- name: Ensure /var/lib/cni exists
+  file:
+    path: /var/lib/cni
+    state: directory
+    mode: 0755
+
+- import_tasks: install.yml
+  tags:
+    - kubelet
+
+- import_tasks: loadbalancer/kube-vip.yml
+  when:
+    - is_kube_master
+    - kube_vip_enabled
+  tags:
+    - kube-vip
+
+- import_tasks: loadbalancer/nginx-proxy.yml
+  when:
+    - not is_kube_master or kube_apiserver_bind_address != '0.0.0.0'
+    - loadbalancer_apiserver_localhost
+    - loadbalancer_apiserver_type == 'nginx'
+  tags:
+    - nginx
+
+- import_tasks: loadbalancer/haproxy.yml
+  when:
+    - not is_kube_master or kube_apiserver_bind_address != '0.0.0.0'
+    - loadbalancer_apiserver_localhost
+    - loadbalancer_apiserver_type == 'haproxy'
+  tags:
+    - haproxy
+
+- name: Ensure nodePort range is reserved
+  sysctl:
+    name: net.ipv4.ip_local_reserved_ports
+    value: "{{ kube_apiserver_node_port_range }}"
+    sysctl_set: yes
+    sysctl_file: "{{ sysctl_file_path }}"
+    state: present
+    reload: yes
+  when: kube_apiserver_node_port_range is defined
+  tags:
+    - kube-proxy
+
+- name: Verify if br_netfilter module exists
+  command: "modinfo br_netfilter"
+  environment:
+    PATH: "{{ ansible_env.PATH }}:/sbin"  # Make sure we can workaround RH's conservative path management
+  register: modinfo_br_netfilter
+  failed_when: modinfo_br_netfilter.rc not in [0, 1]
+  changed_when: false
+  check_mode: no
+
+- name: Verify br_netfilter module path exists
+  file:
+    path: /etc/modules-load.d
+    state: directory
+    mode: 0755
+
+- name: Enable br_netfilter module
+  modprobe:
+    name: br_netfilter
+    state: present
+  when: modinfo_br_netfilter.rc == 0
+
+- name: Persist br_netfilter module
+  copy:
+    dest: /etc/modules-load.d/kubespray-br_netfilter.conf
+    content: br_netfilter
+    mode: 0644
+  when: modinfo_br_netfilter.rc == 0
+
+# kube-proxy needs net.bridge.bridge-nf-call-iptables enabled when found if br_netfilter is not a module
+- name: Check if bridge-nf-call-iptables key exists
+  command: "sysctl net.bridge.bridge-nf-call-iptables"
+  failed_when: false
+  changed_when: false
+  check_mode: no
+  register: sysctl_bridge_nf_call_iptables
+
+- name: Enable bridge-nf-call tables
+  sysctl:
+    name: "{{ item }}"
+    state: present
+    sysctl_file: "{{ sysctl_file_path }}"
+    value: "1"
+    reload: yes
+  when: sysctl_bridge_nf_call_iptables.rc == 0
+  with_items:
+    - net.bridge.bridge-nf-call-iptables
+    - net.bridge.bridge-nf-call-arptables
+    - net.bridge.bridge-nf-call-ip6tables
+
+- name: Modprobe Kernel Module for IPVS
+  modprobe:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - ip_vs
+    - ip_vs_rr
+    - ip_vs_wrr
+    - ip_vs_sh
+  when: kube_proxy_mode == 'ipvs'
+  tags:
+    - kube-proxy
+
+- name: Modprobe nf_conntrack_ipv4
+  modprobe:
+    name: nf_conntrack_ipv4
+    state: present
+  register: modprobe_nf_conntrack_ipv4
+  ignore_errors: true  # noqa ignore-errors
+  when:
+    - kube_proxy_mode == 'ipvs'
+  tags:
+    - kube-proxy
+
+- name: Persist ip_vs modules
+  copy:
+    dest: /etc/modules-load.d/kube_proxy-ipvs.conf
+    mode: 0644
+    content: |
+      ip_vs
+      ip_vs_rr
+      ip_vs_wrr
+      ip_vs_sh
+      {% if modprobe_nf_conntrack_ipv4 is success -%}
+      nf_conntrack_ipv4
+      {%-   endif -%}
+  when: kube_proxy_mode == 'ipvs'
+  tags:
+    - kube-proxy
+
+- include_tasks: "cloud-credentials/{{ cloud_provider }}-credential-check.yml"
+  when:
+    - cloud_provider is defined
+    - cloud_provider in [ 'openstack', 'azure', 'vsphere' ]
+  tags:
+    - cloud-provider
+    - facts
+
+- name: Test if openstack_cacert is a base64 string
+  set_fact:
+    openstack_cacert_is_base64: "{% if openstack_cacert is search ('^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)?$') %}true{% else %}false{% endif %}"
+  when:
+    - cloud_provider is defined
+    - cloud_provider == 'openstack'
+    - openstack_cacert is defined
+    - openstack_cacert | length > 0
+
+
+- name: Write cacert file
+  copy:
+    src: "{{ openstack_cacert if not openstack_cacert_is_base64 else omit }}"
+    content: "{{ openstack_cacert | b64decode if openstack_cacert_is_base64 else omit }}"
+    dest: "{{ kube_config_dir }}/openstack-cacert.pem"
+    group: "{{ kube_cert_group }}"
+    mode: 0640
+  when:
+    - cloud_provider is defined
+    - cloud_provider == 'openstack'
+    - openstack_cacert is defined
+    - openstack_cacert | length > 0
+  tags:
+    - cloud-provider
+
+- name: Write cloud-config
+  template:
+    src: "cloud-configs/{{ cloud_provider }}-cloud-config.j2"
+    dest: "{{ kube_config_dir }}/cloud_config"
+    group: "{{ kube_cert_group }}"
+    mode: 0640
+  when:
+    - cloud_provider is defined
+    - cloud_provider in [ 'openstack', 'azure', 'vsphere', 'aws', 'gce' ]
+  notify: Node | restart kubelet
+  tags:
+    - cloud-provider
+
+- import_tasks: kubelet.yml
+  tags:
+    - kubelet
+    - kubeadm

kubespray/extra_playbooks/roles/kubernetes/node/tasks/pre_upgrade.yml

@@ -0,0 +1,48 @@
+---
+- name: "Pre-upgrade | check if kubelet container exists"
+  shell: >-
+    set -o pipefail &&
+    {% if container_manager in ['crio', 'docker'] %}
+    {{ docker_bin_dir }}/docker ps -af name=kubelet | grep kubelet
+    {% elif container_manager == 'containerd' %}
+    {{ bin_dir }}/crictl ps --all --name kubelet | grep kubelet
+    {% endif %}
+  args:
+    executable: /bin/bash
+  failed_when: false
+  changed_when: false
+  check_mode: no
+  register: kubelet_container_check
+
+- name: "Pre-upgrade | copy /var/lib/cni from kubelet"
+  command: >-
+    {% if container_manager in ['crio', 'docker'] %}
+    docker cp kubelet:/var/lib/cni /var/lib/cni
+    {% elif container_manager == 'containerd' %}
+    ctr run --rm --mount type=bind,src=/var/lib/cni,dst=/cnilibdir,options=rbind:rw kubelet kubelet-tmp sh -c 'cp /var/lib/cni/* /cnilibdir/'
+    {% endif %}
+  args:
+    creates: "/var/lib/cni"
+  failed_when: false
+  when: kubelet_container_check.rc == 0
+
+- name: "Pre-upgrade | ensure kubelet container service is stopped if using host deployment"
+  service:
+    name: kubelet
+    state: stopped
+  when: kubelet_container_check.rc == 0
+
+- name: "Pre-upgrade | ensure kubelet container is removed if using host deployment"
+  shell: >-
+    {% if container_manager in ['crio', 'docker'] %}
+    {{ docker_bin_dir }}/docker rm -fv kubelet
+    {% elif container_manager == 'containerd' %}
+    {{ bin_dir }}/crictl stop kubelet && {{ bin_dir }}/crictl rm kubelet
+    {% endif %}
+  failed_when: false
+  changed_when: false
+  register: remove_kubelet_container
+  retries: 4
+  until: remove_kubelet_container.rc == 0
+  delay: 5
+  when: kubelet_container_check.rc == 0