dsk-dev kubespray 이동

2023-12-19 14:31:22 +09:00
parent a35325e16b
commit 5671a92148
2568 changed files with 0 additions and 0 deletions
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/defaults/main.yml
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/defaults/main.yml
@@ -0,0 +1,235 @@
+---
+# advertised host IP for kubelet. This affects network plugin config. Take caution
+kubelet_address: "{{ ip | default(fallback_ips[inventory_hostname]) }}{{ (',' + ip6) if enable_dual_stack_networks and ip6 is defined else '' }}"
+
+# bind address for kubelet. Set to 0.0.0.0 to listen on all interfaces
+kubelet_bind_address: "{{ ip | default('0.0.0.0') }}"
+
+# resolv.conf to base dns config
+kube_resolv_conf: "/etc/resolv.conf"
+
+# Set to empty to avoid cgroup creation
+kubelet_enforce_node_allocatable: "\"\""
+
+# Set runtime and kubelet cgroups when using systemd as cgroup driver (default)
+kubelet_runtime_cgroups: "/systemd/system.slice"
+kubelet_kubelet_cgroups: "/systemd/system.slice"
+
+# Set runtime and kubelet cgroups when using cgroupfs as cgroup driver
+kubelet_runtime_cgroups_cgroupfs: "/system.slice/containerd.service"
+kubelet_kubelet_cgroups_cgroupfs: "/system.slice/kubelet.service"
+
+### fail with swap on (default true)
+kubelet_fail_swap_on: true
+
+# Set systemd service hardening features
+kubelet_systemd_hardening: false
+
+# List of secure IPs for kubelet
+kubelet_secure_addresses: >-
+  {%- for host in groups['kube_control_plane'] -%}
+    {{ hostvars[host]['ip'] | default(fallback_ips[host]) }}{{ ' ' if not loop.last else '' }}
+  {%- endfor -%}
+
+# Reserve this space for kube resources
+kube_memory_reserved: 256Mi
+kube_cpu_reserved: 100m
+# kube_ephemeral_storage_reserved: 2Gi
+# kube_pid_reserved: "1000"
+# Reservation for master hosts
+kube_master_memory_reserved: 512Mi
+kube_master_cpu_reserved: 200m
+# kube_master_ephemeral_storage_reserved: 2Gi
+# kube_master_pid_reserved: "1000"
+
+# Set to true to reserve resources for system daemons
+system_reserved: false
+system_memory_reserved: 512Mi
+system_cpu_reserved: 500m
+# system_ephemeral_storage_reserved: 2Gi
+# system_pid_reserved: "1000"
+# Reservation for master hosts
+system_master_memory_reserved: 256Mi
+system_master_cpu_reserved: 250m
+# system_master_ephemeral_storage_reserved: 2Gi
+# system_master_pid_reserved: "1000"
+
+## Eviction Thresholds to avoid system OOMs
+# https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#eviction-thresholds
+eviction_hard: {}
+eviction_hard_control_plane: {}
+
+kubelet_status_update_frequency: 10s
+
+# kube-vip
+kube_vip_version: v0.5.5
+
+kube_vip_arp_enabled: false
+kube_vip_interface:
+kube_vip_services_interface:
+kube_vip_cidr: 32
+kube_vip_controlplane_enabled: false
+kube_vip_ddns_enabled: false
+kube_vip_services_enabled: false
+kube_vip_leader_election_enabled: "{{ kube_vip_arp_enabled }}"
+kube_vip_bgp_enabled: false
+kube_vip_bgp_routerid:
+kube_vip_local_as: 65000
+kube_vip_bgp_peeraddress:
+kube_vip_bgp_peerpass:
+kube_vip_bgp_peeras: 65000
+kube_vip_bgppeers:
+kube_vip_address:
+
+# Requests for load balancer app
+loadbalancer_apiserver_memory_requests: 32M
+loadbalancer_apiserver_cpu_requests: 25m
+
+loadbalancer_apiserver_keepalive_timeout: 5m
+
+# Uncomment if you need to enable deprecated runtimes
+# kube_api_runtime_config:
+#   - apps/v1beta1=true
+#   - apps/v1beta2=true
+#   - extensions/v1beta1/daemonsets=true
+#   - extensions/v1beta1/deployments=true
+#   - extensions/v1beta1/replicasets=true
+#   - extensions/v1beta1/networkpolicies=true
+#   - extensions/v1beta1/podsecuritypolicies=true
+
+# A port range to reserve for services with NodePort visibility.
+# Inclusive at both ends of the range.
+kube_apiserver_node_port_range: "30000-32767"
+
+# Configure the amount of pods able to run on single node
+# default is equal to application default
+kubelet_max_pods: 110
+
+# Sets the maximum number of processes running per Pod
+# Default value -1 = unlimited
+kubelet_pod_pids_limit: -1
+
+## Support parameters to be passed to kubelet via kubelet-config.yaml
+kubelet_config_extra_args: {}
+
+## Parameters to be passed to kubelet via kubelet-config.yaml when cgroupfs is used as cgroup driver
+kubelet_config_extra_args_cgroupfs:
+  systemCgroups: /system.slice
+  cgroupRoot: /
+
+## Support parameters to be passed to kubelet via kubelet-config.yaml only on nodes, not masters
+kubelet_node_config_extra_args: {}
+
+# Maximum number of container log files that can be present for a container.
+kubelet_logfiles_max_nr: 5
+
+# Maximum size of the container log file before it is rotated
+kubelet_logfiles_max_size: 10Mi
+
+## Support custom flags to be passed to kubelet
+kubelet_custom_flags: []
+
+## Support custom flags to be passed to kubelet only on nodes, not masters
+kubelet_node_custom_flags: []
+
+# If non-empty, will use this string as identification instead of the actual hostname
+kube_override_hostname: >-
+  {%- if cloud_provider is defined and cloud_provider in [ 'aws' ] -%}
+  {%- else -%}
+  {{ inventory_hostname }}
+  {%- endif -%}
+
+# The read-only port for the Kubelet to serve on with no authentication/authorization.
+kube_read_only_port: 0
+
+# Port for healthz for Kubelet
+kubelet_healthz_port: 10248
+
+# Bind address for healthz for Kubelet
+kubelet_healthz_bind_address: 127.0.0.1
+
+# sysctl_file_path to add sysctl conf to
+sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf"
+
+# For the openstack integration kubelet will need credentials to access
+# openstack apis like nova and cinder. Per default this values will be
+# read from the environment.
+openstack_auth_url: "{{ lookup('env','OS_AUTH_URL')  }}"
+openstack_username: "{{ lookup('env','OS_USERNAME')  }}"
+openstack_password: "{{ lookup('env','OS_PASSWORD')  }}"
+openstack_region: "{{ lookup('env','OS_REGION_NAME')  }}"
+openstack_tenant_id: "{{ lookup('env','OS_TENANT_ID')| default(lookup('env','OS_PROJECT_ID')|default(lookup('env','OS_PROJECT_NAME'),true),true) }}"
+openstack_tenant_name: "{{ lookup('env','OS_TENANT_NAME') }}"
+openstack_domain_name: "{{ lookup('env','OS_USER_DOMAIN_NAME') }}"
+openstack_domain_id: "{{ lookup('env','OS_USER_DOMAIN_ID') }}"
+
+# For the vsphere integration, kubelet will need credentials to access
+# vsphere apis
+# Documentation regarding these values can be found
+# https://github.com/kubernetes/kubernetes/blob/master/pkg/cloudprovider/providers/vsphere/vsphere.go#L105
+vsphere_vcenter_ip: "{{ lookup('env', 'VSPHERE_VCENTER') }}"
+vsphere_vcenter_port: "{{ lookup('env', 'VSPHERE_VCENTER_PORT') }}"
+vsphere_user: "{{ lookup('env', 'VSPHERE_USER') }}"
+vsphere_password: "{{ lookup('env', 'VSPHERE_PASSWORD') }}"
+vsphere_datacenter: "{{ lookup('env', 'VSPHERE_DATACENTER') }}"
+vsphere_datastore: "{{ lookup('env', 'VSPHERE_DATASTORE') }}"
+vsphere_working_dir: "{{ lookup('env', 'VSPHERE_WORKING_DIR') }}"
+vsphere_insecure: "{{ lookup('env', 'VSPHERE_INSECURE') }}"
+vsphere_resource_pool: "{{ lookup('env', 'VSPHERE_RESOURCE_POOL') }}"
+
+vsphere_scsi_controller_type: pvscsi
+# vsphere_public_network is name of the network the VMs are joined to
+vsphere_public_network: "{{ lookup('env', 'VSPHERE_PUBLIC_NETWORK')|default('') }}"
+
+## When azure is used, you need to also set the following variables.
+## see docs/azure.md for details on how to get these values
+# azure_tenant_id:
+# azure_subscription_id:
+# azure_aad_client_id:
+# azure_aad_client_secret:
+# azure_resource_group:
+# azure_location:
+# azure_subnet_name:
+# azure_security_group_name:
+# azure_vnet_name:
+# azure_route_table_name:
+# supported values are 'standard' or 'vmss'
+# azure_vmtype: standard
+# Sku of Load Balancer and Public IP. Candidate values are: basic and standard.
+azure_loadbalancer_sku: basic
+# excludes master nodes from standard load balancer.
+azure_exclude_master_from_standard_lb: true
+# disables the outbound SNAT for public load balancer rules
+azure_disable_outbound_snat: false
+# use instance metadata service where possible
+azure_use_instance_metadata: true
+# use specific Azure API endpoints
+azure_cloud: AzurePublicCloud
+
+## Support tls min version, Possible values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
+# tls_min_version: ""
+
+## Support tls cipher suites.
+# tls_cipher_suites:
+#   - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA
+#   - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
+#   - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
+#   - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA
+#   - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+#   - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
+#   - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA
+#   - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA
+#   - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA
+#   - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256
+#   - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
+#   - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA
+#   - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+#   - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305
+#   - TLS_ECDHE_RSA_WITH_RC4_128_SHA
+#   - TLS_RSA_WITH_3DES_EDE_CBC_SHA
+#   - TLS_RSA_WITH_AES_128_CBC_SHA
+#   - TLS_RSA_WITH_AES_128_CBC_SHA256
+#   - TLS_RSA_WITH_AES_128_GCM_SHA256
+#   - TLS_RSA_WITH_AES_256_CBC_SHA
+#   - TLS_RSA_WITH_AES_256_GCM_SHA384
+#   - TLS_RSA_WITH_RC4_128_SHA
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/handlers/main.yml
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/handlers/main.yml
@@ -0,0 +1,15 @@
+---
+- name: Node | restart kubelet
+  command: /bin/true
+  notify:
+    - Kubelet | reload systemd
+    - Kubelet | restart kubelet
+
+- name: Kubelet | reload systemd
+  systemd:
+    daemon_reload: true
+
+- name: Kubelet | restart kubelet
+  service:
+    name: kubelet
+    state: restarted
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/tasks/cloud-credentials/azure-credential-check.yml
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/tasks/cloud-credentials/azure-credential-check.yml
@@ -0,0 +1,82 @@
+---
+- name: check azure_tenant_id value
+  fail:
+    msg: "azure_tenant_id is missing"
+  when: azure_tenant_id is not defined or not azure_tenant_id
+
+- name: check azure_subscription_id value
+  fail:
+    msg: "azure_subscription_id is missing"
+  when: azure_subscription_id is not defined or not azure_subscription_id
+
+- name: check azure_aad_client_id value
+  fail:
+    msg: "azure_aad_client_id is missing"
+  when: azure_aad_client_id is not defined or not azure_aad_client_id
+
+- name: check azure_aad_client_secret value
+  fail:
+    msg: "azure_aad_client_secret is missing"
+  when: azure_aad_client_secret is not defined or not azure_aad_client_secret
+
+- name: check azure_resource_group value
+  fail:
+    msg: "azure_resource_group is missing"
+  when: azure_resource_group is not defined or not azure_resource_group
+
+- name: check azure_location value
+  fail:
+    msg: "azure_location is missing"
+  when: azure_location is not defined or not azure_location
+
+- name: check azure_subnet_name value
+  fail:
+    msg: "azure_subnet_name is missing"
+  when: azure_subnet_name is not defined or not azure_subnet_name
+
+- name: check azure_security_group_name value
+  fail:
+    msg: "azure_security_group_name is missing"
+  when: azure_security_group_name is not defined or not azure_security_group_name
+
+- name: check azure_vnet_name value
+  fail:
+    msg: "azure_vnet_name is missing"
+  when: azure_vnet_name is not defined or not azure_vnet_name
+
+- name: check azure_vnet_resource_group value
+  fail:
+    msg: "azure_vnet_resource_group is missing"
+  when: azure_vnet_resource_group is not defined or not azure_vnet_resource_group
+
+- name: check azure_route_table_name value
+  fail:
+    msg: "azure_route_table_name is missing"
+  when: azure_route_table_name is not defined or not azure_route_table_name
+
+- name: check azure_loadbalancer_sku value
+  fail:
+    msg: "azure_loadbalancer_sku has an invalid value '{{ azure_loadbalancer_sku }}'. Supported values are 'basic', 'standard'"
+  when: azure_loadbalancer_sku not in ["basic", "standard"]
+
+- name: "check azure_exclude_master_from_standard_lb is a bool"
+  assert:
+    that: azure_exclude_master_from_standard_lb |type_debug == 'bool'
+
+- name: "check azure_disable_outbound_snat is a bool"
+  assert:
+    that: azure_disable_outbound_snat |type_debug == 'bool'
+
+- name: "check azure_use_instance_metadata is a bool"
+  assert:
+    that: azure_use_instance_metadata |type_debug == 'bool'
+
+- name: check azure_vmtype value
+  fail:
+    msg: "azure_vmtype is missing. Supported values are 'standard' or 'vmss'"
+  when: azure_vmtype is not defined or not azure_vmtype
+
+- name: check azure_cloud value
+  fail:
+    msg: "azure_cloud has an invalid value '{{ azure_cloud }}'. Supported values are 'AzureChinaCloud', 'AzureGermanCloud', 'AzurePublicCloud', 'AzureUSGovernmentCloud'."
+  when: azure_cloud not in ["AzureChinaCloud", "AzureGermanCloud", "AzurePublicCloud", "AzureUSGovernmentCloud"]
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/tasks/cloud-credentials/openstack-credential-check.yml
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/tasks/cloud-credentials/openstack-credential-check.yml
@@ -0,0 +1,34 @@
+---
+- name: check openstack_auth_url value
+  fail:
+    msg: "openstack_auth_url is missing"
+  when: openstack_auth_url is not defined or not openstack_auth_url
+
+- name: check openstack_username value
+  fail:
+    msg: "openstack_username is missing"
+  when: openstack_username is not defined or not openstack_username
+
+- name: check openstack_password value
+  fail:
+    msg: "openstack_password is missing"
+  when: openstack_password is not defined or not openstack_password
+
+- name: check openstack_region value
+  fail:
+    msg: "openstack_region is missing"
+  when: openstack_region is not defined or not openstack_region
+
+- name: check openstack_tenant_id value
+  fail:
+    msg: "one of openstack_tenant_id or openstack_trust_id must be specified"
+  when:
+    - openstack_tenant_id is not defined or not openstack_tenant_id
+    - openstack_trust_id is not defined
+
+- name: check openstack_trust_id value
+  fail:
+    msg: "one of openstack_tenant_id or openstack_trust_id must be specified"
+  when:
+    - openstack_trust_id is not defined or not openstack_trust_id
+    - openstack_tenant_id is not defined
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/tasks/cloud-credentials/vsphere-credential-check.yml
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/tasks/cloud-credentials/vsphere-credential-check.yml
@@ -0,0 +1,22 @@
+---
+- name: check vsphere environment variables
+  fail:
+    msg: "{{ item.name }} is missing"
+  when: item.value is not defined or not item.value
+  with_items:
+    - name: vsphere_vcenter_ip
+      value: "{{ vsphere_vcenter_ip }}"
+    - name: vsphere_vcenter_port
+      value: "{{ vsphere_vcenter_port }}"
+    - name: vsphere_user
+      value: "{{ vsphere_user }}"
+    - name: vsphere_password
+      value: "{{ vsphere_password }}"
+    - name: vsphere_datacenter
+      value: "{{ vsphere_datacenter }}"
+    - name: vsphere_datastore
+      value: "{{ vsphere_datastore }}"
+    - name: vsphere_working_dir
+      value: "{{ vsphere_working_dir }}"
+    - name: vsphere_insecure
+      value: "{{ vsphere_insecure }}"
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/tasks/facts.yml
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/tasks/facts.yml
@@ -0,0 +1,57 @@
+---
+- block:
+  - name: look up docker cgroup driver
+    shell: "docker info | grep 'Cgroup Driver' | awk -F': ' '{ print $2; }'"
+    register: docker_cgroup_driver_result
+    changed_when: false
+    check_mode: no
+
+  - name: set kubelet_cgroup_driver_detected fact for docker
+    set_fact:
+      kubelet_cgroup_driver_detected: "{{ docker_cgroup_driver_result.stdout }}"
+  when: container_manager == 'docker'
+
+- block:
+  - name: look up crio cgroup driver
+    shell: "{{ bin_dir }}/crio-status info | grep 'cgroup driver' | awk -F': ' '{ print $2; }'"
+    register: crio_cgroup_driver_result
+    changed_when: false
+
+  - name: set kubelet_cgroup_driver_detected fact for crio
+    set_fact:
+      kubelet_cgroup_driver_detected: "{{ crio_cgroup_driver_result.stdout }}"
+  when: container_manager == 'crio'
+
+- name: set kubelet_cgroup_driver_detected fact for containerd
+  set_fact:
+    kubelet_cgroup_driver_detected: >-
+      {%- if containerd_use_systemd_cgroup -%}systemd{%- else -%}cgroupfs{%- endif -%}
+  when: container_manager == 'containerd'
+
+- name: set kubelet_cgroup_driver
+  set_fact:
+    kubelet_cgroup_driver: "{{ kubelet_cgroup_driver_detected }}"
+  when: kubelet_cgroup_driver is undefined
+
+- name: set kubelet_cgroups options when cgroupfs is used
+  set_fact:
+    kubelet_runtime_cgroups: "{{ kubelet_runtime_cgroups_cgroupfs }}"
+    kubelet_kubelet_cgroups: "{{ kubelet_kubelet_cgroups_cgroupfs }}"
+  when: kubelet_cgroup_driver == 'cgroupfs'
+
+- name: set kubelet_config_extra_args options when cgroupfs is used
+  vars:
+  set_fact:
+    kubelet_config_extra_args: "{{ kubelet_config_extra_args | combine(kubelet_config_extra_args_cgroupfs) }}"
+  when: kubelet_cgroup_driver == 'cgroupfs'
+
+- name: os specific vars
+  include_vars: "{{ item }}"
+  with_first_found:
+  - files:
+    - "{{ ansible_distribution|lower }}-{{ ansible_distribution_version|lower|replace('/', '_') }}.yml"
+    - "{{ ansible_distribution|lower }}-{{ ansible_distribution_release }}.yml"
+    - "{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version|lower|replace('/', '_') }}.yml"
+    - "{{ ansible_distribution|lower }}.yml"
+    - "{{ ansible_os_family|lower }}.yml"
+    skip: true
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/tasks/install.yml
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/tasks/install.yml
@@ -0,0 +1,22 @@
+---
+- name: install | Copy kubeadm binary from download dir
+  copy:
+    src: "{{ local_release_dir }}/kubeadm-{{ kubeadm_version }}-{{ image_arch }}"
+    dest: "{{ bin_dir }}/kubeadm"
+    mode: 0755
+    remote_src: true
+  tags:
+    - kubeadm
+  when:
+    - not inventory_hostname in groups['kube_control_plane']
+
+- name: install | Copy kubelet binary from download dir
+  copy:
+    src: "{{ local_release_dir }}/kubelet-{{ kube_version }}-{{ image_arch }}"
+    dest: "{{ bin_dir }}/kubelet"
+    mode: 0755
+    remote_src: true
+  tags:
+    - kubelet
+    - upgrade
+  notify: Node | restart kubelet
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/tasks/kubelet.yml
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/tasks/kubelet.yml
@@ -0,0 +1,52 @@
+---
+- name: Set kubelet api version to v1beta1
+  set_fact:
+    kubeletConfig_api_version: v1beta1
+  tags:
+    - kubelet
+    - kubeadm
+
+- name: Write kubelet environment config file (kubeadm)
+  template:
+    src: "kubelet.env.{{ kubeletConfig_api_version }}.j2"
+    dest: "{{ kube_config_dir }}/kubelet.env"
+    setype: "{{ (preinstall_selinux_state != 'disabled') | ternary('etc_t', omit) }}"
+    backup: yes
+    mode: 0640
+  notify: Node | restart kubelet
+  tags:
+    - kubelet
+    - kubeadm
+
+- name: Write kubelet config file
+  template:
+    src: "kubelet-config.{{ kubeletConfig_api_version }}.yaml.j2"
+    dest: "{{ kube_config_dir }}/kubelet-config.yaml"
+    mode: 0640
+  notify: Kubelet | restart kubelet
+  tags:
+    - kubelet
+    - kubeadm
+
+- name: Write kubelet systemd init file
+  template:
+    src: "kubelet.service.j2"
+    dest: "/etc/systemd/system/kubelet.service"
+    backup: "yes"
+    mode: 0644
+  notify: Node | restart kubelet
+  tags:
+    - kubelet
+    - kubeadm
+
+- name: flush_handlers and reload-systemd
+  meta: flush_handlers
+
+- name: Enable kubelet
+  service:
+    name: kubelet
+    enabled: yes
+    state: started
+  tags:
+    - kubelet
+  notify: Kubelet | restart kubelet
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/tasks/loadbalancer/haproxy.yml
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/tasks/loadbalancer/haproxy.yml
@@ -0,0 +1,34 @@
+---
+- name: haproxy | Cleanup potentially deployed nginx-proxy
+  file:
+    path: "{{ kube_manifest_dir }}/nginx-proxy.yml"
+    state: absent
+
+- name: haproxy | Make haproxy directory
+  file:
+    path: "{{ haproxy_config_dir }}"
+    state: directory
+    mode: 0755
+    owner: root
+
+- name: haproxy | Write haproxy configuration
+  template:
+    src: "loadbalancer/haproxy.cfg.j2"
+    dest: "{{ haproxy_config_dir }}/haproxy.cfg"
+    owner: root
+    mode: 0755
+    backup: yes
+
+- name: haproxy | Get checksum from config
+  stat:
+    path: "{{ haproxy_config_dir }}/haproxy.cfg"
+    get_attributes: no
+    get_checksum: yes
+    get_mime: no
+  register: haproxy_stat
+
+- name: haproxy | Write static pod
+  template:
+    src: manifests/haproxy.manifest.j2
+    dest: "{{ kube_manifest_dir }}/haproxy.yml"
+    mode: 0640
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/tasks/loadbalancer/kube-vip.yml
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/tasks/loadbalancer/kube-vip.yml
@@ -0,0 +1,13 @@
+---
+- name: kube-vip  | Check cluster settings for kube-vip
+  fail:
+    msg: "kube-vip require kube_proxy_strict_arp = true, see https://github.com/kube-vip/kube-vip/blob/main/docs/kubernetes/arp/index.md"
+  when:
+    - kube_proxy_mode == 'ipvs' and not kube_proxy_strict_arp
+    - kube_vip_arp_enabled
+
+- name: kube-vip | Write static pod
+  template:
+    src: manifests/kube-vip.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-vip.yml"
+    mode: 0640
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/tasks/loadbalancer/nginx-proxy.yml
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/tasks/loadbalancer/nginx-proxy.yml
@@ -0,0 +1,34 @@
+---
+- name: haproxy | Cleanup potentially deployed haproxy
+  file:
+    path: "{{ kube_manifest_dir }}/haproxy.yml"
+    state: absent
+
+- name: nginx-proxy | Make nginx directory
+  file:
+    path: "{{ nginx_config_dir }}"
+    state: directory
+    mode: 0700
+    owner: root
+
+- name: nginx-proxy | Write nginx-proxy configuration
+  template:
+    src: "loadbalancer/nginx.conf.j2"
+    dest: "{{ nginx_config_dir }}/nginx.conf"
+    owner: root
+    mode: 0755
+    backup: yes
+
+- name: nginx-proxy | Get checksum from config
+  stat:
+    path: "{{ nginx_config_dir }}/nginx.conf"
+    get_attributes: no
+    get_checksum: yes
+    get_mime: no
+  register: nginx_stat
+
+- name: nginx-proxy | Write static pod
+  template:
+    src: manifests/nginx-proxy.manifest.j2
+    dest: "{{ kube_manifest_dir }}/nginx-proxy.yml"
+    mode: 0640
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/tasks/main.yml
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/tasks/main.yml
@@ -0,0 +1,193 @@
+---
+- import_tasks: facts.yml
+  tags:
+    - facts
+
+- import_tasks: pre_upgrade.yml
+  tags:
+    - kubelet
+
+- name: Ensure /var/lib/cni exists
+  file:
+    path: /var/lib/cni
+    state: directory
+    mode: 0755
+
+- import_tasks: install.yml
+  tags:
+    - kubelet
+
+- import_tasks: loadbalancer/kube-vip.yml
+  when:
+    - is_kube_master
+    - kube_vip_enabled
+  tags:
+    - kube-vip
+
+- import_tasks: loadbalancer/nginx-proxy.yml
+  when:
+    - not is_kube_master or kube_apiserver_bind_address != '0.0.0.0'
+    - loadbalancer_apiserver_localhost
+    - loadbalancer_apiserver_type == 'nginx'
+  tags:
+    - nginx
+
+- import_tasks: loadbalancer/haproxy.yml
+  when:
+    - not is_kube_master or kube_apiserver_bind_address != '0.0.0.0'
+    - loadbalancer_apiserver_localhost
+    - loadbalancer_apiserver_type == 'haproxy'
+  tags:
+    - haproxy
+
+- name: Ensure nodePort range is reserved
+  sysctl:
+    name: net.ipv4.ip_local_reserved_ports
+    value: "{{ kube_apiserver_node_port_range }}"
+    sysctl_set: yes
+    sysctl_file: "{{ sysctl_file_path }}"
+    state: present
+    reload: yes
+  when: kube_apiserver_node_port_range is defined
+  tags:
+    - kube-proxy
+
+- name: Verify if br_netfilter module exists
+  command: "modinfo br_netfilter"
+  environment:
+    PATH: "{{ ansible_env.PATH }}:/sbin"  # Make sure we can workaround RH's conservative path management
+  register: modinfo_br_netfilter
+  failed_when: modinfo_br_netfilter.rc not in [0, 1]
+  changed_when: false
+  check_mode: no
+
+- name: Verify br_netfilter module path exists
+  file:
+    path: /etc/modules-load.d
+    state: directory
+    mode: 0755
+
+- name: Enable br_netfilter module
+  modprobe:
+    name: br_netfilter
+    state: present
+  when: modinfo_br_netfilter.rc == 0
+
+- name: Persist br_netfilter module
+  copy:
+    dest: /etc/modules-load.d/kubespray-br_netfilter.conf
+    content: br_netfilter
+    mode: 0644
+  when: modinfo_br_netfilter.rc == 0
+
+# kube-proxy needs net.bridge.bridge-nf-call-iptables enabled when found if br_netfilter is not a module
+- name: Check if bridge-nf-call-iptables key exists
+  command: "sysctl net.bridge.bridge-nf-call-iptables"
+  failed_when: false
+  changed_when: false
+  check_mode: no
+  register: sysctl_bridge_nf_call_iptables
+
+- name: Enable bridge-nf-call tables
+  sysctl:
+    name: "{{ item }}"
+    state: present
+    sysctl_file: "{{ sysctl_file_path }}"
+    value: "1"
+    reload: yes
+  when: sysctl_bridge_nf_call_iptables.rc == 0
+  with_items:
+    - net.bridge.bridge-nf-call-iptables
+    - net.bridge.bridge-nf-call-arptables
+    - net.bridge.bridge-nf-call-ip6tables
+
+- name: Modprobe Kernel Module for IPVS
+  modprobe:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - ip_vs
+    - ip_vs_rr
+    - ip_vs_wrr
+    - ip_vs_sh
+  when: kube_proxy_mode == 'ipvs'
+  tags:
+    - kube-proxy
+
+- name: Modprobe nf_conntrack_ipv4
+  modprobe:
+    name: nf_conntrack_ipv4
+    state: present
+  register: modprobe_nf_conntrack_ipv4
+  ignore_errors: true  # noqa ignore-errors
+  when:
+    - kube_proxy_mode == 'ipvs'
+  tags:
+    - kube-proxy
+
+- name: Persist ip_vs modules
+  copy:
+    dest: /etc/modules-load.d/kube_proxy-ipvs.conf
+    mode: 0644
+    content: |
+      ip_vs
+      ip_vs_rr
+      ip_vs_wrr
+      ip_vs_sh
+      {% if modprobe_nf_conntrack_ipv4 is success -%}
+      nf_conntrack_ipv4
+      {%-   endif -%}
+  when: kube_proxy_mode == 'ipvs'
+  tags:
+    - kube-proxy
+
+- include_tasks: "cloud-credentials/{{ cloud_provider }}-credential-check.yml"
+  when:
+    - cloud_provider is defined
+    - cloud_provider in [ 'openstack', 'azure', 'vsphere' ]
+  tags:
+    - cloud-provider
+    - facts
+
+- name: Test if openstack_cacert is a base64 string
+  set_fact:
+    openstack_cacert_is_base64: "{% if openstack_cacert is search ('^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)?$') %}true{% else %}false{% endif %}"
+  when:
+    - cloud_provider is defined
+    - cloud_provider == 'openstack'
+    - openstack_cacert is defined
+    - openstack_cacert | length > 0
+
+
+- name: Write cacert file
+  copy:
+    src: "{{ openstack_cacert if not openstack_cacert_is_base64 else omit }}"
+    content: "{{ openstack_cacert | b64decode if openstack_cacert_is_base64 else omit }}"
+    dest: "{{ kube_config_dir }}/openstack-cacert.pem"
+    group: "{{ kube_cert_group }}"
+    mode: 0640
+  when:
+    - cloud_provider is defined
+    - cloud_provider == 'openstack'
+    - openstack_cacert is defined
+    - openstack_cacert | length > 0
+  tags:
+    - cloud-provider
+
+- name: Write cloud-config
+  template:
+    src: "cloud-configs/{{ cloud_provider }}-cloud-config.j2"
+    dest: "{{ kube_config_dir }}/cloud_config"
+    group: "{{ kube_cert_group }}"
+    mode: 0640
+  when:
+    - cloud_provider is defined
+    - cloud_provider in [ 'openstack', 'azure', 'vsphere', 'aws', 'gce' ]
+  notify: Node | restart kubelet
+  tags:
+    - cloud-provider
+
+- import_tasks: kubelet.yml
+  tags:
+    - kubelet
+    - kubeadm
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/tasks/pre_upgrade.yml
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/tasks/pre_upgrade.yml
@@ -0,0 +1,48 @@
+---
+- name: "Pre-upgrade | check if kubelet container exists"
+  shell: >-
+    set -o pipefail &&
+    {% if container_manager in ['crio', 'docker'] %}
+    {{ docker_bin_dir }}/docker ps -af name=kubelet | grep kubelet
+    {% elif container_manager == 'containerd' %}
+    {{ bin_dir }}/crictl ps --all --name kubelet | grep kubelet
+    {% endif %}
+  args:
+    executable: /bin/bash
+  failed_when: false
+  changed_when: false
+  check_mode: no
+  register: kubelet_container_check
+
+- name: "Pre-upgrade | copy /var/lib/cni from kubelet"
+  command: >-
+    {% if container_manager in ['crio', 'docker'] %}
+    docker cp kubelet:/var/lib/cni /var/lib/cni
+    {% elif container_manager == 'containerd' %}
+    ctr run --rm --mount type=bind,src=/var/lib/cni,dst=/cnilibdir,options=rbind:rw kubelet kubelet-tmp sh -c 'cp /var/lib/cni/* /cnilibdir/'
+    {% endif %}
+  args:
+    creates: "/var/lib/cni"
+  failed_when: false
+  when: kubelet_container_check.rc == 0
+
+- name: "Pre-upgrade | ensure kubelet container service is stopped if using host deployment"
+  service:
+    name: kubelet
+    state: stopped
+  when: kubelet_container_check.rc == 0
+
+- name: "Pre-upgrade | ensure kubelet container is removed if using host deployment"
+  shell: >-
+    {% if container_manager in ['crio', 'docker'] %}
+    {{ docker_bin_dir }}/docker rm -fv kubelet
+    {% elif container_manager == 'containerd' %}
+    {{ bin_dir }}/crictl stop kubelet && {{ bin_dir }}/crictl rm kubelet
+    {% endif %}
+  failed_when: false
+  changed_when: false
+  register: remove_kubelet_container
+  retries: 4
+  until: remove_kubelet_container.rc == 0
+  delay: 5
+  when: kubelet_container_check.rc == 0
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/cloud-configs/aws-cloud-config.j2
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/cloud-configs/aws-cloud-config.j2
@@ -0,0 +1,11 @@
+[Global]
+zone={{ aws_zone|default("") }}
+vpc={{ aws_vpc|default("") }}
+subnetId={{ aws_subnet_id|default("") }}
+routeTableId={{ aws_route_table_id|default("") }}
+roleArn={{ aws_role_arn|default("") }}
+kubernetesClusterTag={{ aws_kubernetes_cluster_tag|default("") }}
+kubernetesClusterId={{ aws_kubernetes_cluster_id|default("") }}
+disableSecurityGroupIngress={{ "true" if aws_disable_security_group_ingress|default(False) else "false" }}
+disableStrictZoneCheck={{ "true" if aws_disable_strict_zone_check|default(False) else "false" }}
+elbSecurityGroup={{ aws_elb_security_group|default("") }}
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/cloud-configs/azure-cloud-config.j2
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/cloud-configs/azure-cloud-config.j2
@@ -0,0 +1,26 @@
+{
+  "cloud": "{{ azure_cloud }}",
+  "tenantId": "{{ azure_tenant_id }}",
+  "subscriptionId": "{{ azure_subscription_id }}",
+  "aadClientId": "{{ azure_aad_client_id }}",
+  "aadClientSecret": "{{ azure_aad_client_secret }}",
+  "resourceGroup": "{{ azure_resource_group }}",
+  "location": "{{ azure_location }}",
+  "subnetName": "{{ azure_subnet_name }}",
+  "securityGroupName": "{{ azure_security_group_name }}",
+  "securityGroupResourceGroup": "{{ azure_security_group_resource_group | default(azure_vnet_resource_group) }}",
+  "vnetName": "{{ azure_vnet_name }}",
+  "vnetResourceGroup": "{{ azure_vnet_resource_group }}",
+  "routeTableName": "{{ azure_route_table_name }}",
+  "routeTableResourceGroup": "{{ azure_route_table_resource_group | default(azure_vnet_resource_group) }}",
+  "vmType": "{{ azure_vmtype }}",
+{% if azure_primary_availability_set_name is defined %}
+  "primaryAvailabilitySetName": "{{ azure_primary_availability_set_name }}",
+{%endif%}
+  "useInstanceMetadata": {{azure_use_instance_metadata | lower }},
+{% if azure_loadbalancer_sku == "standard" %}
+  "excludeMasterFromStandardLB": {{ azure_exclude_master_from_standard_lb | lower }},
+  "disableOutboundSNAT": {{ azure_disable_outbound_snat | lower }},
+{% endif%}
+  "loadBalancerSku": "{{ azure_loadbalancer_sku }}"
+}
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/cloud-configs/gce-cloud-config.j2
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/cloud-configs/gce-cloud-config.j2
@@ -0,0 +1,3 @@
+[global]
+node-tags = {{ gce_node_tags }} 
+
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/cloud-configs/openstack-cloud-config.j2
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/cloud-configs/openstack-cloud-config.j2
@@ -0,0 +1,54 @@
+[Global]
+auth-url="{{ openstack_auth_url }}"
+username="{{ openstack_username }}"
+password="{{ openstack_password }}"
+region="{{ openstack_region }}"
+{% if openstack_trust_id is defined and openstack_trust_id != "" %}
+trust-id="{{ openstack_trust_id }}"
+{% else %}
+tenant-id="{{ openstack_tenant_id }}"
+{% endif %}
+{% if openstack_tenant_name is defined and openstack_tenant_name != "" %}
+tenant-name="{{ openstack_tenant_name }}"
+{% endif %}
+{% if openstack_domain_name is defined and openstack_domain_name != "" %}
+domain-name="{{ openstack_domain_name }}"
+{% elif openstack_domain_id is defined and openstack_domain_id != "" %}
+domain-id ="{{ openstack_domain_id }}"
+{% endif %}
+{% if openstack_cacert is defined and openstack_cacert != "" %}
+ca-file="{{ kube_config_dir }}/openstack-cacert.pem"
+{% endif %}
+
+[BlockStorage]
+{% if openstack_blockstorage_version is defined %}
+bs-version={{ openstack_blockstorage_version }}
+{% endif %}
+{% if openstack_blockstorage_ignore_volume_az is defined and openstack_blockstorage_ignore_volume_az|bool %}
+ignore-volume-az={{ openstack_blockstorage_ignore_volume_az }}
+{% endif %}
+{% if node_volume_attach_limit is defined and node_volume_attach_limit != "" %}
+node-volume-attach-limit="{{ node_volume_attach_limit }}"
+{% endif %}
+
+{% if openstack_lbaas_enabled and openstack_lbaas_subnet_id is defined %}
+[LoadBalancer]
+subnet-id={{ openstack_lbaas_subnet_id }}
+{% if openstack_lbaas_floating_network_id is defined %}
+floating-network-id={{ openstack_lbaas_floating_network_id }}
+{% endif %}
+{% if openstack_lbaas_use_octavia is defined %}
+use-octavia={{ openstack_lbaas_use_octavia }}
+{% endif %}
+{% if openstack_lbaas_method is defined %}
+lb-method={{ openstack_lbaas_method }}
+{% endif %}
+{% if openstack_lbaas_provider is defined %}
+lb-provider={{ openstack_lbaas_provider }}
+{% endif %}
+
+create-monitor={{ openstack_lbaas_create_monitor }}
+monitor-delay={{ openstack_lbaas_monitor_delay }}
+monitor-timeout={{ openstack_lbaas_monitor_timeout }}
+monitor-max-retries={{ openstack_lbaas_monitor_max_retries }}
+{% endif %}
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/cloud-configs/vsphere-cloud-config.j2
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/cloud-configs/vsphere-cloud-config.j2
@@ -0,0 +1,36 @@
+[Global]
+user = "{{ vsphere_user }}"
+password = "{{ vsphere_password }}"
+port = {{ vsphere_vcenter_port }}
+insecure-flag = {{ vsphere_insecure }}
+
+datacenters = "{{ vsphere_datacenter }}"
+
+[VirtualCenter "{{ vsphere_vcenter_ip }}"]
+
+
+[Workspace]
+server = "{{ vsphere_vcenter_ip }}"
+datacenter = "{{ vsphere_datacenter }}"
+folder = "{{ vsphere_working_dir }}"
+default-datastore = "{{ vsphere_datastore }}"
+{% if vsphere_resource_pool is defined and vsphere_resource_pool != ""  %}
+resourcepool-path = "{{ vsphere_resource_pool }}"
+{% endif %}
+
+
+[Disk]
+scsicontrollertype = {{ vsphere_scsi_controller_type }}
+
+{% if vsphere_public_network is defined and vsphere_public_network != ""  %}
+[Network]
+public-network = {{ vsphere_public_network }}
+{% endif %}
+
+[Labels]
+{% if vsphere_zone_category is defined and vsphere_zone_category != ""  %}
+zone = {{ vsphere_zone_category }}
+{% endif %}
+{% if vsphere_region_category is defined and vsphere_region_category != ""  %}
+region = {{ vsphere_region_category }}
+{% endif %}
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/http-proxy.conf.j2
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/http-proxy.conf.j2
@@ -0,0 +1,2 @@
+[Service]
+Environment={% if http_proxy %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy %}"NO_PROXY={{ no_proxy }}"{% endif %}
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2
@@ -0,0 +1,151 @@
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+nodeStatusUpdateFrequency: "{{ kubelet_status_update_frequency }}"
+failSwapOn: {{ kubelet_fail_swap_on|default(true) }}
+authentication:
+  anonymous:
+    enabled: false
+  webhook:
+    enabled: {{ kubelet_authentication_token_webhook }}
+  x509:
+    clientCAFile: {{ kube_cert_dir }}/ca.crt
+authorization:
+{% if kubelet_authorization_mode_webhook %}
+  mode: Webhook
+{% else %}
+  mode: AlwaysAllow
+{% endif %}
+{% if kubelet_enforce_node_allocatable is defined and kubelet_enforce_node_allocatable != "\"\"" %}
+{% set kubelet_enforce_node_allocatable_list = kubelet_enforce_node_allocatable.split() %}
+enforceNodeAllocatable:
+{% for item in kubelet_enforce_node_allocatable_list %}
+- {{ item }}
+{% endfor %}
+{% endif %}
+staticPodPath: {{ kube_manifest_dir }}
+cgroupDriver: {{ kubelet_cgroup_driver | default('systemd') }}
+containerLogMaxFiles: {{ kubelet_logfiles_max_nr }}
+containerLogMaxSize: {{ kubelet_logfiles_max_size }}
+maxPods: {{ kubelet_max_pods }}
+podPidsLimit: {{ kubelet_pod_pids_limit }}
+address: {{ kubelet_bind_address }}
+readOnlyPort: {{ kube_read_only_port }}
+healthzPort: {{ kubelet_healthz_port }}
+healthzBindAddress: {{ kubelet_healthz_bind_address }}
+kubeletCgroups: {{ kubelet_kubelet_cgroups }}
+clusterDomain: {{ dns_domain }}
+{% if kubelet_protect_kernel_defaults|bool %}
+protectKernelDefaults: true
+{% endif %}
+{% if kubelet_rotate_certificates|bool %}
+rotateCertificates: true
+{% endif %}
+{% if kubelet_rotate_server_certificates|bool %}
+serverTLSBootstrap: true
+{% endif %}
+{# DNS settings for kubelet #}
+{% if enable_nodelocaldns %}
+{% set kubelet_cluster_dns = [nodelocaldns_ip] %}
+{% elif dns_mode in ['coredns'] %}
+{% set kubelet_cluster_dns = [skydns_server] %}
+{% elif dns_mode == 'coredns_dual' %}
+{% set kubelet_cluster_dns = [skydns_server,skydns_server_secondary] %}
+{% elif dns_mode == 'manual' %}
+{% set kubelet_cluster_dns = [manual_dns_server] %}
+{% else %}
+{% set kubelet_cluster_dns = [] %}
+{% endif %}
+clusterDNS:
+{% for dns_address in kubelet_cluster_dns %}
+- {{ dns_address }}
+{% endfor %}
+{# Node reserved CPU/memory #}
+kubeReserved:
+{% if is_kube_master|bool %}
+  cpu: {{ kube_master_cpu_reserved }}
+  memory: {{ kube_master_memory_reserved }}
+{% if kube_master_ephemeral_storage_reserved is defined %}
+  ephemeral-storage: {{ kube_master_ephemeral_storage_reserved }}
+{% endif %}
+{% if kube_master_pid_reserved is defined %}
+  pid: "{{ kube_master_pid_reserved }}"
+{% endif %}
+{% else %}
+  cpu: {{ kube_cpu_reserved }}
+  memory: {{ kube_memory_reserved }}
+{% if kube_ephemeral_storage_reserved is defined %}
+  ephemeral-storage: {{ kube_ephemeral_storage_reserved }}
+{% endif %}
+{% if kube_pid_reserved is defined %}
+  pid: "{{ kube_pid_reserved }}"
+{% endif %}
+{% endif %}
+{% if system_reserved is defined and system_reserved %}
+systemReserved:
+{% if is_kube_master|bool %}
+  cpu: {{ system_master_cpu_reserved }}
+  memory: {{ system_master_memory_reserved }}
+{% if system_master_ephemeral_storage_reserved is defined %}
+  ephemeral-storage: {{ system_master_ephemeral_storage_reserved }}
+{% endif %}
+{% if system_master_pid_reserved is defined %}
+  pid: "{{ system_master_pid_reserved }}"
+{% endif %}
+{% else %}
+  cpu: {{ system_cpu_reserved }}
+  memory: {{ system_memory_reserved }}
+{% if system_ephemeral_storage_reserved is defined %}
+  ephemeral-storage: {{ system_ephemeral_storage_reserved }}
+{% endif %}
+{% if system_pid_reserved is defined %}
+  pid: "{{ system_pid_reserved }}"
+{% endif %}
+{% endif %}
+{% endif %}
+{% if is_kube_master|bool and eviction_hard_control_plane is defined and eviction_hard_control_plane %}
+evictionHard:
+  {{ eviction_hard_control_plane | to_nice_yaml(indent=2) | indent(2) }}
+{% elif not is_kube_master|bool and eviction_hard is defined and eviction_hard %}
+evictionHard:
+  {{ eviction_hard | to_nice_yaml(indent=2) | indent(2) }}
+{% endif %}
+resolvConf: "{{ kube_resolv_conf }}"
+{% if kubelet_config_extra_args %}
+{{ kubelet_config_extra_args | to_nice_yaml(indent=2) }}
+{% endif %}
+{% if inventory_hostname in groups['kube_node'] and kubelet_node_config_extra_args %}
+{{ kubelet_node_config_extra_args | to_nice_yaml(indent=2) }}
+{% endif %}
+{% if kubelet_feature_gates or kube_feature_gates %}
+featureGates:
+{% for feature in (kubelet_feature_gates | default(kube_feature_gates, true)) %}
+  {{ feature|replace("=", ": ") }}
+{% endfor %}
+{% endif %}
+{% if tls_min_version is defined %}
+tlsMinVersion: {{ tls_min_version }}
+{% endif %}
+{% if tls_cipher_suites is defined %}
+tlsCipherSuites:
+{% for tls in tls_cipher_suites %}
+- {{ tls }}
+{% endfor %}
+{% endif %}
+{% if kubelet_event_record_qps %}
+eventRecordQPS: {{ kubelet_event_record_qps }}
+{% endif %}
+shutdownGracePeriod: {{ kubelet_shutdown_grace_period }}
+shutdownGracePeriodCriticalPods: {{ kubelet_shutdown_grace_period_critical_pods }}
+{% if not kubelet_fail_swap_on|default(true) %}
+memorySwap:
+  swapBehavior: {{ kubelet_swap_behavior|default("LimitedSwap") }}
+{% endif %}
+{% if kubelet_streaming_connection_idle_timeout is defined %}
+streamingConnectionIdleTimeout: {{ kubelet_streaming_connection_idle_timeout }}
+{% endif %}
+{% if kubelet_make_iptables_util_chains is defined %}
+makeIPTablesUtilChains: {{ kubelet_make_iptables_util_chains | bool }}
+{% endif %}
+{% if kubelet_seccomp_default is defined %}
+seccompDefault: {{ kubelet_seccomp_default | bool }}
+{% endif %}
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2
@@ -0,0 +1,43 @@
+KUBE_LOGTOSTDERR="--logtostderr=true"
+KUBE_LOG_LEVEL="--v={{ kube_log_level }}"
+KUBELET_ADDRESS="--node-ip={{ kubelet_address }}"
+{% if kube_override_hostname|default('') %}
+KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
+{% endif %}
+
+{# Base kubelet args #}
+{% set kubelet_args_base -%}
+{# start kubeadm specific settings #}
+--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \
+--config={{ kube_config_dir }}/kubelet-config.yaml \
+--kubeconfig={{ kube_config_dir }}/kubelet.conf \
+{# end kubeadm specific settings #}
+--container-runtime=remote \
+--container-runtime-endpoint={{ cri_socket }} \
+--runtime-cgroups={{ kubelet_runtime_cgroups }} \
+{% endset %}
+
+{# Kubelet node taints for gpu #}
+{% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %}
+{%   if inventory_hostname in nvidia_gpu_nodes and node_taints is defined %}
+{%       set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %}
+{%   elif inventory_hostname in nvidia_gpu_nodes and node_taints is not defined %}
+{%       set node_taints = [] %}
+{%       set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %}
+{%   endif %}
+{% endif %}
+
+KUBELET_ARGS="{{ kubelet_args_base }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube_node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
+{% if kubelet_flexvolumes_plugins_dir is defined %}
+KUBELET_VOLUME_PLUGIN="--volume-plugin-dir={{ kubelet_flexvolumes_plugins_dir }}"
+{% endif %}
+{% if kube_network_plugin is defined and kube_network_plugin == "cloud" %}
+KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet"
+{% endif %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce", "external"] %}
+KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config"
+{% else %}
+KUBELET_CLOUDPROVIDER=""
+{% endif %}
+
+PATH={{ bin_dir }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/kubelet.service.j2
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/kubelet.service.j2
@@ -0,0 +1,34 @@
+[Unit]
+Description=Kubernetes Kubelet Server
+Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+After={{ container_manager }}.service
+{% if container_manager == 'docker' %}
+Wants=docker.socket
+{% else %}
+Wants={{ container_manager }}.service
+{% endif %}
+
+[Service]
+EnvironmentFile=-{{ kube_config_dir }}/kubelet.env
+ExecStart={{ bin_dir }}/kubelet \
+		$KUBE_LOGTOSTDERR \
+		$KUBE_LOG_LEVEL \
+		$KUBELET_API_SERVER \
+		$KUBELET_ADDRESS \
+		$KUBELET_PORT \
+		$KUBELET_HOSTNAME \
+		$KUBELET_ARGS \
+		$DOCKER_SOCKET \
+		$KUBELET_NETWORK_PLUGIN \
+		$KUBELET_VOLUME_PLUGIN \
+		$KUBELET_CLOUDPROVIDER
+Restart=always
+RestartSec=10s
+{% if kubelet_systemd_hardening %}
+# Hardening setup
+IPAddressDeny=any
+IPAddressAllow={{ kubelet_secure_addresses }}
+{% endif %}
+
+[Install]
+WantedBy=multi-user.target
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/loadbalancer/haproxy.cfg.j2
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/loadbalancer/haproxy.cfg.j2
@@ -0,0 +1,43 @@
+global
+    maxconn                 4000
+    log                     127.0.0.1 local0
+
+defaults
+    mode                    http
+    log                     global
+    option                  httplog
+    option                  dontlognull
+    option                  http-server-close
+    option                  redispatch
+    retries                 5
+    timeout http-request    5m
+    timeout queue           5m
+    timeout connect         30s
+    timeout client          {{ loadbalancer_apiserver_keepalive_timeout }}
+    timeout server          15m
+    timeout http-keep-alive 30s
+    timeout check           30s
+    maxconn                 4000
+
+{% if loadbalancer_apiserver_healthcheck_port is defined -%}
+frontend healthz
+  bind *:{{ loadbalancer_apiserver_healthcheck_port }}
+  mode http
+  monitor-uri /healthz
+{% endif %}
+
+frontend kube_api_frontend
+  bind 127.0.0.1:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }}
+  mode tcp
+  option tcplog
+  default_backend kube_api_backend
+
+backend kube_api_backend
+  mode tcp
+  balance leastconn
+  default-server inter 15s downinter 15s rise 2 fall 2 slowstart 60s maxconn 1000 maxqueue 256 weight 100
+  option httpchk GET /healthz
+  http-check expect status 200
+  {% for host in groups['kube_control_plane'] -%}
+  server {{ host }} {{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(fallback_ips[host])) }}:{{ kube_apiserver_port }} check check-ssl verify none
+  {% endfor -%}
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/loadbalancer/nginx.conf.j2
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/loadbalancer/nginx.conf.j2
@@ -0,0 +1,60 @@
+error_log stderr notice;
+
+worker_processes 2;
+worker_rlimit_nofile 130048;
+worker_shutdown_timeout 10s;
+
+events {
+  multi_accept on;
+  use epoll;
+  worker_connections 16384;
+}
+
+stream {
+  upstream kube_apiserver {
+    least_conn;
+    {% for host in groups['kube_control_plane'] -%}
+    server {{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(fallback_ips[host])) }}:{{ kube_apiserver_port }};
+    {% endfor -%}
+  }
+
+  server {
+    listen        127.0.0.1:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }};
+    {% if enable_dual_stack_networks -%}
+    listen        [::]:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }};
+    {% endif -%}
+    proxy_pass    kube_apiserver;
+    proxy_timeout 10m;
+    proxy_connect_timeout 1s;
+  }
+}
+
+http {
+  aio threads;
+  aio_write on;
+  tcp_nopush on;
+  tcp_nodelay on;
+
+  keepalive_timeout {{ loadbalancer_apiserver_keepalive_timeout }};
+  keepalive_requests 100;
+  reset_timedout_connection on;
+  server_tokens off;
+  autoindex off;
+
+  {% if loadbalancer_apiserver_healthcheck_port is defined -%}
+  server {
+    listen {{ loadbalancer_apiserver_healthcheck_port }};
+    {% if enable_dual_stack_networks -%}
+    listen [::]:{{ loadbalancer_apiserver_healthcheck_port }};
+    {% endif -%}
+    location /healthz {
+      access_log off;
+      return 200;
+    }
+    location /stub_status {
+      stub_status on;
+      access_log off;
+    }
+  }
+  {% endif %}
+}
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/manifests/haproxy.manifest.j2
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/manifests/haproxy.manifest.j2
@@ -0,0 +1,42 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: haproxy
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    k8s-app: kube-haproxy
+  annotations:
+    haproxy-cfg-checksum: "{{ haproxy_stat.stat.checksum }}"
+spec:
+  hostNetwork: true
+  dnsPolicy: ClusterFirstWithHostNet
+  nodeSelector:
+    kubernetes.io/os: linux
+  priorityClassName: system-node-critical
+  containers:
+  - name: haproxy
+    image: {{ haproxy_image_repo }}:{{ haproxy_image_tag }}
+    imagePullPolicy: {{ k8s_image_pull_policy }}
+    resources:
+      requests:
+        cpu: {{ loadbalancer_apiserver_cpu_requests }}
+        memory: {{ loadbalancer_apiserver_memory_requests }}
+    {% if loadbalancer_apiserver_healthcheck_port is defined -%}
+    livenessProbe:
+      httpGet:
+        path: /healthz
+        port: {{ loadbalancer_apiserver_healthcheck_port }}
+    readinessProbe:
+      httpGet:
+        path: /healthz
+        port: {{ loadbalancer_apiserver_healthcheck_port }}
+    {% endif -%}
+    volumeMounts:
+    - mountPath: /usr/local/etc/haproxy/
+      name: etc-haproxy
+      readOnly: true
+  volumes:
+  - name: etc-haproxy
+    hostPath:
+      path: {{ haproxy_config_dir }}
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/manifests/kube-vip.manifest.j2
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/manifests/kube-vip.manifest.j2
@@ -0,0 +1,93 @@
+# Inspired by https://github.com/kube-vip/kube-vip/blob/v0.5.5/pkg/kubevip/config_generator.go#L13
+apiVersion: v1
+kind: Pod
+metadata:
+  creationTimestamp: null
+  name: kube-vip
+  namespace: kube-system
+spec:
+  containers:
+  - args:
+    - manager
+    env:
+    - name: vip_arp
+      value: {{ kube_vip_arp_enabled | string | to_json }}
+    - name: port
+      value: {{ kube_apiserver_port | string | to_json }}
+{% if kube_vip_interface %}
+    - name: vip_interface
+      value: {{ kube_vip_interface | string | to_json }}
+{% endif %}
+{% if kube_vip_services_interface %}
+    - name: vip_servicesinterface
+      value: {{ kube_vip_services_interface | string | to_json }}
+{% endif %}
+{% if kube_vip_cidr %}
+    - name: vip_cidr
+      value: {{ kube_vip_cidr | string | to_json }}
+{% endif %}
+{% if kube_vip_controlplane_enabled %}
+    - name: cp_enable
+      value: "true"
+    - name: cp_namespace
+      value: kube-system
+    - name: vip_ddns
+      value: {{ kube_vip_ddns_enabled | string | to_json }}
+{% endif %}
+{% if kube_vip_services_enabled %}
+    - name: svc_enable
+      value: "true"
+{% endif %}
+{% if kube_vip_leader_election_enabled %}
+    - name: vip_leaderelection
+      value: "true"
+    - name: vip_leaseduration
+      value: "5"
+    - name: vip_renewdeadline
+      value: "3"
+    - name: vip_retryperiod
+      value: "1"
+{% endif %}
+{% if kube_vip_bgp_enabled %}
+    - name: bgp_enable
+      value: "true"
+    - name: bgp_routerid
+      value: {{ kube_vip_bgp_routerid | string | to_json }}
+    - name: bgp_as
+      value: {{ kube_vip_local_as | string | to_json }}
+    - name: bgp_peeraddress
+      value: {{ kube_vip_bgp_peeraddress | to_json }}
+    - name: bgp_peerpass
+      value: {{ kube_vip_bgp_peerpass | to_json }}
+    - name: bgp_peeras
+      value: {{ kube_vip_bgp_peeras | string | to_json }}
+{% if kube_vip_bgppeers %}
+    - name: bgp_peers
+      value: {{ kube_vip_bgppeers | join(',')  | to_json }}
+{% endif %}
+{% endif %}
+    - name: address
+      value: {{ kube_vip_address | to_json }}
+    image: {{ kube_vip_image_repo }}:{{ kube_vip_image_tag }}
+    imagePullPolicy: {{ k8s_image_pull_policy }}
+    name: kube-vip
+    resources: {}
+    securityContext:
+      capabilities:
+        add:
+        - NET_ADMIN
+        - NET_RAW
+    volumeMounts:
+    - mountPath: /etc/kubernetes/admin.conf
+      name: kubeconfig
+  hostAliases:
+  - hostnames:
+    - kubernetes
+    ip: 127.0.0.1
+  hostNetwork: true
+  volumes:
+  - hostPath:
+      path: /etc/kubernetes/admin.conf
+    name: kubeconfig
+status: {}
+
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2
@@ -0,0 +1,42 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-proxy
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    k8s-app: kube-nginx
+  annotations:
+    nginx-cfg-checksum: "{{ nginx_stat.stat.checksum }}"
+spec:
+  hostNetwork: true
+  dnsPolicy: ClusterFirstWithHostNet
+  nodeSelector:
+    kubernetes.io/os: linux
+  priorityClassName: system-node-critical
+  containers:
+  - name: nginx-proxy
+    image: {{ nginx_image_repo }}:{{ nginx_image_tag }}
+    imagePullPolicy: {{ k8s_image_pull_policy }}
+    resources:
+      requests:
+        cpu: {{ loadbalancer_apiserver_cpu_requests }}
+        memory: {{ loadbalancer_apiserver_memory_requests }}
+    {% if loadbalancer_apiserver_healthcheck_port is defined -%}
+    livenessProbe:
+      httpGet:
+        path: /healthz
+        port: {{ loadbalancer_apiserver_healthcheck_port }}
+    readinessProbe:
+      httpGet:
+        path: /healthz
+        port: {{ loadbalancer_apiserver_healthcheck_port }}
+    {% endif -%}
+    volumeMounts:
+    - mountPath: /etc/nginx
+      name: etc-nginx
+      readOnly: true
+  volumes:
+  - name: etc-nginx
+    hostPath:
+      path: {{ nginx_config_dir }}
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2
@@ -0,0 +1,19 @@
+---
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    certificate-authority: {{ kube_cert_dir }}/ca.pem
+    server: {{ kube_apiserver_endpoint }}
+users:
+- name: kubelet
+  user:
+    client-certificate: {{ kube_cert_dir }}/node-{{ inventory_hostname }}.pem
+    client-key: {{ kube_cert_dir }}/node-{{ inventory_hostname }}-key.pem
+contexts:
+- context:
+    cluster: local
+    user: kubelet
+  name: kubelet-{{ cluster_name }}
+current-context: kubelet-{{ cluster_name }}
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/vars/fedora.yml
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/vars/fedora.yml
@@ -0,0 +1,2 @@
+---
+kube_resolv_conf: "/run/systemd/resolve/resolv.conf"
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/vars/ubuntu-18.yml
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/vars/ubuntu-18.yml
@@ -0,0 +1,2 @@
+---
+kube_resolv_conf: "/run/systemd/resolve/resolv.conf"
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/vars/ubuntu-20.yml
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/vars/ubuntu-20.yml
@@ -0,0 +1,2 @@
+---
+kube_resolv_conf: "/run/systemd/resolve/resolv.conf"
--- a/ansible/kubespray/extra_playbooks/roles/kubernetes/node/vars/ubuntu-22.yml
+++ b/ansible/kubespray/extra_playbooks/roles/kubernetes/node/vars/ubuntu-22.yml
@@ -0,0 +1,2 @@
+---
+kube_resolv_conf: "/run/systemd/resolve/resolv.conf"