update

kubespray/roles/kubernetes/node/templates/cloud-configs/aws-cloud-config.j2

@@ -0,0 +1,11 @@
+[Global]
+zone={{ aws_zone|default("") }}
+vpc={{ aws_vpc|default("") }}
+subnetId={{ aws_subnet_id|default("") }}
+routeTableId={{ aws_route_table_id|default("") }}
+roleArn={{ aws_role_arn|default("") }}
+kubernetesClusterTag={{ aws_kubernetes_cluster_tag|default("") }}
+kubernetesClusterId={{ aws_kubernetes_cluster_id|default("") }}
+disableSecurityGroupIngress={{ "true" if aws_disable_security_group_ingress|default(False) else "false" }}
+disableStrictZoneCheck={{ "true" if aws_disable_strict_zone_check|default(False) else "false" }}
+elbSecurityGroup={{ aws_elb_security_group|default("") }}

kubespray/roles/kubernetes/node/templates/cloud-configs/azure-cloud-config.j2

@@ -0,0 +1,26 @@
+{
+  "cloud": "{{ azure_cloud }}",
+  "tenantId": "{{ azure_tenant_id }}",
+  "subscriptionId": "{{ azure_subscription_id }}",
+  "aadClientId": "{{ azure_aad_client_id }}",
+  "aadClientSecret": "{{ azure_aad_client_secret }}",
+  "resourceGroup": "{{ azure_resource_group }}",
+  "location": "{{ azure_location }}",
+  "subnetName": "{{ azure_subnet_name }}",
+  "securityGroupName": "{{ azure_security_group_name }}",
+  "securityGroupResourceGroup": "{{ azure_security_group_resource_group | default(azure_vnet_resource_group) }}",
+  "vnetName": "{{ azure_vnet_name }}",
+  "vnetResourceGroup": "{{ azure_vnet_resource_group }}",
+  "routeTableName": "{{ azure_route_table_name }}",
+  "routeTableResourceGroup": "{{ azure_route_table_resource_group | default(azure_vnet_resource_group) }}",
+  "vmType": "{{ azure_vmtype }}",
+{% if azure_primary_availability_set_name is defined %}
+  "primaryAvailabilitySetName": "{{ azure_primary_availability_set_name }}",
+{%endif%}
+  "useInstanceMetadata": {{azure_use_instance_metadata | lower }},
+{% if azure_loadbalancer_sku == "standard" %}
+  "excludeMasterFromStandardLB": {{ azure_exclude_master_from_standard_lb | lower }},
+  "disableOutboundSNAT": {{ azure_disable_outbound_snat | lower }},
+{% endif%}
+  "loadBalancerSku": "{{ azure_loadbalancer_sku }}"
+}

kubespray/roles/kubernetes/node/templates/cloud-configs/gce-cloud-config.j2

@@ -0,0 +1,3 @@
+[global]
+node-tags = {{ gce_node_tags }} 
+

kubespray/roles/kubernetes/node/templates/cloud-configs/openstack-cloud-config.j2

@@ -0,0 +1,54 @@
+[Global]
+auth-url="{{ openstack_auth_url }}"
+username="{{ openstack_username }}"
+password="{{ openstack_password }}"
+region="{{ openstack_region }}"
+{% if openstack_trust_id is defined and openstack_trust_id != "" %}
+trust-id="{{ openstack_trust_id }}"
+{% else %}
+tenant-id="{{ openstack_tenant_id }}"
+{% endif %}
+{% if openstack_tenant_name is defined and openstack_tenant_name != "" %}
+tenant-name="{{ openstack_tenant_name }}"
+{% endif %}
+{% if openstack_domain_name is defined and openstack_domain_name != "" %}
+domain-name="{{ openstack_domain_name }}"
+{% elif openstack_domain_id is defined and openstack_domain_id != "" %}
+domain-id ="{{ openstack_domain_id }}"
+{% endif %}
+{% if openstack_cacert is defined and openstack_cacert != "" %}
+ca-file="{{ kube_config_dir }}/openstack-cacert.pem"
+{% endif %}
+
+[BlockStorage]
+{% if openstack_blockstorage_version is defined %}
+bs-version={{ openstack_blockstorage_version }}
+{% endif %}
+{% if openstack_blockstorage_ignore_volume_az is defined and openstack_blockstorage_ignore_volume_az|bool %}
+ignore-volume-az={{ openstack_blockstorage_ignore_volume_az }}
+{% endif %}
+{% if node_volume_attach_limit is defined and node_volume_attach_limit != "" %}
+node-volume-attach-limit="{{ node_volume_attach_limit }}"
+{% endif %}
+
+{% if openstack_lbaas_enabled and openstack_lbaas_subnet_id is defined %}
+[LoadBalancer]
+subnet-id={{ openstack_lbaas_subnet_id }}
+{% if openstack_lbaas_floating_network_id is defined %}
+floating-network-id={{ openstack_lbaas_floating_network_id }}
+{% endif %}
+{% if openstack_lbaas_use_octavia is defined %}
+use-octavia={{ openstack_lbaas_use_octavia }}
+{% endif %}
+{% if openstack_lbaas_method is defined %}
+lb-method={{ openstack_lbaas_method }}
+{% endif %}
+{% if openstack_lbaas_provider is defined %}
+lb-provider={{ openstack_lbaas_provider }}
+{% endif %}
+
+create-monitor={{ openstack_lbaas_create_monitor }}
+monitor-delay={{ openstack_lbaas_monitor_delay }}
+monitor-timeout={{ openstack_lbaas_monitor_timeout }}
+monitor-max-retries={{ openstack_lbaas_monitor_max_retries }}
+{% endif %}

kubespray/roles/kubernetes/node/templates/cloud-configs/vsphere-cloud-config.j2

@@ -0,0 +1,36 @@
+[Global]
+user = "{{ vsphere_user }}"
+password = "{{ vsphere_password }}"
+port = {{ vsphere_vcenter_port }}
+insecure-flag = {{ vsphere_insecure }}
+
+datacenters = "{{ vsphere_datacenter }}"
+
+[VirtualCenter "{{ vsphere_vcenter_ip }}"]
+
+
+[Workspace]
+server = "{{ vsphere_vcenter_ip }}"
+datacenter = "{{ vsphere_datacenter }}"
+folder = "{{ vsphere_working_dir }}"
+default-datastore = "{{ vsphere_datastore }}"
+{% if vsphere_resource_pool is defined and vsphere_resource_pool != ""  %}
+resourcepool-path = "{{ vsphere_resource_pool }}"
+{% endif %}
+
+
+[Disk]
+scsicontrollertype = {{ vsphere_scsi_controller_type }}
+
+{% if vsphere_public_network is defined and vsphere_public_network != ""  %}
+[Network]
+public-network = {{ vsphere_public_network }}
+{% endif %}
+
+[Labels]
+{% if vsphere_zone_category is defined and vsphere_zone_category != ""  %}
+zone = {{ vsphere_zone_category }}
+{% endif %}
+{% if vsphere_region_category is defined and vsphere_region_category != ""  %}
+region = {{ vsphere_region_category }}
+{% endif %}

kubespray/roles/kubernetes/node/templates/http-proxy.conf.j2

@@ -0,0 +1,2 @@
+[Service]
+Environment={% if http_proxy %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy %}"NO_PROXY={{ no_proxy }}"{% endif %}

kubespray/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2

@@ -0,0 +1,151 @@
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+nodeStatusUpdateFrequency: "{{ kubelet_status_update_frequency }}"
+failSwapOn: {{ kubelet_fail_swap_on|default(true) }}
+authentication:
+  anonymous:
+    enabled: false
+  webhook:
+    enabled: {{ kubelet_authentication_token_webhook }}
+  x509:
+    clientCAFile: {{ kube_cert_dir }}/ca.crt
+authorization:
+{% if kubelet_authorization_mode_webhook %}
+  mode: Webhook
+{% else %}
+  mode: AlwaysAllow
+{% endif %}
+{% if kubelet_enforce_node_allocatable is defined and kubelet_enforce_node_allocatable != "\"\"" %}
+{% set kubelet_enforce_node_allocatable_list = kubelet_enforce_node_allocatable.split() %}
+enforceNodeAllocatable:
+{% for item in kubelet_enforce_node_allocatable_list %}
+- {{ item }}
+{% endfor %}
+{% endif %}
+staticPodPath: {{ kube_manifest_dir }}
+cgroupDriver: {{ kubelet_cgroup_driver | default('systemd') }}
+containerLogMaxFiles: {{ kubelet_logfiles_max_nr }}
+containerLogMaxSize: {{ kubelet_logfiles_max_size }}
+maxPods: {{ kubelet_max_pods }}
+podPidsLimit: {{ kubelet_pod_pids_limit }}
+address: {{ kubelet_bind_address }}
+readOnlyPort: {{ kube_read_only_port }}
+healthzPort: {{ kubelet_healthz_port }}
+healthzBindAddress: {{ kubelet_healthz_bind_address }}
+kubeletCgroups: {{ kubelet_kubelet_cgroups }}
+clusterDomain: {{ dns_domain }}
+{% if kubelet_protect_kernel_defaults|bool %}
+protectKernelDefaults: true
+{% endif %}
+{% if kubelet_rotate_certificates|bool %}
+rotateCertificates: true
+{% endif %}
+{% if kubelet_rotate_server_certificates|bool %}
+serverTLSBootstrap: true
+{% endif %}
+{# DNS settings for kubelet #}
+{% if enable_nodelocaldns %}
+{% set kubelet_cluster_dns = [nodelocaldns_ip] %}
+{% elif dns_mode in ['coredns'] %}
+{% set kubelet_cluster_dns = [skydns_server] %}
+{% elif dns_mode == 'coredns_dual' %}
+{% set kubelet_cluster_dns = [skydns_server,skydns_server_secondary] %}
+{% elif dns_mode == 'manual' %}
+{% set kubelet_cluster_dns = [manual_dns_server] %}
+{% else %}
+{% set kubelet_cluster_dns = [] %}
+{% endif %}
+clusterDNS:
+{% for dns_address in kubelet_cluster_dns %}
+- {{ dns_address }}
+{% endfor %}
+{# Node reserved CPU/memory #}
+kubeReserved:
+{% if is_kube_master|bool %}
+  cpu: {{ kube_master_cpu_reserved }}
+  memory: {{ kube_master_memory_reserved }}
+{% if kube_master_ephemeral_storage_reserved is defined %}
+  ephemeral-storage: {{ kube_master_ephemeral_storage_reserved }}
+{% endif %}
+{% if kube_master_pid_reserved is defined %}
+  pid: "{{ kube_master_pid_reserved }}"
+{% endif %}
+{% else %}
+  cpu: {{ kube_cpu_reserved }}
+  memory: {{ kube_memory_reserved }}
+{% if kube_ephemeral_storage_reserved is defined %}
+  ephemeral-storage: {{ kube_ephemeral_storage_reserved }}
+{% endif %}
+{% if kube_pid_reserved is defined %}
+  pid: "{{ kube_pid_reserved }}"
+{% endif %}
+{% endif %}
+{% if system_reserved is defined and system_reserved %}
+systemReserved:
+{% if is_kube_master|bool %}
+  cpu: {{ system_master_cpu_reserved }}
+  memory: {{ system_master_memory_reserved }}
+{% if system_master_ephemeral_storage_reserved is defined %}
+  ephemeral-storage: {{ system_master_ephemeral_storage_reserved }}
+{% endif %}
+{% if system_master_pid_reserved is defined %}
+  pid: "{{ system_master_pid_reserved }}"
+{% endif %}
+{% else %}
+  cpu: {{ system_cpu_reserved }}
+  memory: {{ system_memory_reserved }}
+{% if system_ephemeral_storage_reserved is defined %}
+  ephemeral-storage: {{ system_ephemeral_storage_reserved }}
+{% endif %}
+{% if system_pid_reserved is defined %}
+  pid: "{{ system_pid_reserved }}"
+{% endif %}
+{% endif %}
+{% endif %}
+{% if is_kube_master|bool and eviction_hard_control_plane is defined and eviction_hard_control_plane %}
+evictionHard:
+  {{ eviction_hard_control_plane | to_nice_yaml(indent=2) | indent(2) }}
+{% elif not is_kube_master|bool and eviction_hard is defined and eviction_hard %}
+evictionHard:
+  {{ eviction_hard | to_nice_yaml(indent=2) | indent(2) }}
+{% endif %}
+resolvConf: "{{ kube_resolv_conf }}"
+{% if kubelet_config_extra_args %}
+{{ kubelet_config_extra_args | to_nice_yaml(indent=2) }}
+{% endif %}
+{% if inventory_hostname in groups['kube_node'] and kubelet_node_config_extra_args %}
+{{ kubelet_node_config_extra_args | to_nice_yaml(indent=2) }}
+{% endif %}
+{% if kubelet_feature_gates or kube_feature_gates %}
+featureGates:
+{% for feature in (kubelet_feature_gates | default(kube_feature_gates, true)) %}
+  {{ feature|replace("=", ": ") }}
+{% endfor %}
+{% endif %}
+{% if tls_min_version is defined %}
+tlsMinVersion: {{ tls_min_version }}
+{% endif %}
+{% if tls_cipher_suites is defined %}
+tlsCipherSuites:
+{% for tls in tls_cipher_suites %}
+- {{ tls }}
+{% endfor %}
+{% endif %}
+{% if kubelet_event_record_qps %}
+eventRecordQPS: {{ kubelet_event_record_qps }}
+{% endif %}
+shutdownGracePeriod: {{ kubelet_shutdown_grace_period }}
+shutdownGracePeriodCriticalPods: {{ kubelet_shutdown_grace_period_critical_pods }}
+{% if not kubelet_fail_swap_on|default(true) %}
+memorySwap:
+  swapBehavior: {{ kubelet_swap_behavior|default("LimitedSwap") }}
+{% endif %}
+{% if kubelet_streaming_connection_idle_timeout is defined %}
+streamingConnectionIdleTimeout: {{ kubelet_streaming_connection_idle_timeout }}
+{% endif %}
+{% if kubelet_make_iptables_util_chains is defined %}
+makeIPTablesUtilChains: {{ kubelet_make_iptables_util_chains | bool }}
+{% endif %}
+{% if kubelet_seccomp_default is defined %}
+seccompDefault: {{ kubelet_seccomp_default | bool }}
+{% endif %}

kubespray/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2

@@ -0,0 +1,43 @@
+KUBE_LOGTOSTDERR="--logtostderr=true"
+KUBE_LOG_LEVEL="--v={{ kube_log_level }}"
+KUBELET_ADDRESS="--node-ip={{ kubelet_address }}"
+{% if kube_override_hostname|default('') %}
+KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
+{% endif %}
+
+{# Base kubelet args #}
+{% set kubelet_args_base -%}
+{# start kubeadm specific settings #}
+--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \
+--config={{ kube_config_dir }}/kubelet-config.yaml \
+--kubeconfig={{ kube_config_dir }}/kubelet.conf \
+{# end kubeadm specific settings #}
+--container-runtime=remote \
+--container-runtime-endpoint={{ cri_socket }} \
+--runtime-cgroups={{ kubelet_runtime_cgroups }} \
+{% endset %}
+
+{# Kubelet node taints for gpu #}
+{% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %}
+{%   if inventory_hostname in nvidia_gpu_nodes and node_taints is defined %}
+{%       set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %}
+{%   elif inventory_hostname in nvidia_gpu_nodes and node_taints is not defined %}
+{%       set node_taints = [] %}
+{%       set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %}
+{%   endif %}
+{% endif %}
+
+KUBELET_ARGS="{{ kubelet_args_base }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube_node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
+{% if kubelet_flexvolumes_plugins_dir is defined %}
+KUBELET_VOLUME_PLUGIN="--volume-plugin-dir={{ kubelet_flexvolumes_plugins_dir }}"
+{% endif %}
+{% if kube_network_plugin is defined and kube_network_plugin == "cloud" %}
+KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet"
+{% endif %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "gce", "external"] %}
+KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config"
+{% else %}
+KUBELET_CLOUDPROVIDER=""
+{% endif %}
+
+PATH={{ bin_dir }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

kubespray/roles/kubernetes/node/templates/kubelet.service.j2

@@ -0,0 +1,34 @@
+[Unit]
+Description=Kubernetes Kubelet Server
+Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+After={{ container_manager }}.service
+{% if container_manager == 'docker' %}
+Wants=docker.socket
+{% else %}
+Wants={{ container_manager }}.service
+{% endif %}
+
+[Service]
+EnvironmentFile=-{{ kube_config_dir }}/kubelet.env
+ExecStart={{ bin_dir }}/kubelet \
+		$KUBE_LOGTOSTDERR \
+		$KUBE_LOG_LEVEL \
+		$KUBELET_API_SERVER \
+		$KUBELET_ADDRESS \
+		$KUBELET_PORT \
+		$KUBELET_HOSTNAME \
+		$KUBELET_ARGS \
+		$DOCKER_SOCKET \
+		$KUBELET_NETWORK_PLUGIN \
+		$KUBELET_VOLUME_PLUGIN \
+		$KUBELET_CLOUDPROVIDER
+Restart=always
+RestartSec=10s
+{% if kubelet_systemd_hardening %}
+# Hardening setup
+IPAddressDeny=any
+IPAddressAllow={{ kubelet_secure_addresses }}
+{% endif %}
+
+[Install]
+WantedBy=multi-user.target

kubespray/roles/kubernetes/node/templates/loadbalancer/haproxy.cfg.j2

@@ -0,0 +1,43 @@
+global
+    maxconn                 4000
+    log                     127.0.0.1 local0
+
+defaults
+    mode                    http
+    log                     global
+    option                  httplog
+    option                  dontlognull
+    option                  http-server-close
+    option                  redispatch
+    retries                 5
+    timeout http-request    5m
+    timeout queue           5m
+    timeout connect         30s
+    timeout client          {{ loadbalancer_apiserver_keepalive_timeout }}
+    timeout server          15m
+    timeout http-keep-alive 30s
+    timeout check           30s
+    maxconn                 4000
+
+{% if loadbalancer_apiserver_healthcheck_port is defined -%}
+frontend healthz
+  bind *:{{ loadbalancer_apiserver_healthcheck_port }}
+  mode http
+  monitor-uri /healthz
+{% endif %}
+
+frontend kube_api_frontend
+  bind 127.0.0.1:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }}
+  mode tcp
+  option tcplog
+  default_backend kube_api_backend
+
+backend kube_api_backend
+  mode tcp
+  balance leastconn
+  default-server inter 15s downinter 15s rise 2 fall 2 slowstart 60s maxconn 1000 maxqueue 256 weight 100
+  option httpchk GET /healthz
+  http-check expect status 200
+  {% for host in groups['kube_control_plane'] -%}
+  server {{ host }} {{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(fallback_ips[host])) }}:{{ kube_apiserver_port }} check check-ssl verify none
+  {% endfor -%}

kubespray/roles/kubernetes/node/templates/loadbalancer/nginx.conf.j2

@@ -0,0 +1,60 @@
+error_log stderr notice;
+
+worker_processes 2;
+worker_rlimit_nofile 130048;
+worker_shutdown_timeout 10s;
+
+events {
+  multi_accept on;
+  use epoll;
+  worker_connections 16384;
+}
+
+stream {
+  upstream kube_apiserver {
+    least_conn;
+    {% for host in groups['kube_control_plane'] -%}
+    server {{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(fallback_ips[host])) }}:{{ kube_apiserver_port }};
+    {% endfor -%}
+  }
+
+  server {
+    listen        127.0.0.1:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }};
+    {% if enable_dual_stack_networks -%}
+    listen        [::]:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }};
+    {% endif -%}
+    proxy_pass    kube_apiserver;
+    proxy_timeout 10m;
+    proxy_connect_timeout 1s;
+  }
+}
+
+http {
+  aio threads;
+  aio_write on;
+  tcp_nopush on;
+  tcp_nodelay on;
+
+  keepalive_timeout {{ loadbalancer_apiserver_keepalive_timeout }};
+  keepalive_requests 100;
+  reset_timedout_connection on;
+  server_tokens off;
+  autoindex off;
+
+  {% if loadbalancer_apiserver_healthcheck_port is defined -%}
+  server {
+    listen {{ loadbalancer_apiserver_healthcheck_port }};
+    {% if enable_dual_stack_networks -%}
+    listen [::]:{{ loadbalancer_apiserver_healthcheck_port }};
+    {% endif -%}
+    location /healthz {
+      access_log off;
+      return 200;
+    }
+    location /stub_status {
+      stub_status on;
+      access_log off;
+    }
+  }
+  {% endif %}
+}

kubespray/roles/kubernetes/node/templates/manifests/haproxy.manifest.j2

@@ -0,0 +1,42 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: haproxy
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    k8s-app: kube-haproxy
+  annotations:
+    haproxy-cfg-checksum: "{{ haproxy_stat.stat.checksum }}"
+spec:
+  hostNetwork: true
+  dnsPolicy: ClusterFirstWithHostNet
+  nodeSelector:
+    kubernetes.io/os: linux
+  priorityClassName: system-node-critical
+  containers:
+  - name: haproxy
+    image: {{ haproxy_image_repo }}:{{ haproxy_image_tag }}
+    imagePullPolicy: {{ k8s_image_pull_policy }}
+    resources:
+      requests:
+        cpu: {{ loadbalancer_apiserver_cpu_requests }}
+        memory: {{ loadbalancer_apiserver_memory_requests }}
+    {% if loadbalancer_apiserver_healthcheck_port is defined -%}
+    livenessProbe:
+      httpGet:
+        path: /healthz
+        port: {{ loadbalancer_apiserver_healthcheck_port }}
+    readinessProbe:
+      httpGet:
+        path: /healthz
+        port: {{ loadbalancer_apiserver_healthcheck_port }}
+    {% endif -%}
+    volumeMounts:
+    - mountPath: /usr/local/etc/haproxy/
+      name: etc-haproxy
+      readOnly: true
+  volumes:
+  - name: etc-haproxy
+    hostPath:
+      path: {{ haproxy_config_dir }}

kubespray/roles/kubernetes/node/templates/manifests/kube-vip.manifest.j2

@@ -0,0 +1,93 @@
+# Inspired by https://github.com/kube-vip/kube-vip/blob/v0.5.5/pkg/kubevip/config_generator.go#L13
+apiVersion: v1
+kind: Pod
+metadata:
+  creationTimestamp: null
+  name: kube-vip
+  namespace: kube-system
+spec:
+  containers:
+  - args:
+    - manager
+    env:
+    - name: vip_arp
+      value: {{ kube_vip_arp_enabled | string | to_json }}
+    - name: port
+      value: {{ kube_apiserver_port | string | to_json }}
+{% if kube_vip_interface %}
+    - name: vip_interface
+      value: {{ kube_vip_interface | string | to_json }}
+{% endif %}
+{% if kube_vip_services_interface %}
+    - name: vip_servicesinterface
+      value: {{ kube_vip_services_interface | string | to_json }}
+{% endif %}
+{% if kube_vip_cidr %}
+    - name: vip_cidr
+      value: {{ kube_vip_cidr | string | to_json }}
+{% endif %}
+{% if kube_vip_controlplane_enabled %}
+    - name: cp_enable
+      value: "true"
+    - name: cp_namespace
+      value: kube-system
+    - name: vip_ddns
+      value: {{ kube_vip_ddns_enabled | string | to_json }}
+{% endif %}
+{% if kube_vip_services_enabled %}
+    - name: svc_enable
+      value: "true"
+{% endif %}
+{% if kube_vip_leader_election_enabled %}
+    - name: vip_leaderelection
+      value: "true"
+    - name: vip_leaseduration
+      value: "5"
+    - name: vip_renewdeadline
+      value: "3"
+    - name: vip_retryperiod
+      value: "1"
+{% endif %}
+{% if kube_vip_bgp_enabled %}
+    - name: bgp_enable
+      value: "true"
+    - name: bgp_routerid
+      value: {{ kube_vip_bgp_routerid | string | to_json }}
+    - name: bgp_as
+      value: {{ kube_vip_local_as | string | to_json }}
+    - name: bgp_peeraddress
+      value: {{ kube_vip_bgp_peeraddress | to_json }}
+    - name: bgp_peerpass
+      value: {{ kube_vip_bgp_peerpass | to_json }}
+    - name: bgp_peeras
+      value: {{ kube_vip_bgp_peeras | string | to_json }}
+{% if kube_vip_bgppeers %}
+    - name: bgp_peers
+      value: {{ kube_vip_bgppeers | join(',')  | to_json }}
+{% endif %}
+{% endif %}
+    - name: address
+      value: {{ kube_vip_address | to_json }}
+    image: {{ kube_vip_image_repo }}:{{ kube_vip_image_tag }}
+    imagePullPolicy: {{ k8s_image_pull_policy }}
+    name: kube-vip
+    resources: {}
+    securityContext:
+      capabilities:
+        add:
+        - NET_ADMIN
+        - NET_RAW
+    volumeMounts:
+    - mountPath: /etc/kubernetes/admin.conf
+      name: kubeconfig
+  hostAliases:
+  - hostnames:
+    - kubernetes
+    ip: 127.0.0.1
+  hostNetwork: true
+  volumes:
+  - hostPath:
+      path: /etc/kubernetes/admin.conf
+    name: kubeconfig
+status: {}
+

kubespray/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2

@@ -0,0 +1,42 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-proxy
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    k8s-app: kube-nginx
+  annotations:
+    nginx-cfg-checksum: "{{ nginx_stat.stat.checksum }}"
+spec:
+  hostNetwork: true
+  dnsPolicy: ClusterFirstWithHostNet
+  nodeSelector:
+    kubernetes.io/os: linux
+  priorityClassName: system-node-critical
+  containers:
+  - name: nginx-proxy
+    image: {{ nginx_image_repo }}:{{ nginx_image_tag }}
+    imagePullPolicy: {{ k8s_image_pull_policy }}
+    resources:
+      requests:
+        cpu: {{ loadbalancer_apiserver_cpu_requests }}
+        memory: {{ loadbalancer_apiserver_memory_requests }}
+    {% if loadbalancer_apiserver_healthcheck_port is defined -%}
+    livenessProbe:
+      httpGet:
+        path: /healthz
+        port: {{ loadbalancer_apiserver_healthcheck_port }}
+    readinessProbe:
+      httpGet:
+        path: /healthz
+        port: {{ loadbalancer_apiserver_healthcheck_port }}
+    {% endif -%}
+    volumeMounts:
+    - mountPath: /etc/nginx
+      name: etc-nginx
+      readOnly: true
+  volumes:
+  - name: etc-nginx
+    hostPath:
+      path: {{ nginx_config_dir }}

kubespray/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2

@@ -0,0 +1,19 @@
+---
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    certificate-authority: {{ kube_cert_dir }}/ca.pem
+    server: {{ kube_apiserver_endpoint }}
+users:
+- name: kubelet
+  user:
+    client-certificate: {{ kube_cert_dir }}/node-{{ inventory_hostname }}.pem
+    client-key: {{ kube_cert_dir }}/node-{{ inventory_hostname }}-key.pem
+contexts:
+- context:
+    cluster: local
+    user: kubelet
+  name: kubelet-{{ cluster_name }}
+current-context: kubelet-{{ cluster_name }}