update

kubespray/extra_playbooks/roles/kubernetes-apps/ansible/tasks/cleanup_dns.yml

@@ -0,0 +1,44 @@
+---
+- name: Kubernetes Apps | Register coredns deployment annotation `createdby`
+  command: "{{ kubectl }} get deploy -n kube-system coredns -o jsonpath='{ .spec.template.metadata.annotations.createdby }'"
+  register: createdby_annotation_deploy
+  changed_when: false
+  check_mode: false
+  ignore_errors: true  # noqa ignore-errors
+  when:
+    - dns_mode in ['coredns', 'coredns_dual']
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Kubernetes Apps | Register coredns service annotation `createdby`
+  command: "{{ kubectl }} get svc -n kube-system coredns -o jsonpath='{ .metadata.annotations.createdby }'"
+  register: createdby_annotation_svc
+  changed_when: false
+  check_mode: false
+  ignore_errors: true  # noqa ignore-errors
+  when:
+    - dns_mode in ['coredns', 'coredns_dual']
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Kubernetes Apps | Delete kubeadm CoreDNS
+  kube:
+    name: "coredns"
+    namespace: "kube-system"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "deploy"
+    state: absent
+  when:
+    - dns_mode in ['coredns', 'coredns_dual']
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - createdby_annotation_deploy.stdout != 'kubespray'
+
+- name: Kubernetes Apps | Delete kubeadm Kube-DNS service
+  kube:
+    name: "kube-dns"
+    namespace: "kube-system"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "svc"
+    state: absent
+  when:
+    - dns_mode in ['coredns', 'coredns_dual']
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - createdby_annotation_svc.stdout != 'kubespray'

kubespray/extra_playbooks/roles/kubernetes-apps/ansible/tasks/coredns.yml

@@ -0,0 +1,44 @@
+---
+- name: Kubernetes Apps | Lay Down CoreDNS templates
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: 0644
+  loop:
+    - { name: coredns, file: coredns-clusterrole.yml, type: clusterrole }
+    - { name: coredns, file: coredns-clusterrolebinding.yml, type: clusterrolebinding }
+    - { name: coredns, file: coredns-config.yml, type: configmap }
+    - { name: coredns, file: coredns-deployment.yml, type: deployment }
+    - { name: coredns, file: coredns-sa.yml, type: sa }
+    - { name: coredns, file: coredns-svc.yml, type: svc }
+    - { name: dns-autoscaler, file: dns-autoscaler.yml, type: deployment }
+    - { name: dns-autoscaler, file: dns-autoscaler-clusterrole.yml, type: clusterrole }
+    - { name: dns-autoscaler, file: dns-autoscaler-clusterrolebinding.yml, type: clusterrolebinding }
+    - { name: dns-autoscaler, file: dns-autoscaler-sa.yml, type: sa }
+  register: coredns_manifests
+  vars:
+    clusterIP: "{{ skydns_server }}"
+  when:
+    - dns_mode in ['coredns', 'coredns_dual']
+    - inventory_hostname == groups['kube_control_plane'][0]
+  tags:
+    - coredns
+
+- name: Kubernetes Apps | Lay Down Secondary CoreDNS Template
+  template:
+    src: "{{ item.src }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: 0644
+  with_items:
+    - { name: coredns, src: coredns-deployment.yml, file: coredns-deployment-secondary.yml, type: deployment }
+    - { name: coredns, src: coredns-svc.yml, file: coredns-svc-secondary.yml, type: svc }
+    - { name: dns-autoscaler, src: dns-autoscaler.yml, file: coredns-autoscaler-secondary.yml, type: deployment }
+  register: coredns_secondary_manifests
+  vars:
+    clusterIP: "{{ skydns_server_secondary }}"
+    coredns_ordinal_suffix: "-secondary"
+  when:
+    - dns_mode == 'coredns_dual'
+    - inventory_hostname == groups['kube_control_plane'][0]
+  tags:
+    - coredns

kubespray/extra_playbooks/roles/kubernetes-apps/ansible/tasks/dashboard.yml

@@ -0,0 +1,21 @@
+---
+- name: Kubernetes Apps | Lay down dashboard template
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: 0644
+  with_items:
+    - { file: dashboard.yml, type: deploy, name: kubernetes-dashboard }
+  register: manifests
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Kubernetes Apps | Start dashboard
+  kube:
+    name: "{{ item.item.name }}"
+    namespace: "{{ dashboard_namespace }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items: "{{ manifests.results }}"
+  when: inventory_hostname == groups['kube_control_plane'][0]

kubespray/extra_playbooks/roles/kubernetes-apps/ansible/tasks/etcd_metrics.yml

@@ -0,0 +1,22 @@
+---
+- name: Kubernetes Apps | Lay down etcd_metrics templates
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: 0644
+  with_items:
+    - { file: etcd_metrics-endpoints.yml, type: endpoints, name: etcd-metrics }
+    - { file: etcd_metrics-service.yml, type: service, name: etcd-metrics }
+  register: manifests
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Kubernetes Apps | Start etcd_metrics
+  kube:
+    name: "{{ item.item.name }}"
+    namespace: kube-system
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items: "{{ manifests.results }}"
+  when: inventory_hostname == groups['kube_control_plane'][0]

kubespray/extra_playbooks/roles/kubernetes-apps/ansible/tasks/main.yml

@@ -0,0 +1,82 @@
+---
+- name: Kubernetes Apps | Wait for kube-apiserver
+  uri:
+    url: "{{ kube_apiserver_endpoint }}/healthz"
+    validate_certs: no
+    client_cert: "{{ kube_apiserver_client_cert }}"
+    client_key: "{{ kube_apiserver_client_key }}"
+  register: result
+  until: result.status == 200
+  retries: 20
+  delay: 1
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Kubernetes Apps | Cleanup DNS
+  import_tasks: cleanup_dns.yml
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+  tags:
+    - upgrade
+    - coredns
+    - nodelocaldns
+
+- name: Kubernetes Apps | CoreDNS
+  import_tasks: "coredns.yml"
+  when:
+    - dns_mode in ['coredns', 'coredns_dual']
+    - inventory_hostname == groups['kube_control_plane'][0]
+  tags:
+    - coredns
+
+- name: Kubernetes Apps | nodelocalDNS
+  import_tasks: "nodelocaldns.yml"
+  when:
+    - enable_nodelocaldns
+    - inventory_hostname == groups['kube_control_plane'] | first
+  tags:
+    - nodelocaldns
+
+- name: Kubernetes Apps | Start Resources
+  kube:
+    name: "{{ item.item.name }}"
+    namespace: "kube-system"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items:
+    - "{{ coredns_manifests.results | default({}) }}"
+    - "{{ coredns_secondary_manifests.results | default({}) }}"
+    - "{{ nodelocaldns_manifests.results | default({}) }}"
+    - "{{ nodelocaldns_second_manifests.results | default({}) }}"
+  when:
+    - dns_mode != 'none'
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - not item is skipped
+  register: resource_result
+  until: resource_result is succeeded
+  retries: 4
+  delay: 5
+  tags:
+    - coredns
+    - nodelocaldns
+  loop_control:
+    label: "{{ item.item.file }}"
+
+- name: Kubernetes Apps | Etcd metrics endpoints
+  import_tasks: etcd_metrics.yml
+  when: etcd_metrics_port is defined and etcd_metrics_service_labels is defined
+  tags:
+    - etcd_metrics
+
+- name: Kubernetes Apps | Netchecker
+  import_tasks: netchecker.yml
+  when: deploy_netchecker
+  tags:
+    - netchecker
+
+- name: Kubernetes Apps | Dashboard
+  import_tasks: dashboard.yml
+  when: dashboard_enabled
+  tags:
+    - dashboard

kubespray/extra_playbooks/roles/kubernetes-apps/ansible/tasks/netchecker.yml

@@ -0,0 +1,56 @@
+---
+- name: Kubernetes Apps | Check AppArmor status
+  command: which apparmor_parser
+  register: apparmor_status
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+  failed_when: false
+
+- name: Kubernetes Apps | Set apparmor_enabled
+  set_fact:
+    apparmor_enabled: "{{ apparmor_status.rc == 0 }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Kubernetes Apps | Netchecker Templates list
+  set_fact:
+    netchecker_templates:
+      - {file: netchecker-ns.yml, type: ns, name: netchecker-namespace}
+      - {file: netchecker-agent-sa.yml, type: sa, name: netchecker-agent}
+      - {file: netchecker-agent-ds.yml, type: ds, name: netchecker-agent}
+      - {file: netchecker-agent-hostnet-ds.yml, type: ds, name: netchecker-agent-hostnet}
+      - {file: netchecker-server-sa.yml, type: sa, name: netchecker-server}
+      - {file: netchecker-server-clusterrole.yml, type: clusterrole, name: netchecker-server}
+      - {file: netchecker-server-clusterrolebinding.yml, type: clusterrolebinding, name: netchecker-server}
+      - {file: netchecker-server-deployment.yml, type: deployment, name: netchecker-server}
+      - {file: netchecker-server-svc.yml, type: svc, name: netchecker-service}
+    netchecker_templates_for_psp:
+      - {file: netchecker-agent-hostnet-psp.yml, type: podsecuritypolicy, name: netchecker-agent-hostnet-policy}
+      - {file: netchecker-agent-hostnet-clusterrole.yml, type: clusterrole, name: netchecker-agent}
+      - {file: netchecker-agent-hostnet-clusterrolebinding.yml, type: clusterrolebinding, name: netchecker-agent}
+
+- name: Kubernetes Apps | Append extra templates to Netchecker Templates list for PodSecurityPolicy
+  set_fact:
+    netchecker_templates: "{{ netchecker_templates_for_psp + netchecker_templates }}"
+  when: podsecuritypolicy_enabled
+
+- name: Kubernetes Apps | Lay Down Netchecker Template
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: 0644
+  with_items: "{{ netchecker_templates }}"
+  register: manifests
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Kubernetes Apps | Start Netchecker Resources
+  kube:
+    name: "{{ item.item.name }}"
+    namespace: "{{ netcheck_namespace }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items: "{{ manifests.results }}"
+  when: inventory_hostname == groups['kube_control_plane'][0] and not item is skipped

kubespray/extra_playbooks/roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml

@@ -0,0 +1,75 @@
+---
+- name: Kubernetes Apps | set up necessary nodelocaldns parameters
+  set_fact:
+    primaryClusterIP: >-
+      {%- if dns_mode in ['coredns', 'coredns_dual'] -%}
+      {{ skydns_server }}
+      {%- elif dns_mode == 'manual' -%}
+      {{ manual_dns_server }}
+      {%- endif -%}
+    secondaryclusterIP: "{{ skydns_server_secondary }}"
+  when:
+    - enable_nodelocaldns
+    - inventory_hostname == groups['kube_control_plane'] | first
+  tags:
+    - nodelocaldns
+    - coredns
+
+- name: Kubernetes Apps | Lay Down nodelocaldns Template
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: 0644
+  with_items:
+    - { name: nodelocaldns, file: nodelocaldns-config.yml, type: configmap }
+    - { name: nodelocaldns, file: nodelocaldns-sa.yml, type: sa }
+    - { name: nodelocaldns, file: nodelocaldns-daemonset.yml, type: daemonset }
+  register: nodelocaldns_manifests
+  vars:
+    forwardTarget: >-
+      {%- if secondaryclusterIP is defined and dns_mode == 'coredns_dual' -%}
+      {{ primaryClusterIP }} {{ secondaryclusterIP }}
+      {%- else -%}
+      {{ primaryClusterIP }}
+      {%- endif -%}
+    upstreamForwardTarget: >-
+      {%- if upstream_dns_servers is defined and upstream_dns_servers|length > 0 -%}
+      {{ upstream_dns_servers|join(' ') }}
+      {%- else -%}
+      /etc/resolv.conf
+      {%- endif -%}
+  when:
+    - enable_nodelocaldns
+    - inventory_hostname == groups['kube_control_plane'] | first
+  tags:
+    - nodelocaldns
+    - coredns
+
+- name: Kubernetes Apps | Lay Down nodelocaldns-secondary Template
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: 0644
+  with_items:
+    - { name: nodelocaldns, file: nodelocaldns-second-daemonset.yml, type: daemonset }
+  register: nodelocaldns_second_manifests
+  vars:
+    forwardTarget: >-
+      {%- if secondaryclusterIP is defined and dns_mode == 'coredns_dual' -%}
+      {{ primaryClusterIP }} {{ secondaryclusterIP }}
+      {%- else -%}
+      {{ primaryClusterIP }}
+      {%- endif -%}
+    upstreamForwardTarget: >-
+      {%- if upstream_dns_servers is defined and upstream_dns_servers|length > 0 -%}
+      {{ upstream_dns_servers|join(' ') }}
+      {%- else -%}
+      /etc/resolv.conf
+      {%- endif -%}
+  when:
+    - enable_nodelocaldns
+    - enable_nodelocaldns_secondary
+    - inventory_hostname == groups['kube_control_plane'] | first
+  tags:
+    - nodelocaldns
+    - coredns