dsk-dev kubespray 이동

ansible/kubespray/roles/network_plugin/calico/tasks/calico_apiserver_certs.yml

@@ -0,0 +1,60 @@
+---
+- name: Calico | Check if calico apiserver exists
+  command: "{{ kubectl }} -n calico-apiserver get secret calico-apiserver-certs"
+  register: calico_apiserver_secret
+  changed_when: false
+  failed_when: false
+
+- name: Calico | Create ns manifests
+  template:
+    src: "calico-apiserver-ns.yml.j2"
+    dest: "{{ kube_config_dir }}/calico-apiserver-ns.yml"
+    mode: 0644
+
+- name: Calico | Apply ns manifests
+  kube:
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/calico-apiserver-ns.yml"
+    state: "latest"
+
+- name: Calico | Ensure calico certs dir
+  file:
+    path: /etc/calico/certs
+    state: directory
+    mode: 0755
+  when: calico_apiserver_secret.rc != 0
+
+- name: Calico | Copy ssl script for apiserver certs
+  template:
+    src: make-ssl-calico.sh.j2
+    dest: "{{ bin_dir }}/make-ssl-apiserver.sh"
+    mode: 0755
+  when: calico_apiserver_secret.rc != 0
+
+- name: Calico | Copy ssl config for apiserver certs
+  copy:
+    src: openssl.conf
+    dest: /etc/calico/certs/openssl.conf
+    mode: 0644
+  when: calico_apiserver_secret.rc != 0
+
+- name: Calico | Generate apiserver certs
+  command: >-
+    {{ bin_dir }}/make-ssl-apiserver.sh
+    -f /etc/calico/certs/openssl.conf
+    -c {{ kube_cert_dir }}
+    -d /etc/calico/certs
+    -s apiserver
+  when: calico_apiserver_secret.rc != 0
+
+- name: Calico | Create calico apiserver generic secrets
+  command: >-
+    {{ kubectl }} -n calico-apiserver
+    create secret generic {{ item.name }}
+    --from-file={{ item.cert }}
+    --from-file={{ item.key }}
+  with_items:
+    - name: calico-apiserver-certs
+      cert: /etc/calico/certs/apiserver.crt
+      key: /etc/calico/certs/apiserver.key
+  when: calico_apiserver_secret.rc != 0

ansible/kubespray/roles/network_plugin/calico/tasks/check.yml

@@ -0,0 +1,194 @@
+---
+- name: Stop if legacy encapsulation variables are detected (ipip)
+  assert:
+    that:
+      - ipip is not defined
+    msg: "'ipip' configuration variable is deprecated, please configure your inventory with 'calico_ipip_mode' set to 'Always' or 'CrossSubnet' according to your specific needs"
+  run_once: True
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: Stop if legacy encapsulation variables are detected (ipip_mode)
+  assert:
+    that:
+      - ipip_mode is not defined
+    msg: "'ipip_mode' configuration variable is deprecated, please configure your inventory with 'calico_ipip_mode' set to 'Always' or 'CrossSubnet' according to your specific needs"
+  run_once: True
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: Stop if legacy encapsulation variables are detected (calcio_ipam_autoallocateblocks)
+  assert:
+    that:
+      - calcio_ipam_autoallocateblocks is not defined
+    msg: "'calcio_ipam_autoallocateblocks' configuration variable is deprecated, it's a typo, please configure your inventory with 'calico_ipam_autoallocateblocks' set to 'true' or 'false' according to your specific needs"
+  run_once: True
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+
+- name: Stop if incompatible network plugin and cloudprovider
+  assert:
+    that:
+      - calico_ipip_mode == 'Never'
+      - calico_vxlan_mode in ['Always', 'CrossSubnet']
+    msg: "When using cloud_provider azure and network_plugin calico calico_ipip_mode must be 'Never' and calico_vxlan_mode 'Always' or 'CrossSubnet'"
+  when:
+    - cloud_provider is defined and cloud_provider == 'azure'
+  run_once: True
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: Stop if supported Calico versions
+  assert:
+    that:
+      - "calico_version in calico_crds_archive_checksums.keys()"
+    msg: "Calico version not supported {{ calico_version }} not in {{ calico_crds_archive_checksums.keys() }}"
+  run_once: True
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: Get current calico cluster version
+  shell: "set -o pipefail && {{ bin_dir }}/calicoctl.sh version  | grep 'Cluster Version:' | awk '{ print $3}'"
+  args:
+    executable: /bin/bash
+  register: calico_version_on_server
+  async: 10
+  poll: 3
+  run_once: True
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  changed_when: false
+  failed_when: false
+
+- name: Check that current calico version is enough for upgrade
+  assert:
+    that:
+      - calico_version_on_server.stdout is version(calico_min_version_required, '>=')
+    msg: >
+      Your version of calico is not fresh enough for upgrade.
+      Minimum version is {{ calico_min_version_required }} supported by the previous kubespray release.
+  when:
+    - 'calico_version_on_server.stdout is defined'
+    - calico_version_on_server.stdout
+    - inventory_hostname == groups['kube_control_plane'][0]
+  run_once: True
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Check that cluster_id is set if calico_rr enabled"
+  assert:
+    that:
+      - cluster_id is defined
+    msg: "A unique cluster_id is required if using calico_rr"
+  when:
+    - peer_with_calico_rr
+    - inventory_hostname == groups['kube_control_plane'][0]
+  run_once: True
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Check that calico_rr nodes are in k8s_cluster group"
+  assert:
+    that:
+      - '"k8s_cluster" in group_names'
+    msg: "calico_rr must be a child group of k8s_cluster group"
+  when:
+    - '"calico_rr" in group_names'
+  run_once: True
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Check vars defined correctly"
+  assert:
+    that:
+      - "calico_pool_name is defined"
+      - "calico_pool_name is match('^[a-zA-Z0-9-_\\\\.]{2,63}$')"
+    msg: "calico_pool_name contains invalid characters"
+  run_once: True
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Check calico network backend defined correctly"
+  assert:
+    that:
+      - "calico_network_backend in ['bird', 'vxlan', 'none']"
+    msg: "calico network backend is not 'bird', 'vxlan' or 'none'"
+  run_once: True
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Check ipip and vxlan mode defined correctly"
+  assert:
+    that:
+      - "calico_ipip_mode in ['Always', 'CrossSubnet', 'Never']"
+      - "calico_vxlan_mode in ['Always', 'CrossSubnet', 'Never']"
+    msg: "calico inter host encapsulation mode is not 'Always', 'CrossSubnet' or 'Never'"
+  run_once: True
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Check ipip and vxlan mode if simultaneously enabled"
+  assert:
+    that:
+      - "calico_vxlan_mode in ['Never']"
+    msg: "IP in IP and VXLAN mode is mutualy exclusive modes"
+  when:
+    - "calico_ipip_mode in ['Always', 'CrossSubnet']"
+  run_once: True
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Check ipip and vxlan mode if simultaneously enabled"
+  assert:
+    that:
+      - "calico_ipip_mode in ['Never']"
+    msg: "IP in IP and VXLAN mode is mutualy exclusive modes"
+  when:
+    - "calico_vxlan_mode in ['Always', 'CrossSubnet']"
+  run_once: True
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Get Calico {{ calico_pool_name }} configuration"
+  command: "{{ bin_dir }}/calicoctl.sh get ipPool {{ calico_pool_name }} -o json"
+  failed_when: False
+  changed_when: False
+  check_mode: no
+  register: calico
+  run_once: True
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Set calico_pool_conf"
+  set_fact:
+    calico_pool_conf: '{{ calico.stdout | from_json }}'
+  when: calico.rc == 0 and calico.stdout
+  run_once: True
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Check if inventory match current cluster configuration"
+  assert:
+    that:
+      - calico_pool_conf.spec.blockSize|int == (calico_pool_blocksize | default(kube_network_node_prefix) | int)
+      - calico_pool_conf.spec.cidr == (calico_pool_cidr | default(kube_pods_subnet))
+      - not calico_pool_conf.spec.ipipMode is defined or calico_pool_conf.spec.ipipMode == calico_ipip_mode
+      - not calico_pool_conf.spec.vxlanMode is defined or calico_pool_conf.spec.vxlanMode == calico_vxlan_mode
+    msg: "Your inventory doesn't match the current cluster configuration"
+  when:
+    - calico_pool_conf is defined
+  run_once: True
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Check kdd calico_datastore if calico_apiserver_enabled"
+  assert:
+    that: calico_datastore == "kdd"
+    msg: "When using calico apiserver you need to use the kubernetes datastore"
+  when:
+    - calico_apiserver_enabled
+  run_once: True
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Check kdd calico_datastore if typha_enabled"
+  assert:
+    that: calico_datastore == "kdd"
+    msg: "When using typha you need to use the kubernetes datastore"
+  when:
+    - typha_enabled
+  run_once: True
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
+- name: "Check ipip mode is Never for calico ipv6"
+  assert:
+    that:
+      - "calico_ipip_mode_ipv6 in ['Never']"
+    msg: "Calico doesn't support ipip tunneling for the IPv6"
+  when:
+    - enable_dual_stack_networks
+  run_once: True
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"

ansible/kubespray/roles/network_plugin/calico/tasks/install.yml

@@ -0,0 +1,475 @@
+---
+- name: Calico | Install Wireguard packages
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items: "{{ calico_wireguard_packages }}"
+  register: calico_package_install
+  until: calico_package_install is succeeded
+  retries: 4
+  when: calico_wireguard_enabled
+
+- name: Calico | Copy calicoctl binary from download dir
+  copy:
+    src: "{{ local_release_dir }}/calicoctl"
+    dest: "{{ bin_dir }}/calicoctl"
+    mode: 0755
+    remote_src: yes
+
+- name: Calico | Write Calico cni config
+  template:
+    src: "cni-calico.conflist.j2"
+    dest: "/etc/cni/net.d/calico.conflist.template"
+    mode: 0644
+    owner: root
+  register: calico_conflist
+  notify: reset_calico_cni
+
+- name: Calico | Create calico certs directory
+  file:
+    dest: "{{ calico_cert_dir }}"
+    state: directory
+    mode: 0750
+    owner: root
+    group: root
+  when: calico_datastore == "etcd"
+
+- name: Calico | Link etcd certificates for calico-node
+  file:
+    src: "{{ etcd_cert_dir }}/{{ item.s }}"
+    dest: "{{ calico_cert_dir }}/{{ item.d }}"
+    state: hard
+    mode: 0640
+    force: yes
+  with_items:
+    - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"}
+    - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"}
+    - {s: "{{ kube_etcd_key_file }}", d: "key.pem"}
+  when: calico_datastore == "etcd"
+
+- name: Calico | Generate typha certs
+  include_tasks: typha_certs.yml
+  when:
+    - typha_secure
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Calico | Generate apiserver certs
+  include_tasks: calico_apiserver_certs.yml
+  when:
+    - calico_apiserver_enabled
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Calico | Install calicoctl wrapper script
+  template:
+    src: "calicoctl.{{ calico_datastore }}.sh.j2"
+    dest: "{{ bin_dir }}/calicoctl.sh"
+    mode: 0755
+    owner: root
+    group: root
+
+- name: Calico | wait for etcd
+  uri:
+    url: "{{ etcd_access_addresses.split(',') | first }}/health"
+    validate_certs: no
+    client_cert: "{{ calico_cert_dir }}/cert.crt"
+    client_key: "{{ calico_cert_dir }}/key.pem"
+  register: result
+  until: result.status == 200 or result.status == 401
+  retries: 10
+  delay: 5
+  run_once: true
+  when: calico_datastore == "etcd"
+
+- name: Calico | Check if calico network pool has already been configured
+  # noqa 306 - grep will exit 1 if no match found
+  shell: >
+    {{ bin_dir }}/calicoctl.sh get ippool | grep -w "{{ calico_pool_cidr | default(kube_pods_subnet) }}" | wc -l
+  args:
+    executable: /bin/bash
+  register: calico_conf
+  retries: 4
+  until: calico_conf.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  changed_when: false
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Calico | Ensure that calico_pool_cidr is within kube_pods_subnet when defined
+  assert:
+    that: "[calico_pool_cidr] | ipaddr(kube_pods_subnet) | length == 1"
+    msg: "{{ calico_pool_cidr }} is not within or equal to {{ kube_pods_subnet }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - 'calico_conf.stdout == "0"'
+    - calico_pool_cidr is defined
+
+- name: Calico | Check if calico IPv6 network pool has already been configured
+  # noqa 306 - grep will exit 1 if no match found
+  shell: >
+    {{ bin_dir }}/calicoctl.sh get ippool | grep -w "{{ calico_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6) }}" | wc -l
+  args:
+    executable: /bin/bash
+  register: calico_conf_ipv6
+  retries: 4
+  until: calico_conf_ipv6.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  changed_when: false
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - enable_dual_stack_networks
+
+- name: Calico | Ensure that calico_pool_cidr_ipv6 is within kube_pods_subnet_ipv6 when defined
+  assert:
+    that: "[calico_pool_cidr_ipv6] | ipaddr(kube_pods_subnet_ipv6) | length == 1"
+    msg: "{{ calico_pool_cidr_ipv6 }} is not within or equal to {{ kube_pods_subnet_ipv6 }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - calico_conf_ipv6.stdout is defined and calico_conf_ipv6.stdout == "0"
+    - calico_pool_cidr_ipv6 is defined
+    - enable_dual_stack_networks
+
+- block:
+    - name: Calico | Check if extra directory is needed
+      stat:
+        path: "{{ local_release_dir }}/calico-{{ calico_version }}-kdd-crds/{{ 'kdd' if (calico_version is version('v3.22.3','<')) else 'crd' }}"
+      register: kdd_path
+    - name: Calico | Set kdd path when calico < v3.22.3
+      set_fact:
+        calico_kdd_path: "{{ local_release_dir }}/calico-{{ calico_version }}-kdd-crds{{ '/kdd' if kdd_path.stat.exists is defined and kdd_path.stat.exists }}"
+      when:
+        - calico_version is version('v3.22.3', '<')
+    - name: Calico | Set kdd path when calico > v3.22.2
+      set_fact:
+        calico_kdd_path: "{{ local_release_dir }}/calico-{{ calico_version }}-kdd-crds{{ '/crd' if kdd_path.stat.exists is defined and kdd_path.stat.exists }}"
+      when:
+        - calico_version is version('v3.22.2', '>')
+    - name: Calico | Create calico manifests for kdd
+      assemble:
+        src: "{{ calico_kdd_path }}"
+        dest: "{{ kube_config_dir }}/kdd-crds.yml"
+        mode: 0644
+        delimiter: "---\n"
+        regexp: ".*\\.yaml"
+        remote_src: true
+
+    - name: Calico | Create Calico Kubernetes datastore resources
+      kube:
+        kubectl: "{{ bin_dir }}/kubectl"
+        filename: "{{ kube_config_dir }}/kdd-crds.yml"
+        state: "latest"
+      when:
+        - inventory_hostname == groups['kube_control_plane'][0]
+  when:
+    - inventory_hostname in groups['kube_control_plane']
+    - calico_datastore == "kdd"
+
+- block:
+    - name: Calico | Get existing FelixConfiguration
+      command: "{{ bin_dir }}/calicoctl.sh get felixconfig default -o json"
+      register: _felix_cmd
+      ignore_errors: True
+      changed_when: False
+
+    - name: Calico | Set kubespray FelixConfiguration
+      set_fact:
+        _felix_config: >
+          {
+            "kind": "FelixConfiguration",
+            "apiVersion": "projectcalico.org/v3",
+            "metadata": {
+              "name": "default",
+            },
+            "spec": {
+              "ipipEnabled": {{ calico_ipip_mode != 'Never' }},
+              "reportingInterval": "{{ calico_felix_reporting_interval }}",
+              "bpfLogLevel": "{{ calico_bpf_log_level }}",
+              "bpfEnabled": {{ calico_bpf_enabled | bool }},
+              "bpfExternalServiceMode": "{{ calico_bpf_service_mode }}",
+              "wireguardEnabled": {{ calico_wireguard_enabled | bool }},
+              "logSeverityScreen": "{{ calico_felix_log_severity_screen }}",
+              "vxlanEnabled": {{ calico_vxlan_mode != 'Never' }},
+              "featureDetectOverride": "{{ calico_feature_detect_override }}"
+            }
+          }
+
+    - name: Calico | Process FelixConfiguration
+      set_fact:
+        _felix_config: "{{ _felix_cmd.stdout | from_json | combine(_felix_config, recursive=True) }}"
+      when:
+        - _felix_cmd is success
+
+    - name: Calico | Configure calico FelixConfiguration
+      command:
+        cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+        stdin: "{{ _felix_config is string | ternary(_felix_config, _felix_config|to_json) }}"
+      changed_when: False
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- block:
+    - name: Calico | Get existing calico network pool
+      command: "{{ bin_dir }}/calicoctl.sh get ippool {{ calico_pool_name }} -o json"
+      register: _calico_pool_cmd
+      ignore_errors: True
+      changed_when: False
+
+    - name: Calico | Set kubespray calico network pool
+      set_fact:
+        _calico_pool: >
+          {
+            "kind": "IPPool",
+            "apiVersion": "projectcalico.org/v3",
+            "metadata": {
+              "name": "{{ calico_pool_name }}",
+            },
+            "spec": {
+              "blockSize": {{ calico_pool_blocksize | default(kube_network_node_prefix) }},
+              "cidr": "{{ calico_pool_cidr | default(kube_pods_subnet) }}",
+              "ipipMode": "{{ calico_ipip_mode }}",
+              "vxlanMode": "{{ calico_vxlan_mode }}",
+              "natOutgoing": {{ nat_outgoing|default(false) }}
+            }
+          }
+
+    - name: Calico | Process calico network pool
+      set_fact:
+        _calico_pool: "{{ _calico_pool_cmd.stdout | from_json | combine(_calico_pool, recursive=True) }}"
+      when:
+        - _calico_pool_cmd is success
+
+    - name: Calico | Configure calico network pool
+      command:
+        cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+        stdin: "{{ _calico_pool is string | ternary(_calico_pool, _calico_pool|to_json) }}"
+      changed_when: False
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- block:
+    - name: Calico | Get existing calico ipv6 network pool
+      command: "{{ bin_dir }}/calicoctl.sh get ippool {{ calico_pool_name }}-ipv6 -o json"
+      register: _calico_pool_ipv6_cmd
+      ignore_errors: True
+      changed_when: False
+
+    - name: Calico | Set kubespray calico network pool
+      set_fact:
+        _calico_pool_ipv6: >
+          {
+            "kind": "IPPool",
+            "apiVersion": "projectcalico.org/v3",
+            "metadata": {
+              "name": "{{ calico_pool_name }}-ipv6",
+            },
+            "spec": {
+              "blockSize": {{ calico_pool_blocksize_ipv6 | default(kube_network_node_prefix_ipv6) }},
+              "cidr": "{{ calico_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6) }}",
+              "ipipMode": "{{ calico_ipip_mode_ipv6 }}",
+              "vxlanMode": "{{ calico_vxlan_mode_ipv6 }}",
+              "natOutgoing": {{ nat_outgoing_ipv6|default(false) }}
+            }
+          }
+
+    - name: Calico | Process calico ipv6 network pool
+      set_fact:
+        _calico_pool_ipv6: "{{ _calico_pool_ipv6_cmd.stdout | from_json | combine(_calico_pool_ipv6, recursive=True) }}"
+      when:
+        - _calico_pool_ipv6_cmd is success
+
+    - name: Calico | Configure calico ipv6 network pool
+      command:
+        cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+        stdin: "{{ _calico_pool_ipv6 is string | ternary(_calico_pool_ipv6, _calico_pool_ipv6|to_json) }}"
+      changed_when: False
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - enable_dual_stack_networks | bool
+
+- name: Populate Service External IPs
+  set_fact:
+    _service_external_ips: "{{ _service_external_ips|default([]) + [ {'cidr': item} ] }}"
+  with_items: "{{ calico_advertise_service_external_ips }}"
+  run_once: yes
+
+- name: Populate Service LoadBalancer IPs
+  set_fact:
+    _service_loadbalancer_ips: "{{ _service_loadbalancer_ips|default([]) + [ {'cidr': item} ] }}"
+  with_items: "{{ calico_advertise_service_loadbalancer_ips }}"
+  run_once: yes
+
+- name: "Determine nodeToNodeMesh needed state"
+  set_fact:
+    nodeToNodeMeshEnabled: "false"
+  when:
+    - peer_with_router|default(false) or peer_with_calico_rr|default(false)
+    - inventory_hostname in groups['k8s_cluster']
+  run_once: yes
+
+- block:
+    - name: Calico | Get existing BGP Configuration
+      command: "{{ bin_dir }}/calicoctl.sh get bgpconfig default -o json"
+      register: _bgp_config_cmd
+      ignore_errors: True
+      changed_when: False
+
+    - name: Calico | Set kubespray BGP Configuration
+      set_fact:
+        _bgp_config: >
+          {
+            "kind": "BGPConfiguration",
+            "apiVersion": "projectcalico.org/v3",
+            "metadata": {
+              "name": "default",
+            },
+            "spec": {
+              "listenPort": {{ calico_bgp_listen_port }},
+              "logSeverityScreen": "Info",
+              {% if not calico_no_global_as_num|default(false) %}"asNumber": {{ global_as_num }},{% endif %}
+              "nodeToNodeMeshEnabled": {{ nodeToNodeMeshEnabled|default('true') }} ,
+              {% if calico_advertise_cluster_ips|default(false) %}
+              "serviceClusterIPs": [{"cidr": "{{ kube_service_addresses }}" } {{ ',{"cidr":"' + kube_service_addresses_ipv6 + '"}' if enable_dual_stack_networks else '' }}],{% endif %}
+              {% if calico_advertise_service_loadbalancer_ips|length > 0  %}"serviceLoadBalancerIPs": {{ _service_loadbalancer_ips }},{% endif %}
+              "serviceExternalIPs": {{ _service_external_ips|default([]) }}
+            }
+          }
+
+    - name: Calico | Process BGP Configuration
+      set_fact:
+        _bgp_config: "{{ _bgp_config_cmd.stdout | from_json | combine(_bgp_config, recursive=True) }}"
+      when:
+        - _bgp_config_cmd is success
+
+    - name: Calico | Set up BGP Configuration
+      command:
+        cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+        stdin: "{{ _bgp_config is string | ternary(_bgp_config, _bgp_config|to_json) }}"
+      changed_when: False
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Calico | Create calico manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: 0644
+  with_items:
+    - {name: calico-config, file: calico-config.yml, type: cm}
+    - {name: calico-node, file: calico-node.yml, type: ds}
+    - {name: calico, file: calico-node-sa.yml, type: sa}
+    - {name: calico, file: calico-cr.yml, type: clusterrole}
+    - {name: calico, file: calico-crb.yml, type: clusterrolebinding}
+    - {name: kubernetes-services-endpoint, file: kubernetes-services-endpoint.yml, type: cm }
+  register: calico_node_manifests
+  when:
+    - inventory_hostname in groups['kube_control_plane']
+    - rbac_enabled or item.type not in rbac_resources
+
+- name: Calico | Create calico manifests for typha
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: 0644
+  with_items:
+    - {name: calico, file: calico-typha.yml, type: typha}
+  register: calico_node_typha_manifest
+  when:
+    - inventory_hostname in groups['kube_control_plane']
+    - typha_enabled
+
+- name: Calico | get calico apiserver caBundle
+  command: "{{ bin_dir }}/kubectl get secret -n calico-apiserver calico-apiserver-certs -o jsonpath='{.data.apiserver\\.crt}'"
+  changed_when: false
+  register: calico_apiserver_cabundle
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - calico_apiserver_enabled
+
+- name: Calico | set calico apiserver caBundle fact
+  set_fact:
+    calico_apiserver_cabundle: "{{ calico_apiserver_cabundle.stdout }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - calico_apiserver_enabled
+
+- name: Calico | Create calico manifests for apiserver
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: 0644
+  with_items:
+    - {name: calico, file: calico-apiserver.yml, type: calico-apiserver}
+  register: calico_apiserver_manifest
+  when:
+    - inventory_hostname in groups['kube_control_plane']
+    - calico_apiserver_enabled
+
+- name: Start Calico resources
+  kube:
+    name: "{{ item.item.name }}"
+    namespace: "kube-system"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items:
+    - "{{ calico_node_manifests.results }}"
+    - "{{ calico_node_typha_manifest.results }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - not item is skipped
+  loop_control:
+    label: "{{ item.item.file }}"
+
+- name: Start Calico apiserver resources
+  kube:
+    name: "{{ item.item.name }}"
+    namespace: "calico-apiserver"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items:
+    - "{{ calico_apiserver_manifest.results }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - not item is skipped
+  loop_control:
+    label: "{{ item.item.file }}"
+
+- name: Wait for calico kubeconfig to be created
+  wait_for:
+    path: /etc/cni/net.d/calico-kubeconfig
+  when:
+    - inventory_hostname not in groups['kube_control_plane']
+    - calico_datastore == "kdd"
+
+- name: Calico | Create Calico ipam manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: 0644
+  with_items:
+    - {name: calico, file: calico-ipamconfig.yml, type: ipam}
+  when:
+    - inventory_hostname in groups['kube_control_plane']
+    - calico_datastore == "kdd"
+
+- name: Calico | Create ipamconfig resources
+  kube:
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/calico-ipamconfig.yml"
+    state: "latest"
+  register: resource_result
+  until: resource_result is succeeded
+  retries: 4
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - calico_datastore == "kdd"
+
+- include_tasks: peer_with_calico_rr.yml
+  when:
+    - peer_with_calico_rr|default(false)
+
+- include_tasks: peer_with_router.yml
+  when:
+    - peer_with_router|default(false)

ansible/kubespray/roles/network_plugin/calico/tasks/main.yml

@@ -0,0 +1,6 @@
+---
+- import_tasks: pre.yml
+
+- import_tasks: repos.yml
+
+- include_tasks: install.yml

ansible/kubespray/roles/network_plugin/calico/tasks/peer_with_calico_rr.yml

@@ -0,0 +1,86 @@
+---
+- name: Calico | Set lable for groups nodes  # noqa 301 305
+  shell: "{{ bin_dir }}/calicoctl.sh label node  {{ inventory_hostname }} calico-group-id={{ calico_group_id }} --overwrite"
+  changed_when: false
+  register: calico_group_id_label
+  until: calico_group_id_label is succeeded
+  delay: "{{ retry_stagger | random + 3 }}"
+  retries: 10
+  when:
+    - calico_group_id is defined
+
+- name: Calico | Configure peering with route reflectors at global scope
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+    # revert when it's already a string
+    stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}"
+  vars:
+    stdin: >
+      {"apiVersion": "projectcalico.org/v3",
+      "kind": "BGPPeer",
+      "metadata": {
+        "name": "{{ calico_rr_id }}-to-node"
+      },
+      "spec": {
+        "peerSelector": "calico-rr-id == '{{ calico_rr_id }}'",
+        "nodeSelector": "calico-group-id == '{{ calico_group_id }}'"
+      }}
+  register: output
+  retries: 4
+  until: output.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  when:
+    - calico_rr_id is defined
+    - calico_group_id is defined
+    - inventory_hostname in groups['calico_rr']
+
+- name: Calico | Configure peering with route reflectors at global scope
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+    # revert when it's already a string
+    stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}"
+  vars:
+    stdin: >
+      {"apiVersion": "projectcalico.org/v3",
+      "kind": "BGPPeer",
+      "metadata": {
+        "name": "peer-to-rrs"
+      },
+      "spec": {
+        "nodeSelector": "!has(i-am-a-route-reflector)",
+        "peerSelector": "has(i-am-a-route-reflector)"
+      }}
+  register: output
+  retries: 4
+  until: output.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  with_items:
+    - "{{ groups['calico_rr'] | default([]) }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - calico_rr_id is not defined or calico_group_id is not defined
+
+- name: Calico | Configure route reflectors to peer with each other
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+    # revert when it's already a string
+    stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}"
+  vars:
+    stdin: >
+      {"apiVersion": "projectcalico.org/v3",
+      "kind": "BGPPeer",
+      "metadata": {
+        "name": "rr-mesh"
+      },
+      "spec": {
+        "nodeSelector": "has(i-am-a-route-reflector)",
+        "peerSelector": "has(i-am-a-route-reflector)"
+      }}
+  register: output
+  retries: 4
+  until: output.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  with_items:
+    - "{{ groups['calico_rr'] | default([]) }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]

ansible/kubespray/roles/network_plugin/calico/tasks/peer_with_router.yml

@@ -0,0 +1,77 @@
+---
+- name: Calico | Configure peering with router(s) at global scope
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+    stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}"
+  vars:
+    stdin: >
+      {"apiVersion": "projectcalico.org/v3",
+      "kind": "BGPPeer",
+      "metadata": {
+        "name": "global-{{ item.name | default(item.router_id|replace(':','-')) }}"
+      },
+      "spec": {
+        "asNumber": "{{ item.as }}",
+        "peerIP": "{{ item.router_id }}"
+      }}
+  register: output
+  retries: 4
+  until: output.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  with_items:
+    - "{{ peers|selectattr('scope','defined')|selectattr('scope','equalto', 'global')|list|default([]) }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Calico | Configure node asNumber for per node peering
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+    stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}"
+  vars:
+    stdin: >
+      {"apiVersion": "projectcalico.org/v3",
+      "kind": "Node",
+      "metadata": {
+        "name": "{{ inventory_hostname }}"
+      },
+      "spec": {
+        "bgp": {
+          "asNumber": "{{ local_as }}"
+        },
+        "orchRefs":[{"nodeName":"{{ inventory_hostname }}","orchestrator":"k8s"}]
+      }}
+  register: output
+  retries: 4
+  until: output.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  when:
+    - inventory_hostname in groups['k8s_cluster']
+    - local_as is defined
+    - groups['calico_rr'] | default([]) | length == 0
+
+- name: Calico | Configure peering with router(s) at node scope
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+    stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}"
+  vars:
+    stdin: >
+      {"apiVersion": "projectcalico.org/v3",
+      "kind": "BGPPeer",
+      "metadata": {
+        "name": "{{ inventory_hostname }}-{{ item.name | default(item.router_id|replace(':','-')) }}"
+      },
+      "spec": {
+        "asNumber": "{{ item.as }}",
+        "node": "{{ inventory_hostname }}",
+        "peerIP": "{{ item.router_id }}",
+        "sourceAddress": "{{ item.sourceaddress|default('UseNodeIP') }}"
+      }}
+  register: output
+  retries: 4
+  until: output.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  with_items:
+    - "{{ peers|selectattr('scope','undefined')|list|default([]) | union(peers|selectattr('scope','defined')|selectattr('scope','equalto', 'node')|list|default([])) }}"
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  when:
+    - inventory_hostname in groups['k8s_cluster']

ansible/kubespray/roles/network_plugin/calico/tasks/pre.yml

@@ -0,0 +1,46 @@
+---
+- name: Slurp CNI config
+  slurp:
+    src: /etc/cni/net.d/10-calico.conflist
+  register: calico_cni_config_slurp
+  failed_when: false
+
+- block:
+  - name: Set fact calico_cni_config from slurped CNI config
+    set_fact:
+      calico_cni_config: "{{ calico_cni_config_slurp['content'] | b64decode | from_json }}"
+  - name: Set fact calico_datastore to etcd if needed
+    set_fact:
+      calico_datastore: etcd
+    when:
+    - "'plugins' in calico_cni_config"
+    - "'etcd_endpoints' in calico_cni_config.plugins.0"
+  when: calico_cni_config_slurp.content is defined
+
+- name: Calico | Get kubelet hostname
+  shell: >-
+    set -o pipefail && {{ kubectl }} get node -o custom-columns='NAME:.metadata.name,INTERNAL-IP:.status.addresses[?(@.type=="InternalIP")].address'
+    | egrep "{{ ansible_all_ipv4_addresses | join('$|') }}$" | cut -d" " -f1
+  args:
+    executable: /bin/bash
+  register: calico_kubelet_name
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  when:
+  - "cloud_provider is defined"
+
+- name: Calico | Gather os specific variables
+  include_vars: "{{ item }}"
+  with_first_found:
+  - files:
+    - "{{ ansible_distribution|lower }}-{{ ansible_distribution_version|lower|replace('/', '_') }}.yml"
+    - "{{ ansible_distribution|lower }}-{{ ansible_distribution_release }}.yml"
+    - "{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version|lower|replace('/', '_') }}.yml"
+    - "{{ ansible_distribution|lower }}.yml"
+    - "{{ ansible_os_family|lower }}-{{ ansible_architecture }}.yml"
+    - "{{ ansible_os_family|lower }}.yml"
+    - defaults.yml
+    paths:
+    - ../vars
+    skip: true
+  tags:
+  - facts

ansible/kubespray/roles/network_plugin/calico/tasks/repos.yml

@@ -0,0 +1,21 @@
+---
+- name: Calico | Add wireguard yum repo
+  when:
+    - calico_wireguard_enabled
+  block:
+
+    - name: Calico | Add wireguard yum repo
+      yum_repository:
+        name: copr:copr.fedorainfracloud.org:jdoss:wireguard
+        file: _copr:copr.fedorainfracloud.org:jdoss:wireguard
+        description: Copr repo for wireguard owned by jdoss
+        baseurl: "{{ calico_wireguard_repo }}"
+        gpgcheck: yes
+        gpgkey: https://download.copr.fedorainfracloud.org/results/jdoss/wireguard/pubkey.gpg
+        skip_if_unavailable: yes
+        enabled: yes
+        repo_gpgcheck: no
+      when:
+        - ansible_os_family in ['RedHat']
+        - ansible_distribution not in ['Fedora']
+        - ansible_facts['distribution_major_version'] | int < 9

ansible/kubespray/roles/network_plugin/calico/tasks/reset.yml

@@ -0,0 +1,30 @@
+---
+- name: reset | check vxlan.calico network device
+  stat:
+    path: /sys/class/net/vxlan.calico
+    get_attributes: no
+    get_checksum: no
+    get_mime: no
+  register: vxlan
+
+- name: reset | remove the network vxlan.calico device created by calico
+  command: ip link del vxlan.calico
+  when: vxlan.stat.exists
+
+- name: reset | check dummy0 network device
+  stat:
+    path: /sys/class/net/dummy0
+    get_attributes: no
+    get_checksum: no
+    get_mime: no
+  register: dummy0
+
+- name: reset | remove the network device created by calico
+  command: ip link del dummy0
+  when: dummy0.stat.exists
+
+- name: reset | get and remove remaining routes set by bird
+  shell: set -o pipefail && ip route show proto bird | xargs -i bash -c "ip route del {} proto bird "
+  args:
+    executable: /bin/bash
+  changed_when: false

ansible/kubespray/roles/network_plugin/calico/tasks/typha_certs.yml

@@ -0,0 +1,51 @@
+---
+- name: Calico | Check if typha-server exists
+  command: "{{ kubectl }} -n kube-system get secret typha-server"
+  register: typha_server_secret
+  changed_when: false
+  failed_when: false
+
+- name: Calico | Ensure calico certs dir
+  file:
+    path: /etc/calico/certs
+    state: directory
+    mode: 0755
+  when: typha_server_secret.rc != 0
+
+- name: Calico | Copy ssl script for typha certs
+  template:
+    src: make-ssl-calico.sh.j2
+    dest: "{{ bin_dir }}/make-ssl-typha.sh"
+    mode: 0755
+  when: typha_server_secret.rc != 0
+
+- name: Calico | Copy ssl config for typha certs
+  copy:
+    src: openssl.conf
+    dest: /etc/calico/certs/openssl.conf
+    mode: 0644
+  when: typha_server_secret.rc != 0
+
+- name: Calico | Generate typha certs
+  command: >-
+    {{ bin_dir }}/make-ssl-typha.sh
+    -f /etc/calico/certs/openssl.conf
+    -c {{ kube_cert_dir }}
+    -d /etc/calico/certs
+    -s typha
+  when: typha_server_secret.rc != 0
+
+- name: Calico | Create typha tls secrets
+  command: >-
+    {{ kubectl }} -n kube-system
+    create secret tls {{ item.name }}
+    --cert {{ item.cert }}
+    --key {{ item.key }}
+  with_items:
+    - name: typha-server
+      cert: /etc/calico/certs/typha-server.crt
+      key: /etc/calico/certs/typha-server.key
+    - name: typha-client
+      cert: /etc/calico/certs/typha-client.crt
+      key: /etc/calico/certs/typha-client.key
+  when: typha_server_secret.rc != 0