dsk-dev kubespray 이동

ansible/kubespray/roles/kubernetes/kubeadm/defaults/main.yml

@@ -0,0 +1,12 @@
+---
+# discovery_timeout modifies the discovery timeout
+# This value must be smaller than kubeadm_join_timeout
+discovery_timeout: 60s
+kubeadm_join_timeout: 120s
+
+# If non-empty, will use this string as identification instead of the actual hostname
+kube_override_hostname: >-
+  {%- if cloud_provider is defined and cloud_provider in [ 'aws' ] -%}
+  {%- else -%}
+  {{ inventory_hostname }}
+  {%- endif -%}

ansible/kubespray/roles/kubernetes/kubeadm/handlers/main.yml

@@ -0,0 +1,15 @@
+---
+- name: Kubeadm | restart kubelet
+  command: /bin/true
+  notify:
+    - Kubeadm | reload systemd
+    - Kubeadm | reload kubelet
+
+- name: Kubeadm | reload systemd
+  systemd:
+    daemon_reload: true
+
+- name: Kubeadm | reload kubelet
+  service:
+    name: kubelet
+    state: restarted

ansible/kubespray/roles/kubernetes/kubeadm/tasks/kubeadm_etcd_node.yml

@@ -0,0 +1,61 @@
+---
+- name: Parse certificate key if not set
+  set_fact:
+    kubeadm_certificate_key: "{{ hostvars[groups['kube_control_plane'][0]]['kubeadm_certificate_key'] }}"
+  when: kubeadm_certificate_key is undefined
+
+- name: Pull control plane certs down
+  shell: >-
+    {{ bin_dir }}/kubeadm join phase
+    control-plane-prepare download-certs
+    --certificate-key {{ kubeadm_certificate_key }}
+    --control-plane
+    --token {{ kubeadm_token }}
+    --discovery-token-unsafe-skip-ca-verification
+    {{ kubeadm_discovery_address }}
+    &&
+    {{ bin_dir }}/kubeadm join phase
+    control-plane-prepare certs
+    --control-plane
+    --token {{ kubeadm_token }}
+    --discovery-token-unsafe-skip-ca-verification
+    {{ kubeadm_discovery_address }}
+  args:
+    creates: "{{ kube_cert_dir }}/apiserver-etcd-client.key"
+
+- name: Delete unneeded certificates
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+    - "{{ kube_cert_dir }}/apiserver.crt"
+    - "{{ kube_cert_dir }}/apiserver.key"
+    - "{{ kube_cert_dir }}/ca.key"
+    - "{{ kube_cert_dir }}/etcd/ca.key"
+    - "{{ kube_cert_dir }}/etcd/healthcheck-client.crt"
+    - "{{ kube_cert_dir }}/etcd/healthcheck-client.key"
+    - "{{ kube_cert_dir }}/etcd/peer.crt"
+    - "{{ kube_cert_dir }}/etcd/peer.key"
+    - "{{ kube_cert_dir }}/etcd/server.crt"
+    - "{{ kube_cert_dir }}/etcd/server.key"
+    - "{{ kube_cert_dir }}/front-proxy-ca.crt"
+    - "{{ kube_cert_dir }}/front-proxy-ca.key"
+    - "{{ kube_cert_dir }}/front-proxy-client.crt"
+    - "{{ kube_cert_dir }}/front-proxy-client.key"
+    - "{{ kube_cert_dir }}/sa.key"
+    - "{{ kube_cert_dir }}/sa.pub"
+
+- name: Calculate etcd cert serial
+  command: "openssl x509 -in {{ kube_cert_dir }}/apiserver-etcd-client.crt -noout -serial"
+  register: "etcd_client_cert_serial_result"
+  changed_when: false
+  when:
+    - inventory_hostname in groups['k8s_cluster']|union(groups['calico_rr']|default([]))|unique|sort
+  tags:
+    - network
+
+- name: Set etcd_client_cert_serial
+  set_fact:
+    etcd_client_cert_serial: "{{ etcd_client_cert_serial_result.stdout.split('=')[1] }}"
+  tags:
+    - network

ansible/kubespray/roles/kubernetes/kubeadm/tasks/main.yml

@@ -0,0 +1,176 @@
+---
+- name: Set kubeadm_discovery_address
+  set_fact:
+    kubeadm_discovery_address: >-
+      {%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%}
+      {{ first_kube_control_plane_address }}:{{ kube_apiserver_port }}
+      {%- else -%}
+      {{ kube_apiserver_endpoint | replace("https://", "") }}
+      {%- endif %}
+  tags:
+    - facts
+
+- name: Check if kubelet.conf exists
+  stat:
+    path: "{{ kube_config_dir }}/kubelet.conf"
+    get_attributes: no
+    get_checksum: no
+    get_mime: no
+  register: kubelet_conf
+
+- name: Check if kubeadm CA cert is accessible
+  stat:
+    path: "{{ kube_cert_dir }}/ca.crt"
+    get_attributes: no
+    get_checksum: no
+    get_mime: no
+  register: kubeadm_ca_stat
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  run_once: true
+
+- name: Calculate kubeadm CA cert hash
+  shell: set -o pipefail && openssl x509 -pubkey -in {{ kube_cert_dir }}/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'
+  args:
+    executable: /bin/bash
+  register: kubeadm_ca_hash
+  when:
+    - kubeadm_ca_stat.stat is defined
+    - kubeadm_ca_stat.stat.exists
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  run_once: true
+  changed_when: false
+
+- name: Create kubeadm token for joining nodes with 24h expiration (default)
+  command: "{{ bin_dir }}/kubeadm token create"
+  register: temp_token
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  when: kubeadm_token is not defined
+  changed_when: false
+
+- name: Set kubeadm_token to generated token
+  set_fact:
+    kubeadm_token: "{{ temp_token.stdout }}"
+  when: kubeadm_token is not defined
+
+- name: Set kubeadm api version to v1beta3
+  set_fact:
+    kubeadmConfig_api_version: v1beta3
+
+- name: Create kubeadm client config
+  template:
+    src: "kubeadm-client.conf.{{ kubeadmConfig_api_version }}.j2"
+    dest: "{{ kube_config_dir }}/kubeadm-client.conf"
+    backup: yes
+    mode: 0640
+  when: not is_kube_master
+
+- name: kubeadm | Create directory to store kubeadm patches
+  file:
+    path: "{{ kubeadm_patches.dest_dir }}"
+    state: directory
+    mode: 0640
+  when: kubeadm_patches is defined and kubeadm_patches.enabled
+
+- name: kubeadm | Copy kubeadm patches from inventory files
+  copy:
+    src: "{{ kubeadm_patches.source_dir }}/"
+    dest: "{{ kubeadm_patches.dest_dir }}"
+    owner: "root"
+    mode: 0644
+  when: kubeadm_patches is defined and kubeadm_patches.enabled
+
+- name: Join to cluster if needed
+  environment:
+    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}:/sbin"
+  when: not is_kube_master and (not kubelet_conf.stat.exists)
+  block:
+
+    - name: Join to cluster
+      command: >-
+        timeout -k {{ kubeadm_join_timeout }} {{ kubeadm_join_timeout }}
+        {{ bin_dir }}/kubeadm join
+        --config {{ kube_config_dir }}/kubeadm-client.conf
+        --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests
+        --skip-phases={{ kubeadm_join_phases_skip | join(',') }}
+      register: kubeadm_join
+      changed_when: kubeadm_join is success
+
+  rescue:
+
+    - name: Join to cluster with ignores
+      command: >-
+        timeout -k {{ kubeadm_join_timeout }} {{ kubeadm_join_timeout }}
+        {{ bin_dir }}/kubeadm join
+        --config {{ kube_config_dir }}/kubeadm-client.conf
+        --ignore-preflight-errors=all
+        --skip-phases={{ kubeadm_join_phases_skip | join(',') }}
+      register: kubeadm_join
+      changed_when: kubeadm_join is success
+
+  always:
+
+    - name: Display kubeadm join stderr if any
+      when: kubeadm_join is failed
+      debug:
+        msg: |
+          Joined with warnings
+          {{ kubeadm_join.stderr_lines }}
+
+- name: Update server field in kubelet kubeconfig
+  lineinfile:
+    dest: "{{ kube_config_dir }}/kubelet.conf"
+    regexp: 'server:'
+    line: '    server: {{ kube_apiserver_endpoint }}'
+    backup: yes
+  when:
+    - kubeadm_config_api_fqdn is not defined
+    - not is_kube_master
+    - kubeadm_discovery_address != kube_apiserver_endpoint | replace("https://", "")
+  notify: Kubeadm | restart kubelet
+
+# FIXME(mattymo): Need to point to localhost, otherwise masters will all point
+#                 incorrectly to first master, creating SPoF.
+- name: Update server field in kube-proxy kubeconfig
+  shell: >-
+    set -o pipefail && {{ kubectl }} get configmap kube-proxy -n kube-system -o yaml
+    | sed 's#server:.*#server: https://127.0.0.1:{{ kube_apiserver_port }}#g'
+    | {{ kubectl }} replace -f -
+  args:
+    executable: /bin/bash
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane']|first }}"
+  delegate_facts: false
+  when:
+    - kubeadm_config_api_fqdn is not defined
+    - kubeadm_discovery_address != kube_apiserver_endpoint | replace("https://", "")
+    - kube_proxy_deployed
+    - loadbalancer_apiserver_localhost
+  tags:
+    - kube-proxy
+
+- name: Set ca.crt file permission
+  file:
+    path: "{{ kube_cert_dir }}/ca.crt"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Restart all kube-proxy pods to ensure that they load the new configmap
+  command: "{{ kubectl }} delete pod -n kube-system -l k8s-app=kube-proxy --force --grace-period=0"
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane']|first }}"
+  delegate_facts: false
+  when:
+    - kubeadm_config_api_fqdn is not defined
+    - kubeadm_discovery_address != kube_apiserver_endpoint | replace("https://", "")
+    - kube_proxy_deployed
+  tags:
+    - kube-proxy
+
+- name: Extract etcd certs from control plane if using etcd kubeadm mode
+  include_tasks: kubeadm_etcd_node.yml
+  when:
+    - etcd_deployment_type == "kubeadm"
+    - inventory_hostname not in groups['kube_control_plane']
+    - kube_network_plugin in ["calico", "flannel", "canal", "cilium"] or cilium_deploy_additionally | default(false) | bool
+    - kube_network_plugin != "calico" or calico_datastore == "etcd"

ansible/kubespray/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2

@@ -0,0 +1,32 @@
+---
+apiVersion: kubeadm.k8s.io/v1beta3
+kind: JoinConfiguration
+discovery:
+  bootstrapToken:
+{% if kubeadm_config_api_fqdn is defined %}
+    apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
+{% else %}
+    apiServerEndpoint: {{ kubeadm_discovery_address }}
+{% endif %}
+    token: {{ kubeadm_token }}
+{% if kubeadm_ca_hash.stdout is defined %}
+    caCertHashes:
+    - sha256:{{ kubeadm_ca_hash.stdout }}
+{% else %}
+    unsafeSkipCAVerification: true
+{% endif %}
+  timeout: {{ discovery_timeout }}
+  tlsBootstrapToken: {{ kubeadm_token }}
+caCertPath: {{ kube_cert_dir }}/ca.crt
+nodeRegistration:
+  name: '{{ kube_override_hostname }}'
+  criSocket: {{ cri_socket }}
+{% if 'calico_rr' in group_names and 'kube_node' not in group_names %}
+  taints:
+  - effect: NoSchedule
+    key: node-role.kubernetes.io/calico-rr
+{% endif %}
+{% if kubeadm_patches is defined and kubeadm_patches.enabled %}
+patches:
+  directory: {{ kubeadm_patches.dest_dir }}
+{% endif %}