kubespray 2.24 추가

roles/container-engine/containerd/defaults/main.yml

@@ -0,0 +1,107 @@
+---
+containerd_storage_dir: "/var/lib/containerd"
+containerd_state_dir: "/run/containerd"
+containerd_systemd_dir: "/etc/systemd/system/containerd.service.d"
+# The default value is not -999 here because containerd's oom_score_adj has been
+# set to the -999 even if containerd_oom_score is 0.
+# Ref: https://github.com/kubernetes-sigs/kubespray/pull/9275#issuecomment-1246499242
+containerd_oom_score: 0
+
+# containerd_default_runtime: "runc"
+# containerd_snapshotter: "native"
+
+containerd_runc_runtime:
+  name: runc
+  type: "io.containerd.runc.v2"
+  engine: ""
+  root: ""
+  base_runtime_spec: cri-base.json
+  options:
+    systemdCgroup: "{{ containerd_use_systemd_cgroup | ternary('true', 'false') }}"
+    binaryName: "{{ bin_dir }}/runc"
+
+containerd_additional_runtimes: []
+# Example for Kata Containers as additional runtime:
+#  - name: kata
+#    type: "io.containerd.kata.v2"
+#    engine: ""
+#    root: ""
+
+containerd_base_runtime_spec_rlimit_nofile: 65535
+
+containerd_default_base_runtime_spec_patch:
+  process:
+    rlimits:
+      - type: RLIMIT_NOFILE
+        hard: "{{ containerd_base_runtime_spec_rlimit_nofile }}"
+        soft: "{{ containerd_base_runtime_spec_rlimit_nofile }}"
+
+containerd_base_runtime_specs:
+  cri-base.json: "{{ containerd_default_base_runtime_spec | combine(containerd_default_base_runtime_spec_patch, recursive=1) }}"
+
+containerd_grpc_max_recv_message_size: 16777216
+containerd_grpc_max_send_message_size: 16777216
+
+containerd_debug_level: "info"
+
+containerd_metrics_address: ""
+
+containerd_metrics_grpc_histogram: false
+
+containerd_registries_mirrors:
+  - prefix: docker.io
+    mirrors:
+      - host: https://registry-1.docker.io
+        capabilities: ["pull", "resolve"]
+        skip_verify: false
+
+containerd_max_container_log_line_size: -1
+
+# If enabled it will allow non root users to use port numbers <1024
+containerd_enable_unprivileged_ports: false
+# If enabled it will allow non root users to use icmp sockets
+containerd_enable_unprivileged_icmp: false
+
+containerd_cfg_dir: /etc/containerd
+
+# Extra config to be put in {{ containerd_cfg_dir }}/config.toml literally
+containerd_extra_args: ''
+
+# Configure registry auth (if applicable to secure/insecure registries)
+containerd_registry_auth: []
+#  - registry: 10.0.0.2:5000
+#    username: user
+#    password: pass
+
+# Configure containerd service
+containerd_limit_proc_num: "infinity"
+containerd_limit_core: "infinity"
+containerd_limit_open_file_num: "infinity"
+containerd_limit_mem_lock: "infinity"
+
+# If enabled it will use config_path and config to be put in {{ containerd_cfg_dir }}/certs.d/
+containerd_use_config_path: false
+
+# OS distributions that already support containerd
+containerd_supported_distributions:
+  - "CentOS"
+  - "OracleLinux"
+  - "RedHat"
+  - "Ubuntu"
+  - "Debian"
+  - "Fedora"
+  - "AlmaLinux"
+  - "Rocky"
+  - "Amazon"
+  - "Flatcar"
+  - "Flatcar Container Linux by Kinvolk"
+  - "Suse"
+  - "openSUSE Leap"
+  - "openSUSE Tumbleweed"
+  - "Kylin Linux Advanced Server"
+  - "UnionTech"
+  - "UniontechOS"
+  - "openEuler"
+
+# Enable container device interface
+enable_cdi: false

roles/container-engine/containerd/handlers/main.yml

@@ -0,0 +1,17 @@
+---
+- name: Containerd | restart containerd
+  systemd:
+    name: containerd
+    state: restarted
+    enabled: yes
+    daemon-reload: yes
+    masked: no
+  listen: Restart containerd
+
+- name: Containerd | wait for containerd
+  command: "{{ containerd_bin_dir }}/ctr images ls -q"
+  register: containerd_ready
+  retries: 8
+  delay: 4
+  until: containerd_ready.rc == 0
+  listen: Restart containerd

roles/container-engine/containerd/handlers/reset.yml

@@ -0,0 +1 @@
+---

roles/container-engine/containerd/meta/main.yml

@@ -0,0 +1,6 @@
+---
+dependencies:
+  - role: container-engine/containerd-common
+  - role: container-engine/runc
+  - role: container-engine/crictl
+  - role: container-engine/nerdctl

roles/container-engine/containerd/molecule/default/converge.yml

@@ -0,0 +1,9 @@
+---
+- name: Converge
+  hosts: all
+  become: true
+  vars:
+    container_manager: containerd
+  roles:
+    - role: kubespray-defaults
+    - role: container-engine/containerd

roles/container-engine/containerd/molecule/default/molecule.yml

@@ -0,0 +1,47 @@
+---
+role_name_check: 1
+driver:
+  name: vagrant
+  provider:
+    name: libvirt
+platforms:
+  - name: ubuntu20
+    box: generic/ubuntu2004
+    cpus: 1
+    memory: 1024
+    groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+    provider_options:
+      driver: kvm
+  - name: debian11
+    box: generic/debian11
+    cpus: 1
+    memory: 1024
+    groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+    provider_options:
+      driver: kvm
+  - name: almalinux8
+    box: almalinux/8
+    cpus: 1
+    memory: 1024
+    groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+    provider_options:
+      driver: kvm
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_ROLES_PATH: ../../../../
+  config_options:
+    defaults:
+      callbacks_enabled: profile_tasks
+      timeout: 120
+verifier:
+  name: testinfra

roles/container-engine/containerd/molecule/default/prepare.yml

@@ -0,0 +1,29 @@
+---
+- name: Prepare
+  hosts: all
+  gather_facts: False
+  become: true
+  vars:
+    ignore_assert_errors: true
+  roles:
+    - role: kubespray-defaults
+    - role: bootstrap-os
+    - role: kubernetes/preinstall
+    - role: adduser
+      user: "{{ addusers.kube }}"
+  tasks:
+    - name: Download CNI
+      include_tasks: "../../../../download/tasks/download_file.yml"
+      vars:
+        download: "{{ download_defaults | combine(downloads.cni) }}"
+
+- name: Prepare CNI
+  hosts: all
+  gather_facts: False
+  become: true
+  vars:
+    ignore_assert_errors: true
+    kube_network_plugin: cni
+  roles:
+    - role: kubespray-defaults
+    - role: network_plugin/cni

roles/container-engine/containerd/molecule/default/tests/test_default.py

@@ -0,0 +1,55 @@
+import os
+import pytest
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_service(host):
+    svc = host.service("containerd")
+    assert svc.is_running
+    assert svc.is_enabled
+
+
+def test_version(host):
+    crictl = "/usr/local/bin/crictl"
+    path = "unix:///var/run/containerd/containerd.sock"
+    with host.sudo():
+        cmd = host.command(crictl + " --runtime-endpoint " + path + " version")
+    assert cmd.rc == 0
+    assert "RuntimeName:  containerd" in cmd.stdout
+
+
+@pytest.mark.parametrize('image, dest', [
+    ('quay.io/kubespray/hello-world:latest', '/tmp/hello-world.tar')
+])
+def test_image_pull_save_load(host, image, dest):
+    nerdctl = "/usr/local/bin/nerdctl"
+    dest_file = host.file(dest)
+
+    with host.sudo():
+        pull_cmd = host.command(nerdctl + " pull " + image)
+    assert pull_cmd.rc ==0
+
+    with host.sudo():
+        save_cmd = host.command(nerdctl + " save -o " + dest + " " + image)
+    assert save_cmd.rc == 0
+    assert dest_file.exists
+
+    with host.sudo():
+        load_cmd = host.command(nerdctl + " load < " + dest)
+    assert load_cmd.rc == 0
+
+
+@pytest.mark.parametrize('image', [
+    ('quay.io/kubespray/hello-world:latest')
+])
+def test_run(host, image):
+    nerdctl = "/usr/local/bin/nerdctl"
+
+    with host.sudo():
+        cmd = host.command(nerdctl + " -n k8s.io run " + image)
+    assert cmd.rc == 0
+    assert "Hello from Docker" in cmd.stdout

roles/container-engine/containerd/tasks/main.yml

@@ -0,0 +1,143 @@
+---
+- name: Fail containerd setup if distribution is not supported
+  fail:
+    msg: "{{ ansible_distribution }} is not supported by containerd."
+  when:
+    - not (allow_unsupported_distribution_setup | default(false)) and (ansible_distribution not in containerd_supported_distributions)
+
+- name: Containerd | Remove any package manager controlled containerd package
+  package:
+    name: "{{ containerd_package }}"
+    state: absent
+  when:
+    - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar"))
+
+- name: Containerd | Remove containerd repository
+  file:
+    path: "{{ yum_repo_dir }}/containerd.repo"
+    state: absent
+  when:
+    - ansible_os_family in ['RedHat']
+
+- name: Containerd | Remove containerd repository
+  apt_repository:
+    repo: "{{ item }}"
+    state: absent
+  with_items: "{{ containerd_repo_info.repos }}"
+  when: ansible_pkg_mgr == 'apt'
+
+- name: Containerd | Download containerd
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.containerd) }}"
+
+- name: Containerd | Unpack containerd archive
+  unarchive:
+    src: "{{ downloads.containerd.dest }}"
+    dest: "{{ containerd_bin_dir }}"
+    mode: 0755
+    remote_src: yes
+    extra_opts:
+      - --strip-components=1
+  notify: Restart containerd
+
+- name: Containerd | Remove orphaned binary
+  file:
+    path: "/usr/bin/{{ item }}"
+    state: absent
+  when:
+    - containerd_bin_dir != "/usr/bin"
+    - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar"))
+  ignore_errors: true  # noqa ignore-errors
+  with_items:
+    - containerd
+    - containerd-shim
+    - containerd-shim-runc-v1
+    - containerd-shim-runc-v2
+    - ctr
+
+- name: Containerd | Generate systemd service for containerd
+  template:
+    src: containerd.service.j2
+    dest: /etc/systemd/system/containerd.service
+    mode: 0644
+    validate: "sh -c '[ -f /usr/bin/systemd/system/factory-reset.target ] || exit 0 && systemd-analyze verify %s:containerd.service'"
+    # FIXME: check that systemd version >= 250 (factory-reset.target was introduced in that release)
+    # Remove once we drop support for systemd < 250
+  notify: Restart containerd
+
+- name: Containerd | Ensure containerd directories exist
+  file:
+    dest: "{{ item }}"
+    state: directory
+    mode: 0755
+    owner: root
+    group: root
+  with_items:
+    - "{{ containerd_systemd_dir }}"
+    - "{{ containerd_cfg_dir }}"
+    - "{{ containerd_storage_dir }}"
+    - "{{ containerd_state_dir }}"
+
+- name: Containerd | Write containerd proxy drop-in
+  template:
+    src: http-proxy.conf.j2
+    dest: "{{ containerd_systemd_dir }}/http-proxy.conf"
+    mode: 0644
+  notify: Restart containerd
+  when: http_proxy is defined or https_proxy is defined
+
+- name: Containerd | Generate default base_runtime_spec
+  register: ctr_oci_spec
+  command: "{{ containerd_bin_dir }}/ctr oci spec"
+  check_mode: false
+  changed_when: false
+
+- name: Containerd | Store generated default base_runtime_spec
+  set_fact:
+    containerd_default_base_runtime_spec: "{{ ctr_oci_spec.stdout | from_json }}"
+
+- name: Containerd | Write base_runtime_specs
+  copy:
+    content: "{{ item.value }}"
+    dest: "{{ containerd_cfg_dir }}/{{ item.key }}"
+    owner: "root"
+    mode: 0644
+  with_dict: "{{ containerd_base_runtime_specs | default({}) }}"
+  notify: Restart containerd
+
+- name: Containerd | Copy containerd config file
+  template:
+    src: config.toml.j2
+    dest: "{{ containerd_cfg_dir }}/config.toml"
+    owner: "root"
+    mode: 0640
+  notify: Restart containerd
+
+- name: Containerd | Configure containerd registries
+  when: containerd_registries_mirrors is defined
+  block:
+    - name: Containerd | Create registry directories
+      file:
+        path: "{{ containerd_cfg_dir }}/certs.d/{{ item.prefix }}"
+        state: directory
+        mode: 0755
+      loop: "{{ containerd_registries_mirrors }}"
+    - name: Containerd | Write hosts.toml file
+      template:
+        src: hosts.toml.j2
+        dest: "{{ containerd_cfg_dir }}/certs.d/{{ item.prefix }}/hosts.toml"
+        mode: 0640
+      loop: "{{ containerd_registries_mirrors }}"
+
+# you can sometimes end up in a state where everything is installed
+# but containerd was not started / enabled
+- name: Containerd | Flush handlers
+  meta: flush_handlers
+
+- name: Containerd | Ensure containerd is started and enabled
+  systemd:
+    name: containerd
+    daemon_reload: yes
+    enabled: yes
+    state: started

roles/container-engine/containerd/tasks/reset.yml

@@ -0,0 +1,40 @@
+---
+- name: Containerd | Remove containerd repository for RedHat os family
+  file:
+    path: "{{ yum_repo_dir }}/containerd.repo"
+    state: absent
+  when:
+    - ansible_os_family in ['RedHat']
+  tags:
+    - reset_containerd
+
+- name: Containerd | Remove containerd repository for Debian os family
+  apt_repository:
+    repo: "{{ item }}"
+    state: absent
+  with_items: "{{ containerd_repo_info.repos }}"
+  when: ansible_pkg_mgr == 'apt'
+  tags:
+    - reset_containerd
+
+- name: Containerd | Stop containerd service
+  service:
+    name: containerd
+    daemon_reload: true
+    enabled: false
+    state: stopped
+  tags:
+    - reset_containerd
+
+- name: Containerd | Remove configuration files
+  file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - /etc/systemd/system/containerd.service
+    - "{{ containerd_systemd_dir }}"
+    - "{{ containerd_cfg_dir }}"
+    - "{{ containerd_storage_dir }}"
+    - "{{ containerd_state_dir }}"
+  tags:
+    - reset_containerd

roles/container-engine/containerd/templates/config.toml.j2

@@ -0,0 +1,96 @@
+version = 2
+root = "{{ containerd_storage_dir }}"
+state = "{{ containerd_state_dir }}"
+oom_score = {{ containerd_oom_score }}
+
+[grpc]
+  max_recv_message_size = {{ containerd_grpc_max_recv_message_size | default(16777216) }}
+  max_send_message_size = {{ containerd_grpc_max_send_message_size | default(16777216) }}
+
+[debug]
+  level = "{{ containerd_debug_level | default('info') }}"
+
+[metrics]
+  address = "{{ containerd_metrics_address | default('') }}"
+  grpc_histogram = {{ containerd_metrics_grpc_histogram | default(false) | lower }}
+
+[plugins]
+  [plugins."io.containerd.grpc.v1.cri"]
+    sandbox_image = "{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}"
+    max_container_log_line_size = {{ containerd_max_container_log_line_size }}
+    enable_unprivileged_ports = {{ containerd_enable_unprivileged_ports | default(false) | lower }}
+    enable_unprivileged_icmp = {{ containerd_enable_unprivileged_icmp | default(false) | lower }}
+{% if enable_cdi %}
+    enable_cdi = true
+    cdi_spec_dirs = ["/etc/cdi", "/var/run/cdi"]
+{% endif %}
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "{{ containerd_default_runtime | default('runc') }}"
+      snapshotter = "{{ containerd_snapshotter | default('overlayfs') }}"
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+{% for runtime in [containerd_runc_runtime] + containerd_additional_runtimes %}
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}]
+          runtime_type = "{{ runtime.type }}"
+          runtime_engine = "{{ runtime.engine }}"
+          runtime_root = "{{ runtime.root }}"
+{% if runtime.base_runtime_spec is defined %}
+          base_runtime_spec = "{{ containerd_cfg_dir }}/{{ runtime.base_runtime_spec }}"
+{% endif %}
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}.options]
+{% for key, value in runtime.options.items() %}
+{% if value | string != "true" and value | string != "false" %}
+            {{ key }} = "{{ value }}"
+{% else %}
+            {{ key }} = {{ value }}
+{% endif %}
+{% endfor %}
+{% endfor %}
+{% if kata_containers_enabled %}
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu]
+          runtime_type = "io.containerd.kata-qemu.v2"
+{% endif %}
+{% if gvisor_enabled %}
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc]
+          runtime_type = "io.containerd.runsc.v1"
+{% endif %}
+    [plugins."io.containerd.grpc.v1.cri".registry]
+{% if containerd_use_config_path is defined and containerd_use_config_path|bool %}
+      config_path = "{{ containerd_cfg_dir }}/certs.d"
+{% else %}
+      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+{% set insecure_registries_addr = [] %}
+{% for registry in containerd_registries_mirrors %}
+        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ registry.prefix }}"]
+{% set endpoint = [] %}
+{% for mirror in registry.mirrors %}
+{% if endpoint.append(mirror.host) %}{% endif %}
+{% if mirror.skip_verify is defined and mirror.skip_verify|bool %}{% if insecure_registries_addr.append(mirror.host | urlsplit('netloc')) %}{% endif %}{% endif %}
+{% endfor %}
+          endpoint = ["{{ ( endpoint | unique ) | join('","') }}"]
+{% endfor %}
+{% for addr in insecure_registries_addr | unique %}
+        [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ addr }}".tls]
+          insecure_skip_verify = true
+{% endfor %}
+{% endif %}
+{% for registry in containerd_registry_auth if registry['registry'] is defined %}
+{% if (registry['username'] is defined and registry['password'] is defined) or registry['auth'] is defined %}
+      [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ registry['registry'] }}".auth]
+{% if registry['username'] is defined and registry['password'] is defined %}
+        password = "{{ registry['password'] }}"
+        username = "{{ registry['username'] }}"
+{% else %}
+        auth = "{{ registry['auth'] }}"
+{% endif %}
+{% endif %}
+{% endfor %}
+
+{% if nri_enabled and containerd_version is version('1.7.0', '>=') %}
+  [plugins."io.containerd.nri.v1.nri"]
+    disable = false
+{% endif %}
+
+{% if containerd_extra_args is defined %}
+{{ containerd_extra_args }}
+{% endif %}

roles/container-engine/containerd/templates/containerd.service.j2

@@ -0,0 +1,45 @@
+# Copyright The containerd Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[Unit]
+Description=containerd container runtime
+Documentation=https://containerd.io
+After=network.target local-fs.target
+
+[Service]
+ExecStartPre=-/sbin/modprobe overlay
+ExecStart={{ containerd_bin_dir }}/containerd
+
+Type=notify
+Delegate=yes
+KillMode=process
+Restart=always
+RestartSec=5
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNPROC={{ containerd_limit_proc_num }}
+LimitCORE={{ containerd_limit_core }}
+LimitNOFILE={{ containerd_limit_open_file_num }}
+LimitMEMLOCK={{ containerd_limit_mem_lock }}
+# Comment TasksMax if your systemd version does not supports it.
+# Only systemd 226 and above support this version.
+TasksMax=infinity
+OOMScoreAdjust=-999
+# Set the cgroup slice of the service so that kube reserved takes effect
+{% if kube_reserved is defined and kube_reserved|bool %}
+Slice={{ kube_reserved_cgroups_for_service_slice }}
+{% endif %}
+
+[Install]
+WantedBy=multi-user.target

roles/container-engine/containerd/templates/hosts.toml.j2

@@ -0,0 +1,7 @@
+server = "https://{{ item.prefix }}"
+{% for mirror in item.mirrors %}
+[host."{{ mirror.host }}"]
+  capabilities = ["{{ ([ mirror.capabilities ] | flatten ) | join('","') }}"]
+  skip_verify = {{ mirror.skip_verify | default('false') | string | lower }}
+  override_path = {{ mirror.override_path | default('false') | string | lower }}
+{% endfor %}

roles/container-engine/containerd/templates/http-proxy.conf.j2

@@ -0,0 +1,2 @@
+[Service]
+Environment={% if http_proxy is defined %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy is defined %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy is defined %}"NO_PROXY={{ no_proxy }}"{% endif %}

roles/container-engine/containerd/vars/debian.yml

@@ -0,0 +1,7 @@
+---
+containerd_repo_info:
+  repos:
+    - >
+      deb {{ containerd_debian_repo_base_url }}
+      {{ ansible_distribution_release | lower }}
+      {{ containerd_debian_repo_component }}

roles/container-engine/containerd/vars/ubuntu.yml

@@ -0,0 +1,7 @@
+---
+containerd_repo_info:
+  repos:
+    - >
+      deb {{ containerd_ubuntu_repo_base_url }}
+      {{ ansible_distribution_release | lower }}
+      {{ containerd_ubuntu_repo_component }}