dsk-dev kubespray 이동

ansible/kubespray/roles/container-engine/containerd-common/defaults/main.yml

@@ -0,0 +1,17 @@
+---
+# We keep these variables around to allow migration from package
+# manager controlled installs to direct download ones.
+containerd_package: 'containerd.io'
+yum_repo_dir: /etc/yum.repos.d
+
+# Keep minimal repo information around for cleanup
+containerd_repo_info:
+  repos:
+
+# Ubuntu docker-ce repo
+containerd_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu"
+containerd_ubuntu_repo_component: "stable"
+
+# Debian docker-ce repo
+containerd_debian_repo_base_url: "https://download.docker.com/linux/debian"
+containerd_debian_repo_component: "stable"

ansible/kubespray/roles/container-engine/containerd-common/meta/main.yml

@@ -0,0 +1,2 @@
+---
+allow_duplicates: true

ansible/kubespray/roles/container-engine/containerd-common/tasks/main.yml

@@ -0,0 +1,31 @@
+---
+- name: containerd-common | check if fedora coreos
+  stat:
+    path: /run/ostree-booted
+    get_attributes: no
+    get_checksum: no
+    get_mime: no
+  register: ostree
+
+- name: containerd-common | set is_ostree
+  set_fact:
+    is_ostree: "{{ ostree.stat.exists }}"
+
+- name: containerd-common | gather os specific variables
+  include_vars: "{{ item }}"
+  with_first_found:
+    - files:
+        - "{{ ansible_distribution|lower }}-{{ ansible_distribution_version|lower|replace('/', '_') }}.yml"
+        - "{{ ansible_distribution|lower }}-{{ ansible_distribution_release|lower }}-{{ host_architecture }}.yml"
+        - "{{ ansible_distribution|lower }}-{{ ansible_distribution_release|lower }}.yml"
+        - "{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version|lower|replace('/', '_') }}.yml"
+        - "{{ ansible_distribution|lower }}-{{ host_architecture }}.yml"
+        - "{{ ansible_distribution|lower }}.yml"
+        - "{{ ansible_os_family|lower }}-{{ host_architecture }}.yml"
+        - "{{ ansible_os_family|lower }}.yml"
+        - defaults.yml
+      paths:
+        - ../vars
+      skip: true
+  tags:
+    - facts

ansible/kubespray/roles/container-engine/containerd-common/vars/amazon.yml

@@ -0,0 +1,2 @@
+---
+containerd_package: containerd

ansible/kubespray/roles/container-engine/containerd-common/vars/suse.yml

@@ -0,0 +1,2 @@
+---
+containerd_package: containerd

ansible/kubespray/roles/container-engine/containerd/defaults/main.yml

@@ -0,0 +1,75 @@
+---
+containerd_storage_dir: "/var/lib/containerd"
+containerd_state_dir: "/run/containerd"
+containerd_systemd_dir: "/etc/systemd/system/containerd.service.d"
+# The default value is not -999 here because containerd's oom_score_adj has been
+# set to the -999 even if containerd_oom_score is 0.
+# Ref: https://github.com/kubernetes-sigs/kubespray/pull/9275#issuecomment-1246499242
+containerd_oom_score: 0
+
+# containerd_default_runtime: "runc"
+# containerd_snapshotter: "native"
+
+containerd_runc_runtime:
+  name: runc
+  type: "io.containerd.runc.v2"
+  engine: ""
+  root: ""
+  base_runtime_spec: cri-base.json
+  options:
+    systemdCgroup: "{{ containerd_use_systemd_cgroup | ternary('true', 'false') }}"
+
+containerd_additional_runtimes: []
+# Example for Kata Containers as additional runtime:
+#  - name: kata
+#    type: "io.containerd.kata.v2"
+#    engine: ""
+#    root: ""
+
+containerd_base_runtime_spec_rlimit_nofile: 65535
+
+containerd_default_base_runtime_spec_patch:
+  process:
+    rlimits:
+      - type: RLIMIT_NOFILE
+        hard: "{{ containerd_base_runtime_spec_rlimit_nofile }}"
+        soft: "{{ containerd_base_runtime_spec_rlimit_nofile }}"
+
+containerd_base_runtime_specs:
+  cri-base.json: "{{ containerd_default_base_runtime_spec | combine(containerd_default_base_runtime_spec_patch,recursive=1) }}"
+
+containerd_grpc_max_recv_message_size: 16777216
+containerd_grpc_max_send_message_size: 16777216
+
+containerd_debug_level: "info"
+
+containerd_metrics_address: ""
+
+containerd_metrics_grpc_histogram: false
+
+containerd_registries:
+  "docker.io": "https://registry-1.docker.io"
+
+containerd_max_container_log_line_size: -1
+
+# If enabled it will allow non root users to use port numbers <1024
+containerd_enable_unprivileged_ports: false
+# If enabled it will allow non root users to use icmp sockets
+containerd_enable_unprivileged_icmp: false
+
+containerd_cfg_dir: /etc/containerd
+
+# Extra config to be put in {{ containerd_cfg_dir }}/config.toml literally
+containerd_extra_args: ''
+
+# Configure registry auth (if applicable to secure/insecure registries)
+containerd_registry_auth: []
+#  - registry: 10.0.0.2:5000
+#    username: user
+#    password: pass
+
+# Configure containerd service
+containerd_limit_proc_num: "infinity"
+containerd_limit_core: "infinity"
+containerd_limit_open_file_num: "infinity"
+containerd_limit_mem_lock: "infinity"

ansible/kubespray/roles/container-engine/containerd/handlers/main.yml

@@ -0,0 +1,21 @@
+---
+- name: restart containerd
+  command: /bin/true
+  notify:
+    - Containerd | restart containerd
+    - Containerd | wait for containerd
+
+- name: Containerd | restart containerd
+  systemd:
+    name: containerd
+    state: restarted
+    enabled: yes
+    daemon-reload: yes
+    masked: no
+
+- name: Containerd | wait for containerd
+  command: "{{ containerd_bin_dir }}/ctr images ls -q"
+  register: containerd_ready
+  retries: 8
+  delay: 4
+  until: containerd_ready.rc == 0

ansible/kubespray/roles/container-engine/containerd/handlers/reset.yml

@@ -0,0 +1 @@
+---

ansible/kubespray/roles/container-engine/containerd/meta/main.yml

@@ -0,0 +1,6 @@
+---
+dependencies:
+  - role: container-engine/containerd-common
+  - role: container-engine/runc
+  - role: container-engine/crictl
+  - role: container-engine/nerdctl

ansible/kubespray/roles/container-engine/containerd/molecule/default/converge.yml

@@ -0,0 +1,9 @@
+---
+- name: Converge
+  hosts: all
+  become: true
+  vars:
+    container_manager: containerd
+  roles:
+    - role: kubespray-defaults
+    - role: container-engine/containerd

ansible/kubespray/roles/container-engine/containerd/molecule/default/molecule.yml

@@ -0,0 +1,49 @@
+---
+driver:
+  name: vagrant
+  provider:
+    name: libvirt
+lint: |
+  set -e
+  yamllint -c ../../../.yamllint .
+platforms:
+  - name: ubuntu20
+    box: generic/ubuntu2004
+    cpus: 1
+    memory: 1024
+    groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+  - name: debian11
+    box: generic/debian11
+    cpus: 1
+    memory: 1024
+    groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+  - name: almalinux8
+    box: almalinux/8
+    cpus: 1
+    memory: 1024
+    groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_ROLES_PATH: ../../../../
+  config_options:
+    defaults:
+      callbacks_enabled: profile_tasks
+      timeout: 120
+  lint:
+    name: ansible-lint
+    options:
+      c: ../../../.ansible-lint
+verifier:
+  name: testinfra
+  lint:
+    name: flake8

ansible/kubespray/roles/container-engine/containerd/molecule/default/prepare.yml

@@ -0,0 +1,28 @@
+---
+- name: Prepare
+  hosts: all
+  gather_facts: False
+  become: true
+  vars:
+    ignore_assert_errors: true
+  roles:
+    - role: kubespray-defaults
+    - role: bootstrap-os
+    - role: kubernetes/preinstall
+    - role: adduser
+      user: "{{ addusers.kube }}"
+  tasks:
+    - include_tasks: "../../../../download/tasks/download_file.yml"
+      vars:
+        download: "{{ download_defaults | combine(downloads.cni) }}"
+
+- name: Prepare CNI
+  hosts: all
+  gather_facts: False
+  become: true
+  vars:
+    ignore_assert_errors: true
+    kube_network_plugin: cni
+  roles:
+    - role: kubespray-defaults
+    - role: network_plugin/cni

ansible/kubespray/roles/container-engine/containerd/molecule/default/tests/test_default.py

@@ -0,0 +1,55 @@
+import os
+import pytest
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_service(host):
+    svc = host.service("containerd")
+    assert svc.is_running
+    assert svc.is_enabled
+
+
+def test_version(host):
+    crictl = "/usr/local/bin/crictl"
+    path = "unix:///var/run/containerd/containerd.sock"
+    with host.sudo():
+        cmd = host.command(crictl + " --runtime-endpoint " + path + " version")
+    assert cmd.rc == 0
+    assert "RuntimeName:  containerd" in cmd.stdout
+
+
+@pytest.mark.parametrize('image, dest', [
+    ('quay.io/kubespray/hello-world:latest', '/tmp/hello-world.tar')
+])
+def test_image_pull_save_load(host, image, dest):
+    nerdctl = "/usr/local/bin/nerdctl"
+    dest_file = host.file(dest)
+
+    with host.sudo():
+        pull_cmd = host.command(nerdctl + " pull " + image)
+    assert pull_cmd.rc ==0
+
+    with host.sudo():
+        save_cmd = host.command(nerdctl + " save -o " + dest + " " + image)
+    assert save_cmd.rc == 0
+    assert dest_file.exists
+
+    with host.sudo():
+        load_cmd = host.command(nerdctl + " load < " + dest)
+    assert load_cmd.rc == 0
+
+
+@pytest.mark.parametrize('image', [
+    ('quay.io/kubespray/hello-world:latest')
+])
+def test_run(host, image):
+    nerdctl = "/usr/local/bin/nerdctl"
+
+    with host.sudo():
+        cmd = host.command(nerdctl + " -n k8s.io run " + image)
+    assert cmd.rc == 0
+    assert "Hello from Docker" in cmd.stdout

ansible/kubespray/roles/container-engine/containerd/tasks/main.yml

@@ -0,0 +1,124 @@
+---
+- name: Fail containerd setup if distribution is not supported
+  fail:
+    msg: "{{ ansible_distribution }} is not supported by containerd."
+  when:
+    - ansible_distribution not in ["CentOS", "OracleLinux", "RedHat", "Ubuntu", "Debian", "Fedora", "AlmaLinux", "Rocky", "Amazon", "Flatcar", "Flatcar Container Linux by Kinvolk", "Suse", "openSUSE Leap", "openSUSE Tumbleweed", "Kylin Linux Advanced Server", "UnionTech", "openEuler"]
+
+- name: containerd | Remove any package manager controlled containerd package
+  package:
+    name: "{{ containerd_package }}"
+    state: absent
+  when:
+    - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar"))
+
+- name: containerd | Remove containerd repository
+  file:
+    path: "{{ yum_repo_dir }}/containerd.repo"
+    state: absent
+  when:
+    - ansible_os_family in ['RedHat']
+
+- name: containerd | Remove containerd repository
+  apt_repository:
+    repo: "{{ item }}"
+    state: absent
+  with_items: "{{ containerd_repo_info.repos }}"
+  when: ansible_pkg_mgr == 'apt'
+
+- name: containerd | Download containerd
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.containerd) }}"
+
+- name: containerd | Unpack containerd archive
+  unarchive:
+    src: "{{ downloads.containerd.dest }}"
+    dest: "{{ containerd_bin_dir }}"
+    mode: 0755
+    remote_src: yes
+    extra_opts:
+      - --strip-components=1
+  notify: restart containerd
+
+- name: containerd | Remove orphaned binary
+  file:
+    path: "/usr/bin/{{ item }}"
+    state: absent
+  when:
+    - containerd_bin_dir != "/usr/bin"
+    - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar"))
+  ignore_errors: true  # noqa ignore-errors
+  with_items:
+    - containerd
+    - containerd-shim
+    - containerd-shim-runc-v1
+    - containerd-shim-runc-v2
+    - ctr
+
+- name: containerd | Generate systemd service for containerd
+  template:
+    src: containerd.service.j2
+    dest: /etc/systemd/system/containerd.service
+    mode: 0644
+  notify: restart containerd
+
+- name: containerd | Ensure containerd directories exist
+  file:
+    dest: "{{ item }}"
+    state: directory
+    mode: 0755
+    owner: root
+    group: root
+  with_items:
+    - "{{ containerd_systemd_dir }}"
+    - "{{ containerd_cfg_dir }}"
+    - "{{ containerd_storage_dir }}"
+    - "{{ containerd_state_dir }}"
+
+- name: containerd | Write containerd proxy drop-in
+  template:
+    src: http-proxy.conf.j2
+    dest: "{{ containerd_systemd_dir }}/http-proxy.conf"
+    mode: 0644
+  notify: restart containerd
+  when: http_proxy is defined or https_proxy is defined
+
+- name: containerd | Generate default base_runtime_spec
+  register: ctr_oci_spec
+  command: "{{ containerd_bin_dir }}/ctr oci spec"
+  check_mode: false
+  changed_when: false
+
+- name: containerd | Store generated default base_runtime_spec
+  set_fact:
+    containerd_default_base_runtime_spec: "{{ ctr_oci_spec.stdout | from_json }}"
+
+- name: containerd | Write base_runtime_specs
+  copy:
+    content: "{{ item.value }}"
+    dest: "{{ containerd_cfg_dir }}/{{ item.key }}"
+    owner: "root"
+    mode: 0644
+  with_dict: "{{ containerd_base_runtime_specs | default({}) }}"
+  notify: restart containerd
+
+- name: containerd | Copy containerd config file
+  template:
+    src: config.toml.j2
+    dest: "{{ containerd_cfg_dir }}/config.toml"
+    owner: "root"
+    mode: 0640
+  notify: restart containerd
+
+# you can sometimes end up in a state where everything is installed
+# but containerd was not started / enabled
+- name: containerd | Flush handlers
+  meta: flush_handlers
+
+- name: containerd | Ensure containerd is started and enabled
+  systemd:
+    name: containerd
+    daemon_reload: yes
+    enabled: yes
+    state: started

ansible/kubespray/roles/container-engine/containerd/tasks/reset.yml

@@ -0,0 +1,41 @@
+---
+- name: containerd | Remove containerd repository for RedHat os family
+  file:
+    path: "{{ yum_repo_dir }}/containerd.repo"
+    state: absent
+  when:
+    - ansible_os_family in ['RedHat']
+  tags:
+    - reset_containerd
+
+- name: containerd | Remove containerd repository for Debian os family
+  apt_repository:
+    repo: "{{ item }}"
+    state: absent
+  with_items: "{{ containerd_repo_info.repos }}"
+  when: ansible_pkg_mgr == 'apt'
+  tags:
+    - reset_containerd
+
+- name: containerd | Stop containerd service
+  service:
+    name: containerd
+    daemon_reload: true
+    enabled: false
+    masked: true
+    state: stopped
+  tags:
+    - reset_containerd
+
+- name: containerd | Remove configuration files
+  file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - /etc/systemd/system/containerd.service
+    - "{{ containerd_systemd_dir }}"
+    - "{{ containerd_cfg_dir }}"
+    - "{{ containerd_storage_dir }}"
+    - "{{ containerd_state_dir }}"
+  tags:
+    - reset_containerd

ansible/kubespray/roles/container-engine/containerd/templates/config.toml.j2

@@ -0,0 +1,79 @@
+version = 2
+root = "{{ containerd_storage_dir }}"
+state = "{{ containerd_state_dir }}"
+oom_score = {{ containerd_oom_score }}
+
+[grpc]
+  max_recv_message_size = {{ containerd_grpc_max_recv_message_size | default(16777216) }}
+  max_send_message_size = {{ containerd_grpc_max_send_message_size | default(16777216) }}
+
+[debug]
+  level = "{{ containerd_debug_level | default('info') }}"
+
+[metrics]
+  address = "{{ containerd_metrics_address | default('') }}"
+  grpc_histogram = {{ containerd_metrics_grpc_histogram | default(false) | lower }}
+
+[plugins]
+  [plugins."io.containerd.grpc.v1.cri"]
+    sandbox_image = "{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}"
+    max_container_log_line_size = {{ containerd_max_container_log_line_size }}
+    enable_unprivileged_ports = {{ containerd_enable_unprivileged_ports | default(false) | lower }}
+    enable_unprivileged_icmp = {{ containerd_enable_unprivileged_icmp | default(false) | lower }}
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "{{ containerd_default_runtime | default('runc') }}"
+      snapshotter = "{{ containerd_snapshotter | default('overlayfs') }}"
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+{% for runtime in [containerd_runc_runtime] + containerd_additional_runtimes %}
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}]
+          runtime_type = "{{ runtime.type }}"
+          runtime_engine = "{{ runtime.engine }}"
+          runtime_root = "{{ runtime.root }}"
+{% if runtime.base_runtime_spec is defined %}
+          base_runtime_spec = "{{ containerd_cfg_dir }}/{{ runtime.base_runtime_spec }}"
+{% endif %}
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}.options]
+{% for key, value in runtime.options.items() %}
+            {{ key }} = {{ value }}
+{% endfor %}
+{% endfor %}
+{% if kata_containers_enabled %}
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu]
+          runtime_type = "io.containerd.kata-qemu.v2"
+{% endif %}
+{% if gvisor_enabled %}
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc]
+          runtime_type = "io.containerd.runsc.v1"
+{% endif %}
+    [plugins."io.containerd.grpc.v1.cri".registry]
+      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+{% for registry, addr in containerd_registries.items() %}
+        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ registry }}"]
+          endpoint = ["{{ ([ addr ] | flatten ) | join('","') }}"]
+{% endfor %}
+{% if containerd_insecure_registries is defined and containerd_insecure_registries|length>0 %}
+{% for registry, addr in containerd_insecure_registries.items() %}
+        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ registry }}"]
+          endpoint = ["{{ ([ addr ] | flatten ) | join('","') }}"]
+{% endfor %}
+{% for addr in containerd_insecure_registries.values() | flatten | unique %}
+        [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ addr }}".tls]
+          insecure_skip_verify = true
+{% endfor %}
+{% endif %}
+{% for registry in containerd_registry_auth if registry['registry'] is defined %}
+{% if (registry['username'] is defined and registry['password'] is defined) or registry['auth'] is defined %}
+      [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ registry['registry'] }}".auth]
+{% if registry['username'] is defined and registry['password'] is defined %}
+        password = "{{ registry['password'] }}"
+        username = "{{ registry['username'] }}"
+{% else %}
+        auth = "{{ registry['auth'] }}"
+{% endif %}
+{% endif %}
+{% endfor %}
+
+{% if containerd_extra_args is defined %}
+{{ containerd_extra_args }}
+{% endif %}

ansible/kubespray/roles/container-engine/containerd/templates/containerd.service.j2

@@ -0,0 +1,41 @@
+# Copyright The containerd Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[Unit]
+Description=containerd container runtime
+Documentation=https://containerd.io
+After=network.target local-fs.target
+
+[Service]
+ExecStartPre=-/sbin/modprobe overlay
+ExecStart={{ containerd_bin_dir }}/containerd
+
+Type=notify
+Delegate=yes
+KillMode=process
+Restart=always
+RestartSec=5
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNPROC={{ containerd_limit_proc_num }}
+LimitCORE={{ containerd_limit_core }}
+LimitNOFILE={{ containerd_limit_open_file_num }}
+LimitMEMLOCK={{ containerd_limit_mem_lock }}
+# Comment TasksMax if your systemd version does not supports it.
+# Only systemd 226 and above support this version.
+TasksMax=infinity
+OOMScoreAdjust=-999
+
+[Install]
+WantedBy=multi-user.target

ansible/kubespray/roles/container-engine/containerd/templates/http-proxy.conf.j2

@@ -0,0 +1,2 @@
+[Service]
+Environment={% if http_proxy is defined %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy is defined %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy is defined %}"NO_PROXY={{ no_proxy }}"{% endif %}

ansible/kubespray/roles/container-engine/containerd/vars/debian.yml

@@ -0,0 +1,7 @@
+---
+containerd_repo_info:
+  repos:
+    - >
+      deb {{ containerd_debian_repo_base_url }}
+      {{ ansible_distribution_release|lower }}
+      {{ containerd_debian_repo_component }}

ansible/kubespray/roles/container-engine/containerd/vars/ubuntu.yml

@@ -0,0 +1,7 @@
+---
+containerd_repo_info:
+  repos:
+    - >
+      deb {{ containerd_ubuntu_repo_base_url }}
+      {{ ansible_distribution_release|lower }}
+      {{ containerd_ubuntu_repo_component }}

ansible/kubespray/roles/container-engine/cri-dockerd/handlers/main.yml

@@ -0,0 +1,35 @@
+---
+- name: restart and enable cri-dockerd
+  command: /bin/true
+  notify:
+    - cri-dockerd | reload systemd
+    - cri-dockerd | restart docker.service
+    - cri-dockerd | reload cri-dockerd.socket
+    - cri-dockerd | reload cri-dockerd.service
+    - cri-dockerd | enable cri-dockerd service
+
+- name: cri-dockerd | reload systemd
+  systemd:
+    name: cri-dockerd
+    daemon_reload: true
+    masked: no
+
+- name: cri-dockerd | restart docker.service
+  service:
+    name: docker.service
+    state: restarted
+
+- name: cri-dockerd | reload cri-dockerd.socket
+  service:
+    name: cri-dockerd.socket
+    state: restarted
+
+- name: cri-dockerd | reload cri-dockerd.service
+  service:
+    name: cri-dockerd.service
+    state: restarted
+
+- name: cri-dockerd | enable cri-dockerd service
+  service:
+    name: cri-dockerd.service
+    enabled: yes

ansible/kubespray/roles/container-engine/cri-dockerd/meta/main.yml

@@ -0,0 +1,4 @@
+---
+dependencies:
+  - role: container-engine/docker
+  - role: container-engine/crictl

ansible/kubespray/roles/container-engine/cri-dockerd/molecule/default/converge.yml

@@ -0,0 +1,9 @@
+---
+- name: Converge
+  hosts: all
+  become: true
+  vars:
+    container_manager: docker
+  roles:
+    - role: kubespray-defaults
+    - role: container-engine/cri-dockerd

ansible/kubespray/roles/container-engine/cri-dockerd/molecule/default/files/10-mynet.conf

@@ -0,0 +1,17 @@
+{
+  "cniVersion": "0.2.0",
+  "name": "mynet",
+  "type": "bridge",
+  "bridge": "cni0",
+  "isGateway": true,
+  "ipMasq": true,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "172.19.0.0/24",
+    "routes": [
+      {
+        "dst": "0.0.0.0/0"
+      }
+    ]
+  }
+}

ansible/kubespray/roles/container-engine/cri-dockerd/molecule/default/files/container.json

@@ -0,0 +1,10 @@
+{
+  "metadata": {
+    "name": "cri-dockerd1"
+  },
+  "image": {
+    "image": "quay.io/kubespray/hello-world:latest"
+  },
+  "log_path": "cri-dockerd1.0.log",
+  "linux": {}
+}

ansible/kubespray/roles/container-engine/cri-dockerd/molecule/default/files/sandbox.json

@@ -0,0 +1,10 @@
+{
+  "metadata": {
+    "name": "cri-dockerd1",
+    "namespace": "default",
+    "attempt": 1,
+    "uid": "hdishd83djaidwnduwk28bcsb"
+  },
+  "linux": {},
+  "log_directory": "/tmp"
+}

ansible/kubespray/roles/container-engine/cri-dockerd/molecule/default/molecule.yml

@@ -0,0 +1,45 @@
+---
+driver:
+  name: vagrant
+  provider:
+    name: libvirt
+    options:
+      driver: kvm
+lint: |
+  set -e
+  yamllint -c ../../../.yamllint .
+platforms:
+  - name: almalinux8
+    box: almalinux/8
+    cpus: 1
+    memory: 1024
+    nested: true
+    groups:
+      - kube_control_plane
+  - name: ubuntu20
+    box: generic/ubuntu2004
+    cpus: 1
+    memory: 1024
+    nested: true
+    groups:
+      - kube_control_plane
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_ROLES_PATH: ../../../../
+  config_options:
+    defaults:
+      callbacks_enabled: profile_tasks
+      timeout: 120
+  lint:
+    name: ansible-lint
+    options:
+      c: ../../../.ansible-lint
+  inventory:
+    group_vars:
+      all:
+        become: true
+verifier:
+  name: testinfra
+  lint:
+    name: flake8

ansible/kubespray/roles/container-engine/cri-dockerd/molecule/default/prepare.yml

@@ -0,0 +1,47 @@
+---
+- name: Prepare
+  hosts: all
+  become: true
+  roles:
+    - role: kubespray-defaults
+    - role: bootstrap-os
+    - role: adduser
+      user: "{{ addusers.kube }}"
+  tasks:
+    - include_tasks: "../../../../download/tasks/download_file.yml"
+      vars:
+        download: "{{ download_defaults | combine(downloads.cni) }}"
+
+- name: Prepare container runtime
+  hosts: all
+  become: true
+  vars:
+    container_manager: containerd
+    kube_network_plugin: cni
+  roles:
+    - role: kubespray-defaults
+    - role: network_plugin/cni
+  tasks:
+    - name: Copy test container files
+      copy:
+        src: "{{ item }}"
+        dest: "/tmp/{{ item }}"
+        owner: root
+        mode: 0644
+      with_items:
+        - container.json
+        - sandbox.json
+    - name: Create /etc/cni/net.d directory
+      file:
+        path: /etc/cni/net.d
+        state: directory
+        owner: "{{ kube_owner }}"
+        mode: 0755
+    - name: Setup CNI
+      copy:
+        src: "{{ item }}"
+        dest: "/etc/cni/net.d/{{ item }}"
+        owner: root
+        mode: 0644
+      with_items:
+        - 10-mynet.conf

ansible/kubespray/roles/container-engine/cri-dockerd/molecule/default/tests/test_default.py

@@ -0,0 +1,19 @@
+import os
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_run_pod(host):
+    run_command = "/usr/local/bin/crictl run --with-pull /tmp/container.json /tmp/sandbox.json"
+    with host.sudo():
+        cmd = host.command(run_command)
+    assert cmd.rc == 0
+
+    with host.sudo():
+      log_f = host.file("/tmp/cri-dockerd1.0.log")
+
+      assert log_f.exists
+      assert b"Hello from Docker" in log_f.content

ansible/kubespray/roles/container-engine/cri-dockerd/tasks/main.yml

@@ -0,0 +1,28 @@
+---
+- name: runc | Download cri-dockerd binary
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.cri_dockerd) }}"
+
+- name: Copy cri-dockerd binary from download dir
+  copy:
+    src: "{{ local_release_dir }}/cri-dockerd"
+    dest: "{{ bin_dir }}/cri-dockerd"
+    mode: 0755
+    remote_src: true
+  notify:
+    - restart and enable cri-dockerd
+
+- name: Generate cri-dockerd systemd unit files
+  template:
+    src: "{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    mode: 0644
+  with_items:
+    - cri-dockerd.service
+    - cri-dockerd.socket
+  notify:
+    - restart and enable cri-dockerd
+
+- name: Flush handlers
+  meta: flush_handlers

ansible/kubespray/roles/container-engine/cri-dockerd/templates/cri-dockerd.service.j2

@@ -0,0 +1,40 @@
+[Unit]
+Description=CRI Interface for Docker Application Container Engine
+Documentation=https://docs.mirantis.com
+After=network-online.target firewalld.service docker.service
+Wants=network-online.target docker.service
+Requires=cri-dockerd.socket
+
+[Service]
+Type=notify
+ExecStart={{ bin_dir }}/cri-dockerd --container-runtime-endpoint {{ cri_socket }} --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --network-plugin=cni --pod-cidr={{ kube_pods_subnet }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_version }} {% if enable_dual_stack_networks %}--ipv6-dual-stack=True{% endif %}
+
+ExecReload=/bin/kill -s HUP $MAINPID
+TimeoutSec=0
+RestartSec=2
+Restart=always
+
+# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
+# Both the old, and new location are accepted by systemd 229 and up, so using the old location
+# to make them work for either version of systemd.
+StartLimitBurst=3
+
+# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
+# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
+# this option work for either version of systemd.
+StartLimitInterval=60s
+
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNOFILE=infinity
+LimitNPROC=infinity
+LimitCORE=infinity
+
+# Comment TasksMax if your systemd version does not support it.
+# Only systemd 226 and above support this option.
+TasksMax=infinity
+Delegate=yes
+KillMode=process
+
+[Install]
+WantedBy=multi-user.target

ansible/kubespray/roles/container-engine/cri-dockerd/templates/cri-dockerd.socket.j2

@@ -0,0 +1,12 @@
+[Unit]
+Description=CRI Docker Socket for the API
+PartOf=cri-dockerd.service
+
+[Socket]
+ListenStream=%t/cri-dockerd.sock
+SocketMode=0660
+SocketUser=root
+SocketGroup=docker
+
+[Install]
+WantedBy=sockets.target

ansible/kubespray/roles/container-engine/cri-o/defaults/main.yml

@@ -0,0 +1,103 @@
+---
+
+crio_cgroup_manager: "{{ kubelet_cgroup_driver | default('systemd') }}"
+crio_conmon: "{{ bin_dir }}/conmon"
+crio_enable_metrics: false
+crio_log_level: "info"
+crio_metrics_port: "9090"
+crio_pause_image: "{{ pod_infra_image_repo }}:{{ pod_infra_version }}"
+
+# Registries defined within cri-o.
+# By default unqualified images are not allowed for security reasons
+crio_registries: []
+#  - prefix: docker.io
+#    insecure: false
+#    blocked: false
+#    location: registry-1.docker.io ## REQUIRED
+#    unqualified: false
+#    mirrors:
+#      - location: 172.20.100.52:5000
+#        insecure: true
+#      - location: mirror.gcr.io
+#        insecure: false
+
+crio_registry_auth: []
+#  - registry: 10.0.0.2:5000
+#    username: user
+#    password: pass
+
+crio_seccomp_profile: ""
+crio_selinux: "{{ (preinstall_selinux_state == 'enforcing')|lower }}"
+crio_signature_policy: "{% if ansible_os_family == 'ClearLinux' %}/usr/share/defaults/crio/policy.json{% endif %}"
+
+# Override system default for storage driver
+# crio_storage_driver: "overlay"
+
+crio_stream_port: "10010"
+
+crio_required_version: "{{ kube_version | regex_replace('^v(?P<major>\\d+).(?P<minor>\\d+).(?P<patch>\\d+)$', '\\g<major>.\\g<minor>') }}"
+
+# The crio_runtimes variable defines a list of OCI compatible runtimes.
+crio_runtimes:
+  - name: runc
+    path: "{{ bin_dir }}/runc"
+    type: oci
+    root: /run/runc
+
+# Kata Containers is an OCI runtime, where containers are run inside lightweight
+# VMs. Kata provides additional isolation towards the host, minimizing the host attack
+# surface and mitigating the consequences of containers breakout.
+kata_runtimes:
+  # Kata Containers with the default configured VMM
+  - name: kata-qemu
+    path: /usr/local/bin/containerd-shim-kata-qemu-v2
+    type: vm
+    root: /run/kata-containers
+    privileged_without_host_devices: true
+
+# crun is a fast and low-memory footprint OCI Container Runtime fully written in C.
+crun_runtime:
+  name: crun
+  path: "{{ bin_dir }}/crun"
+  type: oci
+  root: /run/crun
+
+# youki is an implementation of the OCI runtime-spec in Rust, similar to runc.
+youki_runtime:
+  name: youki
+  path: "{{ youki_bin_dir }}/youki"
+  type: oci
+  root: /run/youki
+
+# TODO(cristicalin): remove this after 2.21
+crio_download_base: "download.opensuse.org/repositories/devel:kubic:libcontainers:stable"
+crio_download_crio: "http://{{ crio_download_base }}:/cri-o:/"
+
+# Configure the cri-o pids limit, increase this for heavily multi-threaded workloads
+# see https://github.com/cri-o/cri-o/issues/1921
+crio_pids_limit: 1024
+
+# Reserve 16M uids and gids for user namespaces (256 pods * 65536 uids/gids)
+# at the end of the uid/gid space
+crio_remap_enable: false
+crio_remap_user: containers
+crio_subuid_start: 2130706432
+crio_subuid_length: 16777216
+crio_subgid_start: 2130706432
+crio_subgid_length: 16777216
+
+# cri-o binary files
+crio_bin_files:
+  - conmon
+  - crio
+  - crio-status
+  - pinns
+
+# cri-o manual files
+crio_man_files:
+  5:
+    - crio.conf
+    - crio.conf.d
+  8:
+    - crio
+    - crio-status

ansible/kubespray/roles/container-engine/cri-o/files/mounts.conf

@@ -0,0 +1 @@
+/usr/share/rhel/secrets:/run/secrets

ansible/kubespray/roles/container-engine/cri-o/handlers/main.yml

@@ -0,0 +1,16 @@
+---
+- name: restart crio
+  command: /bin/true
+  notify:
+    - CRI-O | reload systemd
+    - CRI-O | reload crio
+
+- name: CRI-O | reload systemd
+  systemd:
+    daemon_reload: true
+
+- name: CRI-O | reload crio
+  service:
+    name: crio
+    state: restarted
+    enabled: yes

ansible/kubespray/roles/container-engine/cri-o/meta/main.yml

@@ -0,0 +1,5 @@
+---
+dependencies:
+  - role: container-engine/crictl
+  - role: container-engine/runc
+  - role: container-engine/skopeo

ansible/kubespray/roles/container-engine/cri-o/molecule/default/converge.yml

@@ -0,0 +1,9 @@
+---
+- name: Converge
+  hosts: all
+  become: true
+  vars:
+    container_manager: crio
+  roles:
+    - role: kubespray-defaults
+    - role: container-engine/cri-o

ansible/kubespray/roles/container-engine/cri-o/molecule/default/files/10-mynet.conf

@@ -0,0 +1,17 @@
+{
+  "cniVersion": "0.2.0",
+  "name": "mynet",
+  "type": "bridge",
+  "bridge": "cni0",
+  "isGateway": true,
+  "ipMasq": true,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "172.19.0.0/24",
+    "routes": [
+      {
+        "dst": "0.0.0.0/0"
+      }
+    ]
+  }
+}

ansible/kubespray/roles/container-engine/cri-o/molecule/default/files/container.json

@@ -0,0 +1,10 @@
+{
+  "metadata": {
+    "name": "runc1"
+  },
+  "image": {
+    "image": "quay.io/kubespray/hello-world:latest"
+  },
+  "log_path": "runc1.0.log",
+  "linux": {}
+}

ansible/kubespray/roles/container-engine/cri-o/molecule/default/files/sandbox.json

@@ -0,0 +1,10 @@
+{
+  "metadata": {
+    "name": "runc1",
+    "namespace": "default",
+    "attempt": 1,
+    "uid": "hdishd83djaidwnduwk28bcsb"
+  },
+  "linux": {},
+  "log_directory": "/tmp"
+}

ansible/kubespray/roles/container-engine/cri-o/molecule/default/molecule.yml

@@ -0,0 +1,57 @@
+---
+driver:
+  name: vagrant
+  provider:
+    name: libvirt
+lint: |
+  set -e
+  yamllint -c ../../../.yamllint .
+platforms:
+  - name: ubuntu20
+    box: generic/ubuntu2004
+    cpus: 2
+    memory: 1024
+    groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+  - name: almalinux8
+    box: almalinux/8
+    cpus: 2
+    memory: 1024
+    groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+  - name: fedora
+    box: fedora/36-cloud-base
+    cpus: 2
+    memory: 1024
+    groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+  - name: debian10
+    box: generic/debian10
+    cpus: 2
+    memory: 1024
+    groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_ROLES_PATH: ../../../../
+  config_options:
+    defaults:
+      callbacks_enabled: profile_tasks
+      timeout: 120
+  lint:
+    name: ansible-lint
+    options:
+      c: ../../../.ansible-lint
+verifier:
+  name: testinfra
+  lint:
+    name: flake8

ansible/kubespray/roles/container-engine/cri-o/molecule/default/prepare.yml

@@ -0,0 +1,52 @@
+---
+- name: Prepare
+  hosts: all
+  gather_facts: False
+  become: true
+  vars:
+    ignore_assert_errors: true
+  roles:
+    - role: kubespray-defaults
+    - role: bootstrap-os
+    - role: kubernetes/preinstall
+    - role: adduser
+      user: "{{ addusers.kube }}"
+  tasks:
+    - include_tasks: "../../../../download/tasks/download_file.yml"
+      vars:
+        download: "{{ download_defaults | combine(downloads.cni) }}"
+
+- name: Prepare CNI
+  hosts: all
+  gather_facts: False
+  become: true
+  vars:
+    ignore_assert_errors: true
+    kube_network_plugin: cni
+  roles:
+    - role: kubespray-defaults
+    - role: network_plugin/cni
+  tasks:
+    - name: Copy test container files
+      copy:
+        src: "{{ item }}"
+        dest: "/tmp/{{ item }}"
+        owner: root
+        mode: 0644
+      with_items:
+        - container.json
+        - sandbox.json
+    - name: Create /etc/cni/net.d directory
+      file:
+        path: /etc/cni/net.d
+        state: directory
+        owner: "{{ kube_owner }}"
+        mode: 0755
+    - name: Setup CNI
+      copy:
+        src: "{{ item }}"
+        dest: "/etc/cni/net.d/{{ item }}"
+        owner: root
+        mode: 0644
+      with_items:
+        - 10-mynet.conf

ansible/kubespray/roles/container-engine/cri-o/molecule/default/tests/test_default.py

@@ -0,0 +1,35 @@
+import os
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_service(host):
+    svc = host.service("crio")
+    assert svc.is_running
+    assert svc.is_enabled
+
+
+def test_run(host):
+    crictl = "/usr/local/bin/crictl"
+    path = "unix:///var/run/crio/crio.sock"
+    with host.sudo():
+        cmd = host.command(crictl + " --runtime-endpoint " + path + " version")
+    assert cmd.rc == 0
+    assert "RuntimeName:  cri-o" in cmd.stdout
+
+def test_run_pod(host):
+    runtime = "runc"
+
+    run_command = "/usr/local/bin/crictl run --with-pull --runtime {} /tmp/container.json /tmp/sandbox.json".format(runtime)
+    with host.sudo():
+        cmd = host.command(run_command)
+    assert cmd.rc == 0
+
+    with host.sudo():
+      log_f = host.file("/tmp/runc1.0.log")
+
+      assert log_f.exists
+      assert b"Hello from Docker" in log_f.content

ansible/kubespray/roles/container-engine/cri-o/tasks/cleanup.yaml

@@ -0,0 +1,119 @@
+---
+# TODO(cristicalin): drop this file after 2.21
+- name: CRI-O kubic repo name for debian os family
+  set_fact:
+    crio_kubic_debian_repo_name: "{{ ((ansible_distribution == 'Ubuntu') | ternary('x','')) ~ ansible_distribution ~ '_' ~ ansible_distribution_version }}"
+  when: ansible_os_family == "Debian"
+
+- name: Remove legacy CRI-O kubic apt repo key
+  apt_key:
+    url: "https://{{ crio_download_base }}/{{ crio_kubic_debian_repo_name }}/Release.key"
+    state: absent
+  when: crio_kubic_debian_repo_name is defined
+
+- name: Remove legacy CRI-O kubic apt repo
+  apt_repository:
+    repo: "deb http://{{ crio_download_base }}/{{ crio_kubic_debian_repo_name }}/ /"
+    state: absent
+    filename: devel-kubic-libcontainers-stable
+  when: crio_kubic_debian_repo_name is defined
+
+- name: Remove legacy CRI-O kubic cri-o apt repo
+  apt_repository:
+    repo: "deb {{ crio_download_crio }}{{ crio_version }}/{{ crio_kubic_debian_repo_name }}/ /"
+    state: absent
+    filename: devel-kubic-libcontainers-stable-cri-o
+  when: crio_kubic_debian_repo_name is defined
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: devel_kubic_libcontainers_stable
+    description: Stable Releases of Upstream github.com/containers packages (CentOS_$releasever)
+    baseurl: http://{{ crio_download_base }}/CentOS_{{ ansible_distribution_major_version }}/
+    state: absent
+  when:
+    - ansible_os_family == "RedHat"
+    - ansible_distribution not in ["Amazon", "Fedora"]
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: "devel_kubic_libcontainers_stable_cri-o_{{ crio_version }}"
+    description: "CRI-O {{ crio_version }} (CentOS_$releasever)"
+    baseurl: "{{ crio_download_crio }}{{ crio_version }}/CentOS_{{ ansible_distribution_major_version }}/"
+    state: absent
+  when:
+    - ansible_os_family == "RedHat"
+    - ansible_distribution not in ["Amazon", "Fedora"]
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: devel_kubic_libcontainers_stable
+    description: Stable Releases of Upstream github.com/containers packages
+    baseurl: http://{{ crio_download_base }}/Fedora_{{ ansible_distribution_major_version }}/
+    state: absent
+  when:
+    - ansible_distribution in ["Fedora"]
+    - not is_ostree
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: "devel_kubic_libcontainers_stable_cri-o_{{ crio_version }}"
+    description: "CRI-O {{ crio_version }}"
+    baseurl: "{{ crio_download_crio }}{{ crio_version }}/Fedora_{{ ansible_distribution_major_version }}/"
+    state: absent
+  when:
+    - ansible_distribution in ["Fedora"]
+    - not is_ostree
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: devel_kubic_libcontainers_stable
+    description: Stable Releases of Upstream github.com/containers packages
+    baseurl: http://{{ crio_download_base }}/CentOS_7/
+    state: absent
+  when: ansible_distribution in ["Amazon"]
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: "devel_kubic_libcontainers_stable_cri-o_{{ crio_version }}"
+    description: "CRI-O {{ crio_version }}"
+    baseurl: "{{ crio_download_crio }}{{ crio_version }}/CentOS_7/"
+    state: absent
+  when: ansible_distribution in ["Amazon"]
+
+- name: Disable modular repos for CRI-O
+  ini_file:
+    path: "/etc/yum.repos.d/{{ item.repo }}.repo"
+    section: "{{ item.section }}"
+    option: enabled
+    value: 0
+    mode: 0644
+  become: true
+  when: is_ostree
+  loop:
+    - repo: "fedora-updates-modular"
+      section: "updates-modular"
+    - repo: "fedora-modular"
+      section: "fedora-modular"
+
+# Disable any older module version if we enabled them before
+- name: Disable CRI-O ex module
+  command: "rpm-ostree ex module disable cri-o:{{ item }}"
+  become: true
+  when:
+    - is_ostree
+    - ostree_version is defined and ostree_version.stdout is version('2021.9', '>=')
+  with_items:
+    - 1.22
+    - 1.23
+    - 1.24
+
+- name: cri-o | remove installed packages
+  package:
+    name: "{{ item }}"
+    state: absent
+  when: not is_ostree
+  with_items:
+    - cri-o
+    - cri-o-runc
+    - oci-systemd-hook

ansible/kubespray/roles/container-engine/cri-o/tasks/main.yaml

@@ -0,0 +1,206 @@
+---
+- name: cri-o | check if fedora coreos
+  stat:
+    path: /run/ostree-booted
+    get_attributes: no
+    get_checksum: no
+    get_mime: no
+  register: ostree
+
+- name: cri-o | set is_ostree
+  set_fact:
+    is_ostree: "{{ ostree.stat.exists }}"
+
+- name: cri-o | get ostree version
+  shell: "set -o pipefail && rpm-ostree --version | awk -F\\' '/Version/{print $2}'"
+  args:
+    executable: /bin/bash
+  register: ostree_version
+  when: is_ostree
+
+- name: cri-o | Download cri-o
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.crio) }}"
+
+- name: cri-o | special handling for amazon linux
+  import_tasks: "setup-amazon.yaml"
+  when: ansible_distribution in ["Amazon"]
+
+- name: cri-o | clean up reglacy repos
+  import_tasks: "cleanup.yaml"
+
+- name: cri-o | build a list of crio runtimes with Katacontainers runtimes
+  set_fact:
+    crio_runtimes: "{{ crio_runtimes + kata_runtimes  }}"
+  when:
+    - kata_containers_enabled
+
+- name: cri-o | build a list of crio runtimes with crun runtime
+  set_fact:
+    crio_runtimes: "{{ crio_runtimes + [crun_runtime] }}"
+  when:
+    - crun_enabled
+
+- name: cri-o | build a list of crio runtimes with youki runtime
+  set_fact:
+    crio_runtimes: "{{ crio_runtimes + [youki_runtime] }}"
+  when:
+    - youki_enabled
+
+- name: cri-o | make sure needed folders exist in the system
+  with_items:
+    - /etc/crio
+    - /etc/containers
+    - /etc/systemd/system/crio.service.d
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: 0755
+
+- name: cri-o | install cri-o config
+  template:
+    src: crio.conf.j2
+    dest: /etc/crio/crio.conf
+    mode: 0644
+  register: config_install
+
+- name: cri-o | install config.json
+  template:
+    src: config.json.j2
+    dest: /etc/crio/config.json
+    mode: 0644
+  register: reg_auth_install
+
+- name: cri-o | copy binaries
+  copy:
+    src: "{{ local_release_dir }}/cri-o/bin/{{ item }}"
+    dest: "{{ bin_dir }}/{{ item }}"
+    mode: 0755
+    remote_src: true
+  with_items:
+    - "{{ crio_bin_files }}"
+  notify: restart crio
+
+- name: cri-o | copy service file
+  copy:
+    src: "{{ local_release_dir }}/cri-o/contrib/crio.service"
+    dest: /etc/systemd/system/crio.service
+    mode: 0755
+    remote_src: true
+  notify: restart crio
+
+- name: cri-o | copy default policy
+  copy:
+    src: "{{ local_release_dir }}/cri-o/contrib/policy.json"
+    dest: /etc/containers/policy.json
+    mode: 0755
+    remote_src: true
+  notify: restart crio
+
+- name: cri-o | copy mounts.conf
+  copy:
+    src: mounts.conf
+    dest: /etc/containers/mounts.conf
+    mode: 0644
+  when:
+    - ansible_os_family == 'RedHat'
+  notify: restart crio
+
+- name: cri-o | create directory for oci hooks
+  file:
+    path: /etc/containers/oci/hooks.d
+    state: directory
+    owner: root
+    mode: 0755
+
+- name: cri-o | set overlay driver
+  ini_file:
+    dest: /etc/containers/storage.conf
+    section: storage
+    option: "{{ item.option }}"
+    value: "{{ item.value }}"
+    mode: 0644
+  with_items:
+    - option: driver
+      value: '"overlay"'
+    - option: graphroot
+      value: '"/var/lib/containers/storage"'
+
+# metacopy=on is available since 4.19 and was backported to RHEL 4.18 kernel
+- name: cri-o | set metacopy mount options correctly
+  ini_file:
+    dest: /etc/containers/storage.conf
+    section: storage.options.overlay
+    option: mountopt
+    value: '{{ ''"nodev"'' if ansible_kernel is version_compare(("4.18" if ansible_os_family == "RedHat" else "4.19"), "<") else ''"nodev,metacopy=on"'' }}'
+    mode: 0644
+
+- name: cri-o | create directory registries configs
+  file:
+    path: /etc/containers/registries.conf.d
+    state: directory
+    owner: root
+    mode: 0755
+
+- name: cri-o | write registries configs
+  template:
+    src: registry.conf.j2
+    dest: "/etc/containers/registries.conf.d/10-{{ item.prefix | default(item.location) | regex_replace(':', '_') }}.conf"
+    mode: 0644
+  loop: "{{ crio_registries }}"
+  notify: restart crio
+
+- name: cri-o | configure unqualified registry settings
+  template:
+    src: unqualified.conf.j2
+    dest: "/etc/containers/registries.conf.d/01-unqualified.conf"
+    mode: 0644
+  notify: restart crio
+
+- name: cri-o | write cri-o proxy drop-in
+  template:
+    src: http-proxy.conf.j2
+    dest: /etc/systemd/system/crio.service.d/http-proxy.conf
+    mode: 0644
+  notify: restart crio
+  when: http_proxy is defined or https_proxy is defined
+
+- name: cri-o | configure the uid/gid space for user namespaces
+  lineinfile:
+    path: '{{ item.path }}'
+    line: '{{ item.entry }}'
+    regex: '^\s*{{ crio_remap_user }}:'
+    state: '{{ "present" if crio_remap_enable | bool else "absent" }}'
+  loop:
+    - path: /etc/subuid
+      entry: '{{ crio_remap_user }}:{{ crio_subuid_start }}:{{ crio_subuid_length }}'
+    - path: /etc/subgid
+      entry: '{{ crio_remap_user }}:{{ crio_subgid_start }}:{{ crio_subgid_length }}'
+  loop_control:
+    label: '{{ item.path }}'
+
+- name: cri-o | ensure crio service is started and enabled
+  service:
+    name: crio
+    daemon_reload: true
+    enabled: true
+    state: started
+  register: service_start
+
+- name: cri-o | trigger service restart only when needed
+  service:  # noqa 503
+    name: crio
+    state: restarted
+  when:
+    - config_install.changed
+    - reg_auth_install.changed
+    - not service_start.changed
+
+- name: cri-o | verify that crio is running
+  command: "{{ bin_dir }}/crio-status info"
+  register: get_crio_info
+  until: get_crio_info is succeeded
+  changed_when: false
+  retries: 5
+  delay: "{{ retry_stagger | random + 3 }}"

ansible/kubespray/roles/container-engine/cri-o/tasks/reset.yml

@@ -0,0 +1,101 @@
+---
+- name: CRI-O | Kubic repo name for debian os family
+  set_fact:
+    crio_kubic_debian_repo_name: "{{ ((ansible_distribution == 'Ubuntu') | ternary('x','')) ~ ansible_distribution ~ '_' ~ ansible_distribution_version }}"
+  when: ansible_os_family == "Debian"
+  tags:
+    - reset_crio
+
+- name: CRI-O | Remove kubic apt repo
+  apt_repository:
+    repo: "deb http://{{ crio_download_base }}/{{ crio_kubic_debian_repo_name }}/ /"
+    state: absent
+  when: crio_kubic_debian_repo_name is defined
+  tags:
+    - reset_crio
+
+- name: CRI-O | Remove cri-o apt repo
+  apt_repository:
+    repo: "deb {{ crio_download_crio }}{{ crio_version }}/{{ crio_kubic_debian_repo_name }}/ /"
+    state: present
+    filename: devel-kubic-libcontainers-stable-cri-o
+  when: crio_kubic_debian_repo_name is defined
+  tags:
+    - reset_crio
+
+- name: CRI-O | Remove CRI-O kubic yum repo
+  yum_repository:
+    name: devel_kubic_libcontainers_stable
+    state: absent
+  when: ansible_distribution in ["Amazon"]
+  tags:
+    - reset_crio
+
+- name: CRI-O | Remove CRI-O kubic yum repo
+  yum_repository:
+    name: "devel_kubic_libcontainers_stable_cri-o_{{ crio_version }}"
+    state: absent
+  when:
+    - ansible_os_family == "RedHat"
+    - ansible_distribution not in ["Amazon", "Fedora"]
+  tags:
+    - reset_crio
+
+- name: CRI-O | Run yum-clean-metadata
+  command: yum clean metadata
+  args:
+    warn: no
+  when:
+    - ansible_os_family == "RedHat"
+  tags:
+    - reset_crio
+
+- name: CRI-O | Remove crictl
+  file:
+    name: "{{ item }}"
+    state: absent
+  loop:
+    - /etc/crictl.yaml
+    - "{{ bin_dir }}/crictl"
+  tags:
+    - reset_crio
+
+- name: CRI-O | Stop crio service
+  service:
+    name: crio
+    daemon_reload: true
+    enabled: false
+    masked: true
+    state: stopped
+  tags:
+    - reset_crio
+
+- name: CRI-O | Remove CRI-O configuration files
+  file:
+    name: "{{ item }}"
+    state: absent
+  loop:
+    - /etc/crio
+    - /etc/containers
+    - /etc/systemd/system/crio.service.d
+  tags:
+    - reset_crio
+
+- name: CRI-O | Remove dpkg hold
+  dpkg_selections:
+    name: "{{ item }}"
+    selection: install
+  when: ansible_pkg_mgr == 'apt'
+  changed_when: false
+  with_items: "{{ crio_packages }}"
+  tags:
+    - reset_crio
+
+- name: CRI-O | Uninstall CRI-O package
+  package:
+    name: "{{ item }}"
+    state: absent
+  when: not is_ostree
+  with_items: "{{ crio_packages }}"
+  tags:
+    - reset_crio

ansible/kubespray/roles/container-engine/cri-o/tasks/setup-amazon.yaml

@@ -0,0 +1,38 @@
+---
+- name: Check that amzn2-extras.repo exists
+  stat:
+    path: /etc/yum.repos.d/amzn2-extras.repo
+  register: amzn2_extras_file_stat
+
+- name: Find docker repo in amzn2-extras.repo file
+  lineinfile:
+    dest: /etc/yum.repos.d/amzn2-extras.repo
+    line: "[amzn2extra-docker]"
+  check_mode: yes
+  register: amzn2_extras_docker_repo
+  when:
+    - amzn2_extras_file_stat.stat.exists
+
+- name: Remove docker repository
+  ini_file:
+    dest: /etc/yum.repos.d/amzn2-extras.repo
+    section: amzn2extra-docker
+    option: enabled
+    value: "0"
+    backup: yes
+    mode: 0644
+  when:
+    - amzn2_extras_file_stat.stat.exists
+    - not amzn2_extras_docker_repo.changed
+
+- name: Add container-selinux yum repo
+  yum_repository:
+    name: copr:copr.fedorainfracloud.org:lsm5:container-selinux
+    file: _copr_lsm5-container-selinux.repo
+    description: Copr repo for container-selinux owned by lsm5
+    baseurl: https://download.copr.fedorainfracloud.org/results/lsm5/container-selinux/epel-7-$basearch/
+    gpgcheck: yes
+    gpgkey: https://download.copr.fedorainfracloud.org/results/lsm5/container-selinux/pubkey.gpg
+    skip_if_unavailable: yes
+    enabled: yes
+    repo_gpgcheck: no

ansible/kubespray/roles/container-engine/cri-o/templates/config.json.j2

@@ -0,0 +1,17 @@
+{% if crio_registry_auth is defined and crio_registry_auth|length %}
+{ 
+{% for reg in crio_registry_auth %}
+  "auths": {
+    "{{ reg.registry }}": {
+      "auth": "{{ (reg.username + ':' + reg.password) | string | b64encode }}"
+    }
+{% if not loop.last %}
+  },
+{% else %}
+  }
+{% endif %}
+{% endfor %}
+}
+{% else %}
+{}
+{% endif %}

ansible/kubespray/roles/container-engine/cri-o/templates/crio.conf.j2

@@ -0,0 +1,369 @@
+# The CRI-O configuration file specifies all of the available configuration
+# options and command-line flags for the crio(8) OCI Kubernetes Container Runtime
+# daemon, but in a TOML format that can be more easily modified and versioned.
+#
+# Please refer to crio.conf(5) for details of all configuration options.
+
+# CRI-O supports partial configuration reload during runtime, which can be
+# done by sending SIGHUP to the running process. Currently supported options
+# are explicitly mentioned with: 'This option supports live configuration
+# reload'.
+
+# CRI-O reads its storage defaults from the containers-storage.conf(5) file
+# located at /etc/containers/storage.conf. Modify this storage configuration if
+# you want to change the system's defaults. If you want to modify storage just
+# for CRI-O, you can change the storage configuration options here.
+[crio]
+
+# Path to the "root directory". CRI-O stores all of its data, including
+# containers images, in this directory.
+root = "/var/lib/containers/storage"
+
+# Path to the "run directory". CRI-O stores all of its state in this directory.
+runroot = "/var/run/containers/storage"
+
+# Storage driver used to manage the storage of images and containers. Please
+# refer to containers-storage.conf(5) to see all available storage drivers.
+{% if crio_storage_driver is defined %}
+storage_driver = "{{ crio_storage_driver }}"
+{% endif %}
+
+# List to pass options to the storage driver. Please refer to
+# containers-storage.conf(5) to see all available storage options.
+#storage_option = [
+#]
+
+# The default log directory where all logs will go unless directly specified by
+# the kubelet. The log directory specified must be an absolute directory.
+log_dir = "/var/log/crio/pods"
+
+# Location for CRI-O to lay down the temporary version file.
+# It is used to check if crio wipe should wipe containers, which should
+# always happen on a node reboot
+version_file = "/var/run/crio/version"
+
+# Location for CRI-O to lay down the persistent version file.
+# It is used to check if crio wipe should wipe images, which should
+# only happen when CRI-O has been upgraded
+version_file_persist = "/var/lib/crio/version"
+
+# The crio.api table contains settings for the kubelet/gRPC interface.
+[crio.api]
+
+# Path to AF_LOCAL socket on which CRI-O will listen.
+listen = "/var/run/crio/crio.sock"
+
+# IP address on which the stream server will listen.
+stream_address = "127.0.0.1"
+
+# The port on which the stream server will listen. If the port is set to "0", then
+# CRI-O will allocate a random free port number.
+stream_port = "{{ crio_stream_port }}"
+
+# Enable encrypted TLS transport of the stream server.
+stream_enable_tls = false
+
+# Path to the x509 certificate file used to serve the encrypted stream. This
+# file can change, and CRI-O will automatically pick up the changes within 5
+# minutes.
+stream_tls_cert = ""
+
+# Path to the key file used to serve the encrypted stream. This file can
+# change and CRI-O will automatically pick up the changes within 5 minutes.
+stream_tls_key = ""
+
+# Path to the x509 CA(s) file used to verify and authenticate client
+# communication with the encrypted stream. This file can change and CRI-O will
+# automatically pick up the changes within 5 minutes.
+stream_tls_ca = ""
+
+# Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 16 * 1024 * 1024.
+grpc_max_send_msg_size = 16777216
+
+# Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 16 * 1024 * 1024.
+grpc_max_recv_msg_size = 16777216
+
+# The crio.runtime table contains settings pertaining to the OCI runtime used
+# and options for how to set up and manage the OCI runtime.
+[crio.runtime]
+
+# A list of ulimits to be set in containers by default, specified as
+# "<ulimit name>=<soft limit>:<hard limit>", for example:
+# "nofile=1024:2048"
+# If nothing is set here, settings will be inherited from the CRI-O daemon
+#default_ulimits = [
+#]
+
+# default_runtime is the _name_ of the OCI runtime to be used as the default.
+# The name is matched against the runtimes map below.
+default_runtime = "runc"
+
+# If true, the runtime will not use pivot_root, but instead use MS_MOVE.
+no_pivot = false
+
+# decryption_keys_path is the path where the keys required for
+# image decryption are stored. This option supports live configuration reload.
+decryption_keys_path = "/etc/crio/keys/"
+
+# Path to the conmon binary, used for monitoring the OCI runtime.
+# Will be searched for using $PATH if empty.
+conmon = "{{ crio_conmon }}"
+
+# Cgroup setting for conmon
+{% if crio_cgroup_manager == "cgroupfs" %}
+conmon_cgroup = "pod"
+{% else %}
+conmon_cgroup = "system.slice"
+{% endif %}
+
+# Environment variable list for the conmon process, used for passing necessary
+# environment variables to conmon or the runtime.
+conmon_env = [
+	"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+]
+
+# Additional environment variables to set for all the
+# containers. These are overridden if set in the
+# container image spec or in the container runtime configuration.
+default_env = [
+]
+
+# If true, SELinux will be used for pod separation on the host.
+selinux = {{ crio_selinux }}
+
+# Path to the seccomp.json profile which is used as the default seccomp profile
+# for the runtime. If not specified, then the internal default seccomp profile
+# will be used. This option supports live configuration reload.
+seccomp_profile = "{{ crio_seccomp_profile }}"
+
+# Used to change the name of the default AppArmor profile of CRI-O. The default
+# profile name is "crio-default". This profile only takes effect if the user
+# does not specify a profile via the Kubernetes Pod's metadata annotation. If
+# the profile is set to "unconfined", then this equals to disabling AppArmor.
+# This option supports live configuration reload.
+# apparmor_profile = "crio-default"
+
+# Cgroup management implementation used for the runtime.
+cgroup_manager = "{{ crio_cgroup_manager }}"
+
+# List of default capabilities for containers. If it is empty or commented out,
+# only the capabilities defined in the containers json file by the user/kube
+# will be added.
+default_capabilities = [
+	"CHOWN",
+	"DAC_OVERRIDE",
+	"FSETID",
+	"FOWNER",
+	"NET_RAW",
+	"SETGID",
+	"SETUID",
+	"SETPCAP",
+	"NET_BIND_SERVICE",
+	"SYS_CHROOT",
+	"KILL",
+]
+
+# List of default sysctls. If it is empty or commented out, only the sysctls
+# defined in the container json file by the user/kube will be added.
+default_sysctls = [
+]
+
+# List of additional devices. specified as
+# "<device-on-host>:<device-on-container>:<permissions>", for example: "--device=/dev/sdc:/dev/xvdc:rwm".
+#If it is empty or commented out, only the devices
+# defined in the container json file by the user/kube will be added.
+additional_devices = [
+]
+
+# Path to OCI hooks directories for automatically executed hooks. If one of the
+# directories does not exist, then CRI-O will automatically skip them.
+hooks_dir = [
+	"/usr/share/containers/oci/hooks.d",
+]
+
+# List of default mounts for each container. **Deprecated:** this option will
+# be removed in future versions in favor of default_mounts_file.
+default_mounts = [
+]
+
+# Path to the file specifying the defaults mounts for each container. The
+# format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads
+# its default mounts from the following two files:
+#
+#   1) /etc/containers/mounts.conf (i.e., default_mounts_file): This is the
+#      override file, where users can either add in their own default mounts, or
+#      override the default mounts shipped with the package.
+#
+#   2) /usr/share/containers/mounts.conf: This is the default file read for
+#      mounts. If you want CRI-O to read from a different, specific mounts file,
+#      you can change the default_mounts_file. Note, if this is done, CRI-O will
+#      only add mounts it finds in this file.
+#
+#default_mounts_file = ""
+
+# Maximum number of processes allowed in a container.
+pids_limit = {{ crio_pids_limit }}
+
+# Maximum sized allowed for the container log file. Negative numbers indicate
+# that no size limit is imposed. If it is positive, it must be >= 8192 to
+# match/exceed conmon's read buffer. The file is truncated and re-opened so the
+# limit is never exceeded.
+log_size_max = -1
+
+# Whether container output should be logged to journald in addition to the kuberentes log file
+log_to_journald = false
+
+# Path to directory in which container exit files are written to by conmon.
+container_exits_dir = "/var/run/crio/exits"
+
+# Path to directory for container attach sockets.
+container_attach_socket_dir = "/var/run/crio"
+
+# The prefix to use for the source of the bind mounts.
+bind_mount_prefix = ""
+
+# If set to true, all containers will run in read-only mode.
+read_only = false
+
+# Changes the verbosity of the logs based on the level it is set to. Options
+# are fatal, panic, error, warn, info, debug and trace. This option supports
+# live configuration reload.
+log_level = "{{ crio_log_level }}"
+
+# Filter the log messages by the provided regular expression.
+# This option supports live configuration reload.
+log_filter = ""
+
+# The UID mappings for the user namespace of each container. A range is
+# specified in the form containerUID:HostUID:Size. Multiple ranges must be
+# separated by comma.
+uid_mappings = ""
+
+# The GID mappings for the user namespace of each container. A range is
+# specified in the form containerGID:HostGID:Size. Multiple ranges must be
+# separated by comma.
+gid_mappings = ""
+
+# The minimal amount of time in seconds to wait before issuing a timeout
+# regarding the proper termination of the container. The lowest possible
+# value is 30s, whereas lower values are not considered by CRI-O.
+ctr_stop_timeout = 30
+
+# **DEPRECATED** this option is being replaced by manage_ns_lifecycle, which is described below.
+# manage_network_ns_lifecycle = false
+
+# manage_ns_lifecycle determines whether we pin and remove namespaces
+# and manage their lifecycle
+{% if kata_containers_enabled %}
+manage_ns_lifecycle = true
+{% else %}
+manage_ns_lifecycle = false
+{% endif %}
+
+# The directory where the state of the managed namespaces gets tracked.
+# Only used when manage_ns_lifecycle is true.
+namespaces_dir = "/var/run"
+
+# pinns_path is the path to find the pinns binary, which is needed to manage namespace lifecycle
+pinns_path = ""
+
+# The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes.
+# The runtime to use is picked based on the runtime_handler provided by the CRI.
+# If no runtime_handler is provided, the runtime will be picked based on the level
+# of trust of the workload. Each entry in the table should follow the format:
+#
+#[crio.runtime.runtimes.runtime-handler]
+#  runtime_path = "/path/to/the/executable"
+#  runtime_type = "oci"
+#  runtime_root = "/path/to/the/root"
+#
+# Where:
+# - runtime-handler: name used to identify the runtime
+# - runtime_path (optional, string): absolute path to the runtime executable in
+#   the host filesystem. If omitted, the runtime-handler identifier should match
+#   the runtime executable name, and the runtime executable should be placed
+#   in $PATH.
+# - runtime_type (optional, string): type of runtime, one of: "oci", "vm". If
+#   omitted, an "oci" runtime is assumed.
+# - runtime_root (optional, string): root directory for storage of containers
+#   state.
+
+{% for runtime in crio_runtimes %}
+[crio.runtime.runtimes.{{ runtime.name }}]
+runtime_path = "{{ runtime.path }}"
+runtime_type = "{{ runtime.type }}"
+runtime_root = "{{ runtime.root }}"
+privileged_without_host_devices = {{ runtime.privileged_without_host_devices|default(false)|lower }}
+allowed_annotations = {{ runtime.allowed_annotations|default([])|to_json }}
+{% endfor %}
+
+# Kata Containers with the Firecracker VMM
+#[crio.runtime.runtimes.kata-fc]
+
+# The crio.image table contains settings pertaining to the management of OCI images.
+#
+# CRI-O reads its configured registries defaults from the system wide
+# containers-registries.conf(5) located in /etc/containers/registries.conf. If
+# you want to modify just CRI-O, you can change the registries configuration in
+# this file. Otherwise, leave insecure_registries and registries commented out to
+# use the system's defaults from /etc/containers/registries.conf.
+[crio.image]
+
+# Default transport for pulling images from a remote container storage.
+default_transport = "docker://"
+
+# The path to a file containing credentials necessary for pulling images from
+# secure registries. The file is similar to that of /var/lib/kubelet/config.json
+global_auth_file = "/etc/crio/config.json"
+
+# The image used to instantiate infra containers.
+# This option supports live configuration reload.
+pause_image = "{{ crio_pause_image }}"
+
+# The path to a file containing credentials specific for pulling the pause_image from
+# above. The file is similar to that of /var/lib/kubelet/config.json
+# This option supports live configuration reload.
+pause_image_auth_file = ""
+
+# The command to run to have a container stay in the paused state.
+# When explicitly set to "", it will fallback to the entrypoint and command
+# specified in the pause image. When commented out, it will fallback to the
+# default: "/pause". This option supports live configuration reload.
+pause_command = "/pause"
+
+# Path to the file which decides what sort of policy we use when deciding
+# whether or not to trust an image that we've pulled. It is not recommended that
+# this option be used, as the default behavior of using the system-wide default
+# policy (i.e., /etc/containers/policy.json) is most often preferred. Please
+# refer to containers-policy.json(5) for more details.
+signature_policy = "{{ crio_signature_policy }}"
+
+# Controls how image volumes are handled. The valid values are mkdir, bind and
+# ignore; the latter will ignore volumes entirely.
+image_volumes = "mkdir"
+
+# The crio.network table containers settings pertaining to the management of
+# CNI plugins.
+[crio.network]
+
+# The default CNI network name to be selected. If not set or "", then
+# CRI-O will pick-up the first one found in network_dir.
+# cni_default_network = ""
+
+# Path to the directory where CNI configuration files are located.
+network_dir = "/etc/cni/net.d/"
+
+# Paths to directories where CNI plugin binaries are located.
+plugin_dirs = [
+	"/opt/cni/bin",
+	"/usr/libexec/cni",
+]
+
+# A necessary configuration for Prometheus based metrics retrieval
+[crio.metrics]
+
+# Globally enable or disable metrics support.
+enable_metrics = {{ crio_enable_metrics | bool | lower }}
+
+# The port on which the metrics server will listen.
+metrics_port = {{ crio_metrics_port }}

ansible/kubespray/roles/container-engine/cri-o/templates/http-proxy.conf.j2

@@ -0,0 +1,2 @@
+[Service]
+Environment={% if http_proxy is defined %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy is defined %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy is defined %}"NO_PROXY={{ no_proxy }}"{% endif %}

ansible/kubespray/roles/container-engine/cri-o/templates/registry.conf.j2

@@ -0,0 +1,13 @@
+[[registry]]
+prefix = "{{ item.prefix | default(item.location) }}"
+insecure = {{ item.insecure | default('false') | string | lower }}
+blocked = {{ item.blocked | default('false') | string | lower }}
+location = "{{ item.location }}"
+{% if item.mirrors is defined %}
+{% for mirror in item.mirrors %}
+
+[[registry.mirror]]
+location = "{{ mirror.location }}"
+insecure = {{ mirror.insecure | default('false') | string | lower }}
+{% endfor %}
+{% endif %}

ansible/kubespray/roles/container-engine/cri-o/templates/unqualified.conf.j2

@@ -0,0 +1,10 @@
+{%- set _unqualified_registries = [] -%}
+{% for _registry in crio_registries if _registry.unqualified -%}
+{% if _registry.prefix is defined -%}
+{{ _unqualified_registries.append(_registry.prefix) }}
+{% else %}
+{{ _unqualified_registries.append(_registry.location) }}
+{%- endif %}
+{%- endfor %}
+
+unqualified-search-registries = {{ _unqualified_registries | string }}

ansible/kubespray/roles/container-engine/crictl/handlers/main.yml

@@ -0,0 +1,12 @@
+---
+- name: Get crictl completion
+  command: "{{ bin_dir }}/crictl completion"
+  changed_when: False
+  register: cri_completion
+  check_mode: false
+
+- name: Install crictl completion
+  copy:
+    dest: /etc/bash_completion.d/crictl
+    content: "{{ cri_completion.stdout }}"
+    mode: 0644

ansible/kubespray/roles/container-engine/crictl/tasks/crictl.yml

@@ -0,0 +1,22 @@
+---
+- name: crictl | Download crictl
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.crictl) }}"
+
+- name: Install crictl config
+  template:
+    src: crictl.yaml.j2
+    dest: /etc/crictl.yaml
+    owner: root
+    mode: 0644
+
+- name: Copy crictl binary from download dir
+  copy:
+    src: "{{ local_release_dir }}/crictl"
+    dest: "{{ bin_dir }}/crictl"
+    mode: 0755
+    remote_src: true
+  notify:
+    - Get crictl completion
+    - Install crictl completion

ansible/kubespray/roles/container-engine/crictl/tasks/main.yml

@@ -0,0 +1,3 @@
+---
+- name: install crictĺ
+  include_tasks: crictl.yml

ansible/kubespray/roles/container-engine/crictl/templates/crictl.yaml.j2

@@ -0,0 +1,4 @@
+runtime-endpoint: {{ cri_socket }}
+image-endpoint: {{ cri_socket }}
+timeout: 30
+debug: false

ansible/kubespray/roles/container-engine/crun/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+
+crun_bin_dir: /usr/bin/

ansible/kubespray/roles/container-engine/crun/tasks/main.yml

@@ -0,0 +1,12 @@
+---
+- name: crun | Download crun binary
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.crun) }}"
+
+- name: Copy crun binary from download dir
+  copy:
+    src: "{{ local_release_dir }}/crun"
+    dest: "{{ crun_bin_dir }}/crun"
+    mode: 0755
+    remote_src: true

ansible/kubespray/roles/container-engine/docker-storage/defaults/main.yml

@@ -0,0 +1,19 @@
+---
+docker_container_storage_setup_repository: https://github.com/projectatomic/container-storage-setup.git
+docker_container_storage_setup_version: v0.6.0
+docker_container_storage_setup_profile_name: kubespray
+docker_container_storage_setup_storage_driver: devicemapper
+docker_container_storage_setup_container_thinpool: docker-pool
+# It must be define a disk path for docker_container_storage_setup_devs.
+# Otherwise docker-storage-setup will be executed incorrectly.
+# docker_container_storage_setup_devs: /dev/vdb
+docker_container_storage_setup_data_size: 40%FREE
+docker_container_storage_setup_min_data_size: 2G
+docker_container_storage_setup_chunk_size: 512K
+docker_container_storage_setup_growpart: "false"
+docker_container_storage_setup_auto_extend_pool: "yes"
+docker_container_storage_setup_pool_autoextend_threshold: 60
+docker_container_storage_setup_pool_autoextend_percent: 20
+docker_container_storage_setup_device_wait_timeout: 60
+docker_container_storage_setup_wipe_signatures: "false"
+docker_container_storage_setup_container_root_lv_size: 40%FREE

ansible/kubespray/roles/container-engine/docker-storage/files/install_container_storage_setup.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+set -e
+
+repository=${1:-https://github.com/projectatomic/container-storage-setup.git}
+version=${2:-master}
+profile_name=${3:-kubespray}
+dir=`mktemp -d`
+export GIT_DIR=$dir/.git
+export GIT_WORK_TREE=$dir
+
+git init
+git fetch --depth 1 $repository $version
+git merge FETCH_HEAD
+make -C $dir install
+rm -rf /var/lib/container-storage-setup/$profile_name $dir
+
+set +e
+
+/usr/bin/container-storage-setup create $profile_name /etc/sysconfig/docker-storage-setup && /usr/bin/container-storage-setup activate $profile_name
+# FIXME: exit status can be 1 for both fatal and non fatal errors in current release,
+# could be improved by matching error strings
+exit 0

ansible/kubespray/roles/container-engine/docker-storage/tasks/main.yml

@@ -0,0 +1,48 @@
+---
+
+- name: docker-storage-setup | install git and make
+  with_items: [git, make]
+  package:
+    pkg: "{{ item }}"
+    state: present
+
+- name: docker-storage-setup | docker-storage-setup sysconfig template
+  template:
+    src: docker-storage-setup.j2
+    dest: /etc/sysconfig/docker-storage-setup
+    mode: 0644
+
+- name: docker-storage-override-directory | docker service storage-setup override dir
+  file:
+    dest: /etc/systemd/system/docker.service.d
+    mode: 0755
+    owner: root
+    group: root
+    state: directory
+
+- name: docker-storage-override | docker service storage-setup override file
+  copy:
+    dest: /etc/systemd/system/docker.service.d/override.conf
+    content: |-
+      ### This file is managed by Ansible
+      [Service]
+      EnvironmentFile=-/etc/sysconfig/docker-storage
+
+    owner: root
+    group: root
+    mode: 0644
+
+# https://docs.docker.com/engine/installation/linux/docker-ce/centos/#install-using-the-repository
+- name: docker-storage-setup | install lvm2
+  package:
+    name: lvm2
+    state: present
+
+- name: docker-storage-setup | install and run container-storage-setup
+  become: yes
+  script: |
+    install_container_storage_setup.sh \
+      {{ docker_container_storage_setup_repository }} \
+      {{ docker_container_storage_setup_version }} \
+      {{ docker_container_storage_setup_profile_name }}
+  notify: Docker | reload systemd

ansible/kubespray/roles/container-engine/docker-storage/templates/docker-storage-setup.j2

@@ -0,0 +1,35 @@
+{%if docker_container_storage_setup_storage_driver is defined%}STORAGE_DRIVER={{docker_container_storage_setup_storage_driver}}{%endif%}
+
+{%if docker_container_storage_setup_extra_storage_options is defined%}EXTRA_STORAGE_OPTIONS={{docker_container_storage_setup_extra_storage_options}}{%endif%}
+
+{%if docker_container_storage_setup_devs is defined%}DEVS={{docker_container_storage_setup_devs}}{%endif%}
+
+{%if docker_container_storage_setup_container_thinpool is defined%}CONTAINER_THINPOOL={{docker_container_storage_setup_container_thinpool}}{%endif%}
+
+{%if docker_container_storage_setup_vg is defined%}VG={{docker_container_storage_setup_vg}}{%endif%}
+
+{%if docker_container_storage_setup_root_size is defined%}ROOT_SIZE={{docker_container_storage_setup_root_size}}{%endif%}
+
+{%if docker_container_storage_setup_data_size is defined%}DATA_SIZE={{docker_container_storage_setup_data_size}}{%endif%}
+
+{%if docker_container_storage_setup_min_data_size is defined%}MIN_DATA_SIZE={{docker_container_storage_setup_min_data_size}}{%endif%}
+
+{%if docker_container_storage_setup_chunk_size is defined%}CHUNK_SIZE={{docker_container_storage_setup_chunk_size}}{%endif%}
+
+{%if docker_container_storage_setup_growpart is defined%}GROWPART={{docker_container_storage_setup_growpart}}{%endif%}
+
+{%if docker_container_storage_setup_auto_extend_pool is defined%}AUTO_EXTEND_POOL={{docker_container_storage_setup_auto_extend_pool}}{%endif%}
+
+{%if docker_container_storage_setup_pool_autoextend_threshold is defined%}POOL_AUTOEXTEND_THRESHOLD={{docker_container_storage_setup_pool_autoextend_threshold}}{%endif%}
+
+{%if docker_container_storage_setup_pool_autoextend_percent is defined%}POOL_AUTOEXTEND_PERCENT={{docker_container_storage_setup_pool_autoextend_percent}}{%endif%}
+
+{%if docker_container_storage_setup_device_wait_timeout is defined%}DEVICE_WAIT_TIMEOUT={{docker_container_storage_setup_device_wait_timeout}}{%endif%}
+
+{%if docker_container_storage_setup_wipe_signatures is defined%}WIPE_SIGNATURES={{docker_container_storage_setup_wipe_signatures}}{%endif%}
+
+{%if docker_container_storage_setup_container_root_lv_name is defined%}CONTAINER_ROOT_LV_NAME={{docker_container_storage_setup_container_root_lv_name}}{%endif%}
+
+{%if docker_container_storage_setup_container_root_lv_size is defined%}CONTAINER_ROOT_LV_SIZE={{docker_container_storage_setup_container_root_lv_size}}{%endif%}
+
+{%if docker_container_storage_setup_container_root_lv_mount_path is defined%}CONTAINER_ROOT_LV_MOUNT_PATH={{docker_container_storage_setup_container_root_lv_mount_path}}{%endif%}

ansible/kubespray/roles/container-engine/docker/defaults/main.yml

@@ -0,0 +1,64 @@
+---
+docker_version: '20.10'
+docker_cli_version: "{{ docker_version }}"
+
+docker_package_info:
+  pkgs:
+
+docker_repo_key_info:
+  repo_keys:
+
+docker_repo_info:
+  repos:
+
+docker_cgroup_driver: systemd
+
+docker_bin_dir: "/usr/bin"
+
+# flag to enable/disable docker cleanup
+docker_orphan_clean_up: false
+
+# old docker package names to be removed
+docker_remove_packages_yum:
+  - docker
+  - docker-common
+  - docker-engine
+  - docker-selinux.noarch
+  - docker-client
+  - docker-client-latest
+  - docker-latest
+  - docker-latest-logrotate
+  - docker-logrotate
+  - docker-engine-selinux.noarch
+
+# remove podman to avoid containerd.io confliction
+podman_remove_packages_yum:
+  - podman
+
+docker_remove_packages_apt:
+  - docker
+  - docker-engine
+  - docker.io
+
+# Docker specific repos should be part of the docker role not containerd-common anymore
+# Optional values for containerd apt repo
+containerd_package_info:
+  pkgs:
+
+# Fedora docker-ce repo
+docker_fedora_repo_base_url: 'https://download.docker.com/linux/fedora/{{ ansible_distribution_major_version }}/$basearch/stable'
+docker_fedora_repo_gpgkey: 'https://download.docker.com/linux/fedora/gpg'
+
+# CentOS/RedHat docker-ce repo
+docker_rh_repo_base_url: 'https://download.docker.com/linux/centos/{{ ansible_distribution_major_version }}/$basearch/stable'
+docker_rh_repo_gpgkey: 'https://download.docker.com/linux/centos/gpg'
+
+# Ubuntu docker-ce repo
+docker_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu"
+docker_ubuntu_repo_gpgkey: 'https://download.docker.com/linux/ubuntu/gpg'
+docker_ubuntu_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'
+
+# Debian docker-ce repo
+docker_debian_repo_base_url: "https://download.docker.com/linux/debian"
+docker_debian_repo_gpgkey: 'https://download.docker.com/linux/debian/gpg'
+docker_debian_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'

ansible/kubespray/roles/container-engine/docker/files/cleanup-docker-orphans.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+list_descendants ()
+{
+  local children=$(ps -o pid= --ppid "$1")
+  for pid in $children
+  do
+    list_descendants "$pid"
+  done
+  [[ -n "$children" ]] && echo "$children"
+}
+
+shim_search="^docker-containerd-shim|^containerd-shim"
+count_shim_processes=$(pgrep -f $shim_search | wc -l)
+
+if [ ${count_shim_processes} -gt 0 ]; then
+        # Find all container pids from shims
+        orphans=$(pgrep -P $(pgrep -d ',' -f $shim_search) |\
+        # Filter out valid docker pids, leaving the orphans
+        egrep -v $(docker ps -q | xargs docker inspect --format '{{.State.Pid}}' | awk '{printf "%s%s",sep,$1; sep="|"}'))
+
+        if [[ -n "$orphans" && -n "$(ps -o ppid= $orphans)" ]]
+        then
+                # Get shim pids of orphans
+                orphan_shim_pids=$(ps -o pid= $(ps -o ppid= $orphans))
+
+                # Find all orphaned container PIDs
+                orphan_container_pids=$(for pid in $orphan_shim_pids; do list_descendants $pid; done)
+
+                # Recursively kill all child PIDs of orphan shims
+                echo -e "Killing orphan container PIDs and descendants: \n$(ps -O ppid= $orphan_container_pids)"
+                kill -9 $orphan_container_pids || true
+
+        else
+                echo "No orphaned containers found"
+        fi
+else
+        echo "The node doesn't have any shim processes."
+fi

ansible/kubespray/roles/container-engine/docker/handlers/main.yml

@@ -0,0 +1,32 @@
+---
+- name: restart docker
+  command: /bin/true
+  notify:
+    - Docker | reload systemd
+    - Docker | reload docker.socket
+    - Docker | reload docker
+    - Docker | wait for docker
+
+- name: Docker | reload systemd
+  systemd:
+    name: docker
+    daemon_reload: true
+    masked: no
+
+- name: Docker | reload docker.socket
+  service:
+    name: docker.socket
+    state: restarted
+  when: ansible_os_family in ['Flatcar', 'Flatcar Container Linux by Kinvolk'] or is_fedora_coreos
+
+- name: Docker | reload docker
+  service:
+    name: docker
+    state: restarted
+
+- name: Docker | wait for docker
+  command: "{{ docker_bin_dir }}/docker images"
+  register: docker_ready
+  retries: 20
+  delay: 1
+  until: docker_ready.rc == 0

ansible/kubespray/roles/container-engine/docker/meta/main.yml

@@ -0,0 +1,5 @@
+---
+dependencies:
+  - role: container-engine/containerd-common
+  - role: container-engine/docker-storage
+    when: docker_container_storage_setup and ansible_os_family == "RedHat"

ansible/kubespray/roles/container-engine/docker/tasks/docker_plugin.yml

@@ -0,0 +1,8 @@
+---
+- name: Install Docker plugin
+  command: docker plugin install --grant-all-permissions {{ docker_plugin | quote }}
+  when: docker_plugin is defined
+  register: docker_plugin_status
+  failed_when:
+    - docker_plugin_status.failed
+    - '"already exists" not in docker_plugin_status.stderr'

ansible/kubespray/roles/container-engine/docker/tasks/main.yml

@@ -0,0 +1,177 @@
+---
+- name: check if fedora coreos
+  stat:
+    path: /run/ostree-booted
+    get_attributes: no
+    get_checksum: no
+    get_mime: no
+  register: ostree
+
+- name: set is_ostree
+  set_fact:
+    is_ostree: "{{ ostree.stat.exists }}"
+
+- name: set docker_version for openEuler
+  set_fact:
+    docker_version: '19.03'
+  when: ansible_distribution == "openEuler"
+  tags:
+    - facts
+
+- name: gather os specific variables
+  include_vars: "{{ item }}"
+  with_first_found:
+    - files:
+        - "{{ ansible_distribution|lower }}-{{ ansible_distribution_version|lower|replace('/', '_') }}.yml"
+        - "{{ ansible_distribution|lower }}-{{ ansible_distribution_release|lower }}-{{ host_architecture }}.yml"
+        - "{{ ansible_distribution|lower }}-{{ ansible_distribution_release|lower }}.yml"
+        - "{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version|lower|replace('/', '_') }}.yml"
+        - "{{ ansible_distribution|lower }}-{{ host_architecture }}.yml"
+        - "{{ ansible_distribution|lower }}.yml"
+        - "{{ ansible_distribution.split(' ')[0]|lower }}.yml"
+        - "{{ ansible_os_family|lower }}-{{ ansible_distribution_major_version|lower|replace('/', '_') }}.yml"
+        - "{{ ansible_os_family|lower }}-{{ host_architecture }}.yml"
+        - "{{ ansible_os_family|lower }}.yml"
+        - defaults.yml
+      paths:
+        - ../vars
+      skip: true
+  tags:
+    - facts
+
+- name: Warn about Docker version on SUSE
+  debug:
+    msg: "SUSE distributions always install Docker from the distro repos"
+  when: ansible_pkg_mgr == 'zypper'
+
+- include_tasks: set_facts_dns.yml
+  when: dns_mode != 'none' and resolvconf_mode == 'docker_dns'
+  tags:
+    - facts
+
+- import_tasks: pre-upgrade.yml
+
+- name: ensure docker-ce repository public key is installed
+  apt_key:
+    id: "{{ item }}"
+    url: "{{ docker_repo_key_info.url }}"
+    state: present
+  register: keyserver_task_result
+  until: keyserver_task_result is succeeded
+  retries: 4
+  delay: "{{ retry_stagger | d(3) }}"
+  with_items: "{{ docker_repo_key_info.repo_keys }}"
+  environment: "{{ proxy_env }}"
+  when: ansible_pkg_mgr == 'apt'
+
+- name: ensure docker-ce repository is enabled
+  apt_repository:
+    repo: "{{ item }}"
+    state: present
+  with_items: "{{ docker_repo_info.repos }}"
+  when: ansible_pkg_mgr == 'apt'
+
+- name: Configure docker repository on Fedora
+  template:
+    src: "fedora_docker.repo.j2"
+    dest: "{{ yum_repo_dir }}/docker.repo"
+    mode: 0644
+  when: ansible_distribution == "Fedora" and not is_ostree
+
+- name: Configure docker repository on RedHat/CentOS/OracleLinux/AlmaLinux/KylinLinux
+  template:
+    src: "rh_docker.repo.j2"
+    dest: "{{ yum_repo_dir }}/docker-ce.repo"
+    mode: 0644
+  when:
+    - ansible_os_family == "RedHat"
+    - ansible_distribution != "Fedora"
+    - not is_ostree
+
+- name: Remove dpkg hold
+  dpkg_selections:
+    name: "{{ item }}"
+    selection: install
+  when: ansible_pkg_mgr == 'apt'
+  changed_when: false
+  with_items:
+    - "{{ containerd_package }}"
+    - docker-ce
+    - docker-ce-cli
+
+- name: ensure docker packages are installed
+  package:
+    name: "{{ docker_package_info.pkgs }}"
+    state: "{{ docker_package_info.state | default('present') }}"
+  module_defaults:
+    apt:
+      update_cache: true
+    dnf:
+      enablerepo: "{{ docker_package_info.enablerepo | default(omit) }}"
+      disablerepo: "{{ docker_package_info.disablerepo | default(omit) }}"
+    yum:
+      enablerepo: "{{ docker_package_info.enablerepo | default(omit) }}"
+    zypper:
+      update_cache: true
+  register: docker_task_result
+  until: docker_task_result is succeeded
+  retries: 4
+  delay: "{{ retry_stagger | d(3) }}"
+  notify: restart docker
+  when:
+    - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
+    - not is_ostree
+    - docker_package_info.pkgs|length > 0
+
+# This is required to ensure any apt upgrade will not break kubernetes
+- name: Tell Debian hosts not to change the docker version with apt upgrade
+  dpkg_selections:
+    name: "{{ item }}"
+    selection: hold
+  when: ansible_pkg_mgr == 'apt'
+  changed_when: false
+  with_items:
+    - "{{ containerd_package }}"
+    - docker-ce
+    - docker-ce-cli
+
+- name: ensure docker started, remove our config if docker start failed and try again
+  block:
+    - name: ensure service is started if docker packages are already present
+      service:
+        name: docker
+        state: started
+      when: docker_task_result is not changed
+  rescue:
+    - debug:  # noqa unnamed-task
+        msg: "Docker start failed. Try to remove our config"
+    - name: remove kubespray generated config
+      file:
+        path: "{{ item }}"
+        state: absent
+      with_items:
+        - /etc/systemd/system/docker.service.d/http-proxy.conf
+        - /etc/systemd/system/docker.service.d/docker-options.conf
+        - /etc/systemd/system/docker.service.d/docker-dns.conf
+        - /etc/systemd/system/docker.service.d/docker-orphan-cleanup.conf
+      notify: restart docker
+
+- name: flush handlers so we can wait for docker to come up
+  meta: flush_handlers
+
+# Install each plugin using a looped include to make error handling in the included task simpler.
+- include_tasks: docker_plugin.yml
+  loop: "{{ docker_plugins }}"
+  loop_control:
+    loop_var: docker_plugin
+
+- name: Set docker systemd config
+  import_tasks: systemd.yml
+
+- name: ensure docker service is started and enabled
+  service:
+    name: "{{ item }}"
+    enabled: yes
+    state: started
+  with_items:
+    - docker

ansible/kubespray/roles/container-engine/docker/tasks/pre-upgrade.yml

@@ -0,0 +1,36 @@
+---
+- name: Remove legacy docker repo file
+  file:
+    path: "{{ yum_repo_dir }}/docker.repo"
+    state: absent
+  when:
+    - ansible_os_family == 'RedHat'
+    - not is_ostree
+
+- name: Ensure old versions of Docker are not installed. | Debian
+  apt:
+    name: '{{ docker_remove_packages_apt }}'
+    state: absent
+  when:
+    - ansible_os_family == 'Debian'
+    - (docker_versioned_pkg[docker_version | string] is search('docker-ce'))
+
+
+- name: Ensure podman not installed. | RedHat
+  package:
+    name: '{{ podman_remove_packages_yum }}'
+    state: absent
+  when:
+    - ansible_os_family == 'RedHat'
+    - (docker_versioned_pkg[docker_version | string] is search('docker-ce'))
+    - not is_ostree
+
+
+- name: Ensure old versions of Docker are not installed. | RedHat
+  package:
+    name: '{{ docker_remove_packages_yum }}'
+    state: absent
+  when:
+    - ansible_os_family == 'RedHat'
+    - (docker_versioned_pkg[docker_version | string] is search('docker-ce'))
+    - not is_ostree

ansible/kubespray/roles/container-engine/docker/tasks/reset.yml

@@ -0,0 +1,106 @@
+---
+
+- name: Docker | Get package facts
+  package_facts:
+    manager: auto
+
+- name: Docker | Find docker packages
+  set_fact:
+    docker_packages_list: "{{ ansible_facts.packages.keys() | select('search', '^docker*') }}"
+    containerd_package: "{{ ansible_facts.packages.keys() | select('search', '^containerd*') }}"
+
+- name: Docker | Stop all running container
+  shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -q | xargs -r {{ docker_bin_dir }}/docker kill"
+  args:
+    executable: /bin/bash
+  register: stop_all_containers
+  retries: 5
+  until: stop_all_containers.rc == 0
+  changed_when: true
+  delay: 5
+  ignore_errors: true  # noqa ignore-errors
+  when: docker_packages_list|length>0
+
+- name: reset | remove all containers
+  shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -aq | xargs -r docker rm -fv"
+  args:
+    executable: /bin/bash
+  register: remove_all_containers
+  retries: 4
+  until: remove_all_containers.rc == 0
+  delay: 5
+  when: docker_packages_list|length>0
+
+- name: Docker | Stop docker service
+  service:
+    name: "{{ item }}"
+    enabled: false
+    state: stopped
+  loop:
+    - docker
+    - docker.socket
+    - containerd
+  when: docker_packages_list|length>0
+
+- name: Docker | Remove dpkg hold
+  dpkg_selections:
+    name: "{{ item }}"
+    selection: install
+  when: ansible_pkg_mgr == 'apt'
+  changed_when: false
+  with_items:
+    - "{{ docker_packages_list }}"
+    - "{{ containerd_package }}"
+
+- name: Docker | Remove docker package
+  package:
+    name: "{{ item }}"
+    state: absent
+  changed_when: false
+  with_items:
+    - "{{ docker_packages_list }}"
+    - "{{ containerd_package }}"
+  when:
+    - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
+    - not is_ostree
+    - docker_packages_list|length > 0
+
+- name: Docker | ensure docker-ce repository is removed
+  apt_repository:
+    repo: "{{ item }}"
+    state: absent
+  with_items: "{{ docker_repo_info.repos }}"
+  when: ansible_pkg_mgr == 'apt'
+
+- name: Docker | Remove docker repository on Fedora
+  file:
+    name: "{{ yum_repo_dir }}/docker.repo"
+    state: absent
+  when: ansible_distribution == "Fedora" and not is_ostree
+
+- name: Docker | Remove docker repository on RedHat/CentOS/Oracle/AlmaLinux Linux
+  file:
+    name: "{{ yum_repo_dir }}/docker-ce.repo"
+    state: absent
+  when:
+    - ansible_os_family == "RedHat"
+    - ansible_distribution != "Fedora"
+    - not is_ostree
+
+- name: Docker | Remove docker configuration files
+  file:
+    name: "{{ item }}"
+    state: absent
+  loop:
+    - /etc/systemd/system/docker.service.d/
+    - /etc/systemd/system/docker.socket
+    - /etc/systemd/system/docker.service
+    - /etc/systemd/system/containerd.service
+    - /etc/systemd/system/containerd.service.d
+    - /var/lib/docker
+    - /etc/docker
+  ignore_errors: true  # noqa ignore-errors
+
+- name: Docker | systemctl daemon-reload  # noqa 503
+  systemd:
+    daemon_reload: true

ansible/kubespray/roles/container-engine/docker/tasks/set_facts_dns.yml

@@ -0,0 +1,66 @@
+---
+
+- name: set dns server for docker
+  set_fact:
+    docker_dns_servers: "{{ dns_servers }}"
+
+- name: show docker_dns_servers
+  debug:
+    msg: "{{ docker_dns_servers }}"
+
+- name: add upstream dns servers
+  set_fact:
+    docker_dns_servers: "{{ docker_dns_servers + upstream_dns_servers|default([]) }}"
+  when: dns_mode in ['coredns', 'coredns_dual']
+
+- name: add global searchdomains
+  set_fact:
+    docker_dns_search_domains: "{{ docker_dns_search_domains + searchdomains|default([]) }}"
+
+- name: check system nameservers
+  shell: set -o pipefail && grep "^nameserver" /etc/resolv.conf | sed -r 's/^nameserver\s*([^#\s]+)\s*(#.*)?/\1/'
+  args:
+    executable: /bin/bash
+  changed_when: False
+  register: system_nameservers
+  check_mode: no
+
+- name: check system search domains
+  # noqa 306 - if resolf.conf has no search domain, grep will exit 1 which would force us to add failed_when: false
+  # Therefore -o pipefail is not applicable in this specific instance
+  shell: grep "^search" /etc/resolv.conf | sed -r 's/^search\s*([^#]+)\s*(#.*)?/\1/'
+  args:
+    executable: /bin/bash
+  changed_when: False
+  register: system_search_domains
+  check_mode: no
+
+- name: add system nameservers to docker options
+  set_fact:
+    docker_dns_servers: "{{ docker_dns_servers | union(system_nameservers.stdout_lines) | unique }}"
+  when: system_nameservers.stdout
+
+- name: add system search domains to docker options
+  set_fact:
+    docker_dns_search_domains: "{{ docker_dns_search_domains | union(system_search_domains.stdout.split()|default([])) | unique }}"
+  when: system_search_domains.stdout
+
+- name: check number of nameservers
+  fail:
+    msg: "Too many nameservers. You can relax this check by set docker_dns_servers_strict=false in docker.yml and we will only use the first 3."
+  when: docker_dns_servers|length > 3 and docker_dns_servers_strict|bool
+
+- name: rtrim number of nameservers to 3
+  set_fact:
+    docker_dns_servers: "{{ docker_dns_servers[0:3] }}"
+  when: docker_dns_servers|length > 3 and not docker_dns_servers_strict|bool
+
+- name: check number of search domains
+  fail:
+    msg: "Too many search domains"
+  when: docker_dns_search_domains|length > 6
+
+- name: check length of search domains
+  fail:
+    msg: "Search domains exceeded limit of 256 characters"
+  when: docker_dns_search_domains|join(' ')|length > 256

ansible/kubespray/roles/container-engine/docker/tasks/systemd.yml

@@ -0,0 +1,68 @@
+---
+- name: Create docker service systemd directory if it doesn't exist
+  file:
+    path: /etc/systemd/system/docker.service.d
+    state: directory
+    mode: 0755
+
+- name: Write docker proxy drop-in
+  template:
+    src: http-proxy.conf.j2
+    dest: /etc/systemd/system/docker.service.d/http-proxy.conf
+    mode: 0644
+  notify: restart docker
+  when: http_proxy is defined or https_proxy is defined
+
+- name: get systemd version
+  # noqa 303 - systemctl is called intentionally here
+  shell: set -o pipefail && systemctl --version | head -n 1 | cut -d " " -f 2
+  args:
+    executable: /bin/bash
+  register: systemd_version
+  when: not is_ostree
+  changed_when: false
+  check_mode: false
+
+- name: Write docker.service systemd file
+  template:
+    src: docker.service.j2
+    dest: /etc/systemd/system/docker.service
+    mode: 0644
+  register: docker_service_file
+  notify: restart docker
+  when:
+    - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
+    - not is_fedora_coreos
+
+- name: Write docker options systemd drop-in
+  template:
+    src: docker-options.conf.j2
+    dest: "/etc/systemd/system/docker.service.d/docker-options.conf"
+    mode: 0644
+  notify: restart docker
+
+- name: Write docker dns systemd drop-in
+  template:
+    src: docker-dns.conf.j2
+    dest: "/etc/systemd/system/docker.service.d/docker-dns.conf"
+    mode: 0644
+  notify: restart docker
+  when: dns_mode != 'none' and resolvconf_mode == 'docker_dns'
+
+- name: Copy docker orphan clean up script to the node
+  copy:
+    src: cleanup-docker-orphans.sh
+    dest: "{{ bin_dir }}/cleanup-docker-orphans.sh"
+    mode: 0755
+  when: docker_orphan_clean_up | bool
+
+- name: Write docker orphan clean up systemd drop-in
+  template:
+    src: docker-orphan-cleanup.conf.j2
+    dest: "/etc/systemd/system/docker.service.d/docker-orphan-cleanup.conf"
+    mode: 0644
+  notify: restart docker
+  when: docker_orphan_clean_up | bool
+
+- name: Flush handlers
+  meta: flush_handlers

ansible/kubespray/roles/container-engine/docker/templates/docker-dns.conf.j2

@@ -0,0 +1,6 @@
+[Service]
+Environment="DOCKER_DNS_OPTIONS=\
+    {% for d in docker_dns_servers %}--dns {{ d }} {% endfor %} \
+    {% for d in docker_dns_search_domains %}--dns-search {{ d }} {% endfor %} \
+    {% for o in docker_dns_options %}--dns-opt {{ o }} {% endfor %} \
+"

ansible/kubespray/roles/container-engine/docker/templates/docker-options.conf.j2

@@ -0,0 +1,11 @@
+[Service]
+Environment="DOCKER_OPTS={{ docker_options|default('') }} --iptables={{ docker_iptables_enabled | default('false') }} \
+--exec-opt native.cgroupdriver={{ docker_cgroup_driver }} \
+{% for i in docker_insecure_registries %}--insecure-registry={{ i }} {% endfor %} \
+{% for i in docker_registry_mirrors %}--registry-mirror={{ i }} {% endfor %} \
+--data-root={{ docker_daemon_graph }} \
+{% if ansible_os_family not in ["openSUSE Leap", "openSUSE Tumbleweed", "Suse"] %}{{ docker_log_opts }}{% endif %}"
+
+{% if docker_mount_flags is defined and docker_mount_flags != "" %}
+MountFlags={{ docker_mount_flags }}
+{% endif %}

ansible/kubespray/roles/container-engine/docker/templates/docker-orphan-cleanup.conf.j2

@@ -0,0 +1,2 @@
+[Service]
+ExecStartPost=-{{ bin_dir }}/cleanup-docker-orphans.sh

ansible/kubespray/roles/container-engine/docker/templates/docker.service.j2

@@ -0,0 +1,47 @@
+[Unit]
+Description=Docker Application Container Engine
+Documentation=http://docs.docker.com
+{% if ansible_os_family == "RedHat" %}
+After=network.target {{ ' docker-storage-setup.service' if docker_container_storage_setup else '' }} containerd.service
+BindsTo=containerd.service
+{{ 'Wants=docker-storage-setup.service' if docker_container_storage_setup else '' }}
+{% elif ansible_os_family == "Debian" %}
+After=network.target docker.socket containerd.service
+BindsTo=containerd.service
+Wants=docker.socket
+{% elif ansible_os_family == "Suse" %}
+After=network.target lvm2-monitor.service SuSEfirewall2.service
+# After=network.target containerd.service
+# BindsTo=containerd.service
+{% endif %}
+
+[Service]
+Type=notify
+{% if docker_storage_options is defined %}
+Environment="DOCKER_STORAGE_OPTIONS={{ docker_storage_options }}"
+{% endif %}
+Environment=GOTRACEBACK=crash
+ExecReload=/bin/kill -s HUP $MAINPID
+Delegate=yes
+KillMode=process
+ExecStart={{ docker_bin_dir }}/dockerd \
+{% if ansible_os_family == "Suse" %}
+          --add-runtime oci=/usr/sbin/docker-runc \
+{% endif %}
+          $DOCKER_OPTS \
+          $DOCKER_STORAGE_OPTIONS \
+          $DOCKER_DNS_OPTIONS
+{% if not is_ostree and systemd_version.stdout|int >= 226 %}
+TasksMax=infinity
+{% endif %}
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitCORE=infinity
+TimeoutStartSec=1min
+# restart the docker process if it exits prematurely
+Restart=on-failure
+StartLimitBurst=3
+StartLimitInterval=60s
+
+[Install]
+WantedBy=multi-user.target

ansible/kubespray/roles/container-engine/docker/templates/fedora_docker.repo.j2

@@ -0,0 +1,7 @@
+[docker-ce]
+name=Docker-CE Repository
+baseurl={{ docker_fedora_repo_base_url }}
+enabled=1
+gpgcheck={{ '1' if docker_fedora_repo_gpgkey else '0' }}
+gpgkey={{ docker_fedora_repo_gpgkey }}
+{% if http_proxy is defined %}proxy={{ http_proxy }}{% endif %}

ansible/kubespray/roles/container-engine/docker/templates/http-proxy.conf.j2

@@ -0,0 +1,2 @@
+[Service]
+Environment={% if http_proxy is defined %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy is defined %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy is defined %}"NO_PROXY={{ no_proxy }}"{% endif %}

ansible/kubespray/roles/container-engine/docker/templates/rh_docker.repo.j2

@@ -0,0 +1,10 @@
+[docker-ce]
+name=Docker-CE Repository
+baseurl={{ docker_rh_repo_base_url }}
+enabled=0
+gpgcheck={{ '1' if docker_rh_repo_gpgkey else '0' }}
+keepcache={{ docker_rpm_keepcache | default('1') }}
+gpgkey={{ docker_rh_repo_gpgkey }}
+{% if http_proxy is defined %}
+proxy={{ http_proxy }}
+{% endif %}

ansible/kubespray/roles/container-engine/docker/vars/amazon.yml

@@ -0,0 +1,15 @@
+---
+# https://docs.aws.amazon.com/en_us/AmazonECS/latest/developerguide/docker-basics.html
+
+docker_versioned_pkg:
+  'latest': docker
+  '18.09': docker-18.09.9ce-2.amzn2
+  '19.03': docker-19.03.13ce-1.amzn2
+  '20.10': docker-20.10.7-5.amzn2
+
+docker_version: "latest"
+
+docker_package_info:
+  pkgs:
+    - "{{ docker_versioned_pkg[docker_version | string] }}"
+  enablerepo: amzn2extra-docker

ansible/kubespray/roles/container-engine/docker/vars/clearlinux.yml

@@ -0,0 +1,4 @@
+---
+docker_package_info:
+  pkgs:
+    - "containers-basic"

ansible/kubespray/roles/container-engine/docker/vars/debian-stretch.yml

@@ -0,0 +1,45 @@
+---
+# containerd versions are only relevant for docker
+containerd_versioned_pkg:
+  'latest': "{{ containerd_package }}"
+  '1.3.7': "{{ containerd_package }}=1.3.7-1"
+  '1.3.9': "{{ containerd_package }}=1.3.9-1"
+  '1.4.3': "{{ containerd_package }}=1.4.3-1"
+  'stable': "{{ containerd_package }}=1.4.3-1"
+  'edge': "{{ containerd_package }}=1.4.3-1"
+
+docker_version: 19.03
+docker_cli_version: 19.03
+
+# https://download.docker.com/linux/debian/
+docker_versioned_pkg:
+  'latest': docker-ce
+  '18.09': docker-ce=5:18.09.9~3-0~debian-{{ ansible_distribution_release|lower }}
+  '19.03': docker-ce=5:19.03.15~3-0~debian-{{ ansible_distribution_release|lower }}
+  'stable': docker-ce=5:19.03.15~3-0~debian-{{ ansible_distribution_release|lower }}
+  'edge': docker-ce=5:19.03.15~3-0~debian-{{ ansible_distribution_release|lower }}
+
+docker_cli_versioned_pkg:
+  'latest': docker-ce-cli
+  '18.09': docker-ce-cli=5:18.09.9~3-0~debian-{{ ansible_distribution_release|lower }}
+  '19.03': docker-ce-cli=5:19.03.15~3-0~debian-{{ ansible_distribution_release|lower }}
+  'stable': docker-ce-cli=5:19.03.15~3-0~debian-{{ ansible_distribution_release|lower }}
+  'edge': docker-ce-cli=5:19.03.15~3-0~debian-{{ ansible_distribution_release|lower }}
+
+docker_package_info:
+  pkgs:
+    - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}"
+    - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}"
+    - "{{ docker_versioned_pkg[docker_version | string] }}"
+
+docker_repo_key_info:
+  url: '{{ docker_debian_repo_gpgkey }}'
+  repo_keys:
+    - '{{ docker_debian_repo_repokey }}'
+
+docker_repo_info:
+  repos:
+    - >
+      deb {{ docker_debian_repo_base_url }}
+      {{ ansible_distribution_release|lower }}
+      stable

ansible/kubespray/roles/container-engine/docker/vars/debian.yml

@@ -0,0 +1,49 @@
+---
+# containerd package info is only relevant for docker
+containerd_versioned_pkg:
+  'latest': "{{ containerd_package }}"
+  '1.3.7': "{{ containerd_package }}=1.3.7-1"
+  '1.3.9': "{{ containerd_package }}=1.3.9-1"
+  '1.4.3': "{{ containerd_package }}=1.4.3-2"
+  '1.4.4': "{{ containerd_package }}=1.4.4-1"
+  '1.4.6': "{{ containerd_package }}=1.4.6-1"
+  '1.4.9': "{{ containerd_package }}=1.4.9-1"
+  '1.4.12': "{{ containerd_package }}=1.4.12-1"
+  '1.6.4': "{{ containerd_package }}=1.6.4-1"
+  'stable': "{{ containerd_package }}=1.6.4-1"
+  'edge': "{{ containerd_package }}=1.6.4-1"
+
+# https://download.docker.com/linux/debian/
+docker_versioned_pkg:
+  'latest': docker-ce
+  '18.09': docker-ce=5:18.09.9~3-0~debian-{{ ansible_distribution_release|lower }}
+  '19.03': docker-ce=5:19.03.15~3-0~debian-{{ ansible_distribution_release|lower }}
+  '20.10': docker-ce=5:20.10.20~3-0~debian-{{ ansible_distribution_release|lower }}
+  'stable': docker-ce=5:20.10.20~3-0~debian-{{ ansible_distribution_release|lower }}
+  'edge': docker-ce=5:20.10.20~3-0~debian-{{ ansible_distribution_release|lower }}
+
+docker_cli_versioned_pkg:
+  'latest': docker-ce-cli
+  '18.09': docker-ce-cli=5:18.09.9~3-0~debian-{{ ansible_distribution_release|lower }}
+  '19.03': docker-ce-cli=5:19.03.15~3-0~debian-{{ ansible_distribution_release|lower }}
+  '20.10': docker-ce-cli=5:20.10.20~3-0~debian-{{ ansible_distribution_release|lower }}
+  'stable': docker-ce-cli=5:20.10.20~3-0~debian-{{ ansible_distribution_release|lower }}
+  'edge': docker-ce-cli=5:20.10.20~3-0~debian-{{ ansible_distribution_release|lower }}
+
+docker_package_info:
+  pkgs:
+    - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}"
+    - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}"
+    - "{{ docker_versioned_pkg[docker_version | string] }}"
+
+docker_repo_key_info:
+  url: '{{ docker_debian_repo_gpgkey }}'
+  repo_keys:
+    - '{{ docker_debian_repo_repokey }}'
+
+docker_repo_info:
+  repos:
+    - >
+      deb {{ docker_debian_repo_base_url }}
+      {{ ansible_distribution_release|lower }}
+      stable

ansible/kubespray/roles/container-engine/docker/vars/fedora.yml

@@ -0,0 +1,37 @@
+---
+# containerd versions are only relevant for docker
+containerd_versioned_pkg:
+  'latest': "{{ containerd_package }}"
+  '1.3.7': "{{ containerd_package }}-1.3.7-3.1.fc{{ ansible_distribution_major_version }}"
+  '1.3.9': "{{ containerd_package }}-1.3.9-3.1.fc{{ ansible_distribution_major_version }}"
+  '1.4.3': "{{ containerd_package }}-1.4.3-3.2.fc{{ ansible_distribution_major_version }}"
+  '1.4.4': "{{ containerd_package }}-1.4.4-3.1.fc{{ ansible_distribution_major_version }}"
+  '1.4.6': "{{ containerd_package }}-1.4.6-3.1.fc{{ ansible_distribution_major_version }}"
+  '1.4.9': "{{ containerd_package }}-1.4.9-3.1.fc{{ ansible_distribution_major_version }}"
+  '1.4.12': "{{ containerd_package }}-1.4.12-3.1.fc{{ ansible_distribution_major_version }}"
+  '1.6.4': "{{ containerd_package }}-1.6.4-3.1.fc{{ ansible_distribution_major_version }}"
+  'stable': "{{ containerd_package }}-1.6.4-3.1.fc{{ ansible_distribution_major_version }}"
+  'edge': "{{ containerd_package }}-1.6.4-3.1.fc{{ ansible_distribution_major_version }}"
+
+# https://docs.docker.com/install/linux/docker-ce/fedora/
+# https://download.docker.com/linux/fedora/<fedora-version>/x86_64/stable/Packages/
+docker_versioned_pkg:
+  'latest': docker-ce
+  '19.03': docker-ce-19.03.15-3.fc{{ ansible_distribution_major_version }}
+  '20.10': docker-ce-20.10.20-3.fc{{ ansible_distribution_major_version }}
+  'stable': docker-ce-20.10.20-3.fc{{ ansible_distribution_major_version }}
+  'edge': docker-ce-20.10.20-3.fc{{ ansible_distribution_major_version }}
+
+docker_cli_versioned_pkg:
+  'latest': docker-ce-cli
+  '19.03': docker-ce-cli-19.03.15-3.fc{{ ansible_distribution_major_version }}
+  '20.10': docker-ce-cli-20.10.20-3.fc{{ ansible_distribution_major_version }}
+  'stable': docker-ce-cli-20.10.20-3.fc{{ ansible_distribution_major_version }}
+  'edge': docker-ce-cli-20.10.20-3.fc{{ ansible_distribution_major_version }}
+
+docker_package_info:
+  enablerepo: "docker-ce"
+  pkgs:
+    - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}"
+    - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}"
+    - "{{ docker_versioned_pkg[docker_version | string] }}"

ansible/kubespray/roles/container-engine/docker/vars/kylin.yml

@@ -0,0 +1,41 @@
+---
+# containerd versions are only relevant for docker
+containerd_versioned_pkg:
+  'latest': "{{ containerd_package }}"
+  '1.3.7': "{{ containerd_package }}-1.3.7-3.1.el{{ ansible_distribution_major_version }}"
+  '1.3.9': "{{ containerd_package }}-1.3.9-3.1.el{{ ansible_distribution_major_version }}"
+  '1.4.3': "{{ containerd_package }}-1.4.3-3.2.el{{ ansible_distribution_major_version }}"
+  '1.4.4': "{{ containerd_package }}-1.4.4-3.1.el{{ ansible_distribution_major_version }}"
+  '1.4.6': "{{ containerd_package }}-1.4.6-3.1.el{{ ansible_distribution_major_version }}"
+  '1.4.9': "{{ containerd_package }}-1.4.9-3.1.el{{ ansible_distribution_major_version }}"
+  '1.4.12': "{{ containerd_package }}-1.4.12-3.1.el{{ ansible_distribution_major_version }}"
+  '1.6.4': "{{ containerd_package }}-1.6.4-3.1.el{{ ansible_distribution_major_version }}"
+  'stable': "{{ containerd_package }}-1.6.4-3.1.el{{ ansible_distribution_major_version }}"
+  'edge': "{{ containerd_package }}-1.6.4-3.1.el{{ ansible_distribution_major_version }}"
+
+docker_version: 19.03
+docker_cli_version: 19.03
+
+# https://docs.docker.com/engine/installation/linux/centos/#install-from-a-package
+# https://download.docker.com/linux/centos/<centos_version>>/x86_64/stable/Packages/
+# or do 'yum --showduplicates list docker-engine'
+docker_versioned_pkg:
+  'latest': docker-ce
+  '18.09': docker-ce-3:18.09.9-3.el7
+  '19.03': docker-ce-3:19.03.15-3.el{{ ansible_distribution_major_version }}
+  'stable': docker-ce-3:19.03.15-3.el{{ ansible_distribution_major_version }}
+  'edge': docker-ce-3:19.03.15-3.el{{ ansible_distribution_major_version }}
+
+docker_cli_versioned_pkg:
+  'latest': docker-ce-cli
+  '18.09': docker-ce-cli-1:18.09.9-3.el7
+  '19.03': docker-ce-cli-1:19.03.15-3.el{{ ansible_distribution_major_version }}
+  'stable': docker-ce-cli-1:19.03.15-3.el{{ ansible_distribution_major_version }}
+  'edge': docker-ce-cli-1:19.03.15-3.el{{ ansible_distribution_major_version }}
+
+docker_package_info:
+  enablerepo: "docker-ce"
+  pkgs:
+    - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}"
+    - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}"
+    - "{{ docker_versioned_pkg[docker_version | string] }}"

ansible/kubespray/roles/container-engine/docker/vars/redhat-7.yml

@@ -0,0 +1,40 @@
+---
+# containerd versions are only relevant for docker
+containerd_versioned_pkg:
+  'latest': "{{ containerd_package }}"
+  '1.3.7': "{{ containerd_package }}-1.3.7-3.1.el7"
+  '1.3.9': "{{ containerd_package }}-1.3.9-3.1.el7"
+  '1.4.3': "{{ containerd_package }}-1.4.3-3.2.el7"
+  '1.4.4': "{{ containerd_package }}-1.4.4-3.1.el7"
+  '1.4.6': "{{ containerd_package }}-1.4.6-3.1.el7"
+  '1.4.9': "{{ containerd_package }}-1.4.9-3.1.el7"
+  '1.4.12': "{{ containerd_package }}-1.4.12-3.1.el7"
+  '1.6.4': "{{ containerd_package }}-1.6.4-3.1.el7"
+  'stable': "{{ containerd_package }}-1.6.4-3.1.el7"
+  'edge': "{{ containerd_package }}-1.6.4-3.1.el7"
+
+# https://docs.docker.com/engine/installation/linux/centos/#install-from-a-package
+# https://download.docker.com/linux/centos/<centos_version>>/x86_64/stable/Packages/
+# or do 'yum --showduplicates list docker-engine'
+docker_versioned_pkg:
+  'latest': docker-ce
+  '18.09': docker-ce-18.09.9-3.el7
+  '19.03': docker-ce-19.03.15-3.el7
+  '20.10': docker-ce-20.10.20-3.el7
+  'stable': docker-ce-20.10.20-3.el7
+  'edge': docker-ce-20.10.20-3.el7
+
+docker_cli_versioned_pkg:
+  'latest': docker-ce-cli
+  '18.09': docker-ce-cli-18.09.9-3.el7
+  '19.03': docker-ce-cli-19.03.15-3.el7
+  '20.10': docker-ce-cli-20.10.20-3.el7
+  'stable': docker-ce-cli-20.10.20-3.el7
+  'edge': docker-ce-cli-20.10.20-3.el7
+
+docker_package_info:
+  enablerepo: "docker-ce"
+  pkgs:
+    - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}"
+    - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}"
+    - "{{ docker_versioned_pkg[docker_version | string] }}"

ansible/kubespray/roles/container-engine/docker/vars/redhat.yml

@@ -0,0 +1,40 @@
+---
+# containerd versions are only relevant for docker
+containerd_versioned_pkg:
+  'latest': "{{ containerd_package }}"
+  '1.3.7': "{{ containerd_package }}-1.3.7-3.1.el{{ ansible_distribution_major_version }}"
+  '1.3.9': "{{ containerd_package }}-1.3.9-3.1.el{{ ansible_distribution_major_version }}"
+  '1.4.3': "{{ containerd_package }}-1.4.3-3.2.el{{ ansible_distribution_major_version }}"
+  '1.4.4': "{{ containerd_package }}-1.4.4-3.1.el{{ ansible_distribution_major_version }}"
+  '1.4.6': "{{ containerd_package }}-1.4.6-3.1.el{{ ansible_distribution_major_version }}"
+  '1.4.9': "{{ containerd_package }}-1.4.9-3.1.el{{ ansible_distribution_major_version }}"
+  '1.4.12': "{{ containerd_package }}-1.4.12-3.1.el{{ ansible_distribution_major_version }}"
+  '1.6.4': "{{ containerd_package }}-1.6.4-3.1.el{{ ansible_distribution_major_version }}"
+  'stable': "{{ containerd_package }}-1.6.4-3.1.el{{ ansible_distribution_major_version }}"
+  'edge': "{{ containerd_package }}-1.6.4-3.1.el{{ ansible_distribution_major_version }}"
+
+# https://docs.docker.com/engine/installation/linux/centos/#install-from-a-package
+# https://download.docker.com/linux/centos/<centos_version>>/x86_64/stable/Packages/
+# or do 'yum --showduplicates list docker-engine'
+docker_versioned_pkg:
+  'latest': docker-ce
+  '18.09': docker-ce-3:18.09.9-3.el7
+  '19.03': docker-ce-3:19.03.15-3.el{{ ansible_distribution_major_version }}
+  '20.10': docker-ce-3:20.10.20-3.el{{ ansible_distribution_major_version }}
+  'stable': docker-ce-3:20.10.20-3.el{{ ansible_distribution_major_version }}
+  'edge': docker-ce-3:20.10.20-3.el{{ ansible_distribution_major_version }}
+
+docker_cli_versioned_pkg:
+  'latest': docker-ce-cli
+  '18.09': docker-ce-cli-1:18.09.9-3.el7
+  '19.03': docker-ce-cli-1:19.03.15-3.el{{ ansible_distribution_major_version }}
+  '20.10': docker-ce-cli-1:20.10.20-3.el{{ ansible_distribution_major_version }}
+  'stable': docker-ce-cli-1:20.10.20-3.el{{ ansible_distribution_major_version }}
+  'edge': docker-ce-cli-1:20.10.20-3.el{{ ansible_distribution_major_version }}
+
+docker_package_info:
+  enablerepo: "docker-ce"
+  pkgs:
+    - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}"
+    - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}"
+    - "{{ docker_versioned_pkg[docker_version | string] }}"

ansible/kubespray/roles/container-engine/docker/vars/suse.yml

@@ -0,0 +1,6 @@
+---
+docker_package_info:
+  state: latest
+  pkgs:
+    - docker
+    - containerd

ansible/kubespray/roles/container-engine/docker/vars/ubuntu-16.yml

@@ -0,0 +1,46 @@
+---
+# containerd versions are only relevant for docker
+containerd_versioned_pkg:
+  'latest': "{{ containerd_package }}"
+  '1.3.7': "{{ containerd_package }}=1.3.7-1"
+  '1.3.9': "{{ containerd_package }}=1.3.9-1"
+  '1.4.3': "{{ containerd_package }}=1.4.3-2"
+  '1.4.4': "{{ containerd_package }}=1.4.4-1"
+  '1.4.6': "{{ containerd_package }}=1.4.6-1"
+  'stable': "{{ containerd_package }}=1.4.6-1"
+  'edge': "{{ containerd_package }}=1.4.6-1"
+
+# https://download.docker.com/linux/ubuntu/
+docker_versioned_pkg:
+  'latest': docker-ce
+  '18.09': docker-ce=5:18.09.9~3-0~ubuntu-{{ ansible_distribution_release|lower }}
+  '19.03': docker-ce=5:19.03.15~3-0~ubuntu-{{ ansible_distribution_release|lower }}
+  '20.10': docker-ce=5:20.10.7~3-0~ubuntu-{{ ansible_distribution_release|lower }}
+  'stable': docker-ce=5:20.10.7~3-0~ubuntu-{{ ansible_distribution_release|lower }}
+  'edge': docker-ce=5:20.10.7~3-0~ubuntu-{{ ansible_distribution_release|lower }}
+
+docker_cli_versioned_pkg:
+  'latest': docker-ce-cli
+  '18.09': docker-ce-cli=5:18.09.9~3-0~ubuntu-{{ ansible_distribution_release|lower }}
+  '19.03': docker-ce-cli=5:19.03.15~3-0~ubuntu-{{ ansible_distribution_release|lower }}
+  '20.10': docker-ce-cli=5:20.10.7~3-0~ubuntu-{{ ansible_distribution_release|lower }}
+  'stable': docker-ce-cli=5:20.10.7~3-0~ubuntu-{{ ansible_distribution_release|lower }}
+  'edge': docker-ce-cli=5:20.10.7~3-0~ubuntu-{{ ansible_distribution_release|lower }}
+
+docker_package_info:
+  pkgs:
+    - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}"
+    - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}"
+    - "{{ docker_versioned_pkg[docker_version | string] }}"
+
+docker_repo_key_info:
+  url: '{{ docker_ubuntu_repo_gpgkey }}'
+  repo_keys:
+    - '{{ docker_ubuntu_repo_repokey }}'
+
+docker_repo_info:
+  repos:
+    - >
+      deb [arch={{ host_architecture }}] {{ docker_ubuntu_repo_base_url }}
+      {{ ansible_distribution_release|lower }}
+      stable

ansible/kubespray/roles/container-engine/docker/vars/ubuntu.yml

@@ -0,0 +1,49 @@
+---
+# containerd versions are only relevant for docker
+containerd_versioned_pkg:
+  'latest': "{{ containerd_package }}"
+  '1.3.7': "{{ containerd_package }}=1.3.7-1"
+  '1.3.9': "{{ containerd_package }}=1.3.9-1"
+  '1.4.3': "{{ containerd_package }}=1.4.3-2"
+  '1.4.4': "{{ containerd_package }}=1.4.4-1"
+  '1.4.6': "{{ containerd_package }}=1.4.6-1"
+  '1.4.9': "{{ containerd_package }}=1.4.9-1"
+  '1.4.12': "{{ containerd_package }}=1.4.12-1"
+  '1.6.4': "{{ containerd_package }}=1.6.4-1"
+  'stable': "{{ containerd_package }}=1.6.4-1"
+  'edge': "{{ containerd_package }}=1.6.4-1"
+
+# https://download.docker.com/linux/ubuntu/
+docker_versioned_pkg:
+  'latest': docker-ce
+  '18.09': docker-ce=5:18.09.9~3-0~ubuntu-{{ ansible_distribution_release|lower }}
+  '19.03': docker-ce=5:19.03.15~3-0~ubuntu-{{ ansible_distribution_release|lower }}
+  '20.10': docker-ce=5:20.10.20~3-0~ubuntu-{{ ansible_distribution_release|lower }}
+  'stable': docker-ce=5:20.10.20~3-0~ubuntu-{{ ansible_distribution_release|lower }}
+  'edge': docker-ce=5:20.10.20~3-0~ubuntu-{{ ansible_distribution_release|lower }}
+
+docker_cli_versioned_pkg:
+  'latest': docker-ce-cli
+  '18.09': docker-ce-cli=5:18.09.9~3-0~ubuntu-{{ ansible_distribution_release|lower }}
+  '19.03': docker-ce-cli=5:19.03.15~3-0~ubuntu-{{ ansible_distribution_release|lower }}
+  '20.10': docker-ce-cli=5:20.10.20~3-0~ubuntu-{{ ansible_distribution_release|lower }}
+  'stable': docker-ce-cli=5:20.10.20~3-0~ubuntu-{{ ansible_distribution_release|lower }}
+  'edge': docker-ce-cli=5:20.10.20~3-0~ubuntu-{{ ansible_distribution_release|lower }}
+
+docker_package_info:
+  pkgs:
+    - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}"
+    - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}"
+    - "{{ docker_versioned_pkg[docker_version | string] }}"
+
+docker_repo_key_info:
+  url: '{{ docker_ubuntu_repo_gpgkey }}'
+  repo_keys:
+    - '{{ docker_ubuntu_repo_repokey }}'
+
+docker_repo_info:
+  repos:
+    - >
+      deb [arch={{ host_architecture }}] {{ docker_ubuntu_repo_base_url }}
+      {{ ansible_distribution_release|lower }}
+      stable

ansible/kubespray/roles/container-engine/docker/vars/uniontech.yml

@@ -0,0 +1,45 @@
+---
+# containerd versions are only relevant for docker
+containerd_versioned_pkg:
+  'latest': "{{ containerd_package }}"
+  '1.3.7': "{{ containerd_package }}-1.3.7-3.1.el{{ ansible_distribution_major_version }}"
+  '1.3.9': "{{ containerd_package }}-1.3.9-3.1.el{{ ansible_distribution_major_version }}"
+  '1.4.3': "{{ containerd_package }}-1.4.3-3.2.el{{ ansible_distribution_major_version }}"
+  '1.4.4': "{{ containerd_package }}-1.4.4-3.1.el{{ ansible_distribution_major_version }}"
+  '1.4.6': "{{ containerd_package }}-1.4.6-3.1.el{{ ansible_distribution_major_version }}"
+  '1.4.9': "{{ containerd_package }}-1.4.9-3.1.el{{ ansible_distribution_major_version }}"
+  '1.4.12': "{{ containerd_package }}-1.4.12-3.1.el{{ ansible_distribution_major_version }}"
+  '1.6.4': "{{ containerd_package }}-1.6.4-3.1.el{{ ansible_distribution_major_version }}"
+  '1.6.8': "{{ containerd_package }}-1.6.8-3.1.el{{ ansible_distribution_major_version }}"
+  'stable': "{{ containerd_package }}-1.6.8-3.1.el{{ ansible_distribution_major_version }}"
+  'edge': "{{ containerd_package }}-1.6.8-3.1.el{{ ansible_distribution_major_version }}"
+
+docker_version: 19.03
+docker_cli_version: 19.03
+
+# https://docs.docker.com/engine/installation/linux/centos/#install-from-a-package
+# https://download.docker.com/linux/centos/<centos_version>>/x86_64/stable/Packages/
+# or do 'yum --showduplicates list docker-engine'
+docker_versioned_pkg:
+  'latest': docker-ce
+  '18.09': docker-ce-3:18.09.9-3.el7
+  '19.03': docker-ce-3:19.03.15-3.el{{ ansible_distribution_major_version }}
+  '20.10': docker-ce-3:20.10.17-3.el{{ ansible_distribution_major_version }}
+  'stable': docker-ce-3:20.10.17-3.el{{ ansible_distribution_major_version }}
+  'edge': docker-ce-3:20.10.17-3.el{{ ansible_distribution_major_version }}
+
+docker_cli_versioned_pkg:
+  'latest': docker-ce-cli
+  '18.09': docker-ce-cli-1:18.09.9-3.el7
+  '19.03': docker-ce-cli-1:19.03.15-3.el{{ ansible_distribution_major_version }}
+  '20.10': docker-ce-cli-1:20.10.17-3.el{{ ansible_distribution_major_version }}
+  'stable': docker-ce-cli-1:20.10.17-3.el{{ ansible_distribution_major_version }}
+  'edge': docker-ce-cli-1:20.10.17-3.el{{ ansible_distribution_major_version }}
+
+docker_package_info:
+  enablerepo: "docker-ce"
+  disablerepo: "UniontechOS-20-AppStream"
+  pkgs:
+    - "{{ containerd_versioned_pkg[docker_containerd_version | string] }}"
+    - "{{ docker_cli_versioned_pkg[docker_cli_version | string] }}"
+    - "{{ docker_versioned_pkg[docker_version | string] }}"

ansible/kubespray/roles/container-engine/gvisor/molecule/default/converge.yml

@@ -0,0 +1,11 @@
+---
+- name: Converge
+  hosts: all
+  become: true
+  vars:
+    gvisor_enabled: true
+    container_manager: containerd
+  roles:
+    - role: kubespray-defaults
+    - role: container-engine/containerd
+    - role: container-engine/gvisor

ansible/kubespray/roles/container-engine/gvisor/molecule/default/files/10-mynet.conf

@@ -0,0 +1,17 @@
+{
+  "cniVersion": "0.2.0",
+  "name": "mynet",
+  "type": "bridge",
+  "bridge": "cni0",
+  "isGateway": true,
+  "ipMasq": true,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "172.19.0.0/24",
+    "routes": [
+      {
+        "dst": "0.0.0.0/0"
+      }
+    ]
+  }
+}

ansible/kubespray/roles/container-engine/gvisor/molecule/default/files/container.json

@@ -0,0 +1,10 @@
+{
+  "metadata": {
+    "name": "gvisor1"
+  },
+  "image": {
+    "image": "quay.io/kubespray/hello-world:latest"
+  },
+  "log_path": "gvisor1.0.log",
+  "linux": {}
+}

ansible/kubespray/roles/container-engine/gvisor/molecule/default/files/sandbox.json

@@ -0,0 +1,10 @@
+{
+  "metadata": {
+    "name": "gvisor1",
+    "namespace": "default",
+    "attempt": 1,
+    "uid": "hdishd83djaidwnduwk28bcsb"
+  },
+  "linux": {},
+  "log_directory": "/tmp"
+}

ansible/kubespray/roles/container-engine/gvisor/molecule/default/molecule.yml

@@ -0,0 +1,45 @@
+---
+driver:
+  name: vagrant
+  provider:
+    name: libvirt
+    options:
+      driver: kvm
+lint: |
+  set -e
+  yamllint -c ../../../.yamllint .
+platforms:
+  - name: ubuntu20
+    box: generic/ubuntu2004
+    cpus: 1
+    memory: 1024
+    nested: true
+    groups:
+      - kube_control_plane
+  - name: almalinux8
+    box: almalinux/8
+    cpus: 1
+    memory: 1024
+    nested: true
+    groups:
+      - kube_control_plane
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_ROLES_PATH: ../../../../
+  config_options:
+    defaults:
+      callbacks_enabled: profile_tasks
+      timeout: 120
+  lint:
+    name: ansible-lint
+    options:
+      c: ../../../.ansible-lint
+  inventory:
+    group_vars:
+      all:
+        become: true
+verifier:
+  name: testinfra
+  lint:
+    name: flake8

ansible/kubespray/roles/container-engine/gvisor/molecule/default/prepare.yml

@@ -0,0 +1,48 @@
+---
+- name: Prepare generic
+  hosts: all
+  become: true
+  roles:
+    - role: kubespray-defaults
+    - role: bootstrap-os
+    - role: adduser
+      user: "{{ addusers.kube }}"
+  tasks:
+    - include_tasks: "../../../../download/tasks/download_file.yml"
+      vars:
+        download: "{{ download_defaults | combine(downloads.cni) }}"
+
+- name: Prepare container runtime
+  hosts: all
+  become: true
+  vars:
+    container_manager: containerd
+    kube_network_plugin: cni
+  roles:
+    - role: kubespray-defaults
+    - role: network_plugin/cni
+    - role: container-engine/crictl
+  tasks:
+    - name: Copy test container files
+      copy:
+        src: "{{ item }}"
+        dest: "/tmp/{{ item }}"
+        owner: root
+        mode: 0644
+      with_items:
+        - container.json
+        - sandbox.json
+    - name: Create /etc/cni/net.d directory
+      file:
+        path: /etc/cni/net.d
+        state: directory
+        owner: root
+        mode: 0755
+    - name: Setup CNI
+      copy:
+        src: "{{ item }}"
+        dest: "/etc/cni/net.d/{{ item }}"
+        owner: root
+        mode: 0644
+      with_items:
+        - 10-mynet.conf

ansible/kubespray/roles/container-engine/gvisor/molecule/default/tests/test_default.py

@@ -0,0 +1,29 @@
+import os
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_run(host):
+    gvisorruntime = "/usr/local/bin/runsc"
+    with host.sudo():
+        cmd = host.command(gvisorruntime + " --version")
+    assert cmd.rc == 0
+    assert "runsc version" in cmd.stdout
+
+
+def test_run_pod(host):
+    runtime = "runsc"
+
+    run_command = "/usr/local/bin/crictl run --with-pull --runtime {} /tmp/container.json /tmp/sandbox.json".format(runtime)
+    with host.sudo():
+        cmd = host.command(run_command)
+    assert cmd.rc == 0
+
+    with host.sudo():
+      log_f = host.file("/tmp/gvisor1.0.log")
+
+      assert log_f.exists
+      assert b"Hello from Docker" in log_f.content

ansible/kubespray/roles/container-engine/gvisor/tasks/main.yml

@@ -0,0 +1,20 @@
+---
+- name: gVisor | Download runsc binary
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.gvisor_runsc) }}"
+
+- name: gVisor | Download containerd-shim-runsc-v1 binary
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.gvisor_containerd_shim) }}"
+
+- name: gVisor | Copy binaries
+  copy:
+    src: "{{ local_release_dir }}/gvisor-{{ item }}"
+    dest: "{{ bin_dir }}/{{ item }}"
+    mode: 0755
+    remote_src: yes
+  with_items:
+    - runsc
+    - containerd-shim-runsc-v1

ansible/kubespray/roles/container-engine/kata-containers/OWNERS

@@ -0,0 +1,6 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+  - pasqualet
+reviewers:
+  - pasqualet