update

kubespray/extra_playbooks/roles/adduser/defaults/main.yml

@@ -0,0 +1,27 @@
+---
+kube_owner: kube
+kube_cert_group: kube-cert
+etcd_data_dir: "/var/lib/etcd"
+
+addusers:
+  etcd:
+    name: etcd
+    comment: "Etcd user"
+    create_home: no
+    system: yes
+    shell: /sbin/nologin
+  kube:
+    name: kube
+    comment: "Kubernetes user"
+    create_home: no
+    system: yes
+    shell: /sbin/nologin
+    group: "{{ kube_cert_group }}"
+
+adduser:
+  name: "{{ user.name }}"
+  group: "{{ user.name|default(None) }}"
+  comment: "{{ user.comment|default(None) }}"
+  shell: "{{ user.shell|default(None) }}"
+  system: "{{ user.system|default(None) }}"
+  create_home: "{{ user.create_home|default(None) }}"

kubespray/extra_playbooks/roles/adduser/molecule/default/converge.yml

@@ -0,0 +1,10 @@
+---
+- name: Converge
+  hosts: all
+  become: true
+  gather_facts: false
+  roles:
+    - role: adduser
+  vars:
+    user:
+      name: foo

kubespray/extra_playbooks/roles/adduser/molecule/default/molecule.yml

@@ -0,0 +1,27 @@
+---
+dependency:
+  name: galaxy
+lint: |
+  set -e
+  yamllint -c ../../.yamllint .
+driver:
+  name: vagrant
+  provider:
+    name: libvirt
+platforms:
+  - name: adduser-01
+    box: generic/ubuntu2004
+    cpus: 1
+    memory: 512
+provisioner:
+  name: ansible
+  config_options:
+    defaults:
+      callbacks_enabled: profile_tasks
+      timeout: 120
+  lint:
+    name: ansible-lint
+verifier:
+  name: testinfra
+  lint:
+    name: flake8

kubespray/extra_playbooks/roles/adduser/molecule/default/tests/test_default.py

@@ -0,0 +1,37 @@
+import os
+import yaml
+import glob
+import testinfra.utils.ansible_runner
+from ansible.playbook import Playbook
+from ansible.cli.playbook import PlaybookCLI
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+def read_playbook(playbook):
+    cli_args = [os.path.realpath(playbook), testinfra_hosts]
+    cli = PlaybookCLI(cli_args)
+    cli.parse()
+    loader, inventory, variable_manager = cli._play_prereqs()
+
+    pb = Playbook.load(cli.args[0], variable_manager, loader)
+
+    for play in pb.get_plays():
+        yield variable_manager.get_vars(play)
+
+def get_playbook():
+    with open(os.path.realpath(' '.join(map(str,glob.glob('molecule.*')))), 'r') as yamlfile:
+        data = yaml.load(yamlfile, Loader=yaml.FullLoader)
+        if 'playbooks' in data['provisioner'].keys():
+            if 'converge' in data['provisioner']['playbooks'].keys():
+                return data['provisioner']['playbooks']['converge']
+        else:
+            return ' '.join(map(str,glob.glob('converge.*')))
+
+def test_user(host):
+    for vars in read_playbook(get_playbook()):
+        assert host.user(vars['user']['name']).exists
+        if 'group' in vars['user'].keys():
+            assert host.group(vars['user']['group']).exists
+        else:
+            assert host.group(vars['user']['name']).exists

kubespray/extra_playbooks/roles/adduser/tasks/main.yml

@@ -0,0 +1,16 @@
+---
+- name: User | Create User Group
+  group:
+    name: "{{ user.group|default(user.name) }}"
+    system: "{{ user.system|default(omit) }}"
+
+- name: User | Create User
+  user:
+    comment: "{{ user.comment|default(omit) }}"
+    create_home: "{{ user.create_home|default(omit) }}"
+    group: "{{ user.group|default(user.name) }}"
+    home: "{{ user.home|default(omit) }}"
+    shell: "{{ user.shell|default(omit) }}"
+    name: "{{ user.name }}"
+    system: "{{ user.system|default(omit) }}"
+  when: user.name != "root"

kubespray/extra_playbooks/roles/adduser/vars/coreos.yml

@@ -0,0 +1,8 @@
+---
+addusers:
+  - name: kube
+    comment: "Kubernetes user"
+    shell: /sbin/nologin
+    system: yes
+    group: "{{ kube_cert_group }}"
+    create_home: no

kubespray/extra_playbooks/roles/adduser/vars/debian.yml

@@ -0,0 +1,15 @@
+---
+addusers:
+  - name: etcd
+    comment: "Etcd user"
+    create_home: yes
+    home: "{{ etcd_data_dir }}"
+    system: yes
+    shell: /sbin/nologin
+
+  - name: kube
+    comment: "Kubernetes user"
+    create_home: no
+    system: yes
+    shell: /sbin/nologin
+    group: "{{ kube_cert_group }}"

kubespray/extra_playbooks/roles/adduser/vars/redhat.yml

@@ -0,0 +1,15 @@
+---
+addusers:
+  - name: etcd
+    comment: "Etcd user"
+    create_home: yes
+    home: "{{ etcd_data_dir }}"
+    system: yes
+    shell: /sbin/nologin
+
+  - name: kube
+    comment: "Kubernetes user"
+    create_home: no
+    system: yes
+    shell: /sbin/nologin
+    group: "{{ kube_cert_group }}"

kubespray/extra_playbooks/roles/bastion-ssh-config/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+ssh_bastion_confing__name: ssh-bastion.conf

kubespray/extra_playbooks/roles/bastion-ssh-config/molecule/default/converge.yml

@@ -0,0 +1,15 @@
+---
+- name: Converge
+  hosts: all
+  become: true
+  gather_facts: false
+  roles:
+    - role: bastion-ssh-config
+  tasks:
+    - name: Copy config to remote host
+      copy:
+        src: "{{ playbook_dir }}/{{ ssh_bastion_confing__name }}"
+        dest: "{{ ssh_bastion_confing__name }}"
+        owner: "{{ ansible_user }}"
+        group: "{{ ansible_user }}"
+        mode: 0644

kubespray/extra_playbooks/roles/bastion-ssh-config/molecule/default/molecule.yml

@@ -0,0 +1,35 @@
+---
+dependency:
+  name: galaxy
+lint: |
+  set -e
+  yamllint -c ../../.yamllint .
+driver:
+  name: vagrant
+  provider:
+    name: libvirt
+platforms:
+  - name: bastion-01
+    box: generic/ubuntu2004
+    cpus: 1
+    memory: 512
+provisioner:
+  name: ansible
+  config_options:
+    defaults:
+      callbacks_enabled: profile_tasks
+      timeout: 120
+  lint:
+    name: ansible-lint
+  inventory:
+    hosts:
+      all:
+        hosts:
+        children:
+          bastion:
+            hosts:
+              bastion-01:
+verifier:
+  name: testinfra
+  lint:
+    name: flake8

kubespray/extra_playbooks/roles/bastion-ssh-config/molecule/default/tests/test_default.py

@@ -0,0 +1,34 @@
+import os
+import yaml
+import glob
+import testinfra.utils.ansible_runner
+from ansible.playbook import Playbook
+from ansible.cli.playbook import PlaybookCLI
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+def read_playbook(playbook):
+    cli_args = [os.path.realpath(playbook), testinfra_hosts]
+    cli = PlaybookCLI(cli_args)
+    cli.parse()
+    loader, inventory, variable_manager = cli._play_prereqs()
+
+    pb = Playbook.load(cli.args[0], variable_manager, loader)
+
+    for play in pb.get_plays():
+        yield variable_manager.get_vars(play)
+
+def get_playbook():
+    with open(os.path.realpath(' '.join(map(str,glob.glob('molecule.*')))), 'r') as yamlfile:
+        data = yaml.load(yamlfile, Loader=yaml.FullLoader)
+        if 'playbooks' in data['provisioner'].keys():
+            if 'converge' in data['provisioner']['playbooks'].keys():
+                return data['provisioner']['playbooks']['converge']
+        else:
+            return ' '.join(map(str,glob.glob('converge.*')))
+
+def test_ssh_config(host):
+    for vars in read_playbook(get_playbook()):
+        assert host.file(vars['ssh_bastion_confing__name']).exists
+        assert host.file(vars['ssh_bastion_confing__name']).is_file

kubespray/extra_playbooks/roles/bastion-ssh-config/tasks/main.yml

@@ -0,0 +1,22 @@
+---
+- name: set bastion host IP and port
+  set_fact:
+    bastion_ip: "{{ hostvars[groups['bastion'][0]]['ansible_host'] | d(hostvars[groups['bastion'][0]]['ansible_ssh_host']) }}"
+    bastion_port: "{{ hostvars[groups['bastion'][0]]['ansible_port'] | d(hostvars[groups['bastion'][0]]['ansible_ssh_port']) | d(22) }}"
+  delegate_to: localhost
+  connection: local
+
+# As we are actually running on localhost, the ansible_ssh_user is your local user when you try to use it directly
+# To figure out the real ssh user, we delegate this task to the bastion and store the ansible_user in real_user
+- name: Store the current ansible_user in the real_user fact
+  set_fact:
+    real_user: "{{ ansible_user }}"
+
+- name: create ssh bastion conf
+  become: false
+  delegate_to: localhost
+  connection: local
+  template:
+    src: "{{ ssh_bastion_confing__name }}.j2"
+    dest: "{{ playbook_dir }}/{{ ssh_bastion_confing__name }}"
+    mode: 0640

kubespray/extra_playbooks/roles/bastion-ssh-config/templates/ssh-bastion.conf.j2

@@ -0,0 +1,18 @@
+{% set vars={'hosts': ''} %}
+{% set user='' %}
+
+{% for h in groups['all'] %}
+{% if h not in groups['bastion'] %}
+{% if vars.update({'hosts': vars['hosts'] + ' ' + (hostvars[h].get('ansible_ssh_host') or hostvars[h]['ansible_host'])}) %}{% endif %}
+{% endif %}
+{% endfor %}
+
+Host {{ bastion_ip }}
+  Hostname {{ bastion_ip }}
+  StrictHostKeyChecking no
+  ControlMaster auto
+  ControlPath ~/.ssh/ansible-%r@%h:%p
+  ControlPersist 5m
+
+Host {{ vars['hosts'] }}
+  ProxyCommand ssh -F /dev/null -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p -p {{ bastion_port }} {{ real_user }}@{{ bastion_ip }} {% if ansible_ssh_private_key_file is defined %}-i {{ ansible_ssh_private_key_file }}{% endif %}

kubespray/extra_playbooks/roles/bootstrap-os/defaults/main.yml

@@ -0,0 +1,32 @@
+---
+## CentOS/RHEL/AlmaLinux specific variables
+# Use the fastestmirror yum plugin
+centos_fastestmirror_enabled: false
+
+## Flatcar Container Linux specific variables
+# Disable locksmithd or leave it in its current state
+coreos_locksmithd_disable: false
+
+## Oracle Linux specific variables
+# Install public repo on Oracle Linux
+use_oracle_public_repo: true
+
+fedora_coreos_packages:
+  - python
+  - python3-libselinux
+  - ethtool                 # required in kubeadm preflight phase for verifying the environment
+  - ipset                   # required in kubeadm preflight phase for verifying the environment
+  - conntrack-tools         # required by kube-proxy
+
+## General
+# Set the hostname to inventory_hostname
+override_system_hostname: true
+
+is_fedora_coreos: false
+
+skip_http_proxy_on_os_packages: false
+
+# If this is true, debug information will be displayed but
+# may contain some private data, so it is recommended to set it to false
+# in the production environment.
+unsafe_show_logs: false

kubespray/extra_playbooks/roles/bootstrap-os/files/bootstrap.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+set -e
+
+BINDIR="/opt/bin"
+if [[ -e $BINDIR/.bootstrapped ]]; then
+  exit 0
+fi
+
+ARCH=$(uname -m)
+case $ARCH in
+  "x86_64")
+    PYPY_ARCH=linux64
+    PYPI_HASH=46818cb3d74b96b34787548343d266e2562b531ddbaf330383ba930ff1930ed5
+    ;;
+  "aarch64")
+    PYPY_ARCH=aarch64
+    PYPI_HASH=2e1ae193d98bc51439642a7618d521ea019f45b8fb226940f7e334c548d2b4b9
+    ;;
+  *)
+    echo "Unsupported Architecture: ${ARCH}"
+    exit 1
+esac
+
+PYTHON_VERSION=3.9
+PYPY_VERSION=7.3.9
+PYPY_FILENAME="pypy${PYTHON_VERSION}-v${PYPY_VERSION}-${PYPY_ARCH}"
+PYPI_URL="https://downloads.python.org/pypy/${PYPY_FILENAME}.tar.bz2"
+
+mkdir -p $BINDIR
+
+cd $BINDIR
+
+TAR_FILE=pyp.tar.bz2
+wget -O "${TAR_FILE}" "${PYPI_URL}"
+echo "${PYPI_HASH} ${TAR_FILE}" | sha256sum -c -
+tar -xjf "${TAR_FILE}" && rm "${TAR_FILE}"
+mv -n "${PYPY_FILENAME}" pypy3
+
+ln -s ./pypy3/bin/pypy3 python
+$BINDIR/python --version
+
+touch $BINDIR/.bootstrapped

kubespray/extra_playbooks/roles/bootstrap-os/handlers/main.yml

@@ -0,0 +1,4 @@
+---
+- name: RHEL auto-attach subscription
+  command: /sbin/subscription-manager attach --auto
+  become: true

kubespray/extra_playbooks/roles/bootstrap-os/molecule/default/converge.yml

@@ -0,0 +1,6 @@
+---
+- name: Converge
+  hosts: all
+  gather_facts: no
+  roles:
+    - role: bootstrap-os

kubespray/extra_playbooks/roles/bootstrap-os/molecule/default/molecule.yml

@@ -0,0 +1,57 @@
+---
+dependency:
+  name: galaxy
+lint: |
+  set -e
+  yamllint -c ../../.yamllint .
+driver:
+  name: vagrant
+  provider:
+    name: libvirt
+platforms:
+  - name: ubuntu16
+    box: generic/ubuntu1604
+    cpus: 1
+    memory: 512
+  - name: ubuntu18
+    box: generic/ubuntu1804
+    cpus: 1
+    memory: 512
+  - name: ubuntu20
+    box: generic/ubuntu2004
+    cpus: 1
+    memory: 512
+  - name: centos7
+    box: centos/7
+    cpus: 1
+    memory: 512
+  - name: almalinux8
+    box: almalinux/8
+    cpus: 1
+    memory: 512
+  - name: debian9
+    box: generic/debian9
+    cpus: 1
+    memory: 512
+  - name: debian10
+    box: generic/debian10
+    cpus: 1
+    memory: 512
+provisioner:
+  name: ansible
+  config_options:
+    defaults:
+      callbacks_enabled: profile_tasks
+      timeout: 120
+  lint:
+    name: ansible-lint
+  inventory:
+    group_vars:
+      all:
+        user:
+          name: foo
+          comment: My test comment
+verifier:
+  name: testinfra
+  lint:
+    name: flake8

kubespray/extra_playbooks/roles/bootstrap-os/molecule/default/tests/test_default.py

@@ -0,0 +1,11 @@
+import os
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']
+).get_hosts('all')
+
+
+def test_python(host):
+    assert host.exists('python3') or host.exists('python')

kubespray/extra_playbooks/roles/bootstrap-os/tasks/bootstrap-amazon.yml

@@ -0,0 +1,13 @@
+---
+- name: Enable EPEL repo for Amazon Linux
+  yum_repository:
+    name: epel
+    file: epel
+    description: Extra Packages for Enterprise Linux 7 - $basearch
+    baseurl: http://download.fedoraproject.org/pub/epel/7/$basearch
+    gpgcheck: yes
+    gpgkey: http://download.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7
+    skip_if_unavailable: yes
+    enabled: yes
+    repo_gpgcheck: no
+  when: epel_enabled

kubespray/extra_playbooks/roles/bootstrap-os/tasks/bootstrap-centos.yml

@@ -0,0 +1,117 @@
+---
+- name: Gather host facts to get ansible_distribution_version ansible_distribution_major_version
+  setup:
+    gather_subset: '!all'
+    filter: ansible_distribution_*version
+
+- name: Add proxy to yum.conf or dnf.conf if http_proxy is defined
+  ini_file:
+    path: "{{ ( (ansible_distribution_major_version | int) < 8) | ternary('/etc/yum.conf','/etc/dnf/dnf.conf') }}"
+    section: main
+    option: proxy
+    value: "{{ http_proxy | default(omit) }}"
+    state: "{{ http_proxy | default(False) | ternary('present', 'absent') }}"
+    no_extra_spaces: true
+    mode: 0644
+  become: true
+  when: not skip_http_proxy_on_os_packages
+
+# For Oracle Linux install public repo
+- name: Download Oracle Linux public yum repo
+  get_url:
+    url: https://yum.oracle.com/public-yum-ol7.repo
+    dest: /etc/yum.repos.d/public-yum-ol7.repo
+  when:
+    - use_oracle_public_repo|default(true)
+    - '''ID="ol"'' in os_release.stdout_lines'
+    - (ansible_distribution_version | float) < 7.6
+  environment: "{{ proxy_env }}"
+
+- name: Enable Oracle Linux repo
+  ini_file:
+    dest: /etc/yum.repos.d/public-yum-ol7.repo
+    section: "{{ item }}"
+    option: enabled
+    value: "1"
+    mode: 0644
+  with_items:
+    - ol7_latest
+    - ol7_addons
+    - ol7_developer_EPEL
+  when:
+    - use_oracle_public_repo|default(true)
+    - '''ID="ol"'' in os_release.stdout_lines'
+    - (ansible_distribution_version | float) < 7.6
+
+- name: Install EPEL for Oracle Linux repo package
+  package:
+    name: "oracle-epel-release-el{{ ansible_distribution_major_version }}"
+    state: present
+  when:
+    - use_oracle_public_repo|default(true)
+    - '''ID="ol"'' in os_release.stdout_lines'
+    - (ansible_distribution_version | float) >= 7.6
+
+- name: Enable Oracle Linux repo
+  ini_file:
+    dest: "/etc/yum.repos.d/oracle-linux-ol{{ ansible_distribution_major_version }}.repo"
+    section: "ol{{ ansible_distribution_major_version }}_addons"
+    option: "{{ item.option }}"
+    value: "{{ item.value }}"
+    mode: 0644
+  with_items:
+    - { option: "name", value: "ol{{ ansible_distribution_major_version }}_addons" }
+    - { option: "enabled", value: "1" }
+    - { option: "baseurl", value: "http://yum.oracle.com/repo/OracleLinux/OL{{ ansible_distribution_major_version }}/addons/$basearch/" }
+  when:
+    - use_oracle_public_repo|default(true)
+    - '''ID="ol"'' in os_release.stdout_lines'
+    - (ansible_distribution_version | float) >= 7.6
+
+- name: Enable Centos extra repo for Oracle Linux
+  ini_file:
+    dest: "/etc/yum.repos.d/centos-extras.repo"
+    section: "extras"
+    option: "{{ item.option }}"
+    value: "{{ item.value }}"
+    mode: 0644
+  with_items:
+    - { option: "name", value: "CentOS-{{ ansible_distribution_major_version }} - Extras" }
+    - { option: "enabled", value: "1" }
+    - { option: "gpgcheck", value: "0" }
+    - { option: "baseurl", value: "http://mirror.centos.org/centos/{{ ansible_distribution_major_version }}/extras/$basearch/{% if ansible_distribution_major_version|int > 7 %}os/{% endif %}" }
+  when:
+    - use_oracle_public_repo|default(true)
+    - '''ID="ol"'' in os_release.stdout_lines'
+    - (ansible_distribution_version | float) >= 7.6
+    - (ansible_distribution_version | float) < 9
+
+# CentOS ships with python installed
+
+- name: Check presence of fastestmirror.conf
+  stat:
+    path: /etc/yum/pluginconf.d/fastestmirror.conf
+    get_attributes: no
+    get_checksum: no
+    get_mime: no
+  register: fastestmirror
+
+# the fastestmirror plugin can actually slow down Ansible deployments
+- name: Disable fastestmirror plugin if requested
+  lineinfile:
+    dest: /etc/yum/pluginconf.d/fastestmirror.conf
+    regexp: "^enabled=.*"
+    line: "enabled=0"
+    state: present
+  become: true
+  when:
+    - fastestmirror.stat.exists
+    - not centos_fastestmirror_enabled
+
+# libselinux-python is required on SELinux enabled hosts
+# See https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#managed-node-requirements
+- name: Install libselinux python package
+  package:
+    name: "{{ ( (ansible_distribution_major_version | int) < 8) | ternary('libselinux-python','python3-libselinux') }}"
+    state: present
+  become: true

kubespray/extra_playbooks/roles/bootstrap-os/tasks/bootstrap-clearlinux.yml

@@ -0,0 +1,16 @@
+---
+# ClearLinux ships with Python installed
+
+- name: Install basic package to run containers
+  package:
+    name: containers-basic
+    state: present
+
+- name: Make sure docker service is enabled
+  systemd:
+    name: docker
+    masked: false
+    enabled: true
+    daemon_reload: true
+    state: started
+  become: true

kubespray/extra_playbooks/roles/bootstrap-os/tasks/bootstrap-coreos.yml

@@ -0,0 +1,37 @@
+---
+# CoreOS ships without Python installed
+
+- name: Check if bootstrap is needed
+  raw: stat /opt/bin/.bootstrapped
+  register: need_bootstrap
+  failed_when: false
+  changed_when: false
+  tags:
+    - facts
+
+- name: Force binaries directory for Container Linux by CoreOS and Flatcar
+  set_fact:
+    bin_dir: "/opt/bin"
+  tags:
+    - facts
+
+- name: Run bootstrap.sh
+  script: bootstrap.sh
+  become: true
+  environment: "{{ proxy_env }}"
+  when:
+    - need_bootstrap.rc != 0
+
+- name: Set the ansible_python_interpreter fact
+  set_fact:
+    ansible_python_interpreter: "{{ bin_dir }}/python"
+  tags:
+    - facts
+
+- name: Disable auto-upgrade
+  systemd:
+    name: locksmithd.service
+    masked: true
+    state: stopped
+  when:
+    - coreos_locksmithd_disable

kubespray/extra_playbooks/roles/bootstrap-os/tasks/bootstrap-debian.yml

@@ -0,0 +1,76 @@
+---
+# Some Debian based distros ship without Python installed
+
+- name: Check if bootstrap is needed
+  raw: which python3
+  register: need_bootstrap
+  failed_when: false
+  changed_when: false
+  # This command should always run, even in check mode
+  check_mode: false
+  tags:
+    - facts
+
+- name: Check http::proxy in apt configuration files
+  raw: apt-config dump | grep -qsi 'Acquire::http::proxy'
+  register: need_http_proxy
+  failed_when: false
+  changed_when: false
+  # This command should always run, even in check mode
+  check_mode: false
+
+- name: Add http_proxy to /etc/apt/apt.conf if http_proxy is defined
+  raw: echo 'Acquire::http::proxy "{{ http_proxy }}";' >> /etc/apt/apt.conf
+  become: true
+  when:
+    - http_proxy is defined
+    - need_http_proxy.rc != 0
+    - not skip_http_proxy_on_os_packages
+
+- name: Check https::proxy in apt configuration files
+  raw: apt-config dump | grep -qsi 'Acquire::https::proxy'
+  register: need_https_proxy
+  failed_when: false
+  changed_when: false
+  # This command should always run, even in check mode
+  check_mode: false
+
+- name: Add https_proxy to /etc/apt/apt.conf if https_proxy is defined
+  raw: echo 'Acquire::https::proxy "{{ https_proxy }}";' >> /etc/apt/apt.conf
+  become: true
+  when:
+    - https_proxy is defined
+    - need_https_proxy.rc != 0
+    - not skip_http_proxy_on_os_packages
+
+- name: Install python3
+  raw:
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y python3-minimal
+  become: true
+  when:
+    - need_bootstrap.rc != 0
+
+- name: Update Apt cache
+  raw: apt-get update --allow-releaseinfo-change
+  become: true
+  when:
+    - '''ID=debian'' in os_release.stdout_lines'
+    - '''VERSION_ID="10"'' in os_release.stdout_lines or ''VERSION_ID="11"'' in os_release.stdout_lines'
+  register: bootstrap_update_apt_result
+  changed_when:
+    - '"changed its" in bootstrap_update_apt_result.stdout'
+    - '"value from" in bootstrap_update_apt_result.stdout'
+  ignore_errors: true
+
+- name: Set the ansible_python_interpreter fact
+  set_fact:
+    ansible_python_interpreter: "/usr/bin/python3"
+
+# Workaround for https://github.com/ansible/ansible/issues/25543
+- name: Install dbus for the hostname module
+  package:
+    name: dbus
+    state: present
+    use: apt
+  become: true

kubespray/extra_playbooks/roles/bootstrap-os/tasks/bootstrap-fedora-coreos.yml

@@ -0,0 +1,46 @@
+---
+
+- name: Check if bootstrap is needed
+  raw: which python
+  register: need_bootstrap
+  failed_when: false
+  changed_when: false
+  tags:
+    - facts
+
+- name: Remove podman network cni
+  raw: "podman network rm podman"
+  become: true
+  ignore_errors: true  # noqa ignore-errors
+  when: need_bootstrap.rc != 0
+
+- name: Clean up possible pending packages on fedora coreos
+  raw: "export http_proxy={{ http_proxy | default('') }};rpm-ostree cleanup -p }}"
+  become: true
+  when: need_bootstrap.rc != 0
+
+- name: Install required packages on fedora coreos
+  raw: "export http_proxy={{ http_proxy | default('') }};rpm-ostree install --allow-inactive {{ fedora_coreos_packages|join(' ') }}"
+  become: true
+  when: need_bootstrap.rc != 0
+
+- name: Reboot immediately for updated ostree
+  raw: "nohup bash -c 'sleep 5s && shutdown -r now'"
+  become: true
+  ignore_errors: true  # noqa ignore-errors
+  ignore_unreachable: yes
+  when: need_bootstrap.rc != 0
+
+- name: Wait for the reboot to complete
+  wait_for_connection:
+    timeout: 240
+    connect_timeout: 20
+    delay: 5
+    sleep: 5
+  when: need_bootstrap.rc != 0
+
+- name: Store the fact if this is an fedora core os host
+  set_fact:
+    is_fedora_coreos: True
+  tags:
+    - facts

kubespray/extra_playbooks/roles/bootstrap-os/tasks/bootstrap-fedora.yml

@@ -0,0 +1,36 @@
+---
+# Some Fedora based distros ship without Python installed
+
+- name: Check if bootstrap is needed
+  raw: which python
+  register: need_bootstrap
+  failed_when: false
+  changed_when: false
+  tags:
+    - facts
+
+- name: Add proxy to dnf.conf if http_proxy is defined
+  ini_file:
+    path: "/etc/dnf/dnf.conf"
+    section: main
+    option: proxy
+    value: "{{ http_proxy | default(omit) }}"
+    state: "{{ http_proxy | default(False) | ternary('present', 'absent') }}"
+    no_extra_spaces: true
+    mode: 0644
+  become: true
+  when: not skip_http_proxy_on_os_packages
+
+- name: Install python3 on fedora
+  raw: "dnf install --assumeyes --quiet python3"
+  become: true
+  when:
+    - need_bootstrap.rc != 0
+
+# libselinux-python3 is required on SELinux enabled hosts
+# See https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#managed-node-requirements
+- name: Install libselinux-python3
+  package:
+    name: libselinux-python3
+    state: present
+  become: true

kubespray/extra_playbooks/roles/bootstrap-os/tasks/bootstrap-flatcar.yml

@@ -0,0 +1,37 @@
+---
+# Flatcar Container Linux ships without Python installed
+
+- name: Check if bootstrap is needed
+  raw: stat /opt/bin/.bootstrapped
+  register: need_bootstrap
+  failed_when: false
+  changed_when: false
+  tags:
+    - facts
+
+- name: Force binaries directory for Flatcar Container Linux by Kinvolk
+  set_fact:
+    bin_dir: "/opt/bin"
+  tags:
+    - facts
+
+- name: Run bootstrap.sh
+  script: bootstrap.sh
+  become: true
+  environment: "{{ proxy_env }}"
+  when:
+    - need_bootstrap.rc != 0
+
+- name: Set the ansible_python_interpreter fact
+  set_fact:
+    ansible_python_interpreter: "{{ bin_dir }}/python"
+  tags:
+    - facts
+
+- name: Disable auto-upgrade
+  systemd:
+    name: locksmithd.service
+    masked: true
+    state: stopped
+  when:
+    - coreos_locksmithd_disable

kubespray/extra_playbooks/roles/bootstrap-os/tasks/bootstrap-opensuse.yml

@@ -0,0 +1,85 @@
+---
+# OpenSUSE ships with Python installed
+- name: Gather host facts to get ansible_distribution_version ansible_distribution_major_version
+  setup:
+    gather_subset: '!all'
+    filter: ansible_distribution_*version
+
+- name: Check that /etc/sysconfig/proxy file exists
+  stat:
+    path: /etc/sysconfig/proxy
+    get_attributes: no
+    get_checksum: no
+    get_mime: no
+  register: stat_result
+
+- name: Create the /etc/sysconfig/proxy empty file
+  file:  # noqa risky-file-permissions
+    path: /etc/sysconfig/proxy
+    state: touch
+  when:
+    - http_proxy is defined or https_proxy is defined
+    - not stat_result.stat.exists
+
+- name: Set the http_proxy in /etc/sysconfig/proxy
+  lineinfile:
+    path: /etc/sysconfig/proxy
+    regexp: '^HTTP_PROXY='
+    line: 'HTTP_PROXY="{{ http_proxy }}"'
+  become: true
+  when:
+    - http_proxy is defined
+
+- name: Set the https_proxy in /etc/sysconfig/proxy
+  lineinfile:
+    path: /etc/sysconfig/proxy
+    regexp: '^HTTPS_PROXY='
+    line: 'HTTPS_PROXY="{{ https_proxy }}"'
+  become: true
+  when:
+    - https_proxy is defined
+
+- name: Enable proxies
+  lineinfile:
+    path: /etc/sysconfig/proxy
+    regexp: '^PROXY_ENABLED='
+    line: 'PROXY_ENABLED="yes"'
+  become: true
+  when:
+    - http_proxy is defined or https_proxy is defined
+
+# Required for zypper module
+- name: Install python-xml
+  shell: zypper refresh && zypper --non-interactive install python-xml
+  changed_when: false
+  become: true
+  tags:
+    - facts
+
+# Without this package, the get_url module fails when trying to handle https
+- name: Install python-cryptography
+  zypper:
+    name: python-cryptography
+    state: present
+    update_cache: true
+  become: true
+  when:
+    - ansible_distribution_version is version('15.4', '<')
+
+- name: Install python3-cryptography
+  zypper:
+    name: python3-cryptography
+    state: present
+    update_cache: true
+  become: true
+  when:
+    - ansible_distribution_version is version('15.4', '>=')
+
+# Nerdctl needs some basic packages to get an environment up
+- name: Install basic dependencies
+  zypper:
+    name:
+      - iptables
+      - apparmor-parser
+    state: present
+  become: true

kubespray/extra_playbooks/roles/bootstrap-os/tasks/bootstrap-redhat.yml

@@ -0,0 +1,121 @@
+---
+- name: Gather host facts to get ansible_distribution_version ansible_distribution_major_version
+  setup:
+    gather_subset: '!all'
+    filter: ansible_distribution_*version
+
+- name: Add proxy to yum.conf or dnf.conf if http_proxy is defined
+  ini_file:
+    path: "{{ ( (ansible_distribution_major_version | int) < 8) | ternary('/etc/yum.conf','/etc/dnf/dnf.conf') }}"
+    section: main
+    option: proxy
+    value: "{{ http_proxy | default(omit) }}"
+    state: "{{ http_proxy | default(False) | ternary('present', 'absent') }}"
+    no_extra_spaces: true
+    mode: 0644
+  become: true
+  when: not skip_http_proxy_on_os_packages
+
+- name: Add proxy to RHEL subscription-manager if http_proxy is defined
+  command: /sbin/subscription-manager config --server.proxy_hostname={{ http_proxy | regex_replace(':\d+$') }} --server.proxy_port={{ http_proxy | regex_replace('^.*:') }}
+  become: true
+  when:
+    - not skip_http_proxy_on_os_packages
+    - http_proxy is defined
+
+- name: Check RHEL subscription-manager status
+  command: /sbin/subscription-manager status
+  register: rh_subscription_status
+  changed_when: "rh_subscription_status != 0"
+  ignore_errors: true  # noqa ignore-errors
+  become: true
+
+- name: RHEL subscription Organization ID/Activation Key registration
+  redhat_subscription:
+    state: present
+    org_id: "{{ rh_subscription_org_id }}"
+    activationkey: "{{ rh_subscription_activation_key }}"
+    auto_attach: true
+    force_register: true
+    syspurpose:
+      usage: "{{ rh_subscription_usage }}"
+      role: "{{ rh_subscription_role }}"
+      service_level_agreement: "{{ rh_subscription_sla }}"
+      sync: true
+  notify: RHEL auto-attach subscription
+  ignore_errors: true  # noqa ignore-errors
+  become: true
+  when:
+    - rh_subscription_org_id is defined
+    - rh_subscription_status.changed
+
+# this task has no_log set to prevent logging security sensitive information such as subscription passwords
+- name: RHEL subscription Username/Password registration
+  redhat_subscription:
+    state: present
+    username: "{{ rh_subscription_username }}"
+    password: "{{ rh_subscription_password }}"
+    auto_attach: true
+    force_register: true
+    syspurpose:
+      usage: "{{ rh_subscription_usage }}"
+      role: "{{ rh_subscription_role }}"
+      service_level_agreement: "{{ rh_subscription_sla }}"
+      sync: true
+  notify: RHEL auto-attach subscription
+  ignore_errors: true  # noqa ignore-errors
+  become: true
+  no_log: "{{ not (unsafe_show_logs|bool) }}"
+  when:
+    - rh_subscription_username is defined
+    - rh_subscription_status.changed
+
+# container-selinux is in extras repo
+- name: Enable RHEL 7 repos
+  rhsm_repository:
+    name:
+      - "rhel-7-server-rpms"
+      - "rhel-7-server-extras-rpms"
+    state: enabled
+  when:
+    - rhel_enable_repos | default(True) | bool
+    - ansible_distribution_major_version == "7"
+
+# container-selinux is in appstream repo
+- name: Enable RHEL 8 repos
+  rhsm_repository:
+    name:
+      - "rhel-8-for-*-baseos-rpms"
+      - "rhel-8-for-*-appstream-rpms"
+    state: enabled
+  when:
+    - rhel_enable_repos | default(True) | bool
+    - ansible_distribution_major_version == "8"
+
+- name: Check presence of fastestmirror.conf
+  stat:
+    path: /etc/yum/pluginconf.d/fastestmirror.conf
+    get_attributes: no
+    get_checksum: no
+    get_mime: no
+  register: fastestmirror
+
+# the fastestmirror plugin can actually slow down Ansible deployments
+- name: Disable fastestmirror plugin if requested
+  lineinfile:
+    dest: /etc/yum/pluginconf.d/fastestmirror.conf
+    regexp: "^enabled=.*"
+    line: "enabled=0"
+    state: present
+  become: true
+  when:
+    - fastestmirror.stat.exists
+    - not centos_fastestmirror_enabled
+
+# libselinux-python is required on SELinux enabled hosts
+# See https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#managed-node-requirements
+- name: Install libselinux python package
+  package:
+    name: "{{ ( (ansible_distribution_major_version | int) < 8) | ternary('libselinux-python','python3-libselinux') }}"
+    state: present
+  become: true

kubespray/extra_playbooks/roles/bootstrap-os/tasks/main.yml

@@ -0,0 +1,100 @@
+---
+- name: Fetch /etc/os-release
+  raw: cat /etc/os-release
+  register: os_release
+  changed_when: false
+  # This command should always run, even in check mode
+  check_mode: false
+
+- include_tasks: bootstrap-centos.yml
+  when: '''ID="centos"'' in os_release.stdout_lines or ''ID="ol"'' in os_release.stdout_lines or ''ID="almalinux"'' in os_release.stdout_lines or ''ID="rocky"'' in os_release.stdout_lines or ''ID="kylin"'' in os_release.stdout_lines  or ''ID="uos"'' in os_release.stdout_lines or ''ID="openEuler"'' in os_release.stdout_lines'
+
+- include_tasks: bootstrap-amazon.yml
+  when: '''ID="amzn"'' in os_release.stdout_lines'
+
+- include_tasks: bootstrap-redhat.yml
+  when: '''ID="rhel"'' in os_release.stdout_lines'
+
+- include_tasks: bootstrap-clearlinux.yml
+  when: '''ID=clear-linux-os'' in os_release.stdout_lines'
+
+# Fedora CoreOS
+- include_tasks: bootstrap-fedora-coreos.yml
+  when:
+    - '''ID=fedora'' in os_release.stdout_lines'
+    - '''VARIANT_ID=coreos'' in os_release.stdout_lines'
+
+- include_tasks: bootstrap-flatcar.yml
+  when: '''ID=flatcar'' in os_release.stdout_lines'
+
+- include_tasks: bootstrap-debian.yml
+  when: '''ID=debian'' in os_release.stdout_lines or ''ID=ubuntu'' in os_release.stdout_lines'
+
+# Fedora "classic"
+- include_tasks: bootstrap-fedora.yml
+  when:
+    - '''ID=fedora'' in os_release.stdout_lines'
+    - '''VARIANT_ID=coreos'' not in os_release.stdout_lines'
+
+- include_tasks: bootstrap-opensuse.yml
+  when: '''ID="opensuse-leap"'' in os_release.stdout_lines or ''ID="opensuse-tumbleweed"'' in os_release.stdout_lines'
+
+- name: Create remote_tmp for it is used by another module
+  file:
+    path: "{{ ansible_remote_tmp | default('~/.ansible/tmp') }}"
+    state: directory
+    mode: 0700
+
+# Workaround for https://github.com/ansible/ansible/issues/42726
+# (1/3)
+- name: Gather host facts to get ansible_os_family
+  setup:
+    gather_subset: '!all'
+    filter: ansible_*
+
+- name: Assign inventory name to unconfigured hostnames (non-CoreOS, non-Flatcar, Suse and ClearLinux, non-Fedora)
+  hostname:
+    name: "{{ inventory_hostname }}"
+  when:
+    - override_system_hostname
+    - ansible_os_family not in ['Suse', 'Flatcar', 'Flatcar Container Linux by Kinvolk', 'ClearLinux']
+    - not ansible_distribution == "Fedora"
+    - not is_fedora_coreos
+
+# (2/3)
+- name: Assign inventory name to unconfigured hostnames (CoreOS, Flatcar, Suse, ClearLinux and Fedora only)
+  command: "hostnamectl set-hostname {{ inventory_hostname }}"
+  register: hostname_changed
+  become: true
+  changed_when: false
+  when: >
+    override_system_hostname
+    and (ansible_os_family in ['Suse', 'Flatcar', 'Flatcar Container Linux by Kinvolk', 'ClearLinux']
+    or is_fedora_coreos
+    or ansible_distribution == "Fedora")
+
+# (3/3)
+- name: Update hostname fact (CoreOS, Flatcar, Suse, ClearLinux and Fedora only)
+  setup:
+    gather_subset: '!all'
+    filter: ansible_hostname
+  when: >
+    override_system_hostname
+    and (ansible_os_family in ['Suse', 'Flatcar', 'Flatcar Container Linux by Kinvolk', 'ClearLinux']
+    or is_fedora_coreos
+    or ansible_distribution == "Fedora")
+
+- name: Install ceph-commmon package
+  package:
+    name:
+      - ceph-common
+    state: present
+  when: rbd_provisioner_enabled|default(false)
+
+- name: Ensure bash_completion.d folder exists
+  file:
+    name: /etc/bash_completion.d/
+    state: directory
+    owner: root
+    group: root
+    mode: 0755

kubespray/extra_playbooks/roles/container-engine/containerd-common/defaults/main.yml

@@ -0,0 +1,17 @@
+---
+# We keep these variables around to allow migration from package
+# manager controlled installs to direct download ones.
+containerd_package: 'containerd.io'
+yum_repo_dir: /etc/yum.repos.d
+
+# Keep minimal repo information around for cleanup
+containerd_repo_info:
+  repos:
+
+# Ubuntu docker-ce repo
+containerd_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu"
+containerd_ubuntu_repo_component: "stable"
+
+# Debian docker-ce repo
+containerd_debian_repo_base_url: "https://download.docker.com/linux/debian"
+containerd_debian_repo_component: "stable"

kubespray/extra_playbooks/roles/container-engine/containerd-common/meta/main.yml

@@ -0,0 +1,2 @@
+---
+allow_duplicates: true

kubespray/extra_playbooks/roles/container-engine/containerd-common/tasks/main.yml

@@ -0,0 +1,31 @@
+---
+- name: containerd-common | check if fedora coreos
+  stat:
+    path: /run/ostree-booted
+    get_attributes: no
+    get_checksum: no
+    get_mime: no
+  register: ostree
+
+- name: containerd-common | set is_ostree
+  set_fact:
+    is_ostree: "{{ ostree.stat.exists }}"
+
+- name: containerd-common | gather os specific variables
+  include_vars: "{{ item }}"
+  with_first_found:
+    - files:
+        - "{{ ansible_distribution|lower }}-{{ ansible_distribution_version|lower|replace('/', '_') }}.yml"
+        - "{{ ansible_distribution|lower }}-{{ ansible_distribution_release|lower }}-{{ host_architecture }}.yml"
+        - "{{ ansible_distribution|lower }}-{{ ansible_distribution_release|lower }}.yml"
+        - "{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version|lower|replace('/', '_') }}.yml"
+        - "{{ ansible_distribution|lower }}-{{ host_architecture }}.yml"
+        - "{{ ansible_distribution|lower }}.yml"
+        - "{{ ansible_os_family|lower }}-{{ host_architecture }}.yml"
+        - "{{ ansible_os_family|lower }}.yml"
+        - defaults.yml
+      paths:
+        - ../vars
+      skip: true
+  tags:
+    - facts

kubespray/extra_playbooks/roles/container-engine/containerd-common/vars/amazon.yml

@@ -0,0 +1,2 @@
+---
+containerd_package: containerd

kubespray/extra_playbooks/roles/container-engine/containerd-common/vars/suse.yml

@@ -0,0 +1,2 @@
+---
+containerd_package: containerd

kubespray/extra_playbooks/roles/container-engine/containerd/defaults/main.yml

@@ -0,0 +1,75 @@
+---
+containerd_storage_dir: "/var/lib/containerd"
+containerd_state_dir: "/run/containerd"
+containerd_systemd_dir: "/etc/systemd/system/containerd.service.d"
+# The default value is not -999 here because containerd's oom_score_adj has been
+# set to the -999 even if containerd_oom_score is 0.
+# Ref: https://github.com/kubernetes-sigs/kubespray/pull/9275#issuecomment-1246499242
+containerd_oom_score: 0
+
+# containerd_default_runtime: "runc"
+# containerd_snapshotter: "native"
+
+containerd_runc_runtime:
+  name: runc
+  type: "io.containerd.runc.v2"
+  engine: ""
+  root: ""
+  base_runtime_spec: cri-base.json
+  options:
+    systemdCgroup: "{{ containerd_use_systemd_cgroup | ternary('true', 'false') }}"
+
+containerd_additional_runtimes: []
+# Example for Kata Containers as additional runtime:
+#  - name: kata
+#    type: "io.containerd.kata.v2"
+#    engine: ""
+#    root: ""
+
+containerd_base_runtime_spec_rlimit_nofile: 65535
+
+containerd_default_base_runtime_spec_patch:
+  process:
+    rlimits:
+      - type: RLIMIT_NOFILE
+        hard: "{{ containerd_base_runtime_spec_rlimit_nofile }}"
+        soft: "{{ containerd_base_runtime_spec_rlimit_nofile }}"
+
+containerd_base_runtime_specs:
+  cri-base.json: "{{ containerd_default_base_runtime_spec | combine(containerd_default_base_runtime_spec_patch,recursive=1) }}"
+
+containerd_grpc_max_recv_message_size: 16777216
+containerd_grpc_max_send_message_size: 16777216
+
+containerd_debug_level: "info"
+
+containerd_metrics_address: ""
+
+containerd_metrics_grpc_histogram: false
+
+containerd_registries:
+  "docker.io": "https://registry-1.docker.io"
+
+containerd_max_container_log_line_size: -1
+
+# If enabled it will allow non root users to use port numbers <1024
+containerd_enable_unprivileged_ports: false
+# If enabled it will allow non root users to use icmp sockets
+containerd_enable_unprivileged_icmp: false
+
+containerd_cfg_dir: /etc/containerd
+
+# Extra config to be put in {{ containerd_cfg_dir }}/config.toml literally
+containerd_extra_args: ''
+
+# Configure registry auth (if applicable to secure/insecure registries)
+containerd_registry_auth: []
+#  - registry: 10.0.0.2:5000
+#    username: user
+#    password: pass
+
+# Configure containerd service
+containerd_limit_proc_num: "infinity"
+containerd_limit_core: "infinity"
+containerd_limit_open_file_num: "infinity"
+containerd_limit_mem_lock: "infinity"

kubespray/extra_playbooks/roles/container-engine/containerd/handlers/main.yml

@@ -0,0 +1,21 @@
+---
+- name: restart containerd
+  command: /bin/true
+  notify:
+    - Containerd | restart containerd
+    - Containerd | wait for containerd
+
+- name: Containerd | restart containerd
+  systemd:
+    name: containerd
+    state: restarted
+    enabled: yes
+    daemon-reload: yes
+    masked: no
+
+- name: Containerd | wait for containerd
+  command: "{{ containerd_bin_dir }}/ctr images ls -q"
+  register: containerd_ready
+  retries: 8
+  delay: 4
+  until: containerd_ready.rc == 0

kubespray/extra_playbooks/roles/container-engine/containerd/handlers/reset.yml

@@ -0,0 +1 @@
+---

kubespray/extra_playbooks/roles/container-engine/containerd/meta/main.yml

@@ -0,0 +1,6 @@
+---
+dependencies:
+  - role: container-engine/containerd-common
+  - role: container-engine/runc
+  - role: container-engine/crictl
+  - role: container-engine/nerdctl

kubespray/extra_playbooks/roles/container-engine/containerd/molecule/default/converge.yml

@@ -0,0 +1,9 @@
+---
+- name: Converge
+  hosts: all
+  become: true
+  vars:
+    container_manager: containerd
+  roles:
+    - role: kubespray-defaults
+    - role: container-engine/containerd

kubespray/extra_playbooks/roles/container-engine/containerd/molecule/default/molecule.yml

@@ -0,0 +1,49 @@
+---
+driver:
+  name: vagrant
+  provider:
+    name: libvirt
+lint: |
+  set -e
+  yamllint -c ../../../.yamllint .
+platforms:
+  - name: ubuntu20
+    box: generic/ubuntu2004
+    cpus: 1
+    memory: 1024
+    groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+  - name: debian11
+    box: generic/debian11
+    cpus: 1
+    memory: 1024
+    groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+  - name: almalinux8
+    box: almalinux/8
+    cpus: 1
+    memory: 1024
+    groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_ROLES_PATH: ../../../../
+  config_options:
+    defaults:
+      callbacks_enabled: profile_tasks
+      timeout: 120
+  lint:
+    name: ansible-lint
+    options:
+      c: ../../../.ansible-lint
+verifier:
+  name: testinfra
+  lint:
+    name: flake8

kubespray/extra_playbooks/roles/container-engine/containerd/molecule/default/prepare.yml

@@ -0,0 +1,28 @@
+---
+- name: Prepare
+  hosts: all
+  gather_facts: False
+  become: true
+  vars:
+    ignore_assert_errors: true
+  roles:
+    - role: kubespray-defaults
+    - role: bootstrap-os
+    - role: kubernetes/preinstall
+    - role: adduser
+      user: "{{ addusers.kube }}"
+  tasks:
+    - include_tasks: "../../../../download/tasks/download_file.yml"
+      vars:
+        download: "{{ download_defaults | combine(downloads.cni) }}"
+
+- name: Prepare CNI
+  hosts: all
+  gather_facts: False
+  become: true
+  vars:
+    ignore_assert_errors: true
+    kube_network_plugin: cni
+  roles:
+    - role: kubespray-defaults
+    - role: network_plugin/cni

kubespray/extra_playbooks/roles/container-engine/containerd/molecule/default/tests/test_default.py

@@ -0,0 +1,55 @@
+import os
+import pytest
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_service(host):
+    svc = host.service("containerd")
+    assert svc.is_running
+    assert svc.is_enabled
+
+
+def test_version(host):
+    crictl = "/usr/local/bin/crictl"
+    path = "unix:///var/run/containerd/containerd.sock"
+    with host.sudo():
+        cmd = host.command(crictl + " --runtime-endpoint " + path + " version")
+    assert cmd.rc == 0
+    assert "RuntimeName:  containerd" in cmd.stdout
+
+
+@pytest.mark.parametrize('image, dest', [
+    ('quay.io/kubespray/hello-world:latest', '/tmp/hello-world.tar')
+])
+def test_image_pull_save_load(host, image, dest):
+    nerdctl = "/usr/local/bin/nerdctl"
+    dest_file = host.file(dest)
+
+    with host.sudo():
+        pull_cmd = host.command(nerdctl + " pull " + image)
+    assert pull_cmd.rc ==0
+
+    with host.sudo():
+        save_cmd = host.command(nerdctl + " save -o " + dest + " " + image)
+    assert save_cmd.rc == 0
+    assert dest_file.exists
+
+    with host.sudo():
+        load_cmd = host.command(nerdctl + " load < " + dest)
+    assert load_cmd.rc == 0
+
+
+@pytest.mark.parametrize('image', [
+    ('quay.io/kubespray/hello-world:latest')
+])
+def test_run(host, image):
+    nerdctl = "/usr/local/bin/nerdctl"
+
+    with host.sudo():
+        cmd = host.command(nerdctl + " -n k8s.io run " + image)
+    assert cmd.rc == 0
+    assert "Hello from Docker" in cmd.stdout

kubespray/extra_playbooks/roles/container-engine/containerd/tasks/main.yml

@@ -0,0 +1,124 @@
+---
+- name: Fail containerd setup if distribution is not supported
+  fail:
+    msg: "{{ ansible_distribution }} is not supported by containerd."
+  when:
+    - ansible_distribution not in ["CentOS", "OracleLinux", "RedHat", "Ubuntu", "Debian", "Fedora", "AlmaLinux", "Rocky", "Amazon", "Flatcar", "Flatcar Container Linux by Kinvolk", "Suse", "openSUSE Leap", "openSUSE Tumbleweed", "Kylin Linux Advanced Server", "UnionTech", "openEuler"]
+
+- name: containerd | Remove any package manager controlled containerd package
+  package:
+    name: "{{ containerd_package }}"
+    state: absent
+  when:
+    - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar"))
+
+- name: containerd | Remove containerd repository
+  file:
+    path: "{{ yum_repo_dir }}/containerd.repo"
+    state: absent
+  when:
+    - ansible_os_family in ['RedHat']
+
+- name: containerd | Remove containerd repository
+  apt_repository:
+    repo: "{{ item }}"
+    state: absent
+  with_items: "{{ containerd_repo_info.repos }}"
+  when: ansible_pkg_mgr == 'apt'
+
+- name: containerd | Download containerd
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.containerd) }}"
+
+- name: containerd | Unpack containerd archive
+  unarchive:
+    src: "{{ downloads.containerd.dest }}"
+    dest: "{{ containerd_bin_dir }}"
+    mode: 0755
+    remote_src: yes
+    extra_opts:
+      - --strip-components=1
+  notify: restart containerd
+
+- name: containerd | Remove orphaned binary
+  file:
+    path: "/usr/bin/{{ item }}"
+    state: absent
+  when:
+    - containerd_bin_dir != "/usr/bin"
+    - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar"))
+  ignore_errors: true  # noqa ignore-errors
+  with_items:
+    - containerd
+    - containerd-shim
+    - containerd-shim-runc-v1
+    - containerd-shim-runc-v2
+    - ctr
+
+- name: containerd | Generate systemd service for containerd
+  template:
+    src: containerd.service.j2
+    dest: /etc/systemd/system/containerd.service
+    mode: 0644
+  notify: restart containerd
+
+- name: containerd | Ensure containerd directories exist
+  file:
+    dest: "{{ item }}"
+    state: directory
+    mode: 0755
+    owner: root
+    group: root
+  with_items:
+    - "{{ containerd_systemd_dir }}"
+    - "{{ containerd_cfg_dir }}"
+    - "{{ containerd_storage_dir }}"
+    - "{{ containerd_state_dir }}"
+
+- name: containerd | Write containerd proxy drop-in
+  template:
+    src: http-proxy.conf.j2
+    dest: "{{ containerd_systemd_dir }}/http-proxy.conf"
+    mode: 0644
+  notify: restart containerd
+  when: http_proxy is defined or https_proxy is defined
+
+- name: containerd | Generate default base_runtime_spec
+  register: ctr_oci_spec
+  command: "{{ containerd_bin_dir }}/ctr oci spec"
+  check_mode: false
+  changed_when: false
+
+- name: containerd | Store generated default base_runtime_spec
+  set_fact:
+    containerd_default_base_runtime_spec: "{{ ctr_oci_spec.stdout | from_json }}"
+
+- name: containerd | Write base_runtime_specs
+  copy:
+    content: "{{ item.value }}"
+    dest: "{{ containerd_cfg_dir }}/{{ item.key }}"
+    owner: "root"
+    mode: 0644
+  with_dict: "{{ containerd_base_runtime_specs | default({}) }}"
+  notify: restart containerd
+
+- name: containerd | Copy containerd config file
+  template:
+    src: config.toml.j2
+    dest: "{{ containerd_cfg_dir }}/config.toml"
+    owner: "root"
+    mode: 0640
+  notify: restart containerd
+
+# you can sometimes end up in a state where everything is installed
+# but containerd was not started / enabled
+- name: containerd | Flush handlers
+  meta: flush_handlers
+
+- name: containerd | Ensure containerd is started and enabled
+  systemd:
+    name: containerd
+    daemon_reload: yes
+    enabled: yes
+    state: started

kubespray/extra_playbooks/roles/container-engine/containerd/tasks/reset.yml

@@ -0,0 +1,41 @@
+---
+- name: containerd | Remove containerd repository for RedHat os family
+  file:
+    path: "{{ yum_repo_dir }}/containerd.repo"
+    state: absent
+  when:
+    - ansible_os_family in ['RedHat']
+  tags:
+    - reset_containerd
+
+- name: containerd | Remove containerd repository for Debian os family
+  apt_repository:
+    repo: "{{ item }}"
+    state: absent
+  with_items: "{{ containerd_repo_info.repos }}"
+  when: ansible_pkg_mgr == 'apt'
+  tags:
+    - reset_containerd
+
+- name: containerd | Stop containerd service
+  service:
+    name: containerd
+    daemon_reload: true
+    enabled: false
+    masked: true
+    state: stopped
+  tags:
+    - reset_containerd
+
+- name: containerd | Remove configuration files
+  file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - /etc/systemd/system/containerd.service
+    - "{{ containerd_systemd_dir }}"
+    - "{{ containerd_cfg_dir }}"
+    - "{{ containerd_storage_dir }}"
+    - "{{ containerd_state_dir }}"
+  tags:
+    - reset_containerd

kubespray/extra_playbooks/roles/container-engine/containerd/templates/config.toml.j2

@@ -0,0 +1,79 @@
+version = 2
+root = "{{ containerd_storage_dir }}"
+state = "{{ containerd_state_dir }}"
+oom_score = {{ containerd_oom_score }}
+
+[grpc]
+  max_recv_message_size = {{ containerd_grpc_max_recv_message_size | default(16777216) }}
+  max_send_message_size = {{ containerd_grpc_max_send_message_size | default(16777216) }}
+
+[debug]
+  level = "{{ containerd_debug_level | default('info') }}"
+
+[metrics]
+  address = "{{ containerd_metrics_address | default('') }}"
+  grpc_histogram = {{ containerd_metrics_grpc_histogram | default(false) | lower }}
+
+[plugins]
+  [plugins."io.containerd.grpc.v1.cri"]
+    sandbox_image = "{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}"
+    max_container_log_line_size = {{ containerd_max_container_log_line_size }}
+    enable_unprivileged_ports = {{ containerd_enable_unprivileged_ports | default(false) | lower }}
+    enable_unprivileged_icmp = {{ containerd_enable_unprivileged_icmp | default(false) | lower }}
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "{{ containerd_default_runtime | default('runc') }}"
+      snapshotter = "{{ containerd_snapshotter | default('overlayfs') }}"
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+{% for runtime in [containerd_runc_runtime] + containerd_additional_runtimes %}
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}]
+          runtime_type = "{{ runtime.type }}"
+          runtime_engine = "{{ runtime.engine }}"
+          runtime_root = "{{ runtime.root }}"
+{% if runtime.base_runtime_spec is defined %}
+          base_runtime_spec = "{{ containerd_cfg_dir }}/{{ runtime.base_runtime_spec }}"
+{% endif %}
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}.options]
+{% for key, value in runtime.options.items() %}
+            {{ key }} = {{ value }}
+{% endfor %}
+{% endfor %}
+{% if kata_containers_enabled %}
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu]
+          runtime_type = "io.containerd.kata-qemu.v2"
+{% endif %}
+{% if gvisor_enabled %}
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc]
+          runtime_type = "io.containerd.runsc.v1"
+{% endif %}
+    [plugins."io.containerd.grpc.v1.cri".registry]
+      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+{% for registry, addr in containerd_registries.items() %}
+        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ registry }}"]
+          endpoint = ["{{ ([ addr ] | flatten ) | join('","') }}"]
+{% endfor %}
+{% if containerd_insecure_registries is defined and containerd_insecure_registries|length>0 %}
+{% for registry, addr in containerd_insecure_registries.items() %}
+        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ registry }}"]
+          endpoint = ["{{ ([ addr ] | flatten ) | join('","') }}"]
+{% endfor %}
+{% for addr in containerd_insecure_registries.values() | flatten | unique %}
+        [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ addr }}".tls]
+          insecure_skip_verify = true
+{% endfor %}
+{% endif %}
+{% for registry in containerd_registry_auth if registry['registry'] is defined %}
+{% if (registry['username'] is defined and registry['password'] is defined) or registry['auth'] is defined %}
+      [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ registry['registry'] }}".auth]
+{% if registry['username'] is defined and registry['password'] is defined %}
+        password = "{{ registry['password'] }}"
+        username = "{{ registry['username'] }}"
+{% else %}
+        auth = "{{ registry['auth'] }}"
+{% endif %}
+{% endif %}
+{% endfor %}
+
+{% if containerd_extra_args is defined %}
+{{ containerd_extra_args }}
+{% endif %}

kubespray/extra_playbooks/roles/container-engine/containerd/templates/containerd.service.j2

@@ -0,0 +1,41 @@
+# Copyright The containerd Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[Unit]
+Description=containerd container runtime
+Documentation=https://containerd.io
+After=network.target local-fs.target
+
+[Service]
+ExecStartPre=-/sbin/modprobe overlay
+ExecStart={{ containerd_bin_dir }}/containerd
+
+Type=notify
+Delegate=yes
+KillMode=process
+Restart=always
+RestartSec=5
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNPROC={{ containerd_limit_proc_num }}
+LimitCORE={{ containerd_limit_core }}
+LimitNOFILE={{ containerd_limit_open_file_num }}
+LimitMEMLOCK={{ containerd_limit_mem_lock }}
+# Comment TasksMax if your systemd version does not supports it.
+# Only systemd 226 and above support this version.
+TasksMax=infinity
+OOMScoreAdjust=-999
+
+[Install]
+WantedBy=multi-user.target

kubespray/extra_playbooks/roles/container-engine/containerd/templates/http-proxy.conf.j2

@@ -0,0 +1,2 @@
+[Service]
+Environment={% if http_proxy is defined %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy is defined %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy is defined %}"NO_PROXY={{ no_proxy }}"{% endif %}

kubespray/extra_playbooks/roles/container-engine/containerd/vars/debian.yml

@@ -0,0 +1,7 @@
+---
+containerd_repo_info:
+  repos:
+    - >
+      deb {{ containerd_debian_repo_base_url }}
+      {{ ansible_distribution_release|lower }}
+      {{ containerd_debian_repo_component }}

kubespray/extra_playbooks/roles/container-engine/containerd/vars/ubuntu.yml

@@ -0,0 +1,7 @@
+---
+containerd_repo_info:
+  repos:
+    - >
+      deb {{ containerd_ubuntu_repo_base_url }}
+      {{ ansible_distribution_release|lower }}
+      {{ containerd_ubuntu_repo_component }}

kubespray/extra_playbooks/roles/container-engine/cri-dockerd/handlers/main.yml

@@ -0,0 +1,35 @@
+---
+- name: restart and enable cri-dockerd
+  command: /bin/true
+  notify:
+    - cri-dockerd | reload systemd
+    - cri-dockerd | restart docker.service
+    - cri-dockerd | reload cri-dockerd.socket
+    - cri-dockerd | reload cri-dockerd.service
+    - cri-dockerd | enable cri-dockerd service
+
+- name: cri-dockerd | reload systemd
+  systemd:
+    name: cri-dockerd
+    daemon_reload: true
+    masked: no
+
+- name: cri-dockerd | restart docker.service
+  service:
+    name: docker.service
+    state: restarted
+
+- name: cri-dockerd | reload cri-dockerd.socket
+  service:
+    name: cri-dockerd.socket
+    state: restarted
+
+- name: cri-dockerd | reload cri-dockerd.service
+  service:
+    name: cri-dockerd.service
+    state: restarted
+
+- name: cri-dockerd | enable cri-dockerd service
+  service:
+    name: cri-dockerd.service
+    enabled: yes

kubespray/extra_playbooks/roles/container-engine/cri-dockerd/meta/main.yml

@@ -0,0 +1,4 @@
+---
+dependencies:
+  - role: container-engine/docker
+  - role: container-engine/crictl

kubespray/extra_playbooks/roles/container-engine/cri-dockerd/molecule/default/converge.yml

@@ -0,0 +1,9 @@
+---
+- name: Converge
+  hosts: all
+  become: true
+  vars:
+    container_manager: docker
+  roles:
+    - role: kubespray-defaults
+    - role: container-engine/cri-dockerd

kubespray/extra_playbooks/roles/container-engine/cri-dockerd/molecule/default/files/10-mynet.conf

@@ -0,0 +1,17 @@
+{
+  "cniVersion": "0.2.0",
+  "name": "mynet",
+  "type": "bridge",
+  "bridge": "cni0",
+  "isGateway": true,
+  "ipMasq": true,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "172.19.0.0/24",
+    "routes": [
+      {
+        "dst": "0.0.0.0/0"
+      }
+    ]
+  }
+}

kubespray/extra_playbooks/roles/container-engine/cri-dockerd/molecule/default/files/container.json

@@ -0,0 +1,10 @@
+{
+  "metadata": {
+    "name": "cri-dockerd1"
+  },
+  "image": {
+    "image": "quay.io/kubespray/hello-world:latest"
+  },
+  "log_path": "cri-dockerd1.0.log",
+  "linux": {}
+}

kubespray/extra_playbooks/roles/container-engine/cri-dockerd/molecule/default/files/sandbox.json

@@ -0,0 +1,10 @@
+{
+  "metadata": {
+    "name": "cri-dockerd1",
+    "namespace": "default",
+    "attempt": 1,
+    "uid": "hdishd83djaidwnduwk28bcsb"
+  },
+  "linux": {},
+  "log_directory": "/tmp"
+}

kubespray/extra_playbooks/roles/container-engine/cri-dockerd/molecule/default/molecule.yml

@@ -0,0 +1,45 @@
+---
+driver:
+  name: vagrant
+  provider:
+    name: libvirt
+    options:
+      driver: kvm
+lint: |
+  set -e
+  yamllint -c ../../../.yamllint .
+platforms:
+  - name: almalinux8
+    box: almalinux/8
+    cpus: 1
+    memory: 1024
+    nested: true
+    groups:
+      - kube_control_plane
+  - name: ubuntu20
+    box: generic/ubuntu2004
+    cpus: 1
+    memory: 1024
+    nested: true
+    groups:
+      - kube_control_plane
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_ROLES_PATH: ../../../../
+  config_options:
+    defaults:
+      callbacks_enabled: profile_tasks
+      timeout: 120
+  lint:
+    name: ansible-lint
+    options:
+      c: ../../../.ansible-lint
+  inventory:
+    group_vars:
+      all:
+        become: true
+verifier:
+  name: testinfra
+  lint:
+    name: flake8

kubespray/extra_playbooks/roles/container-engine/cri-dockerd/molecule/default/prepare.yml

@@ -0,0 +1,47 @@
+---
+- name: Prepare
+  hosts: all
+  become: true
+  roles:
+    - role: kubespray-defaults
+    - role: bootstrap-os
+    - role: adduser
+      user: "{{ addusers.kube }}"
+  tasks:
+    - include_tasks: "../../../../download/tasks/download_file.yml"
+      vars:
+        download: "{{ download_defaults | combine(downloads.cni) }}"
+
+- name: Prepare container runtime
+  hosts: all
+  become: true
+  vars:
+    container_manager: containerd
+    kube_network_plugin: cni
+  roles:
+    - role: kubespray-defaults
+    - role: network_plugin/cni
+  tasks:
+    - name: Copy test container files
+      copy:
+        src: "{{ item }}"
+        dest: "/tmp/{{ item }}"
+        owner: root
+        mode: 0644
+      with_items:
+        - container.json
+        - sandbox.json
+    - name: Create /etc/cni/net.d directory
+      file:
+        path: /etc/cni/net.d
+        state: directory
+        owner: "{{ kube_owner }}"
+        mode: 0755
+    - name: Setup CNI
+      copy:
+        src: "{{ item }}"
+        dest: "/etc/cni/net.d/{{ item }}"
+        owner: root
+        mode: 0644
+      with_items:
+        - 10-mynet.conf

kubespray/extra_playbooks/roles/container-engine/cri-dockerd/molecule/default/tests/test_default.py

@@ -0,0 +1,19 @@
+import os
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_run_pod(host):
+    run_command = "/usr/local/bin/crictl run --with-pull /tmp/container.json /tmp/sandbox.json"
+    with host.sudo():
+        cmd = host.command(run_command)
+    assert cmd.rc == 0
+
+    with host.sudo():
+      log_f = host.file("/tmp/cri-dockerd1.0.log")
+
+      assert log_f.exists
+      assert b"Hello from Docker" in log_f.content

kubespray/extra_playbooks/roles/container-engine/cri-dockerd/tasks/main.yml

@@ -0,0 +1,28 @@
+---
+- name: runc | Download cri-dockerd binary
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.cri_dockerd) }}"
+
+- name: Copy cri-dockerd binary from download dir
+  copy:
+    src: "{{ local_release_dir }}/cri-dockerd"
+    dest: "{{ bin_dir }}/cri-dockerd"
+    mode: 0755
+    remote_src: true
+  notify:
+    - restart and enable cri-dockerd
+
+- name: Generate cri-dockerd systemd unit files
+  template:
+    src: "{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    mode: 0644
+  with_items:
+    - cri-dockerd.service
+    - cri-dockerd.socket
+  notify:
+    - restart and enable cri-dockerd
+
+- name: Flush handlers
+  meta: flush_handlers

kubespray/extra_playbooks/roles/container-engine/cri-dockerd/templates/cri-dockerd.service.j2

@@ -0,0 +1,40 @@
+[Unit]
+Description=CRI Interface for Docker Application Container Engine
+Documentation=https://docs.mirantis.com
+After=network-online.target firewalld.service docker.service
+Wants=network-online.target docker.service
+Requires=cri-dockerd.socket
+
+[Service]
+Type=notify
+ExecStart={{ bin_dir }}/cri-dockerd --container-runtime-endpoint {{ cri_socket }} --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --network-plugin=cni --pod-cidr={{ kube_pods_subnet }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_version }} {% if enable_dual_stack_networks %}--ipv6-dual-stack=True{% endif %}
+
+ExecReload=/bin/kill -s HUP $MAINPID
+TimeoutSec=0
+RestartSec=2
+Restart=always
+
+# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
+# Both the old, and new location are accepted by systemd 229 and up, so using the old location
+# to make them work for either version of systemd.
+StartLimitBurst=3
+
+# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
+# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
+# this option work for either version of systemd.
+StartLimitInterval=60s
+
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNOFILE=infinity
+LimitNPROC=infinity
+LimitCORE=infinity
+
+# Comment TasksMax if your systemd version does not support it.
+# Only systemd 226 and above support this option.
+TasksMax=infinity
+Delegate=yes
+KillMode=process
+
+[Install]
+WantedBy=multi-user.target

kubespray/extra_playbooks/roles/container-engine/cri-dockerd/templates/cri-dockerd.socket.j2

@@ -0,0 +1,12 @@
+[Unit]
+Description=CRI Docker Socket for the API
+PartOf=cri-dockerd.service
+
+[Socket]
+ListenStream=%t/cri-dockerd.sock
+SocketMode=0660
+SocketUser=root
+SocketGroup=docker
+
+[Install]
+WantedBy=sockets.target

kubespray/extra_playbooks/roles/container-engine/cri-o/defaults/main.yml

@@ -0,0 +1,103 @@
+---
+
+crio_cgroup_manager: "{{ kubelet_cgroup_driver | default('systemd') }}"
+crio_conmon: "{{ bin_dir }}/conmon"
+crio_enable_metrics: false
+crio_log_level: "info"
+crio_metrics_port: "9090"
+crio_pause_image: "{{ pod_infra_image_repo }}:{{ pod_infra_version }}"
+
+# Registries defined within cri-o.
+# By default unqualified images are not allowed for security reasons
+crio_registries: []
+#  - prefix: docker.io
+#    insecure: false
+#    blocked: false
+#    location: registry-1.docker.io ## REQUIRED
+#    unqualified: false
+#    mirrors:
+#      - location: 172.20.100.52:5000
+#        insecure: true
+#      - location: mirror.gcr.io
+#        insecure: false
+
+crio_registry_auth: []
+#  - registry: 10.0.0.2:5000
+#    username: user
+#    password: pass
+
+crio_seccomp_profile: ""
+crio_selinux: "{{ (preinstall_selinux_state == 'enforcing')|lower }}"
+crio_signature_policy: "{% if ansible_os_family == 'ClearLinux' %}/usr/share/defaults/crio/policy.json{% endif %}"
+
+# Override system default for storage driver
+# crio_storage_driver: "overlay"
+
+crio_stream_port: "10010"
+
+crio_required_version: "{{ kube_version | regex_replace('^v(?P<major>\\d+).(?P<minor>\\d+).(?P<patch>\\d+)$', '\\g<major>.\\g<minor>') }}"
+
+# The crio_runtimes variable defines a list of OCI compatible runtimes.
+crio_runtimes:
+  - name: runc
+    path: "{{ bin_dir }}/runc"
+    type: oci
+    root: /run/runc
+
+# Kata Containers is an OCI runtime, where containers are run inside lightweight
+# VMs. Kata provides additional isolation towards the host, minimizing the host attack
+# surface and mitigating the consequences of containers breakout.
+kata_runtimes:
+  # Kata Containers with the default configured VMM
+  - name: kata-qemu
+    path: /usr/local/bin/containerd-shim-kata-qemu-v2
+    type: vm
+    root: /run/kata-containers
+    privileged_without_host_devices: true
+
+# crun is a fast and low-memory footprint OCI Container Runtime fully written in C.
+crun_runtime:
+  name: crun
+  path: "{{ bin_dir }}/crun"
+  type: oci
+  root: /run/crun
+
+# youki is an implementation of the OCI runtime-spec in Rust, similar to runc.
+youki_runtime:
+  name: youki
+  path: "{{ youki_bin_dir }}/youki"
+  type: oci
+  root: /run/youki
+
+# TODO(cristicalin): remove this after 2.21
+crio_download_base: "download.opensuse.org/repositories/devel:kubic:libcontainers:stable"
+crio_download_crio: "http://{{ crio_download_base }}:/cri-o:/"
+
+# Configure the cri-o pids limit, increase this for heavily multi-threaded workloads
+# see https://github.com/cri-o/cri-o/issues/1921
+crio_pids_limit: 1024
+
+# Reserve 16M uids and gids for user namespaces (256 pods * 65536 uids/gids)
+# at the end of the uid/gid space
+crio_remap_enable: false
+crio_remap_user: containers
+crio_subuid_start: 2130706432
+crio_subuid_length: 16777216
+crio_subgid_start: 2130706432
+crio_subgid_length: 16777216
+
+# cri-o binary files
+crio_bin_files:
+  - conmon
+  - crio
+  - crio-status
+  - pinns
+
+# cri-o manual files
+crio_man_files:
+  5:
+    - crio.conf
+    - crio.conf.d
+  8:
+    - crio
+    - crio-status

kubespray/extra_playbooks/roles/container-engine/cri-o/files/mounts.conf

@@ -0,0 +1 @@
+/usr/share/rhel/secrets:/run/secrets

kubespray/extra_playbooks/roles/container-engine/cri-o/handlers/main.yml

@@ -0,0 +1,16 @@
+---
+- name: restart crio
+  command: /bin/true
+  notify:
+    - CRI-O | reload systemd
+    - CRI-O | reload crio
+
+- name: CRI-O | reload systemd
+  systemd:
+    daemon_reload: true
+
+- name: CRI-O | reload crio
+  service:
+    name: crio
+    state: restarted
+    enabled: yes

kubespray/extra_playbooks/roles/container-engine/cri-o/meta/main.yml

@@ -0,0 +1,5 @@
+---
+dependencies:
+  - role: container-engine/crictl
+  - role: container-engine/runc
+  - role: container-engine/skopeo

kubespray/extra_playbooks/roles/container-engine/cri-o/molecule/default/converge.yml

@@ -0,0 +1,9 @@
+---
+- name: Converge
+  hosts: all
+  become: true
+  vars:
+    container_manager: crio
+  roles:
+    - role: kubespray-defaults
+    - role: container-engine/cri-o

kubespray/extra_playbooks/roles/container-engine/cri-o/molecule/default/files/10-mynet.conf

@@ -0,0 +1,17 @@
+{
+  "cniVersion": "0.2.0",
+  "name": "mynet",
+  "type": "bridge",
+  "bridge": "cni0",
+  "isGateway": true,
+  "ipMasq": true,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "172.19.0.0/24",
+    "routes": [
+      {
+        "dst": "0.0.0.0/0"
+      }
+    ]
+  }
+}

kubespray/extra_playbooks/roles/container-engine/cri-o/molecule/default/files/container.json

@@ -0,0 +1,10 @@
+{
+  "metadata": {
+    "name": "runc1"
+  },
+  "image": {
+    "image": "quay.io/kubespray/hello-world:latest"
+  },
+  "log_path": "runc1.0.log",
+  "linux": {}
+}

kubespray/extra_playbooks/roles/container-engine/cri-o/molecule/default/files/sandbox.json

@@ -0,0 +1,10 @@
+{
+  "metadata": {
+    "name": "runc1",
+    "namespace": "default",
+    "attempt": 1,
+    "uid": "hdishd83djaidwnduwk28bcsb"
+  },
+  "linux": {},
+  "log_directory": "/tmp"
+}

kubespray/extra_playbooks/roles/container-engine/cri-o/molecule/default/molecule.yml

@@ -0,0 +1,57 @@
+---
+driver:
+  name: vagrant
+  provider:
+    name: libvirt
+lint: |
+  set -e
+  yamllint -c ../../../.yamllint .
+platforms:
+  - name: ubuntu20
+    box: generic/ubuntu2004
+    cpus: 2
+    memory: 1024
+    groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+  - name: almalinux8
+    box: almalinux/8
+    cpus: 2
+    memory: 1024
+    groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+  - name: fedora
+    box: fedora/36-cloud-base
+    cpus: 2
+    memory: 1024
+    groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+  - name: debian10
+    box: generic/debian10
+    cpus: 2
+    memory: 1024
+    groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_ROLES_PATH: ../../../../
+  config_options:
+    defaults:
+      callbacks_enabled: profile_tasks
+      timeout: 120
+  lint:
+    name: ansible-lint
+    options:
+      c: ../../../.ansible-lint
+verifier:
+  name: testinfra
+  lint:
+    name: flake8

kubespray/extra_playbooks/roles/container-engine/cri-o/molecule/default/prepare.yml

@@ -0,0 +1,52 @@
+---
+- name: Prepare
+  hosts: all
+  gather_facts: False
+  become: true
+  vars:
+    ignore_assert_errors: true
+  roles:
+    - role: kubespray-defaults
+    - role: bootstrap-os
+    - role: kubernetes/preinstall
+    - role: adduser
+      user: "{{ addusers.kube }}"
+  tasks:
+    - include_tasks: "../../../../download/tasks/download_file.yml"
+      vars:
+        download: "{{ download_defaults | combine(downloads.cni) }}"
+
+- name: Prepare CNI
+  hosts: all
+  gather_facts: False
+  become: true
+  vars:
+    ignore_assert_errors: true
+    kube_network_plugin: cni
+  roles:
+    - role: kubespray-defaults
+    - role: network_plugin/cni
+  tasks:
+    - name: Copy test container files
+      copy:
+        src: "{{ item }}"
+        dest: "/tmp/{{ item }}"
+        owner: root
+        mode: 0644
+      with_items:
+        - container.json
+        - sandbox.json
+    - name: Create /etc/cni/net.d directory
+      file:
+        path: /etc/cni/net.d
+        state: directory
+        owner: "{{ kube_owner }}"
+        mode: 0755
+    - name: Setup CNI
+      copy:
+        src: "{{ item }}"
+        dest: "/etc/cni/net.d/{{ item }}"
+        owner: root
+        mode: 0644
+      with_items:
+        - 10-mynet.conf

kubespray/extra_playbooks/roles/container-engine/cri-o/molecule/default/tests/test_default.py

@@ -0,0 +1,35 @@
+import os
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_service(host):
+    svc = host.service("crio")
+    assert svc.is_running
+    assert svc.is_enabled
+
+
+def test_run(host):
+    crictl = "/usr/local/bin/crictl"
+    path = "unix:///var/run/crio/crio.sock"
+    with host.sudo():
+        cmd = host.command(crictl + " --runtime-endpoint " + path + " version")
+    assert cmd.rc == 0
+    assert "RuntimeName:  cri-o" in cmd.stdout
+
+def test_run_pod(host):
+    runtime = "runc"
+
+    run_command = "/usr/local/bin/crictl run --with-pull --runtime {} /tmp/container.json /tmp/sandbox.json".format(runtime)
+    with host.sudo():
+        cmd = host.command(run_command)
+    assert cmd.rc == 0
+
+    with host.sudo():
+      log_f = host.file("/tmp/runc1.0.log")
+
+      assert log_f.exists
+      assert b"Hello from Docker" in log_f.content

kubespray/extra_playbooks/roles/container-engine/cri-o/tasks/cleanup.yaml

@@ -0,0 +1,119 @@
+---
+# TODO(cristicalin): drop this file after 2.21
+- name: CRI-O kubic repo name for debian os family
+  set_fact:
+    crio_kubic_debian_repo_name: "{{ ((ansible_distribution == 'Ubuntu') | ternary('x','')) ~ ansible_distribution ~ '_' ~ ansible_distribution_version }}"
+  when: ansible_os_family == "Debian"
+
+- name: Remove legacy CRI-O kubic apt repo key
+  apt_key:
+    url: "https://{{ crio_download_base }}/{{ crio_kubic_debian_repo_name }}/Release.key"
+    state: absent
+  when: crio_kubic_debian_repo_name is defined
+
+- name: Remove legacy CRI-O kubic apt repo
+  apt_repository:
+    repo: "deb http://{{ crio_download_base }}/{{ crio_kubic_debian_repo_name }}/ /"
+    state: absent
+    filename: devel-kubic-libcontainers-stable
+  when: crio_kubic_debian_repo_name is defined
+
+- name: Remove legacy CRI-O kubic cri-o apt repo
+  apt_repository:
+    repo: "deb {{ crio_download_crio }}{{ crio_version }}/{{ crio_kubic_debian_repo_name }}/ /"
+    state: absent
+    filename: devel-kubic-libcontainers-stable-cri-o
+  when: crio_kubic_debian_repo_name is defined
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: devel_kubic_libcontainers_stable
+    description: Stable Releases of Upstream github.com/containers packages (CentOS_$releasever)
+    baseurl: http://{{ crio_download_base }}/CentOS_{{ ansible_distribution_major_version }}/
+    state: absent
+  when:
+    - ansible_os_family == "RedHat"
+    - ansible_distribution not in ["Amazon", "Fedora"]
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: "devel_kubic_libcontainers_stable_cri-o_{{ crio_version }}"
+    description: "CRI-O {{ crio_version }} (CentOS_$releasever)"
+    baseurl: "{{ crio_download_crio }}{{ crio_version }}/CentOS_{{ ansible_distribution_major_version }}/"
+    state: absent
+  when:
+    - ansible_os_family == "RedHat"
+    - ansible_distribution not in ["Amazon", "Fedora"]
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: devel_kubic_libcontainers_stable
+    description: Stable Releases of Upstream github.com/containers packages
+    baseurl: http://{{ crio_download_base }}/Fedora_{{ ansible_distribution_major_version }}/
+    state: absent
+  when:
+    - ansible_distribution in ["Fedora"]
+    - not is_ostree
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: "devel_kubic_libcontainers_stable_cri-o_{{ crio_version }}"
+    description: "CRI-O {{ crio_version }}"
+    baseurl: "{{ crio_download_crio }}{{ crio_version }}/Fedora_{{ ansible_distribution_major_version }}/"
+    state: absent
+  when:
+    - ansible_distribution in ["Fedora"]
+    - not is_ostree
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: devel_kubic_libcontainers_stable
+    description: Stable Releases of Upstream github.com/containers packages
+    baseurl: http://{{ crio_download_base }}/CentOS_7/
+    state: absent
+  when: ansible_distribution in ["Amazon"]
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: "devel_kubic_libcontainers_stable_cri-o_{{ crio_version }}"
+    description: "CRI-O {{ crio_version }}"
+    baseurl: "{{ crio_download_crio }}{{ crio_version }}/CentOS_7/"
+    state: absent
+  when: ansible_distribution in ["Amazon"]
+
+- name: Disable modular repos for CRI-O
+  ini_file:
+    path: "/etc/yum.repos.d/{{ item.repo }}.repo"
+    section: "{{ item.section }}"
+    option: enabled
+    value: 0
+    mode: 0644
+  become: true
+  when: is_ostree
+  loop:
+    - repo: "fedora-updates-modular"
+      section: "updates-modular"
+    - repo: "fedora-modular"
+      section: "fedora-modular"
+
+# Disable any older module version if we enabled them before
+- name: Disable CRI-O ex module
+  command: "rpm-ostree ex module disable cri-o:{{ item }}"
+  become: true
+  when:
+    - is_ostree
+    - ostree_version is defined and ostree_version.stdout is version('2021.9', '>=')
+  with_items:
+    - 1.22
+    - 1.23
+    - 1.24
+
+- name: cri-o | remove installed packages
+  package:
+    name: "{{ item }}"
+    state: absent
+  when: not is_ostree
+  with_items:
+    - cri-o
+    - cri-o-runc
+    - oci-systemd-hook

kubespray/extra_playbooks/roles/container-engine/cri-o/tasks/main.yaml

@@ -0,0 +1,206 @@
+---
+- name: cri-o | check if fedora coreos
+  stat:
+    path: /run/ostree-booted
+    get_attributes: no
+    get_checksum: no
+    get_mime: no
+  register: ostree
+
+- name: cri-o | set is_ostree
+  set_fact:
+    is_ostree: "{{ ostree.stat.exists }}"
+
+- name: cri-o | get ostree version
+  shell: "set -o pipefail && rpm-ostree --version | awk -F\\' '/Version/{print $2}'"
+  args:
+    executable: /bin/bash
+  register: ostree_version
+  when: is_ostree
+
+- name: cri-o | Download cri-o
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.crio) }}"
+
+- name: cri-o | special handling for amazon linux
+  import_tasks: "setup-amazon.yaml"
+  when: ansible_distribution in ["Amazon"]
+
+- name: cri-o | clean up reglacy repos
+  import_tasks: "cleanup.yaml"
+
+- name: cri-o | build a list of crio runtimes with Katacontainers runtimes
+  set_fact:
+    crio_runtimes: "{{ crio_runtimes + kata_runtimes  }}"
+  when:
+    - kata_containers_enabled
+
+- name: cri-o | build a list of crio runtimes with crun runtime
+  set_fact:
+    crio_runtimes: "{{ crio_runtimes + [crun_runtime] }}"
+  when:
+    - crun_enabled
+
+- name: cri-o | build a list of crio runtimes with youki runtime
+  set_fact:
+    crio_runtimes: "{{ crio_runtimes + [youki_runtime] }}"
+  when:
+    - youki_enabled
+
+- name: cri-o | make sure needed folders exist in the system
+  with_items:
+    - /etc/crio
+    - /etc/containers
+    - /etc/systemd/system/crio.service.d
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: 0755
+
+- name: cri-o | install cri-o config
+  template:
+    src: crio.conf.j2
+    dest: /etc/crio/crio.conf
+    mode: 0644
+  register: config_install
+
+- name: cri-o | install config.json
+  template:
+    src: config.json.j2
+    dest: /etc/crio/config.json
+    mode: 0644
+  register: reg_auth_install
+
+- name: cri-o | copy binaries
+  copy:
+    src: "{{ local_release_dir }}/cri-o/bin/{{ item }}"
+    dest: "{{ bin_dir }}/{{ item }}"
+    mode: 0755
+    remote_src: true
+  with_items:
+    - "{{ crio_bin_files }}"
+  notify: restart crio
+
+- name: cri-o | copy service file
+  copy:
+    src: "{{ local_release_dir }}/cri-o/contrib/crio.service"
+    dest: /etc/systemd/system/crio.service
+    mode: 0755
+    remote_src: true
+  notify: restart crio
+
+- name: cri-o | copy default policy
+  copy:
+    src: "{{ local_release_dir }}/cri-o/contrib/policy.json"
+    dest: /etc/containers/policy.json
+    mode: 0755
+    remote_src: true
+  notify: restart crio
+
+- name: cri-o | copy mounts.conf
+  copy:
+    src: mounts.conf
+    dest: /etc/containers/mounts.conf
+    mode: 0644
+  when:
+    - ansible_os_family == 'RedHat'
+  notify: restart crio
+
+- name: cri-o | create directory for oci hooks
+  file:
+    path: /etc/containers/oci/hooks.d
+    state: directory
+    owner: root
+    mode: 0755
+
+- name: cri-o | set overlay driver
+  ini_file:
+    dest: /etc/containers/storage.conf
+    section: storage
+    option: "{{ item.option }}"
+    value: "{{ item.value }}"
+    mode: 0644
+  with_items:
+    - option: driver
+      value: '"overlay"'
+    - option: graphroot
+      value: '"/var/lib/containers/storage"'
+
+# metacopy=on is available since 4.19 and was backported to RHEL 4.18 kernel
+- name: cri-o | set metacopy mount options correctly
+  ini_file:
+    dest: /etc/containers/storage.conf
+    section: storage.options.overlay
+    option: mountopt
+    value: '{{ ''"nodev"'' if ansible_kernel is version_compare(("4.18" if ansible_os_family == "RedHat" else "4.19"), "<") else ''"nodev,metacopy=on"'' }}'
+    mode: 0644
+
+- name: cri-o | create directory registries configs
+  file:
+    path: /etc/containers/registries.conf.d
+    state: directory
+    owner: root
+    mode: 0755
+
+- name: cri-o | write registries configs
+  template:
+    src: registry.conf.j2
+    dest: "/etc/containers/registries.conf.d/10-{{ item.prefix | default(item.location) | regex_replace(':', '_') }}.conf"
+    mode: 0644
+  loop: "{{ crio_registries }}"
+  notify: restart crio
+
+- name: cri-o | configure unqualified registry settings
+  template:
+    src: unqualified.conf.j2
+    dest: "/etc/containers/registries.conf.d/01-unqualified.conf"
+    mode: 0644
+  notify: restart crio
+
+- name: cri-o | write cri-o proxy drop-in
+  template:
+    src: http-proxy.conf.j2
+    dest: /etc/systemd/system/crio.service.d/http-proxy.conf
+    mode: 0644
+  notify: restart crio
+  when: http_proxy is defined or https_proxy is defined
+
+- name: cri-o | configure the uid/gid space for user namespaces
+  lineinfile:
+    path: '{{ item.path }}'
+    line: '{{ item.entry }}'
+    regex: '^\s*{{ crio_remap_user }}:'
+    state: '{{ "present" if crio_remap_enable | bool else "absent" }}'
+  loop:
+    - path: /etc/subuid
+      entry: '{{ crio_remap_user }}:{{ crio_subuid_start }}:{{ crio_subuid_length }}'
+    - path: /etc/subgid
+      entry: '{{ crio_remap_user }}:{{ crio_subgid_start }}:{{ crio_subgid_length }}'
+  loop_control:
+    label: '{{ item.path }}'
+
+- name: cri-o | ensure crio service is started and enabled
+  service:
+    name: crio
+    daemon_reload: true
+    enabled: true
+    state: started
+  register: service_start
+
+- name: cri-o | trigger service restart only when needed
+  service:  # noqa 503
+    name: crio
+    state: restarted
+  when:
+    - config_install.changed
+    - reg_auth_install.changed
+    - not service_start.changed
+
+- name: cri-o | verify that crio is running
+  command: "{{ bin_dir }}/crio-status info"
+  register: get_crio_info
+  until: get_crio_info is succeeded
+  changed_when: false
+  retries: 5
+  delay: "{{ retry_stagger | random + 3 }}"

kubespray/extra_playbooks/roles/container-engine/cri-o/tasks/reset.yml

@@ -0,0 +1,101 @@
+---
+- name: CRI-O | Kubic repo name for debian os family
+  set_fact:
+    crio_kubic_debian_repo_name: "{{ ((ansible_distribution == 'Ubuntu') | ternary('x','')) ~ ansible_distribution ~ '_' ~ ansible_distribution_version }}"
+  when: ansible_os_family == "Debian"
+  tags:
+    - reset_crio
+
+- name: CRI-O | Remove kubic apt repo
+  apt_repository:
+    repo: "deb http://{{ crio_download_base }}/{{ crio_kubic_debian_repo_name }}/ /"
+    state: absent
+  when: crio_kubic_debian_repo_name is defined
+  tags:
+    - reset_crio
+
+- name: CRI-O | Remove cri-o apt repo
+  apt_repository:
+    repo: "deb {{ crio_download_crio }}{{ crio_version }}/{{ crio_kubic_debian_repo_name }}/ /"
+    state: present
+    filename: devel-kubic-libcontainers-stable-cri-o
+  when: crio_kubic_debian_repo_name is defined
+  tags:
+    - reset_crio
+
+- name: CRI-O | Remove CRI-O kubic yum repo
+  yum_repository:
+    name: devel_kubic_libcontainers_stable
+    state: absent
+  when: ansible_distribution in ["Amazon"]
+  tags:
+    - reset_crio
+
+- name: CRI-O | Remove CRI-O kubic yum repo
+  yum_repository:
+    name: "devel_kubic_libcontainers_stable_cri-o_{{ crio_version }}"
+    state: absent
+  when:
+    - ansible_os_family == "RedHat"
+    - ansible_distribution not in ["Amazon", "Fedora"]
+  tags:
+    - reset_crio
+
+- name: CRI-O | Run yum-clean-metadata
+  command: yum clean metadata
+  args:
+    warn: no
+  when:
+    - ansible_os_family == "RedHat"
+  tags:
+    - reset_crio
+
+- name: CRI-O | Remove crictl
+  file:
+    name: "{{ item }}"
+    state: absent
+  loop:
+    - /etc/crictl.yaml
+    - "{{ bin_dir }}/crictl"
+  tags:
+    - reset_crio
+
+- name: CRI-O | Stop crio service
+  service:
+    name: crio
+    daemon_reload: true
+    enabled: false
+    masked: true
+    state: stopped
+  tags:
+    - reset_crio
+
+- name: CRI-O | Remove CRI-O configuration files
+  file:
+    name: "{{ item }}"
+    state: absent
+  loop:
+    - /etc/crio
+    - /etc/containers
+    - /etc/systemd/system/crio.service.d
+  tags:
+    - reset_crio
+
+- name: CRI-O | Remove dpkg hold
+  dpkg_selections:
+    name: "{{ item }}"
+    selection: install
+  when: ansible_pkg_mgr == 'apt'
+  changed_when: false
+  with_items: "{{ crio_packages }}"
+  tags:
+    - reset_crio
+
+- name: CRI-O | Uninstall CRI-O package
+  package:
+    name: "{{ item }}"
+    state: absent
+  when: not is_ostree
+  with_items: "{{ crio_packages }}"
+  tags:
+    - reset_crio

kubespray/extra_playbooks/roles/container-engine/cri-o/tasks/setup-amazon.yaml

@@ -0,0 +1,38 @@
+---
+- name: Check that amzn2-extras.repo exists
+  stat:
+    path: /etc/yum.repos.d/amzn2-extras.repo
+  register: amzn2_extras_file_stat
+
+- name: Find docker repo in amzn2-extras.repo file
+  lineinfile:
+    dest: /etc/yum.repos.d/amzn2-extras.repo
+    line: "[amzn2extra-docker]"
+  check_mode: yes
+  register: amzn2_extras_docker_repo
+  when:
+    - amzn2_extras_file_stat.stat.exists
+
+- name: Remove docker repository
+  ini_file:
+    dest: /etc/yum.repos.d/amzn2-extras.repo
+    section: amzn2extra-docker
+    option: enabled
+    value: "0"
+    backup: yes
+    mode: 0644
+  when:
+    - amzn2_extras_file_stat.stat.exists
+    - not amzn2_extras_docker_repo.changed
+
+- name: Add container-selinux yum repo
+  yum_repository:
+    name: copr:copr.fedorainfracloud.org:lsm5:container-selinux
+    file: _copr_lsm5-container-selinux.repo
+    description: Copr repo for container-selinux owned by lsm5
+    baseurl: https://download.copr.fedorainfracloud.org/results/lsm5/container-selinux/epel-7-$basearch/
+    gpgcheck: yes
+    gpgkey: https://download.copr.fedorainfracloud.org/results/lsm5/container-selinux/pubkey.gpg
+    skip_if_unavailable: yes
+    enabled: yes
+    repo_gpgcheck: no

kubespray/extra_playbooks/roles/container-engine/cri-o/templates/config.json.j2

@@ -0,0 +1,17 @@
+{% if crio_registry_auth is defined and crio_registry_auth|length %}
+{ 
+{% for reg in crio_registry_auth %}
+  "auths": {
+    "{{ reg.registry }}": {
+      "auth": "{{ (reg.username + ':' + reg.password) | string | b64encode }}"
+    }
+{% if not loop.last %}
+  },
+{% else %}
+  }
+{% endif %}
+{% endfor %}
+}
+{% else %}
+{}
+{% endif %}

kubespray/extra_playbooks/roles/container-engine/cri-o/templates/crio.conf.j2

@@ -0,0 +1,369 @@
+# The CRI-O configuration file specifies all of the available configuration
+# options and command-line flags for the crio(8) OCI Kubernetes Container Runtime
+# daemon, but in a TOML format that can be more easily modified and versioned.
+#
+# Please refer to crio.conf(5) for details of all configuration options.
+
+# CRI-O supports partial configuration reload during runtime, which can be
+# done by sending SIGHUP to the running process. Currently supported options
+# are explicitly mentioned with: 'This option supports live configuration
+# reload'.
+
+# CRI-O reads its storage defaults from the containers-storage.conf(5) file
+# located at /etc/containers/storage.conf. Modify this storage configuration if
+# you want to change the system's defaults. If you want to modify storage just
+# for CRI-O, you can change the storage configuration options here.
+[crio]
+
+# Path to the "root directory". CRI-O stores all of its data, including
+# containers images, in this directory.
+root = "/var/lib/containers/storage"
+
+# Path to the "run directory". CRI-O stores all of its state in this directory.
+runroot = "/var/run/containers/storage"
+
+# Storage driver used to manage the storage of images and containers. Please
+# refer to containers-storage.conf(5) to see all available storage drivers.
+{% if crio_storage_driver is defined %}
+storage_driver = "{{ crio_storage_driver }}"
+{% endif %}
+
+# List to pass options to the storage driver. Please refer to
+# containers-storage.conf(5) to see all available storage options.
+#storage_option = [
+#]
+
+# The default log directory where all logs will go unless directly specified by
+# the kubelet. The log directory specified must be an absolute directory.
+log_dir = "/var/log/crio/pods"
+
+# Location for CRI-O to lay down the temporary version file.
+# It is used to check if crio wipe should wipe containers, which should
+# always happen on a node reboot
+version_file = "/var/run/crio/version"
+
+# Location for CRI-O to lay down the persistent version file.
+# It is used to check if crio wipe should wipe images, which should
+# only happen when CRI-O has been upgraded
+version_file_persist = "/var/lib/crio/version"
+
+# The crio.api table contains settings for the kubelet/gRPC interface.
+[crio.api]
+
+# Path to AF_LOCAL socket on which CRI-O will listen.
+listen = "/var/run/crio/crio.sock"
+
+# IP address on which the stream server will listen.
+stream_address = "127.0.0.1"
+
+# The port on which the stream server will listen. If the port is set to "0", then
+# CRI-O will allocate a random free port number.
+stream_port = "{{ crio_stream_port }}"
+
+# Enable encrypted TLS transport of the stream server.
+stream_enable_tls = false
+
+# Path to the x509 certificate file used to serve the encrypted stream. This
+# file can change, and CRI-O will automatically pick up the changes within 5
+# minutes.
+stream_tls_cert = ""
+
+# Path to the key file used to serve the encrypted stream. This file can
+# change and CRI-O will automatically pick up the changes within 5 minutes.
+stream_tls_key = ""
+
+# Path to the x509 CA(s) file used to verify and authenticate client
+# communication with the encrypted stream. This file can change and CRI-O will
+# automatically pick up the changes within 5 minutes.
+stream_tls_ca = ""
+
+# Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 16 * 1024 * 1024.
+grpc_max_send_msg_size = 16777216
+
+# Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 16 * 1024 * 1024.
+grpc_max_recv_msg_size = 16777216
+
+# The crio.runtime table contains settings pertaining to the OCI runtime used
+# and options for how to set up and manage the OCI runtime.
+[crio.runtime]
+
+# A list of ulimits to be set in containers by default, specified as
+# "<ulimit name>=<soft limit>:<hard limit>", for example:
+# "nofile=1024:2048"
+# If nothing is set here, settings will be inherited from the CRI-O daemon
+#default_ulimits = [
+#]
+
+# default_runtime is the _name_ of the OCI runtime to be used as the default.
+# The name is matched against the runtimes map below.
+default_runtime = "runc"
+
+# If true, the runtime will not use pivot_root, but instead use MS_MOVE.
+no_pivot = false
+
+# decryption_keys_path is the path where the keys required for
+# image decryption are stored. This option supports live configuration reload.
+decryption_keys_path = "/etc/crio/keys/"
+
+# Path to the conmon binary, used for monitoring the OCI runtime.
+# Will be searched for using $PATH if empty.
+conmon = "{{ crio_conmon }}"
+
+# Cgroup setting for conmon
+{% if crio_cgroup_manager == "cgroupfs" %}
+conmon_cgroup = "pod"
+{% else %}
+conmon_cgroup = "system.slice"
+{% endif %}
+
+# Environment variable list for the conmon process, used for passing necessary
+# environment variables to conmon or the runtime.
+conmon_env = [
+	"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+]
+
+# Additional environment variables to set for all the
+# containers. These are overridden if set in the
+# container image spec or in the container runtime configuration.
+default_env = [
+]
+
+# If true, SELinux will be used for pod separation on the host.
+selinux = {{ crio_selinux }}
+
+# Path to the seccomp.json profile which is used as the default seccomp profile
+# for the runtime. If not specified, then the internal default seccomp profile
+# will be used. This option supports live configuration reload.
+seccomp_profile = "{{ crio_seccomp_profile }}"
+
+# Used to change the name of the default AppArmor profile of CRI-O. The default
+# profile name is "crio-default". This profile only takes effect if the user
+# does not specify a profile via the Kubernetes Pod's metadata annotation. If
+# the profile is set to "unconfined", then this equals to disabling AppArmor.
+# This option supports live configuration reload.
+# apparmor_profile = "crio-default"
+
+# Cgroup management implementation used for the runtime.
+cgroup_manager = "{{ crio_cgroup_manager }}"
+
+# List of default capabilities for containers. If it is empty or commented out,
+# only the capabilities defined in the containers json file by the user/kube
+# will be added.
+default_capabilities = [
+	"CHOWN",
+	"DAC_OVERRIDE",
+	"FSETID",
+	"FOWNER",
+	"NET_RAW",
+	"SETGID",
+	"SETUID",
+	"SETPCAP",
+	"NET_BIND_SERVICE",
+	"SYS_CHROOT",
+	"KILL",
+]
+
+# List of default sysctls. If it is empty or commented out, only the sysctls
+# defined in the container json file by the user/kube will be added.
+default_sysctls = [
+]
+
+# List of additional devices. specified as
+# "<device-on-host>:<device-on-container>:<permissions>", for example: "--device=/dev/sdc:/dev/xvdc:rwm".
+#If it is empty or commented out, only the devices
+# defined in the container json file by the user/kube will be added.
+additional_devices = [
+]
+
+# Path to OCI hooks directories for automatically executed hooks. If one of the
+# directories does not exist, then CRI-O will automatically skip them.
+hooks_dir = [
+	"/usr/share/containers/oci/hooks.d",
+]
+
+# List of default mounts for each container. **Deprecated:** this option will
+# be removed in future versions in favor of default_mounts_file.
+default_mounts = [
+]
+
+# Path to the file specifying the defaults mounts for each container. The
+# format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads
+# its default mounts from the following two files:
+#
+#   1) /etc/containers/mounts.conf (i.e., default_mounts_file): This is the
+#      override file, where users can either add in their own default mounts, or
+#      override the default mounts shipped with the package.
+#
+#   2) /usr/share/containers/mounts.conf: This is the default file read for
+#      mounts. If you want CRI-O to read from a different, specific mounts file,
+#      you can change the default_mounts_file. Note, if this is done, CRI-O will
+#      only add mounts it finds in this file.
+#
+#default_mounts_file = ""
+
+# Maximum number of processes allowed in a container.
+pids_limit = {{ crio_pids_limit }}
+
+# Maximum sized allowed for the container log file. Negative numbers indicate
+# that no size limit is imposed. If it is positive, it must be >= 8192 to
+# match/exceed conmon's read buffer. The file is truncated and re-opened so the
+# limit is never exceeded.
+log_size_max = -1
+
+# Whether container output should be logged to journald in addition to the kuberentes log file
+log_to_journald = false
+
+# Path to directory in which container exit files are written to by conmon.
+container_exits_dir = "/var/run/crio/exits"
+
+# Path to directory for container attach sockets.
+container_attach_socket_dir = "/var/run/crio"
+
+# The prefix to use for the source of the bind mounts.
+bind_mount_prefix = ""
+
+# If set to true, all containers will run in read-only mode.
+read_only = false
+
+# Changes the verbosity of the logs based on the level it is set to. Options
+# are fatal, panic, error, warn, info, debug and trace. This option supports
+# live configuration reload.
+log_level = "{{ crio_log_level }}"
+
+# Filter the log messages by the provided regular expression.
+# This option supports live configuration reload.
+log_filter = ""
+
+# The UID mappings for the user namespace of each container. A range is
+# specified in the form containerUID:HostUID:Size. Multiple ranges must be
+# separated by comma.
+uid_mappings = ""
+
+# The GID mappings for the user namespace of each container. A range is
+# specified in the form containerGID:HostGID:Size. Multiple ranges must be
+# separated by comma.
+gid_mappings = ""
+
+# The minimal amount of time in seconds to wait before issuing a timeout
+# regarding the proper termination of the container. The lowest possible
+# value is 30s, whereas lower values are not considered by CRI-O.
+ctr_stop_timeout = 30
+
+# **DEPRECATED** this option is being replaced by manage_ns_lifecycle, which is described below.
+# manage_network_ns_lifecycle = false
+
+# manage_ns_lifecycle determines whether we pin and remove namespaces
+# and manage their lifecycle
+{% if kata_containers_enabled %}
+manage_ns_lifecycle = true
+{% else %}
+manage_ns_lifecycle = false
+{% endif %}
+
+# The directory where the state of the managed namespaces gets tracked.
+# Only used when manage_ns_lifecycle is true.
+namespaces_dir = "/var/run"
+
+# pinns_path is the path to find the pinns binary, which is needed to manage namespace lifecycle
+pinns_path = ""
+
+# The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes.
+# The runtime to use is picked based on the runtime_handler provided by the CRI.
+# If no runtime_handler is provided, the runtime will be picked based on the level
+# of trust of the workload. Each entry in the table should follow the format:
+#
+#[crio.runtime.runtimes.runtime-handler]
+#  runtime_path = "/path/to/the/executable"
+#  runtime_type = "oci"
+#  runtime_root = "/path/to/the/root"
+#
+# Where:
+# - runtime-handler: name used to identify the runtime
+# - runtime_path (optional, string): absolute path to the runtime executable in
+#   the host filesystem. If omitted, the runtime-handler identifier should match
+#   the runtime executable name, and the runtime executable should be placed
+#   in $PATH.
+# - runtime_type (optional, string): type of runtime, one of: "oci", "vm". If
+#   omitted, an "oci" runtime is assumed.
+# - runtime_root (optional, string): root directory for storage of containers
+#   state.
+
+{% for runtime in crio_runtimes %}
+[crio.runtime.runtimes.{{ runtime.name }}]
+runtime_path = "{{ runtime.path }}"
+runtime_type = "{{ runtime.type }}"
+runtime_root = "{{ runtime.root }}"
+privileged_without_host_devices = {{ runtime.privileged_without_host_devices|default(false)|lower }}
+allowed_annotations = {{ runtime.allowed_annotations|default([])|to_json }}
+{% endfor %}
+
+# Kata Containers with the Firecracker VMM
+#[crio.runtime.runtimes.kata-fc]
+
+# The crio.image table contains settings pertaining to the management of OCI images.
+#
+# CRI-O reads its configured registries defaults from the system wide
+# containers-registries.conf(5) located in /etc/containers/registries.conf. If
+# you want to modify just CRI-O, you can change the registries configuration in
+# this file. Otherwise, leave insecure_registries and registries commented out to
+# use the system's defaults from /etc/containers/registries.conf.
+[crio.image]
+
+# Default transport for pulling images from a remote container storage.
+default_transport = "docker://"
+
+# The path to a file containing credentials necessary for pulling images from
+# secure registries. The file is similar to that of /var/lib/kubelet/config.json
+global_auth_file = "/etc/crio/config.json"
+
+# The image used to instantiate infra containers.
+# This option supports live configuration reload.
+pause_image = "{{ crio_pause_image }}"
+
+# The path to a file containing credentials specific for pulling the pause_image from
+# above. The file is similar to that of /var/lib/kubelet/config.json
+# This option supports live configuration reload.
+pause_image_auth_file = ""
+
+# The command to run to have a container stay in the paused state.
+# When explicitly set to "", it will fallback to the entrypoint and command
+# specified in the pause image. When commented out, it will fallback to the
+# default: "/pause". This option supports live configuration reload.
+pause_command = "/pause"
+
+# Path to the file which decides what sort of policy we use when deciding
+# whether or not to trust an image that we've pulled. It is not recommended that
+# this option be used, as the default behavior of using the system-wide default
+# policy (i.e., /etc/containers/policy.json) is most often preferred. Please
+# refer to containers-policy.json(5) for more details.
+signature_policy = "{{ crio_signature_policy }}"
+
+# Controls how image volumes are handled. The valid values are mkdir, bind and
+# ignore; the latter will ignore volumes entirely.
+image_volumes = "mkdir"
+
+# The crio.network table containers settings pertaining to the management of
+# CNI plugins.
+[crio.network]
+
+# The default CNI network name to be selected. If not set or "", then
+# CRI-O will pick-up the first one found in network_dir.
+# cni_default_network = ""
+
+# Path to the directory where CNI configuration files are located.
+network_dir = "/etc/cni/net.d/"
+
+# Paths to directories where CNI plugin binaries are located.
+plugin_dirs = [
+	"/opt/cni/bin",
+	"/usr/libexec/cni",
+]
+
+# A necessary configuration for Prometheus based metrics retrieval
+[crio.metrics]
+
+# Globally enable or disable metrics support.
+enable_metrics = {{ crio_enable_metrics | bool | lower }}
+
+# The port on which the metrics server will listen.
+metrics_port = {{ crio_metrics_port }}

kubespray/extra_playbooks/roles/container-engine/cri-o/templates/http-proxy.conf.j2

@@ -0,0 +1,2 @@
+[Service]
+Environment={% if http_proxy is defined %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy is defined %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy is defined %}"NO_PROXY={{ no_proxy }}"{% endif %}

kubespray/extra_playbooks/roles/container-engine/cri-o/templates/registry.conf.j2

@@ -0,0 +1,13 @@
+[[registry]]
+prefix = "{{ item.prefix | default(item.location) }}"
+insecure = {{ item.insecure | default('false') | string | lower }}
+blocked = {{ item.blocked | default('false') | string | lower }}
+location = "{{ item.location }}"
+{% if item.mirrors is defined %}
+{% for mirror in item.mirrors %}
+
+[[registry.mirror]]
+location = "{{ mirror.location }}"
+insecure = {{ mirror.insecure | default('false') | string | lower }}
+{% endfor %}
+{% endif %}

kubespray/extra_playbooks/roles/container-engine/cri-o/templates/unqualified.conf.j2

@@ -0,0 +1,10 @@
+{%- set _unqualified_registries = [] -%}
+{% for _registry in crio_registries if _registry.unqualified -%}
+{% if _registry.prefix is defined -%}
+{{ _unqualified_registries.append(_registry.prefix) }}
+{% else %}
+{{ _unqualified_registries.append(_registry.location) }}
+{%- endif %}
+{%- endfor %}
+
+unqualified-search-registries = {{ _unqualified_registries | string }}

kubespray/extra_playbooks/roles/container-engine/crictl/handlers/main.yml

@@ -0,0 +1,12 @@
+---
+- name: Get crictl completion
+  command: "{{ bin_dir }}/crictl completion"
+  changed_when: False
+  register: cri_completion
+  check_mode: false
+
+- name: Install crictl completion
+  copy:
+    dest: /etc/bash_completion.d/crictl
+    content: "{{ cri_completion.stdout }}"
+    mode: 0644

kubespray/extra_playbooks/roles/container-engine/crictl/tasks/crictl.yml

@@ -0,0 +1,22 @@
+---
+- name: crictl | Download crictl
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.crictl) }}"
+
+- name: Install crictl config
+  template:
+    src: crictl.yaml.j2
+    dest: /etc/crictl.yaml
+    owner: root
+    mode: 0644
+
+- name: Copy crictl binary from download dir
+  copy:
+    src: "{{ local_release_dir }}/crictl"
+    dest: "{{ bin_dir }}/crictl"
+    mode: 0755
+    remote_src: true
+  notify:
+    - Get crictl completion
+    - Install crictl completion

kubespray/extra_playbooks/roles/container-engine/crictl/tasks/main.yml

@@ -0,0 +1,3 @@
+---
+- name: install crictĺ
+  include_tasks: crictl.yml

kubespray/extra_playbooks/roles/container-engine/crictl/templates/crictl.yaml.j2

@@ -0,0 +1,4 @@
+runtime-endpoint: {{ cri_socket }}
+image-endpoint: {{ cri_socket }}
+timeout: 30
+debug: false

kubespray/extra_playbooks/roles/container-engine/crun/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+
+crun_bin_dir: /usr/bin/

kubespray/extra_playbooks/roles/container-engine/crun/tasks/main.yml

@@ -0,0 +1,12 @@
+---
+- name: crun | Download crun binary
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.crun) }}"
+
+- name: Copy crun binary from download dir
+  copy:
+    src: "{{ local_release_dir }}/crun"
+    dest: "{{ crun_bin_dir }}/crun"
+    mode: 0755
+    remote_src: true

kubespray/extra_playbooks/roles/container-engine/docker-storage/defaults/main.yml

@@ -0,0 +1,19 @@
+---
+docker_container_storage_setup_repository: https://github.com/projectatomic/container-storage-setup.git
+docker_container_storage_setup_version: v0.6.0
+docker_container_storage_setup_profile_name: kubespray
+docker_container_storage_setup_storage_driver: devicemapper
+docker_container_storage_setup_container_thinpool: docker-pool
+# It must be define a disk path for docker_container_storage_setup_devs.
+# Otherwise docker-storage-setup will be executed incorrectly.
+# docker_container_storage_setup_devs: /dev/vdb
+docker_container_storage_setup_data_size: 40%FREE
+docker_container_storage_setup_min_data_size: 2G
+docker_container_storage_setup_chunk_size: 512K
+docker_container_storage_setup_growpart: "false"
+docker_container_storage_setup_auto_extend_pool: "yes"
+docker_container_storage_setup_pool_autoextend_threshold: 60
+docker_container_storage_setup_pool_autoextend_percent: 20
+docker_container_storage_setup_device_wait_timeout: 60
+docker_container_storage_setup_wipe_signatures: "false"
+docker_container_storage_setup_container_root_lv_size: 40%FREE

kubespray/extra_playbooks/roles/container-engine/docker-storage/files/install_container_storage_setup.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+set -e
+
+repository=${1:-https://github.com/projectatomic/container-storage-setup.git}
+version=${2:-master}
+profile_name=${3:-kubespray}
+dir=`mktemp -d`
+export GIT_DIR=$dir/.git
+export GIT_WORK_TREE=$dir
+
+git init
+git fetch --depth 1 $repository $version
+git merge FETCH_HEAD
+make -C $dir install
+rm -rf /var/lib/container-storage-setup/$profile_name $dir
+
+set +e
+
+/usr/bin/container-storage-setup create $profile_name /etc/sysconfig/docker-storage-setup && /usr/bin/container-storage-setup activate $profile_name
+# FIXME: exit status can be 1 for both fatal and non fatal errors in current release,
+# could be improved by matching error strings
+exit 0

kubespray/extra_playbooks/roles/container-engine/docker-storage/tasks/main.yml

@@ -0,0 +1,48 @@
+---
+
+- name: docker-storage-setup | install git and make
+  with_items: [git, make]
+  package:
+    pkg: "{{ item }}"
+    state: present
+
+- name: docker-storage-setup | docker-storage-setup sysconfig template
+  template:
+    src: docker-storage-setup.j2
+    dest: /etc/sysconfig/docker-storage-setup
+    mode: 0644
+
+- name: docker-storage-override-directory | docker service storage-setup override dir
+  file:
+    dest: /etc/systemd/system/docker.service.d
+    mode: 0755
+    owner: root
+    group: root
+    state: directory
+
+- name: docker-storage-override | docker service storage-setup override file
+  copy:
+    dest: /etc/systemd/system/docker.service.d/override.conf
+    content: |-
+      ### This file is managed by Ansible
+      [Service]
+      EnvironmentFile=-/etc/sysconfig/docker-storage
+
+    owner: root
+    group: root
+    mode: 0644
+
+# https://docs.docker.com/engine/installation/linux/docker-ce/centos/#install-using-the-repository
+- name: docker-storage-setup | install lvm2
+  package:
+    name: lvm2
+    state: present
+
+- name: docker-storage-setup | install and run container-storage-setup
+  become: yes
+  script: |
+    install_container_storage_setup.sh \
+      {{ docker_container_storage_setup_repository }} \
+      {{ docker_container_storage_setup_version }} \
+      {{ docker_container_storage_setup_profile_name }}
+  notify: Docker | reload systemd

kubespray/extra_playbooks/roles/container-engine/docker-storage/templates/docker-storage-setup.j2

@@ -0,0 +1,35 @@
+{%if docker_container_storage_setup_storage_driver is defined%}STORAGE_DRIVER={{docker_container_storage_setup_storage_driver}}{%endif%}
+
+{%if docker_container_storage_setup_extra_storage_options is defined%}EXTRA_STORAGE_OPTIONS={{docker_container_storage_setup_extra_storage_options}}{%endif%}
+
+{%if docker_container_storage_setup_devs is defined%}DEVS={{docker_container_storage_setup_devs}}{%endif%}
+
+{%if docker_container_storage_setup_container_thinpool is defined%}CONTAINER_THINPOOL={{docker_container_storage_setup_container_thinpool}}{%endif%}
+
+{%if docker_container_storage_setup_vg is defined%}VG={{docker_container_storage_setup_vg}}{%endif%}
+
+{%if docker_container_storage_setup_root_size is defined%}ROOT_SIZE={{docker_container_storage_setup_root_size}}{%endif%}
+
+{%if docker_container_storage_setup_data_size is defined%}DATA_SIZE={{docker_container_storage_setup_data_size}}{%endif%}
+
+{%if docker_container_storage_setup_min_data_size is defined%}MIN_DATA_SIZE={{docker_container_storage_setup_min_data_size}}{%endif%}
+
+{%if docker_container_storage_setup_chunk_size is defined%}CHUNK_SIZE={{docker_container_storage_setup_chunk_size}}{%endif%}
+
+{%if docker_container_storage_setup_growpart is defined%}GROWPART={{docker_container_storage_setup_growpart}}{%endif%}
+
+{%if docker_container_storage_setup_auto_extend_pool is defined%}AUTO_EXTEND_POOL={{docker_container_storage_setup_auto_extend_pool}}{%endif%}
+
+{%if docker_container_storage_setup_pool_autoextend_threshold is defined%}POOL_AUTOEXTEND_THRESHOLD={{docker_container_storage_setup_pool_autoextend_threshold}}{%endif%}
+
+{%if docker_container_storage_setup_pool_autoextend_percent is defined%}POOL_AUTOEXTEND_PERCENT={{docker_container_storage_setup_pool_autoextend_percent}}{%endif%}
+
+{%if docker_container_storage_setup_device_wait_timeout is defined%}DEVICE_WAIT_TIMEOUT={{docker_container_storage_setup_device_wait_timeout}}{%endif%}
+
+{%if docker_container_storage_setup_wipe_signatures is defined%}WIPE_SIGNATURES={{docker_container_storage_setup_wipe_signatures}}{%endif%}
+
+{%if docker_container_storage_setup_container_root_lv_name is defined%}CONTAINER_ROOT_LV_NAME={{docker_container_storage_setup_container_root_lv_name}}{%endif%}
+
+{%if docker_container_storage_setup_container_root_lv_size is defined%}CONTAINER_ROOT_LV_SIZE={{docker_container_storage_setup_container_root_lv_size}}{%endif%}
+
+{%if docker_container_storage_setup_container_root_lv_mount_path is defined%}CONTAINER_ROOT_LV_MOUNT_PATH={{docker_container_storage_setup_container_root_lv_mount_path}}{%endif%}

kubespray/extra_playbooks/roles/container-engine/docker/defaults/main.yml

@@ -0,0 +1,64 @@
+---
+docker_version: '20.10'
+docker_cli_version: "{{ docker_version }}"
+
+docker_package_info:
+  pkgs:
+
+docker_repo_key_info:
+  repo_keys:
+
+docker_repo_info:
+  repos:
+
+docker_cgroup_driver: systemd
+
+docker_bin_dir: "/usr/bin"
+
+# flag to enable/disable docker cleanup
+docker_orphan_clean_up: false
+
+# old docker package names to be removed
+docker_remove_packages_yum:
+  - docker
+  - docker-common
+  - docker-engine
+  - docker-selinux.noarch
+  - docker-client
+  - docker-client-latest
+  - docker-latest
+  - docker-latest-logrotate
+  - docker-logrotate
+  - docker-engine-selinux.noarch
+
+# remove podman to avoid containerd.io confliction
+podman_remove_packages_yum:
+  - podman
+
+docker_remove_packages_apt:
+  - docker
+  - docker-engine
+  - docker.io
+
+# Docker specific repos should be part of the docker role not containerd-common anymore
+# Optional values for containerd apt repo
+containerd_package_info:
+  pkgs:
+
+# Fedora docker-ce repo
+docker_fedora_repo_base_url: 'https://download.docker.com/linux/fedora/{{ ansible_distribution_major_version }}/$basearch/stable'
+docker_fedora_repo_gpgkey: 'https://download.docker.com/linux/fedora/gpg'
+
+# CentOS/RedHat docker-ce repo
+docker_rh_repo_base_url: 'https://download.docker.com/linux/centos/{{ ansible_distribution_major_version }}/$basearch/stable'
+docker_rh_repo_gpgkey: 'https://download.docker.com/linux/centos/gpg'
+
+# Ubuntu docker-ce repo
+docker_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu"
+docker_ubuntu_repo_gpgkey: 'https://download.docker.com/linux/ubuntu/gpg'
+docker_ubuntu_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'
+
+# Debian docker-ce repo
+docker_debian_repo_base_url: "https://download.docker.com/linux/debian"
+docker_debian_repo_gpgkey: 'https://download.docker.com/linux/debian/gpg'
+docker_debian_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'

kubespray/extra_playbooks/roles/container-engine/docker/files/cleanup-docker-orphans.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+list_descendants ()
+{
+  local children=$(ps -o pid= --ppid "$1")
+  for pid in $children
+  do
+    list_descendants "$pid"
+  done
+  [[ -n "$children" ]] && echo "$children"
+}
+
+shim_search="^docker-containerd-shim|^containerd-shim"
+count_shim_processes=$(pgrep -f $shim_search | wc -l)
+
+if [ ${count_shim_processes} -gt 0 ]; then
+        # Find all container pids from shims
+        orphans=$(pgrep -P $(pgrep -d ',' -f $shim_search) |\
+        # Filter out valid docker pids, leaving the orphans
+        egrep -v $(docker ps -q | xargs docker inspect --format '{{.State.Pid}}' | awk '{printf "%s%s",sep,$1; sep="|"}'))
+
+        if [[ -n "$orphans" && -n "$(ps -o ppid= $orphans)" ]]
+        then
+                # Get shim pids of orphans
+                orphan_shim_pids=$(ps -o pid= $(ps -o ppid= $orphans))
+
+                # Find all orphaned container PIDs
+                orphan_container_pids=$(for pid in $orphan_shim_pids; do list_descendants $pid; done)
+
+                # Recursively kill all child PIDs of orphan shims
+                echo -e "Killing orphan container PIDs and descendants: \n$(ps -O ppid= $orphan_container_pids)"
+                kill -9 $orphan_container_pids || true
+
+        else
+                echo "No orphaned containers found"
+        fi
+else
+        echo "The node doesn't have any shim processes."
+fi

kubespray/extra_playbooks/roles/container-engine/docker/handlers/main.yml

@@ -0,0 +1,32 @@
+---
+- name: restart docker
+  command: /bin/true
+  notify:
+    - Docker | reload systemd
+    - Docker | reload docker.socket
+    - Docker | reload docker
+    - Docker | wait for docker
+
+- name: Docker | reload systemd
+  systemd:
+    name: docker
+    daemon_reload: true
+    masked: no
+
+- name: Docker | reload docker.socket
+  service:
+    name: docker.socket
+    state: restarted
+  when: ansible_os_family in ['Flatcar', 'Flatcar Container Linux by Kinvolk'] or is_fedora_coreos
+
+- name: Docker | reload docker
+  service:
+    name: docker
+    state: restarted
+
+- name: Docker | wait for docker
+  command: "{{ docker_bin_dir }}/docker images"
+  register: docker_ready
+  retries: 20
+  delay: 1
+  until: docker_ready.rc == 0

kubespray/extra_playbooks/roles/container-engine/docker/meta/main.yml

@@ -0,0 +1,5 @@
+---
+dependencies:
+  - role: container-engine/containerd-common
+  - role: container-engine/docker-storage
+    when: docker_container_storage_setup and ansible_os_family == "RedHat"

kubespray/extra_playbooks/roles/container-engine/docker/tasks/docker_plugin.yml

@@ -0,0 +1,8 @@
+---
+- name: Install Docker plugin
+  command: docker plugin install --grant-all-permissions {{ docker_plugin | quote }}
+  when: docker_plugin is defined
+  register: docker_plugin_status
+  failed_when:
+    - docker_plugin_status.failed
+    - '"already exists" not in docker_plugin_status.stderr'

kubespray/extra_playbooks/roles/container-engine/docker/tasks/main.yml

@@ -0,0 +1,177 @@
+---
+- name: check if fedora coreos
+  stat:
+    path: /run/ostree-booted
+    get_attributes: no
+    get_checksum: no
+    get_mime: no
+  register: ostree
+
+- name: set is_ostree
+  set_fact:
+    is_ostree: "{{ ostree.stat.exists }}"
+
+- name: set docker_version for openEuler
+  set_fact:
+    docker_version: '19.03'
+  when: ansible_distribution == "openEuler"
+  tags:
+    - facts
+
+- name: gather os specific variables
+  include_vars: "{{ item }}"
+  with_first_found:
+    - files:
+        - "{{ ansible_distribution|lower }}-{{ ansible_distribution_version|lower|replace('/', '_') }}.yml"
+        - "{{ ansible_distribution|lower }}-{{ ansible_distribution_release|lower }}-{{ host_architecture }}.yml"
+        - "{{ ansible_distribution|lower }}-{{ ansible_distribution_release|lower }}.yml"
+        - "{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version|lower|replace('/', '_') }}.yml"
+        - "{{ ansible_distribution|lower }}-{{ host_architecture }}.yml"
+        - "{{ ansible_distribution|lower }}.yml"
+        - "{{ ansible_distribution.split(' ')[0]|lower }}.yml"
+        - "{{ ansible_os_family|lower }}-{{ ansible_distribution_major_version|lower|replace('/', '_') }}.yml"
+        - "{{ ansible_os_family|lower }}-{{ host_architecture }}.yml"
+        - "{{ ansible_os_family|lower }}.yml"
+        - defaults.yml
+      paths:
+        - ../vars
+      skip: true
+  tags:
+    - facts
+
+- name: Warn about Docker version on SUSE
+  debug:
+    msg: "SUSE distributions always install Docker from the distro repos"
+  when: ansible_pkg_mgr == 'zypper'
+
+- include_tasks: set_facts_dns.yml
+  when: dns_mode != 'none' and resolvconf_mode == 'docker_dns'
+  tags:
+    - facts
+
+- import_tasks: pre-upgrade.yml
+
+- name: ensure docker-ce repository public key is installed
+  apt_key:
+    id: "{{ item }}"
+    url: "{{ docker_repo_key_info.url }}"
+    state: present
+  register: keyserver_task_result
+  until: keyserver_task_result is succeeded
+  retries: 4
+  delay: "{{ retry_stagger | d(3) }}"
+  with_items: "{{ docker_repo_key_info.repo_keys }}"
+  environment: "{{ proxy_env }}"
+  when: ansible_pkg_mgr == 'apt'
+
+- name: ensure docker-ce repository is enabled
+  apt_repository:
+    repo: "{{ item }}"
+    state: present
+  with_items: "{{ docker_repo_info.repos }}"
+  when: ansible_pkg_mgr == 'apt'
+
+- name: Configure docker repository on Fedora
+  template:
+    src: "fedora_docker.repo.j2"
+    dest: "{{ yum_repo_dir }}/docker.repo"
+    mode: 0644
+  when: ansible_distribution == "Fedora" and not is_ostree
+
+- name: Configure docker repository on RedHat/CentOS/OracleLinux/AlmaLinux/KylinLinux
+  template:
+    src: "rh_docker.repo.j2"
+    dest: "{{ yum_repo_dir }}/docker-ce.repo"
+    mode: 0644
+  when:
+    - ansible_os_family == "RedHat"
+    - ansible_distribution != "Fedora"
+    - not is_ostree
+
+- name: Remove dpkg hold
+  dpkg_selections:
+    name: "{{ item }}"
+    selection: install
+  when: ansible_pkg_mgr == 'apt'
+  changed_when: false
+  with_items:
+    - "{{ containerd_package }}"
+    - docker-ce
+    - docker-ce-cli
+
+- name: ensure docker packages are installed
+  package:
+    name: "{{ docker_package_info.pkgs }}"
+    state: "{{ docker_package_info.state | default('present') }}"
+  module_defaults:
+    apt:
+      update_cache: true
+    dnf:
+      enablerepo: "{{ docker_package_info.enablerepo | default(omit) }}"
+      disablerepo: "{{ docker_package_info.disablerepo | default(omit) }}"
+    yum:
+      enablerepo: "{{ docker_package_info.enablerepo | default(omit) }}"
+    zypper:
+      update_cache: true
+  register: docker_task_result
+  until: docker_task_result is succeeded
+  retries: 4
+  delay: "{{ retry_stagger | d(3) }}"
+  notify: restart docker
+  when:
+    - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
+    - not is_ostree
+    - docker_package_info.pkgs|length > 0
+
+# This is required to ensure any apt upgrade will not break kubernetes
+- name: Tell Debian hosts not to change the docker version with apt upgrade
+  dpkg_selections:
+    name: "{{ item }}"
+    selection: hold
+  when: ansible_pkg_mgr == 'apt'
+  changed_when: false
+  with_items:
+    - "{{ containerd_package }}"
+    - docker-ce
+    - docker-ce-cli
+
+- name: ensure docker started, remove our config if docker start failed and try again
+  block:
+    - name: ensure service is started if docker packages are already present
+      service:
+        name: docker
+        state: started
+      when: docker_task_result is not changed
+  rescue:
+    - debug:  # noqa unnamed-task
+        msg: "Docker start failed. Try to remove our config"
+    - name: remove kubespray generated config
+      file:
+        path: "{{ item }}"
+        state: absent
+      with_items:
+        - /etc/systemd/system/docker.service.d/http-proxy.conf
+        - /etc/systemd/system/docker.service.d/docker-options.conf
+        - /etc/systemd/system/docker.service.d/docker-dns.conf
+        - /etc/systemd/system/docker.service.d/docker-orphan-cleanup.conf
+      notify: restart docker
+
+- name: flush handlers so we can wait for docker to come up
+  meta: flush_handlers
+
+# Install each plugin using a looped include to make error handling in the included task simpler.
+- include_tasks: docker_plugin.yml
+  loop: "{{ docker_plugins }}"
+  loop_control:
+    loop_var: docker_plugin
+
+- name: Set docker systemd config
+  import_tasks: systemd.yml
+
+- name: ensure docker service is started and enabled
+  service:
+    name: "{{ item }}"
+    enabled: yes
+    state: started
+  with_items:
+    - docker

kubespray/extra_playbooks/roles/container-engine/docker/tasks/pre-upgrade.yml

@@ -0,0 +1,36 @@
+---
+- name: Remove legacy docker repo file
+  file:
+    path: "{{ yum_repo_dir }}/docker.repo"
+    state: absent
+  when:
+    - ansible_os_family == 'RedHat'
+    - not is_ostree
+
+- name: Ensure old versions of Docker are not installed. | Debian
+  apt:
+    name: '{{ docker_remove_packages_apt }}'
+    state: absent
+  when:
+    - ansible_os_family == 'Debian'
+    - (docker_versioned_pkg[docker_version | string] is search('docker-ce'))
+
+
+- name: Ensure podman not installed. | RedHat
+  package:
+    name: '{{ podman_remove_packages_yum }}'
+    state: absent
+  when:
+    - ansible_os_family == 'RedHat'
+    - (docker_versioned_pkg[docker_version | string] is search('docker-ce'))
+    - not is_ostree
+
+
+- name: Ensure old versions of Docker are not installed. | RedHat
+  package:
+    name: '{{ docker_remove_packages_yum }}'
+    state: absent
+  when:
+    - ansible_os_family == 'RedHat'
+    - (docker_versioned_pkg[docker_version | string] is search('docker-ce'))
+    - not is_ostree