update

kubespray/roles/container-engine/cri-o/defaults/main.yml

@@ -0,0 +1,103 @@
+---
+
+crio_cgroup_manager: "{{ kubelet_cgroup_driver | default('systemd') }}"
+crio_conmon: "{{ bin_dir }}/conmon"
+crio_enable_metrics: false
+crio_log_level: "info"
+crio_metrics_port: "9090"
+crio_pause_image: "{{ pod_infra_image_repo }}:{{ pod_infra_version }}"
+
+# Registries defined within cri-o.
+# By default unqualified images are not allowed for security reasons
+crio_registries: []
+#  - prefix: docker.io
+#    insecure: false
+#    blocked: false
+#    location: registry-1.docker.io ## REQUIRED
+#    unqualified: false
+#    mirrors:
+#      - location: 172.20.100.52:5000
+#        insecure: true
+#      - location: mirror.gcr.io
+#        insecure: false
+
+crio_registry_auth: []
+#  - registry: 10.0.0.2:5000
+#    username: user
+#    password: pass
+
+crio_seccomp_profile: ""
+crio_selinux: "{{ (preinstall_selinux_state == 'enforcing')|lower }}"
+crio_signature_policy: "{% if ansible_os_family == 'ClearLinux' %}/usr/share/defaults/crio/policy.json{% endif %}"
+
+# Override system default for storage driver
+# crio_storage_driver: "overlay"
+
+crio_stream_port: "10010"
+
+crio_required_version: "{{ kube_version | regex_replace('^v(?P<major>\\d+).(?P<minor>\\d+).(?P<patch>\\d+)$', '\\g<major>.\\g<minor>') }}"
+
+# The crio_runtimes variable defines a list of OCI compatible runtimes.
+crio_runtimes:
+  - name: runc
+    path: "{{ bin_dir }}/runc"
+    type: oci
+    root: /run/runc
+
+# Kata Containers is an OCI runtime, where containers are run inside lightweight
+# VMs. Kata provides additional isolation towards the host, minimizing the host attack
+# surface and mitigating the consequences of containers breakout.
+kata_runtimes:
+  # Kata Containers with the default configured VMM
+  - name: kata-qemu
+    path: /usr/local/bin/containerd-shim-kata-qemu-v2
+    type: vm
+    root: /run/kata-containers
+    privileged_without_host_devices: true
+
+# crun is a fast and low-memory footprint OCI Container Runtime fully written in C.
+crun_runtime:
+  name: crun
+  path: "{{ bin_dir }}/crun"
+  type: oci
+  root: /run/crun
+
+# youki is an implementation of the OCI runtime-spec in Rust, similar to runc.
+youki_runtime:
+  name: youki
+  path: "{{ youki_bin_dir }}/youki"
+  type: oci
+  root: /run/youki
+
+# TODO(cristicalin): remove this after 2.21
+crio_download_base: "download.opensuse.org/repositories/devel:kubic:libcontainers:stable"
+crio_download_crio: "http://{{ crio_download_base }}:/cri-o:/"
+
+# Configure the cri-o pids limit, increase this for heavily multi-threaded workloads
+# see https://github.com/cri-o/cri-o/issues/1921
+crio_pids_limit: 1024
+
+# Reserve 16M uids and gids for user namespaces (256 pods * 65536 uids/gids)
+# at the end of the uid/gid space
+crio_remap_enable: false
+crio_remap_user: containers
+crio_subuid_start: 2130706432
+crio_subuid_length: 16777216
+crio_subgid_start: 2130706432
+crio_subgid_length: 16777216
+
+# cri-o binary files
+crio_bin_files:
+  - conmon
+  - crio
+  - crio-status
+  - pinns
+
+# cri-o manual files
+crio_man_files:
+  5:
+    - crio.conf
+    - crio.conf.d
+  8:
+    - crio
+    - crio-status

kubespray/roles/container-engine/cri-o/files/mounts.conf

@@ -0,0 +1 @@
+/usr/share/rhel/secrets:/run/secrets

kubespray/roles/container-engine/cri-o/handlers/main.yml

@@ -0,0 +1,16 @@
+---
+- name: restart crio
+  command: /bin/true
+  notify:
+    - CRI-O | reload systemd
+    - CRI-O | reload crio
+
+- name: CRI-O | reload systemd
+  systemd:
+    daemon_reload: true
+
+- name: CRI-O | reload crio
+  service:
+    name: crio
+    state: restarted
+    enabled: yes

kubespray/roles/container-engine/cri-o/meta/main.yml

@@ -0,0 +1,5 @@
+---
+dependencies:
+  - role: container-engine/crictl
+  - role: container-engine/runc
+  - role: container-engine/skopeo

kubespray/roles/container-engine/cri-o/molecule/default/converge.yml

@@ -0,0 +1,9 @@
+---
+- name: Converge
+  hosts: all
+  become: true
+  vars:
+    container_manager: crio
+  roles:
+    - role: kubespray-defaults
+    - role: container-engine/cri-o

kubespray/roles/container-engine/cri-o/molecule/default/files/10-mynet.conf

@@ -0,0 +1,17 @@
+{
+  "cniVersion": "0.2.0",
+  "name": "mynet",
+  "type": "bridge",
+  "bridge": "cni0",
+  "isGateway": true,
+  "ipMasq": true,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "172.19.0.0/24",
+    "routes": [
+      {
+        "dst": "0.0.0.0/0"
+      }
+    ]
+  }
+}

kubespray/roles/container-engine/cri-o/molecule/default/files/container.json

@@ -0,0 +1,10 @@
+{
+  "metadata": {
+    "name": "runc1"
+  },
+  "image": {
+    "image": "quay.io/kubespray/hello-world:latest"
+  },
+  "log_path": "runc1.0.log",
+  "linux": {}
+}

kubespray/roles/container-engine/cri-o/molecule/default/files/sandbox.json

@@ -0,0 +1,10 @@
+{
+  "metadata": {
+    "name": "runc1",
+    "namespace": "default",
+    "attempt": 1,
+    "uid": "hdishd83djaidwnduwk28bcsb"
+  },
+  "linux": {},
+  "log_directory": "/tmp"
+}

kubespray/roles/container-engine/cri-o/molecule/default/molecule.yml

@@ -0,0 +1,57 @@
+---
+driver:
+  name: vagrant
+  provider:
+    name: libvirt
+lint: |
+  set -e
+  yamllint -c ../../../.yamllint .
+platforms:
+  - name: ubuntu20
+    box: generic/ubuntu2004
+    cpus: 2
+    memory: 1024
+    groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+  - name: almalinux8
+    box: almalinux/8
+    cpus: 2
+    memory: 1024
+    groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+  - name: fedora
+    box: fedora/36-cloud-base
+    cpus: 2
+    memory: 1024
+    groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+  - name: debian10
+    box: generic/debian10
+    cpus: 2
+    memory: 1024
+    groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_ROLES_PATH: ../../../../
+  config_options:
+    defaults:
+      callbacks_enabled: profile_tasks
+      timeout: 120
+  lint:
+    name: ansible-lint
+    options:
+      c: ../../../.ansible-lint
+verifier:
+  name: testinfra
+  lint:
+    name: flake8

kubespray/roles/container-engine/cri-o/molecule/default/prepare.yml

@@ -0,0 +1,52 @@
+---
+- name: Prepare
+  hosts: all
+  gather_facts: False
+  become: true
+  vars:
+    ignore_assert_errors: true
+  roles:
+    - role: kubespray-defaults
+    - role: bootstrap-os
+    - role: kubernetes/preinstall
+    - role: adduser
+      user: "{{ addusers.kube }}"
+  tasks:
+    - include_tasks: "../../../../download/tasks/download_file.yml"
+      vars:
+        download: "{{ download_defaults | combine(downloads.cni) }}"
+
+- name: Prepare CNI
+  hosts: all
+  gather_facts: False
+  become: true
+  vars:
+    ignore_assert_errors: true
+    kube_network_plugin: cni
+  roles:
+    - role: kubespray-defaults
+    - role: network_plugin/cni
+  tasks:
+    - name: Copy test container files
+      copy:
+        src: "{{ item }}"
+        dest: "/tmp/{{ item }}"
+        owner: root
+        mode: 0644
+      with_items:
+        - container.json
+        - sandbox.json
+    - name: Create /etc/cni/net.d directory
+      file:
+        path: /etc/cni/net.d
+        state: directory
+        owner: "{{ kube_owner }}"
+        mode: 0755
+    - name: Setup CNI
+      copy:
+        src: "{{ item }}"
+        dest: "/etc/cni/net.d/{{ item }}"
+        owner: root
+        mode: 0644
+      with_items:
+        - 10-mynet.conf

kubespray/roles/container-engine/cri-o/molecule/default/tests/test_default.py

@@ -0,0 +1,35 @@
+import os
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_service(host):
+    svc = host.service("crio")
+    assert svc.is_running
+    assert svc.is_enabled
+
+
+def test_run(host):
+    crictl = "/usr/local/bin/crictl"
+    path = "unix:///var/run/crio/crio.sock"
+    with host.sudo():
+        cmd = host.command(crictl + " --runtime-endpoint " + path + " version")
+    assert cmd.rc == 0
+    assert "RuntimeName:  cri-o" in cmd.stdout
+
+def test_run_pod(host):
+    runtime = "runc"
+
+    run_command = "/usr/local/bin/crictl run --with-pull --runtime {} /tmp/container.json /tmp/sandbox.json".format(runtime)
+    with host.sudo():
+        cmd = host.command(run_command)
+    assert cmd.rc == 0
+
+    with host.sudo():
+      log_f = host.file("/tmp/runc1.0.log")
+
+      assert log_f.exists
+      assert b"Hello from Docker" in log_f.content

kubespray/roles/container-engine/cri-o/tasks/cleanup.yaml

@@ -0,0 +1,119 @@
+---
+# TODO(cristicalin): drop this file after 2.21
+- name: CRI-O kubic repo name for debian os family
+  set_fact:
+    crio_kubic_debian_repo_name: "{{ ((ansible_distribution == 'Ubuntu') | ternary('x','')) ~ ansible_distribution ~ '_' ~ ansible_distribution_version }}"
+  when: ansible_os_family == "Debian"
+
+- name: Remove legacy CRI-O kubic apt repo key
+  apt_key:
+    url: "https://{{ crio_download_base }}/{{ crio_kubic_debian_repo_name }}/Release.key"
+    state: absent
+  when: crio_kubic_debian_repo_name is defined
+
+- name: Remove legacy CRI-O kubic apt repo
+  apt_repository:
+    repo: "deb http://{{ crio_download_base }}/{{ crio_kubic_debian_repo_name }}/ /"
+    state: absent
+    filename: devel-kubic-libcontainers-stable
+  when: crio_kubic_debian_repo_name is defined
+
+- name: Remove legacy CRI-O kubic cri-o apt repo
+  apt_repository:
+    repo: "deb {{ crio_download_crio }}{{ crio_version }}/{{ crio_kubic_debian_repo_name }}/ /"
+    state: absent
+    filename: devel-kubic-libcontainers-stable-cri-o
+  when: crio_kubic_debian_repo_name is defined
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: devel_kubic_libcontainers_stable
+    description: Stable Releases of Upstream github.com/containers packages (CentOS_$releasever)
+    baseurl: http://{{ crio_download_base }}/CentOS_{{ ansible_distribution_major_version }}/
+    state: absent
+  when:
+    - ansible_os_family == "RedHat"
+    - ansible_distribution not in ["Amazon", "Fedora"]
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: "devel_kubic_libcontainers_stable_cri-o_{{ crio_version }}"
+    description: "CRI-O {{ crio_version }} (CentOS_$releasever)"
+    baseurl: "{{ crio_download_crio }}{{ crio_version }}/CentOS_{{ ansible_distribution_major_version }}/"
+    state: absent
+  when:
+    - ansible_os_family == "RedHat"
+    - ansible_distribution not in ["Amazon", "Fedora"]
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: devel_kubic_libcontainers_stable
+    description: Stable Releases of Upstream github.com/containers packages
+    baseurl: http://{{ crio_download_base }}/Fedora_{{ ansible_distribution_major_version }}/
+    state: absent
+  when:
+    - ansible_distribution in ["Fedora"]
+    - not is_ostree
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: "devel_kubic_libcontainers_stable_cri-o_{{ crio_version }}"
+    description: "CRI-O {{ crio_version }}"
+    baseurl: "{{ crio_download_crio }}{{ crio_version }}/Fedora_{{ ansible_distribution_major_version }}/"
+    state: absent
+  when:
+    - ansible_distribution in ["Fedora"]
+    - not is_ostree
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: devel_kubic_libcontainers_stable
+    description: Stable Releases of Upstream github.com/containers packages
+    baseurl: http://{{ crio_download_base }}/CentOS_7/
+    state: absent
+  when: ansible_distribution in ["Amazon"]
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: "devel_kubic_libcontainers_stable_cri-o_{{ crio_version }}"
+    description: "CRI-O {{ crio_version }}"
+    baseurl: "{{ crio_download_crio }}{{ crio_version }}/CentOS_7/"
+    state: absent
+  when: ansible_distribution in ["Amazon"]
+
+- name: Disable modular repos for CRI-O
+  ini_file:
+    path: "/etc/yum.repos.d/{{ item.repo }}.repo"
+    section: "{{ item.section }}"
+    option: enabled
+    value: 0
+    mode: 0644
+  become: true
+  when: is_ostree
+  loop:
+    - repo: "fedora-updates-modular"
+      section: "updates-modular"
+    - repo: "fedora-modular"
+      section: "fedora-modular"
+
+# Disable any older module version if we enabled them before
+- name: Disable CRI-O ex module
+  command: "rpm-ostree ex module disable cri-o:{{ item }}"
+  become: true
+  when:
+    - is_ostree
+    - ostree_version is defined and ostree_version.stdout is version('2021.9', '>=')
+  with_items:
+    - 1.22
+    - 1.23
+    - 1.24
+
+- name: cri-o | remove installed packages
+  package:
+    name: "{{ item }}"
+    state: absent
+  when: not is_ostree
+  with_items:
+    - cri-o
+    - cri-o-runc
+    - oci-systemd-hook

kubespray/roles/container-engine/cri-o/tasks/main.yaml

@@ -0,0 +1,206 @@
+---
+- name: cri-o | check if fedora coreos
+  stat:
+    path: /run/ostree-booted
+    get_attributes: no
+    get_checksum: no
+    get_mime: no
+  register: ostree
+
+- name: cri-o | set is_ostree
+  set_fact:
+    is_ostree: "{{ ostree.stat.exists }}"
+
+- name: cri-o | get ostree version
+  shell: "set -o pipefail && rpm-ostree --version | awk -F\\' '/Version/{print $2}'"
+  args:
+    executable: /bin/bash
+  register: ostree_version
+  when: is_ostree
+
+- name: cri-o | Download cri-o
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.crio) }}"
+
+- name: cri-o | special handling for amazon linux
+  import_tasks: "setup-amazon.yaml"
+  when: ansible_distribution in ["Amazon"]
+
+- name: cri-o | clean up reglacy repos
+  import_tasks: "cleanup.yaml"
+
+- name: cri-o | build a list of crio runtimes with Katacontainers runtimes
+  set_fact:
+    crio_runtimes: "{{ crio_runtimes + kata_runtimes  }}"
+  when:
+    - kata_containers_enabled
+
+- name: cri-o | build a list of crio runtimes with crun runtime
+  set_fact:
+    crio_runtimes: "{{ crio_runtimes + [crun_runtime] }}"
+  when:
+    - crun_enabled
+
+- name: cri-o | build a list of crio runtimes with youki runtime
+  set_fact:
+    crio_runtimes: "{{ crio_runtimes + [youki_runtime] }}"
+  when:
+    - youki_enabled
+
+- name: cri-o | make sure needed folders exist in the system
+  with_items:
+    - /etc/crio
+    - /etc/containers
+    - /etc/systemd/system/crio.service.d
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: 0755
+
+- name: cri-o | install cri-o config
+  template:
+    src: crio.conf.j2
+    dest: /etc/crio/crio.conf
+    mode: 0644
+  register: config_install
+
+- name: cri-o | install config.json
+  template:
+    src: config.json.j2
+    dest: /etc/crio/config.json
+    mode: 0644
+  register: reg_auth_install
+
+- name: cri-o | copy binaries
+  copy:
+    src: "{{ local_release_dir }}/cri-o/bin/{{ item }}"
+    dest: "{{ bin_dir }}/{{ item }}"
+    mode: 0755
+    remote_src: true
+  with_items:
+    - "{{ crio_bin_files }}"
+  notify: restart crio
+
+- name: cri-o | copy service file
+  copy:
+    src: "{{ local_release_dir }}/cri-o/contrib/crio.service"
+    dest: /etc/systemd/system/crio.service
+    mode: 0755
+    remote_src: true
+  notify: restart crio
+
+- name: cri-o | copy default policy
+  copy:
+    src: "{{ local_release_dir }}/cri-o/contrib/policy.json"
+    dest: /etc/containers/policy.json
+    mode: 0755
+    remote_src: true
+  notify: restart crio
+
+- name: cri-o | copy mounts.conf
+  copy:
+    src: mounts.conf
+    dest: /etc/containers/mounts.conf
+    mode: 0644
+  when:
+    - ansible_os_family == 'RedHat'
+  notify: restart crio
+
+- name: cri-o | create directory for oci hooks
+  file:
+    path: /etc/containers/oci/hooks.d
+    state: directory
+    owner: root
+    mode: 0755
+
+- name: cri-o | set overlay driver
+  ini_file:
+    dest: /etc/containers/storage.conf
+    section: storage
+    option: "{{ item.option }}"
+    value: "{{ item.value }}"
+    mode: 0644
+  with_items:
+    - option: driver
+      value: '"overlay"'
+    - option: graphroot
+      value: '"/var/lib/containers/storage"'
+
+# metacopy=on is available since 4.19 and was backported to RHEL 4.18 kernel
+- name: cri-o | set metacopy mount options correctly
+  ini_file:
+    dest: /etc/containers/storage.conf
+    section: storage.options.overlay
+    option: mountopt
+    value: '{{ ''"nodev"'' if ansible_kernel is version_compare(("4.18" if ansible_os_family == "RedHat" else "4.19"), "<") else ''"nodev,metacopy=on"'' }}'
+    mode: 0644
+
+- name: cri-o | create directory registries configs
+  file:
+    path: /etc/containers/registries.conf.d
+    state: directory
+    owner: root
+    mode: 0755
+
+- name: cri-o | write registries configs
+  template:
+    src: registry.conf.j2
+    dest: "/etc/containers/registries.conf.d/10-{{ item.prefix | default(item.location) | regex_replace(':', '_') }}.conf"
+    mode: 0644
+  loop: "{{ crio_registries }}"
+  notify: restart crio
+
+- name: cri-o | configure unqualified registry settings
+  template:
+    src: unqualified.conf.j2
+    dest: "/etc/containers/registries.conf.d/01-unqualified.conf"
+    mode: 0644
+  notify: restart crio
+
+- name: cri-o | write cri-o proxy drop-in
+  template:
+    src: http-proxy.conf.j2
+    dest: /etc/systemd/system/crio.service.d/http-proxy.conf
+    mode: 0644
+  notify: restart crio
+  when: http_proxy is defined or https_proxy is defined
+
+- name: cri-o | configure the uid/gid space for user namespaces
+  lineinfile:
+    path: '{{ item.path }}'
+    line: '{{ item.entry }}'
+    regex: '^\s*{{ crio_remap_user }}:'
+    state: '{{ "present" if crio_remap_enable | bool else "absent" }}'
+  loop:
+    - path: /etc/subuid
+      entry: '{{ crio_remap_user }}:{{ crio_subuid_start }}:{{ crio_subuid_length }}'
+    - path: /etc/subgid
+      entry: '{{ crio_remap_user }}:{{ crio_subgid_start }}:{{ crio_subgid_length }}'
+  loop_control:
+    label: '{{ item.path }}'
+
+- name: cri-o | ensure crio service is started and enabled
+  service:
+    name: crio
+    daemon_reload: true
+    enabled: true
+    state: started
+  register: service_start
+
+- name: cri-o | trigger service restart only when needed
+  service:  # noqa 503
+    name: crio
+    state: restarted
+  when:
+    - config_install.changed
+    - reg_auth_install.changed
+    - not service_start.changed
+
+- name: cri-o | verify that crio is running
+  command: "{{ bin_dir }}/crio-status info"
+  register: get_crio_info
+  until: get_crio_info is succeeded
+  changed_when: false
+  retries: 5
+  delay: "{{ retry_stagger | random + 3 }}"

kubespray/roles/container-engine/cri-o/tasks/reset.yml

@@ -0,0 +1,101 @@
+---
+- name: CRI-O | Kubic repo name for debian os family
+  set_fact:
+    crio_kubic_debian_repo_name: "{{ ((ansible_distribution == 'Ubuntu') | ternary('x','')) ~ ansible_distribution ~ '_' ~ ansible_distribution_version }}"
+  when: ansible_os_family == "Debian"
+  tags:
+    - reset_crio
+
+- name: CRI-O | Remove kubic apt repo
+  apt_repository:
+    repo: "deb http://{{ crio_download_base }}/{{ crio_kubic_debian_repo_name }}/ /"
+    state: absent
+  when: crio_kubic_debian_repo_name is defined
+  tags:
+    - reset_crio
+
+- name: CRI-O | Remove cri-o apt repo
+  apt_repository:
+    repo: "deb {{ crio_download_crio }}{{ crio_version }}/{{ crio_kubic_debian_repo_name }}/ /"
+    state: present
+    filename: devel-kubic-libcontainers-stable-cri-o
+  when: crio_kubic_debian_repo_name is defined
+  tags:
+    - reset_crio
+
+- name: CRI-O | Remove CRI-O kubic yum repo
+  yum_repository:
+    name: devel_kubic_libcontainers_stable
+    state: absent
+  when: ansible_distribution in ["Amazon"]
+  tags:
+    - reset_crio
+
+- name: CRI-O | Remove CRI-O kubic yum repo
+  yum_repository:
+    name: "devel_kubic_libcontainers_stable_cri-o_{{ crio_version }}"
+    state: absent
+  when:
+    - ansible_os_family == "RedHat"
+    - ansible_distribution not in ["Amazon", "Fedora"]
+  tags:
+    - reset_crio
+
+- name: CRI-O | Run yum-clean-metadata
+  command: yum clean metadata
+  args:
+    warn: no
+  when:
+    - ansible_os_family == "RedHat"
+  tags:
+    - reset_crio
+
+- name: CRI-O | Remove crictl
+  file:
+    name: "{{ item }}"
+    state: absent
+  loop:
+    - /etc/crictl.yaml
+    - "{{ bin_dir }}/crictl"
+  tags:
+    - reset_crio
+
+- name: CRI-O | Stop crio service
+  service:
+    name: crio
+    daemon_reload: true
+    enabled: false
+    masked: true
+    state: stopped
+  tags:
+    - reset_crio
+
+- name: CRI-O | Remove CRI-O configuration files
+  file:
+    name: "{{ item }}"
+    state: absent
+  loop:
+    - /etc/crio
+    - /etc/containers
+    - /etc/systemd/system/crio.service.d
+  tags:
+    - reset_crio
+
+- name: CRI-O | Remove dpkg hold
+  dpkg_selections:
+    name: "{{ item }}"
+    selection: install
+  when: ansible_pkg_mgr == 'apt'
+  changed_when: false
+  with_items: "{{ crio_packages }}"
+  tags:
+    - reset_crio
+
+- name: CRI-O | Uninstall CRI-O package
+  package:
+    name: "{{ item }}"
+    state: absent
+  when: not is_ostree
+  with_items: "{{ crio_packages }}"
+  tags:
+    - reset_crio

kubespray/roles/container-engine/cri-o/tasks/setup-amazon.yaml

@@ -0,0 +1,38 @@
+---
+- name: Check that amzn2-extras.repo exists
+  stat:
+    path: /etc/yum.repos.d/amzn2-extras.repo
+  register: amzn2_extras_file_stat
+
+- name: Find docker repo in amzn2-extras.repo file
+  lineinfile:
+    dest: /etc/yum.repos.d/amzn2-extras.repo
+    line: "[amzn2extra-docker]"
+  check_mode: yes
+  register: amzn2_extras_docker_repo
+  when:
+    - amzn2_extras_file_stat.stat.exists
+
+- name: Remove docker repository
+  ini_file:
+    dest: /etc/yum.repos.d/amzn2-extras.repo
+    section: amzn2extra-docker
+    option: enabled
+    value: "0"
+    backup: yes
+    mode: 0644
+  when:
+    - amzn2_extras_file_stat.stat.exists
+    - not amzn2_extras_docker_repo.changed
+
+- name: Add container-selinux yum repo
+  yum_repository:
+    name: copr:copr.fedorainfracloud.org:lsm5:container-selinux
+    file: _copr_lsm5-container-selinux.repo
+    description: Copr repo for container-selinux owned by lsm5
+    baseurl: https://download.copr.fedorainfracloud.org/results/lsm5/container-selinux/epel-7-$basearch/
+    gpgcheck: yes
+    gpgkey: https://download.copr.fedorainfracloud.org/results/lsm5/container-selinux/pubkey.gpg
+    skip_if_unavailable: yes
+    enabled: yes
+    repo_gpgcheck: no

kubespray/roles/container-engine/cri-o/templates/config.json.j2

@@ -0,0 +1,17 @@
+{% if crio_registry_auth is defined and crio_registry_auth|length %}
+{ 
+{% for reg in crio_registry_auth %}
+  "auths": {
+    "{{ reg.registry }}": {
+      "auth": "{{ (reg.username + ':' + reg.password) | string | b64encode }}"
+    }
+{% if not loop.last %}
+  },
+{% else %}
+  }
+{% endif %}
+{% endfor %}
+}
+{% else %}
+{}
+{% endif %}

kubespray/roles/container-engine/cri-o/templates/crio.conf.j2

@@ -0,0 +1,369 @@
+# The CRI-O configuration file specifies all of the available configuration
+# options and command-line flags for the crio(8) OCI Kubernetes Container Runtime
+# daemon, but in a TOML format that can be more easily modified and versioned.
+#
+# Please refer to crio.conf(5) for details of all configuration options.
+
+# CRI-O supports partial configuration reload during runtime, which can be
+# done by sending SIGHUP to the running process. Currently supported options
+# are explicitly mentioned with: 'This option supports live configuration
+# reload'.
+
+# CRI-O reads its storage defaults from the containers-storage.conf(5) file
+# located at /etc/containers/storage.conf. Modify this storage configuration if
+# you want to change the system's defaults. If you want to modify storage just
+# for CRI-O, you can change the storage configuration options here.
+[crio]
+
+# Path to the "root directory". CRI-O stores all of its data, including
+# containers images, in this directory.
+root = "/var/lib/containers/storage"
+
+# Path to the "run directory". CRI-O stores all of its state in this directory.
+runroot = "/var/run/containers/storage"
+
+# Storage driver used to manage the storage of images and containers. Please
+# refer to containers-storage.conf(5) to see all available storage drivers.
+{% if crio_storage_driver is defined %}
+storage_driver = "{{ crio_storage_driver }}"
+{% endif %}
+
+# List to pass options to the storage driver. Please refer to
+# containers-storage.conf(5) to see all available storage options.
+#storage_option = [
+#]
+
+# The default log directory where all logs will go unless directly specified by
+# the kubelet. The log directory specified must be an absolute directory.
+log_dir = "/var/log/crio/pods"
+
+# Location for CRI-O to lay down the temporary version file.
+# It is used to check if crio wipe should wipe containers, which should
+# always happen on a node reboot
+version_file = "/var/run/crio/version"
+
+# Location for CRI-O to lay down the persistent version file.
+# It is used to check if crio wipe should wipe images, which should
+# only happen when CRI-O has been upgraded
+version_file_persist = "/var/lib/crio/version"
+
+# The crio.api table contains settings for the kubelet/gRPC interface.
+[crio.api]
+
+# Path to AF_LOCAL socket on which CRI-O will listen.
+listen = "/var/run/crio/crio.sock"
+
+# IP address on which the stream server will listen.
+stream_address = "127.0.0.1"
+
+# The port on which the stream server will listen. If the port is set to "0", then
+# CRI-O will allocate a random free port number.
+stream_port = "{{ crio_stream_port }}"
+
+# Enable encrypted TLS transport of the stream server.
+stream_enable_tls = false
+
+# Path to the x509 certificate file used to serve the encrypted stream. This
+# file can change, and CRI-O will automatically pick up the changes within 5
+# minutes.
+stream_tls_cert = ""
+
+# Path to the key file used to serve the encrypted stream. This file can
+# change and CRI-O will automatically pick up the changes within 5 minutes.
+stream_tls_key = ""
+
+# Path to the x509 CA(s) file used to verify and authenticate client
+# communication with the encrypted stream. This file can change and CRI-O will
+# automatically pick up the changes within 5 minutes.
+stream_tls_ca = ""
+
+# Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 16 * 1024 * 1024.
+grpc_max_send_msg_size = 16777216
+
+# Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 16 * 1024 * 1024.
+grpc_max_recv_msg_size = 16777216
+
+# The crio.runtime table contains settings pertaining to the OCI runtime used
+# and options for how to set up and manage the OCI runtime.
+[crio.runtime]
+
+# A list of ulimits to be set in containers by default, specified as
+# "<ulimit name>=<soft limit>:<hard limit>", for example:
+# "nofile=1024:2048"
+# If nothing is set here, settings will be inherited from the CRI-O daemon
+#default_ulimits = [
+#]
+
+# default_runtime is the _name_ of the OCI runtime to be used as the default.
+# The name is matched against the runtimes map below.
+default_runtime = "runc"
+
+# If true, the runtime will not use pivot_root, but instead use MS_MOVE.
+no_pivot = false
+
+# decryption_keys_path is the path where the keys required for
+# image decryption are stored. This option supports live configuration reload.
+decryption_keys_path = "/etc/crio/keys/"
+
+# Path to the conmon binary, used for monitoring the OCI runtime.
+# Will be searched for using $PATH if empty.
+conmon = "{{ crio_conmon }}"
+
+# Cgroup setting for conmon
+{% if crio_cgroup_manager == "cgroupfs" %}
+conmon_cgroup = "pod"
+{% else %}
+conmon_cgroup = "system.slice"
+{% endif %}
+
+# Environment variable list for the conmon process, used for passing necessary
+# environment variables to conmon or the runtime.
+conmon_env = [
+	"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+]
+
+# Additional environment variables to set for all the
+# containers. These are overridden if set in the
+# container image spec or in the container runtime configuration.
+default_env = [
+]
+
+# If true, SELinux will be used for pod separation on the host.
+selinux = {{ crio_selinux }}
+
+# Path to the seccomp.json profile which is used as the default seccomp profile
+# for the runtime. If not specified, then the internal default seccomp profile
+# will be used. This option supports live configuration reload.
+seccomp_profile = "{{ crio_seccomp_profile }}"
+
+# Used to change the name of the default AppArmor profile of CRI-O. The default
+# profile name is "crio-default". This profile only takes effect if the user
+# does not specify a profile via the Kubernetes Pod's metadata annotation. If
+# the profile is set to "unconfined", then this equals to disabling AppArmor.
+# This option supports live configuration reload.
+# apparmor_profile = "crio-default"
+
+# Cgroup management implementation used for the runtime.
+cgroup_manager = "{{ crio_cgroup_manager }}"
+
+# List of default capabilities for containers. If it is empty or commented out,
+# only the capabilities defined in the containers json file by the user/kube
+# will be added.
+default_capabilities = [
+	"CHOWN",
+	"DAC_OVERRIDE",
+	"FSETID",
+	"FOWNER",
+	"NET_RAW",
+	"SETGID",
+	"SETUID",
+	"SETPCAP",
+	"NET_BIND_SERVICE",
+	"SYS_CHROOT",
+	"KILL",
+]
+
+# List of default sysctls. If it is empty or commented out, only the sysctls
+# defined in the container json file by the user/kube will be added.
+default_sysctls = [
+]
+
+# List of additional devices. specified as
+# "<device-on-host>:<device-on-container>:<permissions>", for example: "--device=/dev/sdc:/dev/xvdc:rwm".
+#If it is empty or commented out, only the devices
+# defined in the container json file by the user/kube will be added.
+additional_devices = [
+]
+
+# Path to OCI hooks directories for automatically executed hooks. If one of the
+# directories does not exist, then CRI-O will automatically skip them.
+hooks_dir = [
+	"/usr/share/containers/oci/hooks.d",
+]
+
+# List of default mounts for each container. **Deprecated:** this option will
+# be removed in future versions in favor of default_mounts_file.
+default_mounts = [
+]
+
+# Path to the file specifying the defaults mounts for each container. The
+# format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads
+# its default mounts from the following two files:
+#
+#   1) /etc/containers/mounts.conf (i.e., default_mounts_file): This is the
+#      override file, where users can either add in their own default mounts, or
+#      override the default mounts shipped with the package.
+#
+#   2) /usr/share/containers/mounts.conf: This is the default file read for
+#      mounts. If you want CRI-O to read from a different, specific mounts file,
+#      you can change the default_mounts_file. Note, if this is done, CRI-O will
+#      only add mounts it finds in this file.
+#
+#default_mounts_file = ""
+
+# Maximum number of processes allowed in a container.
+pids_limit = {{ crio_pids_limit }}
+
+# Maximum sized allowed for the container log file. Negative numbers indicate
+# that no size limit is imposed. If it is positive, it must be >= 8192 to
+# match/exceed conmon's read buffer. The file is truncated and re-opened so the
+# limit is never exceeded.
+log_size_max = -1
+
+# Whether container output should be logged to journald in addition to the kuberentes log file
+log_to_journald = false
+
+# Path to directory in which container exit files are written to by conmon.
+container_exits_dir = "/var/run/crio/exits"
+
+# Path to directory for container attach sockets.
+container_attach_socket_dir = "/var/run/crio"
+
+# The prefix to use for the source of the bind mounts.
+bind_mount_prefix = ""
+
+# If set to true, all containers will run in read-only mode.
+read_only = false
+
+# Changes the verbosity of the logs based on the level it is set to. Options
+# are fatal, panic, error, warn, info, debug and trace. This option supports
+# live configuration reload.
+log_level = "{{ crio_log_level }}"
+
+# Filter the log messages by the provided regular expression.
+# This option supports live configuration reload.
+log_filter = ""
+
+# The UID mappings for the user namespace of each container. A range is
+# specified in the form containerUID:HostUID:Size. Multiple ranges must be
+# separated by comma.
+uid_mappings = ""
+
+# The GID mappings for the user namespace of each container. A range is
+# specified in the form containerGID:HostGID:Size. Multiple ranges must be
+# separated by comma.
+gid_mappings = ""
+
+# The minimal amount of time in seconds to wait before issuing a timeout
+# regarding the proper termination of the container. The lowest possible
+# value is 30s, whereas lower values are not considered by CRI-O.
+ctr_stop_timeout = 30
+
+# **DEPRECATED** this option is being replaced by manage_ns_lifecycle, which is described below.
+# manage_network_ns_lifecycle = false
+
+# manage_ns_lifecycle determines whether we pin and remove namespaces
+# and manage their lifecycle
+{% if kata_containers_enabled %}
+manage_ns_lifecycle = true
+{% else %}
+manage_ns_lifecycle = false
+{% endif %}
+
+# The directory where the state of the managed namespaces gets tracked.
+# Only used when manage_ns_lifecycle is true.
+namespaces_dir = "/var/run"
+
+# pinns_path is the path to find the pinns binary, which is needed to manage namespace lifecycle
+pinns_path = ""
+
+# The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes.
+# The runtime to use is picked based on the runtime_handler provided by the CRI.
+# If no runtime_handler is provided, the runtime will be picked based on the level
+# of trust of the workload. Each entry in the table should follow the format:
+#
+#[crio.runtime.runtimes.runtime-handler]
+#  runtime_path = "/path/to/the/executable"
+#  runtime_type = "oci"
+#  runtime_root = "/path/to/the/root"
+#
+# Where:
+# - runtime-handler: name used to identify the runtime
+# - runtime_path (optional, string): absolute path to the runtime executable in
+#   the host filesystem. If omitted, the runtime-handler identifier should match
+#   the runtime executable name, and the runtime executable should be placed
+#   in $PATH.
+# - runtime_type (optional, string): type of runtime, one of: "oci", "vm". If
+#   omitted, an "oci" runtime is assumed.
+# - runtime_root (optional, string): root directory for storage of containers
+#   state.
+
+{% for runtime in crio_runtimes %}
+[crio.runtime.runtimes.{{ runtime.name }}]
+runtime_path = "{{ runtime.path }}"
+runtime_type = "{{ runtime.type }}"
+runtime_root = "{{ runtime.root }}"
+privileged_without_host_devices = {{ runtime.privileged_without_host_devices|default(false)|lower }}
+allowed_annotations = {{ runtime.allowed_annotations|default([])|to_json }}
+{% endfor %}
+
+# Kata Containers with the Firecracker VMM
+#[crio.runtime.runtimes.kata-fc]
+
+# The crio.image table contains settings pertaining to the management of OCI images.
+#
+# CRI-O reads its configured registries defaults from the system wide
+# containers-registries.conf(5) located in /etc/containers/registries.conf. If
+# you want to modify just CRI-O, you can change the registries configuration in
+# this file. Otherwise, leave insecure_registries and registries commented out to
+# use the system's defaults from /etc/containers/registries.conf.
+[crio.image]
+
+# Default transport for pulling images from a remote container storage.
+default_transport = "docker://"
+
+# The path to a file containing credentials necessary for pulling images from
+# secure registries. The file is similar to that of /var/lib/kubelet/config.json
+global_auth_file = "/etc/crio/config.json"
+
+# The image used to instantiate infra containers.
+# This option supports live configuration reload.
+pause_image = "{{ crio_pause_image }}"
+
+# The path to a file containing credentials specific for pulling the pause_image from
+# above. The file is similar to that of /var/lib/kubelet/config.json
+# This option supports live configuration reload.
+pause_image_auth_file = ""
+
+# The command to run to have a container stay in the paused state.
+# When explicitly set to "", it will fallback to the entrypoint and command
+# specified in the pause image. When commented out, it will fallback to the
+# default: "/pause". This option supports live configuration reload.
+pause_command = "/pause"
+
+# Path to the file which decides what sort of policy we use when deciding
+# whether or not to trust an image that we've pulled. It is not recommended that
+# this option be used, as the default behavior of using the system-wide default
+# policy (i.e., /etc/containers/policy.json) is most often preferred. Please
+# refer to containers-policy.json(5) for more details.
+signature_policy = "{{ crio_signature_policy }}"
+
+# Controls how image volumes are handled. The valid values are mkdir, bind and
+# ignore; the latter will ignore volumes entirely.
+image_volumes = "mkdir"
+
+# The crio.network table containers settings pertaining to the management of
+# CNI plugins.
+[crio.network]
+
+# The default CNI network name to be selected. If not set or "", then
+# CRI-O will pick-up the first one found in network_dir.
+# cni_default_network = ""
+
+# Path to the directory where CNI configuration files are located.
+network_dir = "/etc/cni/net.d/"
+
+# Paths to directories where CNI plugin binaries are located.
+plugin_dirs = [
+	"/opt/cni/bin",
+	"/usr/libexec/cni",
+]
+
+# A necessary configuration for Prometheus based metrics retrieval
+[crio.metrics]
+
+# Globally enable or disable metrics support.
+enable_metrics = {{ crio_enable_metrics | bool | lower }}
+
+# The port on which the metrics server will listen.
+metrics_port = {{ crio_metrics_port }}

kubespray/roles/container-engine/cri-o/templates/http-proxy.conf.j2

@@ -0,0 +1,2 @@
+[Service]
+Environment={% if http_proxy is defined %}"HTTP_PROXY={{ http_proxy }}"{% endif %} {% if https_proxy is defined %}"HTTPS_PROXY={{ https_proxy }}"{% endif %} {% if no_proxy is defined %}"NO_PROXY={{ no_proxy }}"{% endif %}

kubespray/roles/container-engine/cri-o/templates/registry.conf.j2

@@ -0,0 +1,13 @@
+[[registry]]
+prefix = "{{ item.prefix | default(item.location) }}"
+insecure = {{ item.insecure | default('false') | string | lower }}
+blocked = {{ item.blocked | default('false') | string | lower }}
+location = "{{ item.location }}"
+{% if item.mirrors is defined %}
+{% for mirror in item.mirrors %}
+
+[[registry.mirror]]
+location = "{{ mirror.location }}"
+insecure = {{ mirror.insecure | default('false') | string | lower }}
+{% endfor %}
+{% endif %}

kubespray/roles/container-engine/cri-o/templates/unqualified.conf.j2

@@ -0,0 +1,10 @@
+{%- set _unqualified_registries = [] -%}
+{% for _registry in crio_registries if _registry.unqualified -%}
+{% if _registry.prefix is defined -%}
+{{ _unqualified_registries.append(_registry.prefix) }}
+{% else %}
+{{ _unqualified_registries.append(_registry.location) }}
+{%- endif %}
+{%- endfor %}
+
+unqualified-search-registries = {{ _unqualified_registries | string }}