kubespray 2.24 추가

2024-02-16 17:08:09 +09:00
parent 1fa9b0df4b
commit f69d904725
1423 changed files with 89069 additions and 2 deletions
--- a/.gitlab-ci/build.yml
+++ b/.gitlab-ci/build.yml
@@ -0,0 +1,40 @@
+---
+.build:
+  stage: build
+  image:
+    name: moby/buildkit:rootless
+    entrypoint: [""]
+  variables:
+    BUILDKITD_FLAGS: --oci-worker-no-process-sandbox
+  before_script:
+    - mkdir ~/.docker
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > ~/.docker/config.json
+
+pipeline image:
+  extends: .build
+  script:
+    - |
+      buildctl-daemonless.sh build \
+        --frontend=dockerfile.v0 \
+        --local context=. \
+        --local dockerfile=. \
+        --opt filename=./pipeline.Dockerfile \
+        --output type=image,name=$PIPELINE_IMAGE,push=true \
+        --import-cache type=registry,ref=$CI_REGISTRY_IMAGE/pipeline:cache
+  rules:
+    - if: '$CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH'
+
+pipeline image and build cache:
+  extends: .build
+  script:
+    - |
+      buildctl-daemonless.sh build \
+        --frontend=dockerfile.v0 \
+        --local context=. \
+        --local dockerfile=. \
+        --opt filename=./pipeline.Dockerfile \
+        --output type=image,name=$PIPELINE_IMAGE,push=true \
+        --import-cache type=registry,ref=$CI_REGISTRY_IMAGE/pipeline:cache \
+        --export-cache type=registry,ref=$CI_REGISTRY_IMAGE/pipeline:cache,mode=max
+  rules:
+    - if: '$CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH'
--- a/.gitlab-ci/lint.yml
+++ b/.gitlab-ci/lint.yml
@@ -0,0 +1,118 @@
+---
+yamllint:
+  extends: .job
+  stage: unit-tests
+  tags: [light]
+  variables:
+    LANG: C.UTF-8
+  script:
+    - yamllint --strict .
+  except: ['triggers', 'master']
+
+vagrant-validate:
+  extends: .job
+  stage: unit-tests
+  tags: [light]
+  variables:
+    VAGRANT_VERSION: 2.3.7
+  script:
+    - ./tests/scripts/vagrant-validate.sh
+  except: ['triggers', 'master']
+
+ansible-lint:
+  extends: .job
+  stage: unit-tests
+  tags: [light]
+  script:
+    - ansible-lint -v
+  except: ['triggers', 'master']
+
+jinja-syntax-check:
+  extends: .job
+  stage: unit-tests
+  tags: [light]
+  script:
+    - "find -name '*.j2' -exec tests/scripts/check-templates.py {} +"
+  except: ['triggers', 'master']
+
+syntax-check:
+  extends: .job
+  stage: unit-tests
+  tags: [light]
+  variables:
+    ANSIBLE_INVENTORY: inventory/local-tests.cfg
+    ANSIBLE_REMOTE_USER: root
+    ANSIBLE_BECOME: "true"
+    ANSIBLE_BECOME_USER: root
+    ANSIBLE_VERBOSITY: "3"
+  script:
+    - ansible-playbook --syntax-check cluster.yml
+    - ansible-playbook --syntax-check playbooks/cluster.yml
+    - ansible-playbook --syntax-check upgrade-cluster.yml
+    - ansible-playbook --syntax-check playbooks/upgrade_cluster.yml
+    - ansible-playbook --syntax-check reset.yml
+    - ansible-playbook --syntax-check playbooks/reset.yml
+    - ansible-playbook --syntax-check extra_playbooks/upgrade-only-k8s.yml
+  except: ['triggers', 'master']
+
+collection-build-install-sanity-check:
+  extends: .job
+  stage: unit-tests
+  tags: [light]
+  variables:
+    ANSIBLE_COLLECTIONS_PATH: "./ansible_collections"
+  script:
+    - ansible-galaxy collection build
+    - ansible-galaxy collection install kubernetes_sigs-kubespray-$(grep "^version:" galaxy.yml | awk '{print $2}').tar.gz
+    - ansible-galaxy collection list $(egrep -i '(name:\s+|namespace:\s+)' galaxy.yml | awk '{print $2}' | tr '\n' '.' | sed 's|\.$||g') | grep "^kubernetes_sigs.kubespray"
+    - test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/cluster.yml
+    - test -f ansible_collections/kubernetes_sigs/kubespray/playbooks/reset.yml
+  except: ['triggers', 'master']
+
+tox-inventory-builder:
+  stage: unit-tests
+  tags: [light]
+  extends: .job
+  before_script:
+    - ./tests/scripts/rebase.sh
+  script:
+    - pip3 install tox
+    - cd contrib/inventory_builder && tox
+  except: ['triggers', 'master']
+
+markdownlint:
+  stage: unit-tests
+  tags: [light]
+  image: node
+  before_script:
+    - npm install -g markdownlint-cli@0.22.0
+  script:
+    - markdownlint $(find . -name '*.md' | grep -vF './.git') --ignore docs/_sidebar.md --ignore contrib/dind/README.md
+
+check-readme-versions:
+  stage: unit-tests
+  tags: [light]
+  image: python:3
+  script:
+    - tests/scripts/check_readme_versions.sh
+
+check-galaxy-version:
+  stage: unit-tests
+  tags: [light]
+  image: python:3
+  script:
+    - tests/scripts/check_galaxy_version.sh
+
+check-typo:
+  stage: unit-tests
+  tags: [light]
+  image: python:3
+  script:
+    - tests/scripts/check_typo.sh
+
+ci-matrix:
+  stage: unit-tests
+  tags: [light]
+  image: python:3
+  script:
+    - tests/scripts/md-table/test.sh
--- a/.gitlab-ci/molecule.yml
+++ b/.gitlab-ci/molecule.yml
@@ -0,0 +1,83 @@
+---
+
+.molecule:
+  tags: [c3.small.x86]
+  only: [/^pr-.*$/]
+  except: ['triggers']
+  image: $PIPELINE_IMAGE
+  services: []
+  stage: deploy-part1
+  before_script:
+    - tests/scripts/rebase.sh
+    - ./tests/scripts/vagrant_clean.sh
+  script:
+    - ./tests/scripts/molecule_run.sh
+  after_script:
+    - chronic ./tests/scripts/molecule_logs.sh
+  artifacts:
+    when: always
+    paths:
+      - molecule_logs/
+
+# CI template for periodic CI jobs
+# Enabled when PERIODIC_CI_ENABLED var is set
+.molecule_periodic:
+  only:
+    variables:
+      - $PERIODIC_CI_ENABLED
+  allow_failure: true
+  extends: .molecule
+
+molecule_full:
+  extends: .molecule_periodic
+
+molecule_no_container_engines:
+  extends: .molecule
+  script:
+    - ./tests/scripts/molecule_run.sh -e container-engine
+  when: on_success
+
+molecule_docker:
+  extends: .molecule
+  script:
+    - ./tests/scripts/molecule_run.sh -i container-engine/cri-dockerd
+  when: on_success
+
+molecule_containerd:
+  extends: .molecule
+  script:
+    - ./tests/scripts/molecule_run.sh -i container-engine/containerd
+  when: on_success
+
+molecule_cri-o:
+  extends: .molecule
+  stage: deploy-part2
+  script:
+    - ./tests/scripts/molecule_run.sh -i container-engine/cri-o
+  allow_failure: true
+  when: on_success
+
+# Stage 3 container engines don't get as much attention so allow them to fail
+molecule_kata:
+  extends: .molecule
+  stage: deploy-part3
+  allow_failure: true
+  script:
+    - ./tests/scripts/molecule_run.sh -i container-engine/kata-containers
+  when: on_success
+
+molecule_gvisor:
+  extends: .molecule
+  stage: deploy-part3
+  allow_failure: true
+  script:
+    - ./tests/scripts/molecule_run.sh -i container-engine/gvisor
+  when: on_success
+
+molecule_youki:
+  extends: .molecule
+  stage: deploy-part3
+  allow_failure: true
+  script:
+    - ./tests/scripts/molecule_run.sh -i container-engine/youki
+  when: on_success
--- a/.gitlab-ci/packet.yml
+++ b/.gitlab-ci/packet.yml
@@ -0,0 +1,337 @@
+---
+.packet:
+  extends: .testcases
+  variables:
+    ANSIBLE_TIMEOUT: "120"
+    CI_PLATFORM: packet
+    SSH_USER: kubespray
+  tags:
+    - packet
+  except: [triggers]
+
+# CI template for PRs
+.packet_pr:
+  only: [/^pr-.*$/]
+  extends: .packet
+
+# CI template for periodic CI jobs
+# Enabled when PERIODIC_CI_ENABLED var is set
+.packet_periodic:
+  only:
+    variables:
+      - $PERIODIC_CI_ENABLED
+  allow_failure: true
+  extends: .packet
+
+packet_cleanup_old:
+  stage: deploy-part1
+  extends: .packet_periodic
+  script:
+    - cd tests
+    - make cleanup-packet
+  after_script: []
+
+# The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
+packet_ubuntu20-calico-all-in-one:
+  stage: deploy-part1
+  extends: .packet_pr
+  when: on_success
+  variables:
+    RESET_CHECK: "true"
+
+# ### PR JOBS PART2
+
+packet_ubuntu20-all-in-one-docker:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_ubuntu20-calico-all-in-one-hardening:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_ubuntu22-all-in-one-docker:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_ubuntu22-calico-all-in-one:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_ubuntu22-calico-etcd-datastore:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_centos7-flannel-addons-ha:
+  extends: .packet_pr
+  stage: deploy-part2
+  when: on_success
+
+packet_almalinux8-crio:
+  extends: .packet_pr
+  stage: deploy-part2
+  when: on_success
+  allow_failure: true
+
+packet_ubuntu20-crio:
+  extends: .packet_pr
+  stage: deploy-part2
+  when: manual
+
+packet_fedora37-crio:
+  extends: .packet_pr
+  stage: deploy-part2
+  when: manual
+
+packet_ubuntu20-flannel-ha:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+packet_debian10-cilium-svc-proxy:
+  stage: deploy-part2
+  extends: .packet_periodic
+  when: on_success
+
+packet_debian10-calico:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_debian10-docker:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_debian11-calico:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_debian11-docker:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_debian12-calico:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_debian12-docker:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_debian12-cilium:
+  stage: deploy-part2
+  extends: .packet_periodic
+  when: on_success
+
+packet_centos7-calico-ha-once-localhost:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+  variables:
+    # This will instruct Docker not to start over TLS.
+    DOCKER_TLS_CERTDIR: ""
+  services:
+    - docker:19.03.9-dind
+
+packet_almalinux8-kube-ovn:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_almalinux8-calico:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_rockylinux8-calico:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_rockylinux9-calico:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_rockylinux9-cilium:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+  variables:
+    RESET_CHECK: "true"
+
+packet_almalinux8-docker:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_fedora38-docker-weave:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+  allow_failure: true
+
+packet_opensuse-docker-cilium:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+# ### MANUAL JOBS
+
+packet_ubuntu20-docker-weave-sep:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+packet_ubuntu20-cilium-sep:
+  stage: deploy-special
+  extends: .packet_pr
+  when: manual
+
+packet_ubuntu20-flannel-ha-once:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+# Calico HA eBPF
+packet_almalinux8-calico-ha-ebpf:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+packet_debian10-macvlan:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+packet_centos7-calico-ha:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+packet_centos7-multus-calico:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+packet_fedora38-docker-calico:
+  stage: deploy-part2
+  extends: .packet_periodic
+  when: on_success
+  variables:
+    RESET_CHECK: "true"
+
+packet_fedora37-calico-selinux:
+  stage: deploy-part2
+  extends: .packet_periodic
+  when: on_success
+
+packet_fedora37-calico-swap-selinux:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+packet_amazon-linux-2-all-in-one:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+packet_almalinux8-calico-nodelocaldns-secondary:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+packet_fedora38-kube-ovn:
+  stage: deploy-part2
+  extends: .packet_periodic
+  when: on_success
+
+packet_debian11-custom-cni:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+packet_debian11-kubelet-csr-approver:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+packet_debian12-custom-cni-helm:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+# ### PR JOBS PART3
+# Long jobs (45min+)
+
+packet_centos7-weave-upgrade-ha:
+  stage: deploy-part3
+  extends: .packet_periodic
+  when: on_success
+  variables:
+    UPGRADE_TEST: basic
+
+packet_ubuntu20-calico-etcd-kubeadm-upgrade-ha:
+  stage: deploy-part3
+  extends: .packet_periodic
+  when: on_success
+  variables:
+    UPGRADE_TEST: basic
+
+# Calico HA Wireguard
+packet_ubuntu20-calico-ha-wireguard:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+
+packet_debian11-calico-upgrade:
+  stage: deploy-part3
+  extends: .packet_pr
+  when: on_success
+  variables:
+    UPGRADE_TEST: graceful
+
+packet_almalinux8-calico-remove-node:
+  stage: deploy-part3
+  extends: .packet_pr
+  when: on_success
+  variables:
+    REMOVE_NODE_CHECK: "true"
+    REMOVE_NODE_NAME: "instance-3"
+
+packet_ubuntu20-calico-etcd-kubeadm:
+  stage: deploy-part3
+  extends: .packet_pr
+  when: on_success
+
+packet_debian11-calico-upgrade-once:
+  stage: deploy-part3
+  extends: .packet_periodic
+  when: on_success
+  variables:
+    UPGRADE_TEST: graceful
+
+packet_ubuntu20-calico-ha-recover:
+  stage: deploy-part3
+  extends: .packet_periodic
+  when: on_success
+  variables:
+    RECOVER_CONTROL_PLANE_TEST: "true"
+    RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
+
+packet_ubuntu20-calico-ha-recover-noquorum:
+  stage: deploy-part3
+  extends: .packet_periodic
+  when: on_success
+  variables:
+    RECOVER_CONTROL_PLANE_TEST: "true"
+    RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[1:]:kube_control_plane[1:]"
--- a/.gitlab-ci/shellcheck.yml
+++ b/.gitlab-ci/shellcheck.yml
@@ -0,0 +1,16 @@
+---
+shellcheck:
+  extends: .job
+  stage: unit-tests
+  tags: [light]
+  variables:
+    SHELLCHECK_VERSION: v0.7.1
+  before_script:
+    - ./tests/scripts/rebase.sh
+    - curl --silent --location "https://github.com/koalaman/shellcheck/releases/download/"${SHELLCHECK_VERSION}"/shellcheck-"${SHELLCHECK_VERSION}".linux.x86_64.tar.xz" | tar -xJv
+    - cp shellcheck-"${SHELLCHECK_VERSION}"/shellcheck /usr/bin/
+    - shellcheck --version
+  script:
+    # Run shellcheck for all *.sh
+    - find . -name '*.sh' -not -path './.git/*' | xargs shellcheck --severity error
+  except: ['triggers', 'master']
--- a/.gitlab-ci/terraform.yml
+++ b/.gitlab-ci/terraform.yml
@@ -0,0 +1,233 @@
+---
+# Tests for contrib/terraform/
+.terraform_install:
+  extends: .job
+  before_script:
+    - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
+    - ./tests/scripts/rebase.sh
+    - ./tests/scripts/testcases_prepare.sh
+    - ./tests/scripts/terraform_install.sh
+    # Set Ansible config
+    - cp ansible.cfg ~/.ansible.cfg
+    # Prepare inventory
+    - cp contrib/terraform/$PROVIDER/sample-inventory/cluster.tfvars .
+    - ln -s contrib/terraform/$PROVIDER/hosts
+    - terraform -chdir="contrib/terraform/$PROVIDER" init
+    # Copy SSH keypair
+    - mkdir -p ~/.ssh
+    - echo "$PACKET_PRIVATE_KEY" | base64 -d > ~/.ssh/id_rsa
+    - chmod 400 ~/.ssh/id_rsa
+    - echo "$PACKET_PUBLIC_KEY" | base64 -d > ~/.ssh/id_rsa.pub
+    - mkdir -p contrib/terraform/$PROVIDER/group_vars
+    # Random subnet to avoid routing conflicts
+    - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"
+
+.terraform_validate:
+  extends: .terraform_install
+  stage: unit-tests
+  tags: [light]
+  only: ['master', /^pr-.*$/]
+  script:
+    - terraform -chdir="contrib/terraform/$PROVIDER" validate
+    - terraform -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
+
+.terraform_apply:
+  extends: .terraform_install
+  tags: [light]
+  stage: deploy-part3
+  when: manual
+  only: [/^pr-.*$/]
+  artifacts:
+    when: always
+    paths:
+      - cluster-dump/
+  variables:
+    ANSIBLE_INVENTORY_UNPARSED_FAILED: "true"
+    ANSIBLE_INVENTORY: hosts
+    CI_PLATFORM: tf
+    TF_VAR_ssh_user: $SSH_USER
+    TF_VAR_cluster_name: $CI_JOB_ID
+  script:
+    - tests/scripts/testcases_run.sh
+  after_script:
+    # Cleanup regardless of exit code
+    - chronic ./tests/scripts/testcases_cleanup.sh
+
+tf-validate-openstack:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: openstack
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-equinix:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: equinix
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-aws:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: aws
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-exoscale:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: exoscale
+
+tf-validate-hetzner:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: hetzner
+
+tf-validate-vsphere:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: vsphere
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-upcloud:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: upcloud
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-nifcloud:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: nifcloud
+
+# tf-packet-ubuntu20-default:
+#   extends: .terraform_apply
+#   variables:
+#     TF_VERSION: $TERRAFORM_VERSION
+#     PROVIDER: packet
+#     CLUSTER: $CI_COMMIT_REF_NAME
+#     TF_VAR_number_of_k8s_masters: "1"
+#     TF_VAR_number_of_k8s_nodes: "1"
+#     TF_VAR_plan_k8s_masters: t1.small.x86
+#     TF_VAR_plan_k8s_nodes: t1.small.x86
+#     TF_VAR_metro: am
+#     TF_VAR_public_key_path: ""
+#     TF_VAR_operating_system: ubuntu_20_04
+
+.ovh_variables: &ovh_variables
+  OS_AUTH_URL: https://auth.cloud.ovh.net/v3
+  OS_PROJECT_ID: 8d3cd5d737d74227ace462dee0b903fe
+  OS_PROJECT_NAME: "9361447987648822"
+  OS_USER_DOMAIN_NAME: Default
+  OS_PROJECT_DOMAIN_ID: default
+  OS_USERNAME: 8XuhBMfkKVrk
+  OS_REGION_NAME: UK1
+  OS_INTERFACE: public
+  OS_IDENTITY_API_VERSION: "3"
+
+# Elastx is generously donating resources for Kubespray on Openstack CI
+# Contacts: @gix @bl0m1
+.elastx_variables: &elastx_variables
+  OS_AUTH_URL: https://ops.elastx.cloud:5000
+  OS_PROJECT_ID: 564c6b461c6b44b1bb19cdb9c2d928e4
+  OS_PROJECT_NAME: kubespray_ci
+  OS_USER_DOMAIN_NAME: Default
+  OS_PROJECT_DOMAIN_ID: default
+  OS_USERNAME: kubespray@root314.com
+  OS_REGION_NAME: se-sto
+  OS_INTERFACE: public
+  OS_IDENTITY_API_VERSION: "3"
+  TF_VAR_router_id: "ab95917c-41fb-4881-b507-3a6dfe9403df"
+
+tf-elastx_cleanup:
+  stage: unit-tests
+  tags: [light]
+  image: python
+  variables:
+    <<: *elastx_variables
+  before_script:
+    - pip install -r scripts/openstack-cleanup/requirements.txt
+  script:
+    - ./scripts/openstack-cleanup/main.py
+
+tf-elastx_ubuntu20-calico:
+  extends: .terraform_apply
+  stage: deploy-part3
+  when: on_success
+  allow_failure: true
+  variables:
+    <<: *elastx_variables
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: openstack
+    CLUSTER: $CI_COMMIT_REF_NAME
+    ANSIBLE_TIMEOUT: "60"
+    SSH_USER: ubuntu
+    TF_VAR_number_of_k8s_masters: "1"
+    TF_VAR_number_of_k8s_masters_no_floating_ip: "0"
+    TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
+    TF_VAR_number_of_etcd: "0"
+    TF_VAR_number_of_k8s_nodes: "1"
+    TF_VAR_number_of_k8s_nodes_no_floating_ip: "0"
+    TF_VAR_number_of_gfs_nodes_no_floating_ip: "0"
+    TF_VAR_number_of_bastions: "0"
+    TF_VAR_number_of_k8s_masters_no_etcd: "0"
+    TF_VAR_floatingip_pool: "elx-public1"
+    TF_VAR_dns_nameservers: '["1.1.1.1", "8.8.8.8", "8.8.4.4"]'
+    TF_VAR_use_access_ip: "0"
+    TF_VAR_external_net: "600b8501-78cb-4155-9c9f-23dfcba88828"
+    TF_VAR_network_name: "ci-$CI_JOB_ID"
+    TF_VAR_az_list: '["sto1"]'
+    TF_VAR_az_list_node: '["sto1"]'
+    TF_VAR_flavor_k8s_master: 3f73fc93-ec61-4808-88df-2580d94c1a9b    # v1-standard-2
+    TF_VAR_flavor_k8s_node: 3f73fc93-ec61-4808-88df-2580d94c1a9b      # v1-standard-2
+    TF_VAR_image: ubuntu-20.04-server-latest
+    TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'
+
+# OVH voucher expired, commenting job until things are sorted  out
+
+# tf-ovh_cleanup:
+#  stage: unit-tests
+#  tags: [light]
+#  image: python
+#  environment: ovh
+#  variables:
+#    <<: *ovh_variables
+#  before_script:
+#    - pip install -r scripts/openstack-cleanup/requirements.txt
+#  script:
+#    - ./scripts/openstack-cleanup/main.py
+
+# tf-ovh_ubuntu20-calico:
+#  extends: .terraform_apply
+#  when: on_success
+#  environment: ovh
+#  variables:
+#    <<: *ovh_variables
+#    TF_VERSION: $TERRAFORM_VERSION
+#    PROVIDER: openstack
+#    CLUSTER: $CI_COMMIT_REF_NAME
+#    ANSIBLE_TIMEOUT: "60"
+#    SSH_USER: ubuntu
+#    TF_VAR_number_of_k8s_masters: "0"
+#    TF_VAR_number_of_k8s_masters_no_floating_ip: "1"
+#    TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
+#    TF_VAR_number_of_etcd: "0"
+#    TF_VAR_number_of_k8s_nodes: "0"
+#    TF_VAR_number_of_k8s_nodes_no_floating_ip: "1"
+#    TF_VAR_number_of_gfs_nodes_no_floating_ip: "0"
+#    TF_VAR_number_of_bastions: "0"
+#    TF_VAR_number_of_k8s_masters_no_etcd: "0"
+#    TF_VAR_use_neutron: "0"
+#    TF_VAR_floatingip_pool: "Ext-Net"
+#    TF_VAR_external_net: "6011fbc9-4cbf-46a4-8452-6890a340b60b"
+#    TF_VAR_network_name: "Ext-Net"
+#    TF_VAR_flavor_k8s_master: "defa64c3-bd46-43b4-858a-d93bbae0a229"    # s1-8
+#    TF_VAR_flavor_k8s_node: "defa64c3-bd46-43b4-858a-d93bbae0a229"      # s1-8
+#    TF_VAR_image: "Ubuntu 20.04"
+#    TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'
--- a/.gitlab-ci/vagrant.yml
+++ b/.gitlab-ci/vagrant.yml
@@ -0,0 +1,63 @@
+---
+
+.vagrant:
+  extends: .testcases
+  variables:
+    CI_PLATFORM: "vagrant"
+    SSH_USER: "vagrant"
+    VAGRANT_DEFAULT_PROVIDER: "libvirt"
+    KUBESPRAY_VAGRANT_CONFIG: tests/files/${CI_JOB_NAME}.rb
+  tags: [c3.small.x86]
+  only: [/^pr-.*$/]
+  except: ['triggers']
+  image: $PIPELINE_IMAGE
+  services: []
+  before_script:
+    - ./tests/scripts/vagrant_clean.sh
+  script:
+    - ./tests/scripts/testcases_run.sh
+  after_script:
+    - chronic ./tests/scripts/testcases_cleanup.sh
+  allow_failure: true
+
+vagrant_ubuntu20-calico-dual-stack:
+  stage: deploy-part2
+  extends: .vagrant
+  when: on_success
+
+vagrant_ubuntu20-weave-medium:
+  stage: deploy-part2
+  extends: .vagrant
+  when: manual
+
+vagrant_ubuntu20-flannel:
+  stage: deploy-part2
+  extends: .vagrant
+  when: on_success
+  allow_failure: false
+
+vagrant_ubuntu20-flannel-collection:
+  stage: deploy-part2
+  extends: .vagrant
+  when: on_success
+
+vagrant_ubuntu20-kube-router-sep:
+  stage: deploy-part2
+  extends: .vagrant
+  when: manual
+
+# Service proxy test fails connectivity testing
+vagrant_ubuntu20-kube-router-svc-proxy:
+  stage: deploy-part2
+  extends: .vagrant
+  when: manual
+
+vagrant_fedora37-kube-router:
+  stage: deploy-part2
+  extends: .vagrant
+  when: on_success
+
+vagrant_centos7-kube-router:
+  stage: deploy-part2
+  extends: .vagrant
+  when: manual