update

kubespray/contrib/aws_iam/kubernetes-master-policy.json

@@ -0,0 +1,27 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["ec2:*"],
+      "Resource": ["*"]
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["elasticloadbalancing:*"],
+      "Resource": ["*"]
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["route53:*"],
+      "Resource": ["*"]
+    },
+    {
+      "Effect": "Allow",
+      "Action": "s3:*",
+      "Resource": [
+        "arn:aws:s3:::kubernetes-*"
+      ]
+    }
+  ]
+}

kubespray/contrib/aws_iam/kubernetes-master-role.json

@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com"},
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}

kubespray/contrib/aws_iam/kubernetes-minion-policy.json

@@ -0,0 +1,45 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": "s3:*",
+      "Resource": [
+        "arn:aws:s3:::kubernetes-*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": "ec2:Describe*",
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": "ec2:AttachVolume",
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": "ec2:DetachVolume",
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["route53:*"],
+      "Resource": ["*"]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ecr:GetAuthorizationToken",
+        "ecr:BatchCheckLayerAvailability",
+        "ecr:GetDownloadUrlForLayer",
+        "ecr:GetRepositoryPolicy",
+        "ecr:DescribeRepositories",
+        "ecr:ListImages",
+        "ecr:BatchGetImage"
+      ],
+      "Resource": "*"
+    }
+  ]
+}

kubespray/contrib/aws_iam/kubernetes-minion-role.json

@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com"},
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}

kubespray/contrib/aws_inventory/kubespray-aws-inventory.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+import boto3
+import os
+import argparse
+import json
+
+class SearchEC2Tags(object):
+
+  def __init__(self):
+    self.parse_args()
+    if self.args.list:
+      self.search_tags()
+    if self.args.host:
+      data = {}
+      print(json.dumps(data, indent=2))
+
+  def parse_args(self):
+
+    ##Check if VPC_VISIBILITY is set, if not default to private
+    if "VPC_VISIBILITY" in os.environ:
+      self.vpc_visibility = os.environ['VPC_VISIBILITY']
+    else:
+      self.vpc_visibility = "private"
+
+    ##Support --list and --host flags. We largely ignore the host one.
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--list', action='store_true', default=False, help='List instances')
+    parser.add_argument('--host', action='store_true', help='Get all the variables about a specific instance')
+    self.args = parser.parse_args()
+
+  def search_tags(self):
+    hosts = {}
+    hosts['_meta'] = { 'hostvars': {} }
+
+    ##Search ec2 three times to find nodes of each group type. Relies on kubespray-role key/value.
+    for group in ["kube_control_plane", "kube_node", "etcd"]:
+      hosts[group] = []
+      tag_key = "kubespray-role"
+      tag_value = ["*"+group+"*"]
+      region = os.environ['REGION']
+
+      ec2 = boto3.resource('ec2', region)
+      filters = [{'Name': 'tag:'+tag_key, 'Values': tag_value}, {'Name': 'instance-state-name', 'Values': ['running']}]
+      cluster_name = os.getenv('CLUSTER_NAME')
+      if cluster_name:
+        filters.append({'Name': 'tag-key', 'Values': ['kubernetes.io/cluster/'+cluster_name]})
+      instances = ec2.instances.filter(Filters=filters)
+      for instance in instances:
+
+        ##Suppose default vpc_visibility is private
+        dns_name = instance.private_dns_name
+        ansible_host = {
+          'ansible_ssh_host': instance.private_ip_address
+        }
+
+        ##Override when vpc_visibility actually is public
+        if self.vpc_visibility == "public":
+          dns_name = instance.public_dns_name
+          ansible_host = {
+            'ansible_ssh_host': instance.public_ip_address
+          }
+
+        ##Set when instance actually has node_labels
+        node_labels_tag = list(filter(lambda t: t['Key'] == 'kubespray-node-labels', instance.tags))
+        if node_labels_tag:
+          ansible_host['node_labels'] = dict([ label.strip().split('=') for label in node_labels_tag[0]['Value'].split(',') ])
+
+        hosts[group].append(dns_name)
+        hosts['_meta']['hostvars'][dns_name] = ansible_host
+
+    hosts['k8s_cluster'] = {'children':['kube_control_plane', 'kube_node']}
+    print(json.dumps(hosts, sort_keys=True, indent=2))
+
+SearchEC2Tags()

kubespray/contrib/aws_inventory/requirements.txt

@@ -0,0 +1 @@
+boto3  # Apache-2.0

kubespray/contrib/azurerm/.gitignore

@@ -0,0 +1,2 @@
+.generated
+/inventory

kubespray/contrib/azurerm/README.md

@@ -0,0 +1,67 @@
+# Kubernetes on Azure with Azure Resource Group Templates
+
+Provision the base infrastructure for a Kubernetes cluster by using [Azure Resource Group Templates](https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-group-authoring-templates)
+
+## Status
+
+This will provision the base infrastructure (vnet, vms, nics, ips, ...) needed for Kubernetes in Azure into the specified
+Resource Group. It will not install Kubernetes itself, this has to be done in a later step by yourself (using kubespray of course).
+
+## Requirements
+
+- [Install azure-cli](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest)
+- [Login with azure-cli](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli?view=azure-cli-latest)
+- Dedicated Resource Group created in the Azure Portal or through azure-cli
+
+## Configuration through group_vars/all
+
+You have to modify at least two variables in group_vars/all. The one is the **cluster_name** variable, it must be globally
+unique due to some restrictions in Azure. The other one is the **ssh_public_keys** variable, it must be your ssh public
+key to access your azure virtual machines. Most other variables should be self explanatory if you have some basic Kubernetes
+experience.
+
+## Bastion host
+
+You can enable the use of a Bastion Host by changing **use_bastion** in group_vars/all to **true**. The generated
+templates will then include an additional bastion VM which can then be used to connect to the masters and nodes. The option
+also removes all public IPs from all other VMs.
+
+## Generating and applying
+
+To generate and apply the templates, call:
+
+```shell
+./apply-rg.sh <resource_group_name>
+```
+
+If you change something in the configuration (e.g. number of nodes) later, you can call this again and Azure will
+take care about creating/modifying whatever is needed.
+
+## Clearing a resource group
+
+If you need to delete all resources from a resource group, simply call:
+
+```shell
+./clear-rg.sh <resource_group_name>
+```
+
+**WARNING** this really deletes everything from your resource group, including everything that was later created by you!
+
+## Installing Ansible and the dependencies
+
+Install Ansible according to [Ansible installation guide](/docs/ansible.md#installing-ansible)
+
+## Generating an inventory for kubespray
+
+After you have applied the templates, you can generate an inventory with this call:
+
+```shell
+./generate-inventory.sh <resource_group_name>
+```
+
+It will create the file ./inventory which can then be used with kubespray, e.g.:
+
+```shell
+cd kubespray-root-dir
+ansible-playbook -i contrib/azurerm/inventory -u devops --become -e "@inventory/sample/group_vars/all/all.yml" cluster.yml
+```

kubespray/contrib/azurerm/apply-rg.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -e
+
+AZURE_RESOURCE_GROUP="$1"
+
+if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
+    echo "AZURE_RESOURCE_GROUP is missing"
+    exit 1
+fi
+
+ansible-playbook generate-templates.yml
+
+az deployment group create --template-file ./.generated/network.json -g $AZURE_RESOURCE_GROUP
+az deployment group create --template-file ./.generated/storage.json -g $AZURE_RESOURCE_GROUP
+az deployment group create --template-file ./.generated/availability-sets.json -g $AZURE_RESOURCE_GROUP
+az deployment group create --template-file ./.generated/bastion.json -g $AZURE_RESOURCE_GROUP
+az deployment group create --template-file ./.generated/masters.json -g $AZURE_RESOURCE_GROUP
+az deployment group create --template-file ./.generated/minions.json -g $AZURE_RESOURCE_GROUP

kubespray/contrib/azurerm/clear-rg.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -e
+
+AZURE_RESOURCE_GROUP="$1"
+
+if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
+    echo "AZURE_RESOURCE_GROUP is missing"
+    exit 1
+fi
+
+ansible-playbook generate-templates.yml
+
+az group deployment create -g "$AZURE_RESOURCE_GROUP" --template-file ./.generated/clear-rg.json --mode Complete

kubespray/contrib/azurerm/generate-inventory.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -e
+
+AZURE_RESOURCE_GROUP="$1"
+
+if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
+    echo "AZURE_RESOURCE_GROUP is missing"
+    exit 1
+fi
+# check if azure cli 2.0 exists else use azure cli 1.0
+if az &>/dev/null; then
+    ansible-playbook generate-inventory_2.yml -e azure_resource_group="$AZURE_RESOURCE_GROUP"
+elif azure &>/dev/null; then
+    ansible-playbook generate-inventory.yml -e azure_resource_group="$AZURE_RESOURCE_GROUP"
+else
+    echo "Azure cli not found"
+fi

kubespray/contrib/azurerm/generate-inventory.yml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  gather_facts: False
+  roles:
+    - generate-inventory

kubespray/contrib/azurerm/generate-inventory_2.yml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  gather_facts: False
+  roles:
+    - generate-inventory_2

kubespray/contrib/azurerm/generate-templates.yml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  gather_facts: False
+  roles:
+    - generate-templates

kubespray/contrib/azurerm/group_vars/all

@@ -0,0 +1,51 @@
+
+# Due to some Azure limitations (ex:- Storage Account's name must be unique),
+# this name must be globally unique - it will be used as a prefix for azure components
+cluster_name: example
+
+# Set this to true if you do not want to have public IPs for your masters and minions. This will provision a bastion
+# node that can be used to access the masters and minions
+use_bastion: false
+
+# Set this to a preferred name that will be used as the first part of the dns name for your bastotion host. For example: k8s-bastion.<azureregion>.cloudapp.azure.com.
+# This is convenient when exceptions have to be configured on a firewall to allow ssh to the given bastion host.
+# bastion_domain_prefix: k8s-bastion
+
+number_of_k8s_masters: 3
+number_of_k8s_nodes: 3
+
+masters_vm_size: Standard_A2
+masters_os_disk_size: 1000
+
+minions_vm_size: Standard_A2
+minions_os_disk_size: 1000
+
+admin_username: devops
+admin_password: changeme
+
+# MAKE SURE TO CHANGE THIS TO YOUR PUBLIC KEY to access your azure machines
+ssh_public_keys:
+ - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLRzcxbsFDdEibiyXCSdIFh7bKbXso1NqlKjEyPTptf3aBXHEhVil0lJRjGpTlpfTy7PHvXFbXIOCdv9tOmeH1uxWDDeZawgPFV6VSZ1QneCL+8bxzhjiCn8133wBSPZkN8rbFKd9eEUUBfx8ipCblYblF9FcidylwtMt5TeEmXk8yRVkPiCuEYuDplhc2H0f4PsK3pFb5aDVdaDT3VeIypnOQZZoUxHWqm6ThyHrzLJd3SrZf+RROFWW1uInIDf/SZlXojczUYoffxgT1lERfOJCHJXsqbZWugbxQBwqsVsX59+KPxFFo6nV88h3UQr63wbFx52/MXkX4WrCkAHzN ablock-vwfs@dell-lappy"
+
+# Disable using ssh using password. Change it to false to allow to connect to ssh by password
+disablePasswordAuthentication: true
+
+# Azure CIDRs
+azure_vnet_cidr: 10.0.0.0/8
+azure_admin_cidr: 10.241.2.0/24
+azure_masters_cidr: 10.0.4.0/24
+azure_minions_cidr: 10.240.0.0/16
+
+# Azure loadbalancer port to use to access your cluster
+kube_apiserver_port: 6443
+
+# Azure Netwoking and storage naming to use with inventory/all.yml
+#azure_virtual_network_name: KubeVNET
+#azure_subnet_admin_name: ad-subnet
+#azure_subnet_masters_name: master-subnet
+#azure_subnet_minions_name: minion-subnet
+#azure_route_table_name: routetable
+#azure_security_group_name: secgroup
+
+# Storage types available are: "Standard_LRS","Premium_LRS"
+#azure_storage_account_type: Standard_LRS

kubespray/contrib/azurerm/roles/generate-inventory/tasks/main.yml

@@ -0,0 +1,15 @@
+---
+
+- name: Query Azure VMs  # noqa 301
+  command: azure vm list-ip-address --json {{ azure_resource_group }}
+  register: vm_list_cmd
+
+- name: Set vm_list
+  set_fact:
+    vm_list: "{{ vm_list_cmd.stdout }}"
+
+- name: Generate inventory
+  template:
+    src: inventory.j2
+    dest: "{{ playbook_dir }}/inventory"
+    mode: 0644

kubespray/contrib/azurerm/roles/generate-inventory/templates/inventory.j2

@@ -0,0 +1,33 @@
+
+{% for vm in vm_list %}
+{% if not use_bastion or vm.name == 'bastion' %}
+{{ vm.name }} ansible_ssh_host={{ vm.networkProfile.networkInterfaces[0].expanded.ipConfigurations[0].publicIPAddress.expanded.ipAddress }} ip={{ vm.networkProfile.networkInterfaces[0].expanded.ipConfigurations[0].privateIPAddress }}
+{% else %}
+{{ vm.name }} ansible_ssh_host={{ vm.networkProfile.networkInterfaces[0].expanded.ipConfigurations[0].privateIPAddress }}
+{% endif %}
+{% endfor %}
+
+[kube_control_plane]
+{% for vm in vm_list %}
+{% if 'kube_control_plane' in vm.tags.roles %}
+{{ vm.name }}
+{% endif %}
+{% endfor %}
+
+[etcd]
+{% for vm in vm_list %}
+{% if 'etcd' in vm.tags.roles %}
+{{ vm.name }}
+{% endif %}
+{% endfor %}
+
+[kube_node]
+{% for vm in vm_list %}
+{% if 'kube_node' in vm.tags.roles %}
+{{ vm.name }}
+{% endif %}
+{% endfor %}
+
+[k8s_cluster:children]
+kube_node
+kube_control_plane

kubespray/contrib/azurerm/roles/generate-inventory_2/tasks/main.yml

@@ -0,0 +1,31 @@
+---
+
+- name: Query Azure VMs IPs  # noqa 301
+  command: az vm list-ip-addresses -o json --resource-group {{ azure_resource_group }}
+  register: vm_ip_list_cmd
+
+- name: Query Azure VMs Roles  # noqa 301
+  command: az vm list -o json --resource-group {{ azure_resource_group }}
+  register: vm_list_cmd
+
+- name: Query Azure Load Balancer Public IP  # noqa 301
+  command: az network public-ip show -o json -g {{ azure_resource_group }} -n kubernetes-api-pubip
+  register: lb_pubip_cmd
+
+- name: Set VM IP, roles lists and load balancer public IP
+  set_fact:
+    vm_ip_list: "{{ vm_ip_list_cmd.stdout }}"
+    vm_roles_list: "{{ vm_list_cmd.stdout }}"
+    lb_pubip: "{{ lb_pubip_cmd.stdout }}"
+
+- name: Generate inventory
+  template:
+    src: inventory.j2
+    dest: "{{ playbook_dir }}/inventory"
+    mode: 0644
+
+- name: Generate Load Balancer variables
+  template:
+    src: loadbalancer_vars.j2
+    dest: "{{ playbook_dir }}/loadbalancer_vars.yml"
+    mode: 0644

kubespray/contrib/azurerm/roles/generate-inventory_2/templates/inventory.j2

@@ -0,0 +1,34 @@
+
+{% for vm in vm_ip_list %}
+{% if not use_bastion or vm.virtualMachine.name == 'bastion' %}
+{{ vm.virtualMachine.name }} ansible_ssh_host={{ vm.virtualMachine.network.publicIpAddresses[0].ipAddress }} ip={{ vm.virtualMachine.network.privateIpAddresses[0] }}
+{% else %}
+{{ vm.virtualMachine.name }} ansible_ssh_host={{  vm.virtualMachine.network.privateIpAddresses[0] }}
+{% endif %}
+{% endfor %}
+
+[kube_control_plane]
+{% for vm in vm_roles_list %}
+{% if 'kube_control_plane' in vm.tags.roles %}
+{{ vm.name }}
+{% endif %}
+{% endfor %}
+
+[etcd]
+{% for vm in vm_roles_list %}
+{% if 'etcd' in vm.tags.roles %}
+{{ vm.name }}
+{% endif %}
+{% endfor %}
+
+[kube_node]
+{% for vm in vm_roles_list %}
+{% if 'kube_node' in vm.tags.roles %}
+{{ vm.name }}
+{% endif %}
+{% endfor %}
+
+[k8s_cluster:children]
+kube_node
+kube_control_plane
+

kubespray/contrib/azurerm/roles/generate-inventory_2/templates/loadbalancer_vars.j2

@@ -0,0 +1,8 @@
+## External LB example config
+apiserver_loadbalancer_domain_name: {{ lb_pubip.dnsSettings.fqdn }}
+loadbalancer_apiserver:
+  address: {{ lb_pubip.ipAddress }}
+  port: 6443
+
+## Internal loadbalancers for apiservers
+loadbalancer_apiserver_localhost: false

kubespray/contrib/azurerm/roles/generate-templates/defaults/main.yml

@@ -0,0 +1,37 @@
+---
+apiVersion: "2015-06-15"
+
+virtualNetworkName: "{{ azure_virtual_network_name | default('KubeVNET') }}"
+
+subnetAdminName: "{{ azure_subnet_admin_name | default('ad-subnet') }}"
+subnetMastersName: "{{ azure_subnet_masters_name | default('master-subnet') }}"
+subnetMinionsName: "{{ azure_subnet_minions_name | default('minion-subnet') }}"
+
+routeTableName: "{{ azure_route_table_name | default('routetable') }}"
+securityGroupName: "{{ azure_security_group_name | default('secgroup') }}"
+
+nameSuffix: "{{ cluster_name }}"
+
+availabilitySetMasters: "master-avs"
+availabilitySetMinions: "minion-avs"
+
+faultDomainCount: 3
+updateDomainCount: 10
+
+bastionVmSize: Standard_A0
+bastionVMName: bastion
+bastionIPAddressName: bastion-pubip
+
+disablePasswordAuthentication: true
+
+sshKeyPath: "/home/{{admin_username}}/.ssh/authorized_keys"
+
+imageReference:
+  publisher: "OpenLogic"
+  offer: "CentOS"
+  sku: "7.5"
+  version: "latest"
+imageReferenceJson: "{{imageReference|to_json}}"
+
+storageAccountName: "sa{{nameSuffix | replace('-', '')}}"
+storageAccountType: "{{ azure_storage_account_type | default('Standard_LRS') }}"

kubespray/contrib/azurerm/roles/generate-templates/tasks/main.yml

@@ -0,0 +1,25 @@
+---
+- name: Set base_dir
+  set_fact:
+    base_dir: "{{ playbook_dir }}/.generated/"
+
+- name: Create base_dir
+  file:
+    path: "{{ base_dir }}"
+    state: directory
+    recurse: true
+    mode: 0755
+
+- name: Store json files in base_dir
+  template:
+    src: "{{ item }}"
+    dest: "{{ base_dir }}/{{ item }}"
+    mode: 0644
+  with_items:
+    - network.json
+    - storage.json
+    - availability-sets.json
+    - bastion.json
+    - masters.json
+    - minions.json
+    - clear-rg.json

kubespray/contrib/azurerm/roles/generate-templates/templates/availability-sets.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+  },
+  "resources": [
+    {
+      "type": "Microsoft.Compute/availabilitySets",
+      "name": "{{availabilitySetMasters}}",
+      "apiVersion": "{{apiVersion}}",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "PlatformFaultDomainCount": "{{faultDomainCount}}",
+        "PlatformUpdateDomainCount": "{{updateDomainCount}}"
+      }
+    },
+    {
+      "type": "Microsoft.Compute/availabilitySets",
+      "name": "{{availabilitySetMinions}}",
+      "apiVersion": "{{apiVersion}}",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "PlatformFaultDomainCount": "{{faultDomainCount}}",
+        "PlatformUpdateDomainCount": "{{updateDomainCount}}"
+      }
+    }
+  ]
+}

kubespray/contrib/azurerm/roles/generate-templates/templates/bastion.json

@@ -0,0 +1,106 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+    "vnetID": "[resourceId('Microsoft.Network/virtualNetworks', '{{virtualNetworkName}}')]",
+    "subnetAdminRef": "[concat(variables('vnetID'),'/subnets/', '{{subnetAdminName}}')]"
+  },
+  "resources": [
+    {% if use_bastion %}
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/publicIPAddresses",
+      "name": "{{bastionIPAddressName}}",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "publicIPAllocationMethod": "Static",
+        "dnsSettings": {
+          {% if bastion_domain_prefix %}
+          "domainNameLabel": "{{ bastion_domain_prefix }}"
+          {% endif %}
+        }
+      }
+    },
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/networkInterfaces",
+      "name": "{{bastionVMName}}-nic",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        "[concat('Microsoft.Network/publicIPAddresses/', '{{bastionIPAddressName}}')]"
+      ],
+      "properties": {
+        "ipConfigurations": [
+          {
+            "name": "BastionIpConfig",
+            "properties": {
+              "privateIPAllocationMethod": "Dynamic",
+              "publicIPAddress": {
+                "id": "[resourceId('Microsoft.Network/publicIPAddresses', '{{bastionIPAddressName}}')]"
+              },
+              "subnet": {
+                "id": "[variables('subnetAdminRef')]"
+              }
+            }
+          }
+        ]
+      }
+    },
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Compute/virtualMachines",
+      "name": "{{bastionVMName}}",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        "[concat('Microsoft.Network/networkInterfaces/', '{{bastionVMName}}-nic')]"
+      ],
+      "tags": {
+        "roles": "bastion"
+      },
+      "properties": {
+        "hardwareProfile": {
+          "vmSize": "{{bastionVmSize}}"
+        },
+        "osProfile": {
+          "computerName": "{{bastionVMName}}",
+          "adminUsername": "{{admin_username}}",
+          "adminPassword": "{{admin_password}}",
+          "linuxConfiguration": {
+            "disablePasswordAuthentication": "true",
+            "ssh": {
+              "publicKeys": [
+                {% for key in ssh_public_keys %}
+                {
+                  "path": "{{sshKeyPath}}",
+                  "keyData": "{{key}}"
+                }{% if loop.index < ssh_public_keys | length %},{% endif %}
+                {% endfor %}
+              ]
+            }
+          }
+        },
+        "storageProfile": {
+          "imageReference": {{imageReferenceJson}},
+          "osDisk": {
+            "name": "osdisk",
+            "vhd": {
+              "uri": "[concat('http://', '{{storageAccountName}}', '.blob.core.windows.net/vhds/', '{{bastionVMName}}', '-osdisk.vhd')]"
+            },
+            "caching": "ReadWrite",
+            "createOption": "FromImage"
+          }
+        },
+        "networkProfile": {
+          "networkInterfaces": [
+            {
+              "id": "[resourceId('Microsoft.Network/networkInterfaces', '{{bastionVMName}}-nic')]"
+            }
+          ]
+        }
+      }
+    }
+    {% endif %}
+  ]
+}

kubespray/contrib/azurerm/roles/generate-templates/templates/clear-rg.json

@@ -0,0 +1,8 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {},
+  "variables": {},
+  "resources": [],
+  "outputs": {}
+}

kubespray/contrib/azurerm/roles/generate-templates/templates/masters.json

@@ -0,0 +1,198 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+    "lbDomainName": "{{nameSuffix}}-api",
+    "lbPublicIPAddressName": "kubernetes-api-pubip",
+    "lbPublicIPAddressType": "Static",
+    "lbPublicIPAddressID": "[resourceId('Microsoft.Network/publicIPAddresses',variables('lbPublicIPAddressName'))]",
+    "lbName": "kubernetes-api",
+    "lbID": "[resourceId('Microsoft.Network/loadBalancers',variables('lbName'))]",
+
+    "vnetID": "[resourceId('Microsoft.Network/virtualNetworks', '{{virtualNetworkName}}')]",
+    "kubeMastersSubnetRef": "[concat(variables('vnetID'),'/subnets/', '{{subnetMastersName}}')]"
+  },
+  "resources": [
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/publicIPAddresses",
+      "name": "[variables('lbPublicIPAddressName')]",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "publicIPAllocationMethod": "[variables('lbPublicIPAddressType')]",
+        "dnsSettings": {
+          "domainNameLabel": "[variables('lbDomainName')]"
+        }
+      }
+    },
+    {
+      "apiVersion": "{{apiVersion}}",
+      "name": "[variables('lbName')]",
+      "type": "Microsoft.Network/loadBalancers",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        "[concat('Microsoft.Network/publicIPAddresses/', variables('lbPublicIPAddressName'))]"
+      ],
+      "properties": {
+        "frontendIPConfigurations": [
+          {
+            "name": "kube-api-frontend",
+            "properties": {
+              "publicIPAddress": {
+                "id": "[variables('lbPublicIPAddressID')]"
+              }
+            }
+          }
+        ],
+        "backendAddressPools": [
+          {
+            "name": "kube-api-backend"
+          }
+        ],
+        "loadBalancingRules": [
+          {
+            "name": "kube-api",
+            "properties": {
+              "frontendIPConfiguration": {
+                "id": "[concat(variables('lbID'), '/frontendIPConfigurations/kube-api-frontend')]"
+              },
+              "backendAddressPool": {
+                "id": "[concat(variables('lbID'), '/backendAddressPools/kube-api-backend')]"
+              },
+              "protocol": "tcp",
+              "frontendPort": "{{kube_apiserver_port}}",
+              "backendPort": "{{kube_apiserver_port}}",
+              "enableFloatingIP": false,
+              "idleTimeoutInMinutes": 5,
+              "probe": {
+                "id": "[concat(variables('lbID'), '/probes/kube-api')]"
+              }
+            }
+          }
+        ],
+        "probes": [
+          {
+            "name": "kube-api",
+            "properties": {
+              "protocol": "tcp",
+              "port": "{{kube_apiserver_port}}",
+              "intervalInSeconds": 5,
+              "numberOfProbes": 2
+            }
+          }
+        ]
+      }
+    },
+    {% for i in range(number_of_k8s_masters) %}
+    {% if not use_bastion %}
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/publicIPAddresses",
+      "name": "master-{{i}}-pubip",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "publicIPAllocationMethod": "Static"
+      }
+    },
+    {% endif %}
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/networkInterfaces",
+      "name": "master-{{i}}-nic",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        {% if not use_bastion %}
+        "[concat('Microsoft.Network/publicIPAddresses/', 'master-{{i}}-pubip')]",
+        {% endif %}
+        "[concat('Microsoft.Network/loadBalancers/', variables('lbName'))]"
+      ],
+      "properties": {
+        "ipConfigurations": [
+          {
+            "name": "MastersIpConfig",
+            "properties": {
+              "privateIPAllocationMethod": "Dynamic",
+              {% if not use_bastion %}
+              "publicIPAddress": {
+                "id": "[resourceId('Microsoft.Network/publicIPAddresses', 'master-{{i}}-pubip')]"
+              },
+              {% endif %}
+              "subnet": {
+                "id": "[variables('kubeMastersSubnetRef')]"
+              },
+	          "loadBalancerBackendAddressPools": [
+                {
+                  "id": "[concat(variables('lbID'), '/backendAddressPools/kube-api-backend')]"
+                }
+              ]
+            }
+          }
+        ],
+        "networkSecurityGroup": {
+          "id": "[resourceId('Microsoft.Network/networkSecurityGroups', '{{securityGroupName}}')]"
+        },
+        "enableIPForwarding": true
+      }
+    },
+    {
+      "type": "Microsoft.Compute/virtualMachines",
+      "name": "master-{{i}}",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        "[concat('Microsoft.Network/networkInterfaces/', 'master-{{i}}-nic')]"
+      ],
+      "tags": {
+        "roles": "kube_control_plane,etcd"
+      },
+      "apiVersion": "{{apiVersion}}",
+      "properties": {
+        "availabilitySet": {
+          "id": "[resourceId('Microsoft.Compute/availabilitySets', '{{availabilitySetMasters}}')]"
+        },
+        "hardwareProfile": {
+          "vmSize": "{{masters_vm_size}}"
+        },
+        "osProfile": {
+          "computerName": "master-{{i}}",
+          "adminUsername": "{{admin_username}}",
+          "adminPassword": "{{admin_password}}",
+          "linuxConfiguration": {
+            "disablePasswordAuthentication": "{{disablePasswordAuthentication}}",
+            "ssh": {
+              "publicKeys": [
+                {% for key in ssh_public_keys %}
+                {
+                  "path": "{{sshKeyPath}}",
+                  "keyData": "{{key}}"
+                }{% if loop.index < ssh_public_keys | length %},{% endif %}
+                {% endfor %}
+              ]
+            }
+          }
+        },
+        "storageProfile": {
+          "imageReference": {{imageReferenceJson}},
+          "osDisk": {
+            "name": "ma{{nameSuffix}}{{i}}",
+            "vhd": {
+              "uri": "[concat('http://','{{storageAccountName}}','.blob.core.windows.net/vhds/master-{{i}}.vhd')]"
+            },
+            "caching": "ReadWrite",
+            "createOption": "FromImage",
+            "diskSizeGB": "{{masters_os_disk_size}}"
+          }
+        },
+        "networkProfile": {
+          "networkInterfaces": [
+            {
+              "id": "[resourceId('Microsoft.Network/networkInterfaces', 'master-{{i}}-nic')]"
+            }
+          ]
+        }
+      }
+    } {% if not loop.last %},{% endif %}
+    {% endfor %}
+  ]
+}

kubespray/contrib/azurerm/roles/generate-templates/templates/minions.json

@@ -0,0 +1,115 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+    "vnetID": "[resourceId('Microsoft.Network/virtualNetworks', '{{virtualNetworkName}}')]",
+    "kubeMinionsSubnetRef": "[concat(variables('vnetID'),'/subnets/', '{{subnetMinionsName}}')]"
+  },
+  "resources": [
+    {% for i in range(number_of_k8s_nodes) %}
+    {% if not use_bastion %}
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/publicIPAddresses",
+      "name": "minion-{{i}}-pubip",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "publicIPAllocationMethod": "Static"
+      }
+    },
+    {% endif %}
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/networkInterfaces",
+      "name": "minion-{{i}}-nic",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        {% if not use_bastion %}
+        "[concat('Microsoft.Network/publicIPAddresses/', 'minion-{{i}}-pubip')]"
+        {% endif %}
+      ],
+      "properties": {
+        "ipConfigurations": [
+          {
+            "name": "MinionsIpConfig",
+            "properties": {
+              "privateIPAllocationMethod": "Dynamic",
+              {% if not use_bastion %}
+              "publicIPAddress": {
+                "id": "[resourceId('Microsoft.Network/publicIPAddresses', 'minion-{{i}}-pubip')]"
+              },
+              {% endif %}
+              "subnet": {
+                "id": "[variables('kubeMinionsSubnetRef')]"
+              }
+            }
+          }
+        ],
+        "networkSecurityGroup": {
+          "id": "[resourceId('Microsoft.Network/networkSecurityGroups', '{{securityGroupName}}')]"
+        },
+        "enableIPForwarding": true
+      }
+    },
+    {
+      "type": "Microsoft.Compute/virtualMachines",
+      "name": "minion-{{i}}",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        "[concat('Microsoft.Network/networkInterfaces/', 'minion-{{i}}-nic')]"
+      ],
+      "tags": {
+        "roles": "kube_node"
+      },
+      "apiVersion": "{{apiVersion}}",
+      "properties": {
+        "availabilitySet": {
+          "id": "[resourceId('Microsoft.Compute/availabilitySets', '{{availabilitySetMinions}}')]"
+        },
+        "hardwareProfile": {
+          "vmSize": "{{minions_vm_size}}"
+        },
+        "osProfile": {
+          "computerName": "minion-{{i}}",
+          "adminUsername": "{{admin_username}}",
+          "adminPassword": "{{admin_password}}",
+          "linuxConfiguration": {
+            "disablePasswordAuthentication": "{{disablePasswordAuthentication}}",
+            "ssh": {
+              "publicKeys": [
+                {% for key in ssh_public_keys %}
+                {
+                  "path": "{{sshKeyPath}}",
+                  "keyData": "{{key}}"
+                }{% if loop.index < ssh_public_keys | length %},{% endif %}
+                {% endfor %}
+              ]
+            }
+          }
+        },
+        "storageProfile": {
+          "imageReference": {{imageReferenceJson}},
+          "osDisk": {
+            "name": "mi{{nameSuffix}}{{i}}",
+            "vhd": {
+              "uri": "[concat('http://','{{storageAccountName}}','.blob.core.windows.net/vhds/minion-{{i}}.vhd')]"
+            },
+            "caching": "ReadWrite",
+            "createOption": "FromImage",
+            "diskSizeGB": "{{minions_os_disk_size}}"
+          }
+        },
+        "networkProfile": {
+          "networkInterfaces": [
+            {
+              "id": "[resourceId('Microsoft.Network/networkInterfaces', 'minion-{{i}}-nic')]"
+            }
+          ]
+        }
+      }
+    } {% if not loop.last %},{% endif %}
+    {% endfor %}
+  ]
+}

kubespray/contrib/azurerm/roles/generate-templates/templates/network.json

@@ -0,0 +1,109 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+  },
+  "resources": [
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/routeTables",
+      "name": "{{routeTableName}}",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "routes": [
+        ]
+      }
+    },
+    {
+      "type": "Microsoft.Network/virtualNetworks",
+      "name": "{{virtualNetworkName}}",
+      "location": "[resourceGroup().location]",
+      "apiVersion": "{{apiVersion}}",
+      "dependsOn": [
+        "[concat('Microsoft.Network/routeTables/', '{{routeTableName}}')]"
+      ],
+      "properties": {
+        "addressSpace": {
+          "addressPrefixes": [
+            "{{azure_vnet_cidr}}"
+          ]
+        },
+        "subnets": [
+          {
+            "name": "{{subnetMastersName}}",
+            "properties": {
+              "addressPrefix": "{{azure_masters_cidr}}",
+              "routeTable": {
+                "id": "[resourceId('Microsoft.Network/routeTables', '{{routeTableName}}')]"
+              }
+            }
+          },
+          {
+            "name": "{{subnetMinionsName}}",
+            "properties": {
+              "addressPrefix": "{{azure_minions_cidr}}",
+              "routeTable": {
+                "id": "[resourceId('Microsoft.Network/routeTables', '{{routeTableName}}')]"
+              }
+            }
+          }
+          {% if use_bastion %}
+          ,{
+            "name": "{{subnetAdminName}}",
+            "properties": {
+              "addressPrefix": "{{azure_admin_cidr}}",
+              "routeTable": {
+                "id": "[resourceId('Microsoft.Network/routeTables', '{{routeTableName}}')]"
+              }
+            }
+          }
+          {% endif %}
+        ]
+      }
+    },
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/networkSecurityGroups",
+      "name": "{{securityGroupName}}",
+      "location": "[resourceGroup().location]",
+      "properties": {
+          "securityRules": [
+            {% if not use_bastion %}
+            {
+              "name": "ssh",
+              "properties": {
+                "description": "Allow SSH",
+                "protocol": "Tcp",
+                "sourcePortRange": "*",
+                "destinationPortRange": "22",
+                "sourceAddressPrefix": "Internet",
+                "destinationAddressPrefix": "*",
+                "access": "Allow",
+                "priority": 100,
+                "direction": "Inbound"
+              }
+            },
+            {% endif %}
+            {
+              "name": "kube-api",
+              "properties": {
+                "description": "Allow secure kube-api",
+                "protocol": "Tcp",
+                "sourcePortRange": "*",
+                "destinationPortRange": "{{kube_apiserver_port}}",
+                "sourceAddressPrefix": "Internet",
+                "destinationAddressPrefix": "*",
+                "access": "Allow",
+                "priority": 101,
+                "direction": "Inbound"
+              }
+            }
+          ]
+      },
+      "resources": [],
+      "dependsOn": []
+    }
+  ]
+}

kubespray/contrib/azurerm/roles/generate-templates/templates/storage.json

@@ -0,0 +1,19 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+  },
+  "resources": [
+    {
+      "type": "Microsoft.Storage/storageAccounts",
+      "name": "{{storageAccountName}}",
+      "location": "[resourceGroup().location]",
+      "apiVersion": "{{apiVersion}}",
+      "properties": {
+        "accountType": "{{storageAccountType}}"
+      }
+    }
+  ]
+}

kubespray/contrib/dind/README.md

@@ -0,0 +1,177 @@
+# Kubespray DIND experimental setup
+
+This ansible playbook creates local docker containers
+to serve as Kubernetes "nodes", which in turn will run
+"normal" Kubernetes docker containers, a mode usually
+called DIND (Docker-IN-Docker).
+
+The playbook has two roles:
+
+- dind-host: creates the "nodes" as containers in localhost, with
+  appropriate settings for DIND (privileged, volume mapping for dind
+  storage, etc).
+- dind-cluster: customizes each node container to have required
+  system packages installed, and some utils (swapoff, lsattr)
+  symlinked to /bin/true to ease mimicking a real node.
+
+This playbook has been test with Ubuntu 16.04 as host and ubuntu:16.04
+as docker images (note that dind-cluster has specific customization
+for these images).
+
+The playbook also creates a `/tmp/kubespray.dind.inventory_builder.sh`
+helper (wraps up running `contrib/inventory_builder/inventory.py` with
+node containers IPs and prefix).
+
+## Deploying
+
+See below for a complete successful run:
+
+1. Create the node containers
+
+```shell
+# From the kubespray root dir
+cd contrib/dind
+pip install -r requirements.txt
+
+ansible-playbook -i hosts dind-cluster.yaml
+
+# Back to kubespray root
+cd ../..
+```
+
+NOTE: if the playbook run fails with something like below error
+message, you may need to specifically set `ansible_python_interpreter`,
+see `./hosts` file for an example expanded localhost entry.
+
+```shell
+failed: [localhost] (item=kube-node1) => {"changed": false, "item": "kube-node1", "msg": "Failed to import docker or docker-py - No module named requests.exceptions. Try `pip install docker` or `pip install docker-py` (Python 2.6)"}
+```
+
+2. Customize kubespray-dind.yaml
+
+Note that there's coupling between above created node containers
+and `kubespray-dind.yaml` settings, in particular regarding selected `node_distro`
+(as set in `group_vars/all/all.yaml`), and docker settings.
+
+```shell
+$EDITOR contrib/dind/kubespray-dind.yaml
+```
+
+3. Prepare the inventory and run the playbook
+
+```shell
+INVENTORY_DIR=inventory/local-dind
+mkdir -p ${INVENTORY_DIR}
+rm -f ${INVENTORY_DIR}/hosts.ini
+CONFIG_FILE=${INVENTORY_DIR}/hosts.ini /tmp/kubespray.dind.inventory_builder.sh
+
+ansible-playbook --become -e ansible_ssh_user=debian -i ${INVENTORY_DIR}/hosts.ini cluster.yml --extra-vars @contrib/dind/kubespray-dind.yaml
+```
+
+NOTE: You could also test other distros without editing files by
+passing `--extra-vars` as per below commandline,
+replacing `DISTRO` by either `debian`, `ubuntu`, `centos`, `fedora`:
+
+```shell
+cd contrib/dind
+ansible-playbook -i hosts dind-cluster.yaml --extra-vars node_distro=DISTRO
+
+cd ../..
+CONFIG_FILE=inventory/local-dind/hosts.ini /tmp/kubespray.dind.inventory_builder.sh
+ansible-playbook --become -e ansible_ssh_user=DISTRO -i inventory/local-dind/hosts.ini cluster.yml --extra-vars @contrib/dind/kubespray-dind.yaml --extra-vars bootstrap_os=DISTRO
+```
+
+## Resulting deployment
+
+See below to get an idea on how a completed deployment looks like,
+from the host where you ran kubespray playbooks.
+
+### node_distro: debian
+
+Running from an Ubuntu Xenial host:
+
+```shell
+$ uname -a
+Linux ip-xx-xx-xx-xx 4.4.0-1069-aws #79-Ubuntu SMP Mon Sep 24
+15:01:41 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux
+
+$ docker ps
+CONTAINER ID        IMAGE               COMMAND CREATED             STATUS              PORTS               NAMES
+1835dd183b75        debian:9.5          "sh -c 'apt-get -qy …"   43 minutes ago      Up 43 minutes                           kube-node5
+30b0af8d2924        debian:9.5          "sh -c 'apt-get -qy …"   43 minutes ago      Up 43 minutes                           kube-node4
+3e0d1510c62f        debian:9.5          "sh -c 'apt-get -qy …"   43 minutes ago      Up 43 minutes                           kube-node3
+738993566f94        debian:9.5          "sh -c 'apt-get -qy …"   44 minutes ago      Up 44 minutes                           kube-node2
+c581ef662ed2        debian:9.5          "sh -c 'apt-get -qy …"   44 minutes ago      Up 44 minutes                           kube-node1
+
+$ docker exec kube-node1 kubectl get node
+NAME         STATUS   ROLES         AGE   VERSION
+kube-node1   Ready    master,node   18m   v1.12.1
+kube-node2   Ready    master,node   17m   v1.12.1
+kube-node3   Ready    node          17m   v1.12.1
+kube-node4   Ready    node          17m   v1.12.1
+kube-node5   Ready    node          17m   v1.12.1
+
+$ docker exec kube-node1 kubectl get pod --all-namespaces
+NAMESPACE     NAME                                    READY   STATUS    RESTARTS   AGE
+default       netchecker-agent-67489                  1/1     Running   0          2m51s
+default       netchecker-agent-6qq6s                  1/1     Running   0          2m51s
+default       netchecker-agent-fsw92                  1/1     Running   0          2m51s
+default       netchecker-agent-fw6tl                  1/1     Running   0          2m51s
+default       netchecker-agent-hostnet-8f2zb          1/1     Running   0          3m
+default       netchecker-agent-hostnet-gq7ml          1/1     Running   0          3m
+default       netchecker-agent-hostnet-jfkgv          1/1     Running   0          3m
+default       netchecker-agent-hostnet-kwfwx          1/1     Running   0          3m
+default       netchecker-agent-hostnet-r46nm          1/1     Running   0          3m
+default       netchecker-agent-lxdrn                  1/1     Running   0          2m51s
+default       netchecker-server-864bd4c897-9vstl      1/1     Running   0          2m40s
+default       sh-68fcc6db45-qf55h                     1/1     Running   1          12m
+kube-system   coredns-7598f59475-6vknq                1/1     Running   0          14m
+kube-system   coredns-7598f59475-l5q5x                1/1     Running   0          14m
+kube-system   kube-apiserver-kube-node1               1/1     Running   0          17m
+kube-system   kube-apiserver-kube-node2               1/1     Running   0          18m
+kube-system   kube-controller-manager-kube-node1      1/1     Running   0          18m
+kube-system   kube-controller-manager-kube-node2      1/1     Running   0          18m
+kube-system   kube-proxy-5xx9d                        1/1     Running   0          17m
+kube-system   kube-proxy-cdqq4                        1/1     Running   0          17m
+kube-system   kube-proxy-n64ls                        1/1     Running   0          17m
+kube-system   kube-proxy-pswmj                        1/1     Running   0          18m
+kube-system   kube-proxy-x89qw                        1/1     Running   0          18m
+kube-system   kube-scheduler-kube-node1               1/1     Running   4          17m
+kube-system   kube-scheduler-kube-node2               1/1     Running   4          18m
+kube-system   kubernetes-dashboard-5db4d9f45f-548rl   1/1     Running   0          14m
+kube-system   nginx-proxy-kube-node3                  1/1     Running   4          17m
+kube-system   nginx-proxy-kube-node4                  1/1     Running   4          17m
+kube-system   nginx-proxy-kube-node5                  1/1     Running   4          17m
+kube-system   weave-net-42bfr                         2/2     Running   0          16m
+kube-system   weave-net-6gt8m                         2/2     Running   0          16m
+kube-system   weave-net-88nnc                         2/2     Running   0          16m
+kube-system   weave-net-shckr                         2/2     Running   0          16m
+kube-system   weave-net-xr46t                         2/2     Running   0          16m
+
+$ docker exec kube-node1 curl -s http://localhost:31081/api/v1/connectivity_check
+{"Message":"All 10 pods successfully reported back to the server","Absent":null,"Outdated":null}
+```
+
+## Using ./run-test-distros.sh
+
+You can use `./run-test-distros.sh` to run a set of tests via DIND,
+and excerpt from this script, to get an idea:
+
+```shell
+# The SPEC file(s) must have two arrays as e.g.
+# DISTROS=(debian centos)
+# EXTRAS=(
+#     'kube_network_plugin=calico'
+#     'kube_network_plugin=flannel'
+#     'kube_network_plugin=weave'
+# )
+# that will be tested in a "combinatory" way (e.g. from above there'll be
+# be 6 test runs), creating a sequenced <spec_filename>-nn.out with each output.
+#
+# Each $EXTRAS element will be whitespace split, and passed as --extra-vars
+# to main kubespray ansible-playbook run.
+```
+
+See e.g. `test-some_distros-most_CNIs.env` and
+`test-some_distros-kube_router_combo.env` in particular for a richer
+set of CNI specific `--extra-vars` combo.

kubespray/contrib/dind/dind-cluster.yaml

@@ -0,0 +1,9 @@
+---
+- hosts: localhost
+  gather_facts: False
+  roles:
+    - { role: dind-host }
+
+- hosts: containers
+  roles:
+    - { role: dind-cluster }

kubespray/contrib/dind/group_vars/all/all.yaml

@@ -0,0 +1,3 @@
+---
+# See distro.yaml for supported node_distro images
+node_distro: debian

kubespray/contrib/dind/group_vars/all/distro.yaml

@@ -0,0 +1,41 @@
+---
+distro_settings:
+  debian: &DEBIAN
+    image: "debian:9.5"
+    user: "debian"
+    pid1_exe: /lib/systemd/systemd
+    init: |
+      sh -c "apt-get -qy update && apt-get -qy install systemd-sysv dbus && exec /sbin/init"
+    raw_setup: apt-get -qy update && apt-get -qy install dbus python sudo iproute2
+    raw_setup_done: test -x /usr/bin/sudo
+    agetty_svc: getty@*
+    ssh_service: ssh
+    extra_packages: []
+  ubuntu:
+    <<: *DEBIAN
+    image: "ubuntu:16.04"
+    user: "ubuntu"
+    init: |
+      /sbin/init
+  centos: &CENTOS
+    image: "centos:7"
+    user: "centos"
+    pid1_exe: /usr/lib/systemd/systemd
+    init: |
+      /sbin/init
+    raw_setup: yum -qy install policycoreutils dbus python sudo iproute iptables
+    raw_setup_done: test -x /usr/bin/sudo
+    agetty_svc: getty@* serial-getty@*
+    ssh_service: sshd
+    extra_packages: []
+  fedora:
+    <<: *CENTOS
+    image: "fedora:latest"
+    user: "fedora"
+    raw_setup: yum -qy install policycoreutils dbus python sudo iproute iptables; mkdir -p /etc/modules-load.d
+    extra_packages:
+      - hostname
+      - procps
+      - findutils
+      - kmod
+      - iputils

kubespray/contrib/dind/hosts

@@ -0,0 +1,15 @@
+[local]
+# If you created a virtualenv for ansible, you may need to specify running the
+# python binary from there instead:
+#localhost ansible_connection=local ansible_python_interpreter=/home/user/kubespray/.venv/bin/python
+localhost ansible_connection=local
+
+[containers]
+kube-node1
+kube-node2
+kube-node3
+kube-node4
+kube-node5
+
+[containers:vars]
+ansible_connection=docker

kubespray/contrib/dind/kubespray-dind.yaml

@@ -0,0 +1,22 @@
+---
+# kubespray-dind.yaml: minimal kubespray ansible playbook usable for DIND
+# See contrib/dind/README.md
+kube_api_anonymous_auth: true
+
+kubelet_fail_swap_on: false
+
+# Docker nodes need to have been created with same "node_distro: debian"
+# at contrib/dind/group_vars/all/all.yaml
+bootstrap_os: debian
+
+docker_version: latest
+
+docker_storage_options: -s overlay2 --storage-opt overlay2.override_kernel_check=true -g /dind/docker
+
+dns_mode: coredns
+
+deploy_netchecker: True
+netcheck_agent_image_repo: quay.io/l23network/k8s-netchecker-agent
+netcheck_server_image_repo: quay.io/l23network/k8s-netchecker-server
+netcheck_agent_image_tag: v1.0
+netcheck_server_image_tag: v1.0

kubespray/contrib/dind/requirements.txt

@@ -0,0 +1 @@
+docker

kubespray/contrib/dind/roles/dind-cluster/tasks/main.yaml

@@ -0,0 +1,73 @@
+---
+- name: set_fact distro_setup
+  set_fact:
+    distro_setup: "{{ distro_settings[node_distro] }}"
+
+- name: set_fact other distro settings
+  set_fact:
+    distro_user: "{{ distro_setup['user'] }}"
+    distro_ssh_service: "{{ distro_setup['ssh_service'] }}"
+    distro_extra_packages: "{{ distro_setup['extra_packages'] }}"
+
+- name: Null-ify some linux tools to ease DIND
+  file:
+    src: "/bin/true"
+    dest: "{{ item }}"
+    state: link
+    force: yes
+  with_items:
+    # DIND box may have swap enable, don't bother
+    - /sbin/swapoff
+    # /etc/hosts handling would fail on trying to copy file attributes on edit,
+    # void it by successfully returning nil output
+    - /usr/bin/lsattr
+    # disable selinux-isms, sp needed if running on non-Selinux host
+    - /usr/sbin/semodule
+
+- name: Void installing dpkg docs and man pages on Debian based distros
+  copy:
+    content: |
+      # Delete locales
+      path-exclude=/usr/share/locale/*
+      # Delete man pages
+      path-exclude=/usr/share/man/*
+      # Delete docs
+      path-exclude=/usr/share/doc/*
+      path-include=/usr/share/doc/*/copyright
+    dest: /etc/dpkg/dpkg.cfg.d/01_nodoc
+    mode: 0644
+  when:
+    - ansible_os_family == 'Debian'
+
+- name: Install system packages to better match a full-fledge node
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items: "{{ distro_extra_packages }} + [ 'rsyslog', 'openssh-server' ]"
+
+- name: Start needed services
+  service:
+    name: "{{ item }}"
+    state: started
+  with_items:
+    - rsyslog
+    - "{{ distro_ssh_service }}"
+
+- name: Create distro user "{{ distro_user }}"
+  user:
+    name: "{{ distro_user }}"
+    uid: 1000
+    # groups: sudo
+    append: yes
+
+- name: Allow password-less sudo to "{{ distro_user }}"
+  copy:
+    content: "{{ distro_user }} ALL=(ALL) NOPASSWD:ALL"
+    dest: "/etc/sudoers.d/{{ distro_user }}"
+    mode: 0640
+
+- name: Add my pubkey to "{{ distro_user }}" user authorized keys
+  authorized_key:
+    user: "{{ distro_user }}"
+    state: present
+    key: "{{ lookup('file', lookup('env','HOME') + '/.ssh/id_rsa.pub') }}"

kubespray/contrib/dind/roles/dind-host/tasks/main.yaml

@@ -0,0 +1,88 @@
+---
+- name: set_fact distro_setup
+  set_fact:
+    distro_setup: "{{ distro_settings[node_distro] }}"
+
+- name: set_fact other distro settings
+  set_fact:
+    distro_image: "{{ distro_setup['image'] }}"
+    distro_init: "{{ distro_setup['init'] }}"
+    distro_pid1_exe: "{{ distro_setup['pid1_exe'] }}"
+    distro_raw_setup: "{{ distro_setup['raw_setup'] }}"
+    distro_raw_setup_done: "{{ distro_setup['raw_setup_done'] }}"
+    distro_agetty_svc: "{{ distro_setup['agetty_svc'] }}"
+
+- name: Create dind node containers from "containers" inventory section
+  docker_container:
+    image: "{{ distro_image }}"
+    name: "{{ item }}"
+    state: started
+    hostname: "{{ item }}"
+    command: "{{ distro_init }}"
+    # recreate: yes
+    privileged: true
+    tmpfs:
+      - /sys/module/nf_conntrack/parameters
+    volumes:
+      - /boot:/boot
+      - /lib/modules:/lib/modules
+      - "{{ item }}:/dind/docker"
+  register: containers
+  with_items: "{{ groups.containers }}"
+  tags:
+    - addresses
+
+- name: Gather list of containers IPs
+  set_fact:
+    addresses: "{{ containers.results | map(attribute='ansible_facts') | map(attribute='docker_container') | map(attribute='NetworkSettings') | map(attribute='IPAddress') | list }}"
+  tags:
+    - addresses
+
+- name: Create inventory_builder helper already set with the list of node containers' IPs
+  template:
+    src: inventory_builder.sh.j2
+    dest: /tmp/kubespray.dind.inventory_builder.sh
+    mode: 0755
+  tags:
+    - addresses
+
+- name: Install needed packages into node containers via raw, need to wait for possible systemd packages to finish installing
+  raw: |
+    # agetty processes churn a lot of cpu time failing on inexistent ttys, early STOP them, to rip them in below task
+    pkill -STOP agetty || true
+    {{ distro_raw_setup_done }}  && echo SKIPPED && exit 0
+    until [ "$(readlink /proc/1/exe)" = "{{ distro_pid1_exe }}" ] ; do sleep 1; done
+    {{ distro_raw_setup }}
+  delegate_to: "{{ item._ansible_item_label|default(item.item) }}"
+  with_items: "{{ containers.results }}"
+  register: result
+  changed_when: result.stdout.find("SKIPPED") < 0
+
+- name: Remove gettys from node containers
+  raw: |
+    until test -S /var/run/dbus/system_bus_socket; do sleep 1; done
+    systemctl disable {{ distro_agetty_svc }}
+    systemctl stop {{ distro_agetty_svc }}
+  delegate_to: "{{ item._ansible_item_label|default(item.item) }}"
+  with_items: "{{ containers.results }}"
+  changed_when: false
+
+# Running systemd-machine-id-setup doesn't create a unique id for each node container on Debian,
+# handle manually
+- name: Re-create unique machine-id (as we may just get what comes in the docker image), needed by some CNIs for mac address seeding (notably weave)  # noqa 301
+  raw: |
+    echo {{ item | hash('sha1') }} > /etc/machine-id.new
+    mv -b /etc/machine-id.new /etc/machine-id
+    cmp /etc/machine-id /etc/machine-id~ || true
+    systemctl daemon-reload
+  delegate_to: "{{ item._ansible_item_label|default(item.item) }}"
+  with_items: "{{ containers.results }}"
+
+- name: Early hack image install to adapt for DIND
+  # noqa 302 - this task uses the raw module intentionally
+  raw: |
+    rm -fv /usr/bin/udevadm /usr/sbin/udevadm
+  delegate_to: "{{ item._ansible_item_label|default(item.item) }}"
+  with_items: "{{ containers.results }}"
+  register: result
+  changed_when: result.stdout.find("removed") >= 0

kubespray/contrib/dind/roles/dind-host/templates/inventory_builder.sh.j2

@@ -0,0 +1,3 @@
+#!/bin/bash
+# NOTE: if you change HOST_PREFIX, you also need to edit ./hosts [containers] section
+HOST_PREFIX=kube-node python3 contrib/inventory_builder/inventory.py {% for ip in addresses %} {{ ip }} {% endfor %}

kubespray/contrib/dind/run-test-distros.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+# Q&D test'em all: creates full DIND kubespray deploys
+# for each distro, verifying it via netchecker.
+
+info() {
+    local msg="$*"
+    local date="$(date -Isec)"
+    echo "INFO: [$date] $msg"
+}
+pass_or_fail() {
+    local rc="$?"
+    local msg="$*"
+    local date="$(date -Isec)"
+    [ $rc -eq 0 ] && echo "PASS: [$date] $msg" || echo "FAIL: [$date] $msg"
+    return $rc
+}
+test_distro() {
+    local distro=${1:?};shift
+    local extra="${*:-}"
+    local prefix="${distro[${extra}]}"
+    ansible-playbook -i hosts dind-cluster.yaml -e node_distro=$distro
+    pass_or_fail "$prefix: dind-nodes" || return 1
+    (cd ../..
+        INVENTORY_DIR=inventory/local-dind
+        mkdir -p ${INVENTORY_DIR}
+        rm -f ${INVENTORY_DIR}/hosts.ini
+        CONFIG_FILE=${INVENTORY_DIR}/hosts.ini /tmp/kubespray.dind.inventory_builder.sh
+        # expand $extra with -e in front of each word
+        extra_args=""; for extra_arg in $extra; do extra_args="$extra_args -e $extra_arg"; done
+        ansible-playbook --become -e ansible_ssh_user=$distro -i \
+            ${INVENTORY_DIR}/hosts.ini cluster.yml \
+            -e @contrib/dind/kubespray-dind.yaml -e bootstrap_os=$distro ${extra_args}
+        pass_or_fail "$prefix: kubespray"
+    ) || return 1
+    local node0=${NODES[0]}
+    docker exec ${node0} kubectl get pod --all-namespaces
+    pass_or_fail "$prefix: kube-api" || return 1
+    let retries=60
+    while ((retries--)); do
+        # Some CNI may set NodePort on "main" node interface address (thus no localhost NodePort)
+        # e.g. kube-router: https://github.com/cloudnativelabs/kube-router/pull/217
+        docker exec ${node0} curl -m2 -s http://${NETCHECKER_HOST:?}:31081/api/v1/connectivity_check | grep successfully && break
+        sleep 2
+    done
+    [ $retries -ge 0 ]
+    pass_or_fail "$prefix: netcheck" || return 1
+}
+
+NODES=($(egrep ^kube_node hosts))
+NETCHECKER_HOST=localhost
+
+: ${OUTPUT_DIR:=./out}
+mkdir -p ${OUTPUT_DIR}
+
+# The SPEC file(s) must have two arrays as e.g.
+# DISTROS=(debian centos)
+# EXTRAS=(
+#     'kube_network_plugin=calico'
+#     'kube_network_plugin=flannel'
+#     'kube_network_plugin=weave'
+# )
+# that will be tested in a "combinatory" way (e.g. from above there'll be
+# be 6 test runs), creating a sequenced <spec_filename>-nn.out with each output.
+#
+# Each $EXTRAS element will be whitespace split, and passed as --extra-vars
+# to main kubespray ansible-playbook run.
+
+SPECS=${*:?Missing SPEC files, e.g. test-most_distros-some_CNIs.env}
+for spec in ${SPECS}; do
+    unset DISTROS EXTRAS
+    echo "Loading file=${spec} ..."
+    . ${spec} || continue
+    : ${DISTROS:?} || continue
+    echo "DISTROS:" "${DISTROS[@]}"
+    echo "EXTRAS->"
+    printf "  %s\n" "${EXTRAS[@]}"
+    let n=1
+    for distro in "${DISTROS[@]}"; do
+        for extra in "${EXTRAS[@]:-NULL}"; do
+            # Magic value to let this for run once:
+            [[ ${extra} == NULL ]] && unset extra
+            docker rm -f "${NODES[@]}"
+            printf -v file_out "%s/%s-%02d.out" ${OUTPUT_DIR} ${spec} $((n++))
+            {
+                info "${distro}[${extra}] START: file_out=${file_out}"
+                time test_distro ${distro} ${extra}
+            } |& tee ${file_out}
+            # sleeping for the sake of the human to verify if they want
+            sleep 2m
+        done
+    done
+done
+egrep -H '^(....:|real)' $(ls -tr ${OUTPUT_DIR}/*.out)

kubespray/contrib/dind/test-most_distros-some_CNIs.env

@@ -0,0 +1,11 @@
+# Test spec file: used from ./run-test-distros.sh, will run
+# each distro in $DISTROS overloading main kubespray ansible-playbook run
+# Get all DISTROS from distro.yaml (shame no yaml parsing, but nuff anyway)
+# DISTROS="${*:-$(egrep -o '^  \w+' group_vars/all/distro.yaml|paste -s)}"
+DISTROS=(debian ubuntu centos fedora)
+
+# Each line below will be added as --extra-vars to main playbook run
+EXTRAS=(
+    'kube_network_plugin=calico'
+    'kube_network_plugin=weave'
+)

kubespray/contrib/dind/test-some_distros-kube_router_combo.env

@@ -0,0 +1,6 @@
+DISTROS=(debian centos)
+NETCHECKER_HOST=${NODES[0]}
+EXTRAS=(
+  'kube_network_plugin=kube-router {"kube_router_run_service_proxy":false}'
+  'kube_network_plugin=kube-router {"kube_router_run_service_proxy":true}'
+)

kubespray/contrib/dind/test-some_distros-most_CNIs.env

@@ -0,0 +1,8 @@
+DISTROS=(debian centos)
+EXTRAS=(
+  'kube_network_plugin=calico {}'
+  'kube_network_plugin=canal {}'
+  'kube_network_plugin=cilium {}'
+  'kube_network_plugin=flannel {}'
+  'kube_network_plugin=weave {}'
+)

kubespray/contrib/inventory_builder/inventory.py

@@ -0,0 +1,480 @@
+#!/usr/bin/env python3
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Usage: inventory.py ip1 [ip2 ...]
+# Examples: inventory.py 10.10.1.3 10.10.1.4 10.10.1.5
+#
+# Advanced usage:
+# Add another host after initial creation: inventory.py 10.10.1.5
+# Add range of hosts: inventory.py 10.10.1.3-10.10.1.5
+# Add hosts with different ip and access ip:
+# inventory.py 10.0.0.1,192.168.10.1 10.0.0.2,192.168.10.2 10.0.0.3,192.168.1.3
+# Add hosts with a specific hostname, ip, and optional access ip:
+# inventory.py first,10.0.0.1,192.168.10.1 second,10.0.0.2 last,10.0.0.3
+# Delete a host: inventory.py -10.10.1.3
+# Delete a host by id: inventory.py -node1
+#
+# Load a YAML or JSON file with inventory data: inventory.py load hosts.yaml
+# YAML file should be in the following format:
+#    group1:
+#      host1:
+#        ip: X.X.X.X
+#        var: val
+#    group2:
+#      host2:
+#        ip: X.X.X.X
+
+from collections import OrderedDict
+from ipaddress import ip_address
+from ruamel.yaml import YAML
+
+import os
+import re
+import subprocess
+import sys
+
+ROLES = ['all', 'kube_control_plane', 'kube_node', 'etcd', 'k8s_cluster',
+         'calico_rr']
+PROTECTED_NAMES = ROLES
+AVAILABLE_COMMANDS = ['help', 'print_cfg', 'print_ips', 'print_hostnames',
+                      'load', 'add']
+_boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
+                   '0': False, 'no': False, 'false': False, 'off': False}
+yaml = YAML()
+yaml.Representer.add_representer(OrderedDict, yaml.Representer.represent_dict)
+
+
+def get_var_as_bool(name, default):
+    value = os.environ.get(name, '')
+    return _boolean_states.get(value.lower(), default)
+
+# Configurable as shell vars start
+
+
+CONFIG_FILE = os.environ.get("CONFIG_FILE", "./inventory/sample/hosts.yaml")
+# Remove the reference of KUBE_MASTERS after some deprecation cycles.
+KUBE_CONTROL_HOSTS = int(os.environ.get("KUBE_CONTROL_HOSTS",
+                         os.environ.get("KUBE_MASTERS", 2)))
+# Reconfigures cluster distribution at scale
+SCALE_THRESHOLD = int(os.environ.get("SCALE_THRESHOLD", 50))
+MASSIVE_SCALE_THRESHOLD = int(os.environ.get("MASSIVE_SCALE_THRESHOLD", 200))
+
+DEBUG = get_var_as_bool("DEBUG", True)
+HOST_PREFIX = os.environ.get("HOST_PREFIX", "node")
+USE_REAL_HOSTNAME = get_var_as_bool("USE_REAL_HOSTNAME", False)
+
+# Configurable as shell vars end
+
+
+class KubesprayInventory(object):
+
+    def __init__(self, changed_hosts=None, config_file=None):
+        self.config_file = config_file
+        self.yaml_config = {}
+        loadPreviousConfig = False
+        printHostnames = False
+        # See whether there are any commands to process
+        if changed_hosts and changed_hosts[0] in AVAILABLE_COMMANDS:
+            if changed_hosts[0] == "add":
+                loadPreviousConfig = True
+                changed_hosts = changed_hosts[1:]
+            elif changed_hosts[0] == "print_hostnames":
+                loadPreviousConfig = True
+                printHostnames = True
+            else:
+                self.parse_command(changed_hosts[0], changed_hosts[1:])
+                sys.exit(0)
+
+        # If the user wants to remove a node, we need to load the config anyway
+        if changed_hosts and changed_hosts[0][0] == "-":
+            loadPreviousConfig = True
+
+        if self.config_file and loadPreviousConfig:  # Load previous YAML file
+            try:
+                self.hosts_file = open(config_file, 'r')
+                self.yaml_config = yaml.load(self.hosts_file)
+            except OSError as e:
+                # I am assuming we are catching "cannot open file" exceptions
+                print(e)
+                sys.exit(1)
+
+        if printHostnames:
+            self.print_hostnames()
+            sys.exit(0)
+
+        self.ensure_required_groups(ROLES)
+
+        if changed_hosts:
+            changed_hosts = self.range2ips(changed_hosts)
+            self.hosts = self.build_hostnames(changed_hosts,
+                                              loadPreviousConfig)
+            self.purge_invalid_hosts(self.hosts.keys(), PROTECTED_NAMES)
+            self.set_all(self.hosts)
+            self.set_k8s_cluster()
+            etcd_hosts_count = 3 if len(self.hosts.keys()) >= 3 else 1
+            self.set_etcd(list(self.hosts.keys())[:etcd_hosts_count])
+            if len(self.hosts) >= SCALE_THRESHOLD:
+                self.set_kube_control_plane(list(self.hosts.keys())[
+                    etcd_hosts_count:(etcd_hosts_count + KUBE_CONTROL_HOSTS)])
+            else:
+                self.set_kube_control_plane(
+                  list(self.hosts.keys())[:KUBE_CONTROL_HOSTS])
+            self.set_kube_node(self.hosts.keys())
+            if len(self.hosts) >= SCALE_THRESHOLD:
+                self.set_calico_rr(list(self.hosts.keys())[:etcd_hosts_count])
+        else:  # Show help if no options
+            self.show_help()
+            sys.exit(0)
+
+        self.write_config(self.config_file)
+
+    def write_config(self, config_file):
+        if config_file:
+            with open(self.config_file, 'w') as f:
+                yaml.dump(self.yaml_config, f)
+
+        else:
+            print("WARNING: Unable to save config. Make sure you set "
+                  "CONFIG_FILE env var.")
+
+    def debug(self, msg):
+        if DEBUG:
+            print("DEBUG: {0}".format(msg))
+
+    def get_ip_from_opts(self, optstring):
+        if 'ip' in optstring:
+            return optstring['ip']
+        else:
+            raise ValueError("IP parameter not found in options")
+
+    def ensure_required_groups(self, groups):
+        for group in groups:
+            if group == 'all':
+                self.debug("Adding group {0}".format(group))
+                if group not in self.yaml_config:
+                    all_dict = OrderedDict([('hosts', OrderedDict({})),
+                                            ('children', OrderedDict({}))])
+                    self.yaml_config = {'all': all_dict}
+            else:
+                self.debug("Adding group {0}".format(group))
+                if group not in self.yaml_config['all']['children']:
+                    self.yaml_config['all']['children'][group] = {'hosts': {}}
+
+    def get_host_id(self, host):
+        '''Returns integer host ID (without padding) from a given hostname.'''
+        try:
+            short_hostname = host.split('.')[0]
+            return int(re.findall("\\d+$", short_hostname)[-1])
+        except IndexError:
+            raise ValueError("Host name must end in an integer")
+
+    # Keeps already specified hosts,
+    # and adds or removes the hosts provided as an argument
+    def build_hostnames(self, changed_hosts, loadPreviousConfig=False):
+        existing_hosts = OrderedDict()
+        highest_host_id = 0
+        # Load already existing hosts from the YAML
+        if loadPreviousConfig:
+            try:
+                for host in self.yaml_config['all']['hosts']:
+                    # Read configuration of an existing host
+                    hostConfig = self.yaml_config['all']['hosts'][host]
+                    existing_hosts[host] = hostConfig
+                    # If the existing host seems
+                    # to have been created automatically, detect its ID
+                    if host.startswith(HOST_PREFIX):
+                        host_id = self.get_host_id(host)
+                        if host_id > highest_host_id:
+                            highest_host_id = host_id
+            except Exception as e:
+                # I am assuming we are catching automatically
+                # created hosts without IDs
+                print(e)
+                sys.exit(1)
+
+        # FIXME(mattymo): Fix condition where delete then add reuses highest id
+        next_host_id = highest_host_id + 1
+        next_host = ""
+
+        all_hosts = existing_hosts.copy()
+        for host in changed_hosts:
+            # Delete the host from config the hostname/IP has a "-" prefix
+            if host[0] == "-":
+                realhost = host[1:]
+                if self.exists_hostname(all_hosts, realhost):
+                    self.debug("Marked {0} for deletion.".format(realhost))
+                    all_hosts.pop(realhost)
+                elif self.exists_ip(all_hosts, realhost):
+                    self.debug("Marked {0} for deletion.".format(realhost))
+                    self.delete_host_by_ip(all_hosts, realhost)
+            # Host/Argument starts with a digit,
+            # then we assume its an IP address
+            elif host[0].isdigit():
+                if ',' in host:
+                    ip, access_ip = host.split(',')
+                else:
+                    ip = host
+                    access_ip = host
+                if self.exists_hostname(all_hosts, host):
+                    self.debug("Skipping existing host {0}.".format(host))
+                    continue
+                elif self.exists_ip(all_hosts, ip):
+                    self.debug("Skipping existing host {0}.".format(ip))
+                    continue
+
+                if USE_REAL_HOSTNAME:
+                    cmd = ("ssh -oStrictHostKeyChecking=no "
+                           + access_ip + " 'hostname -s'")
+                    next_host = subprocess.check_output(cmd, shell=True)
+                    next_host = next_host.strip().decode('ascii')
+                else:
+                    # Generates a hostname because we have only an IP address
+                    next_host = "{0}{1}".format(HOST_PREFIX, next_host_id)
+                    next_host_id += 1
+                # Uses automatically generated node name
+                # in case we dont provide it.
+                all_hosts[next_host] = {'ansible_host': access_ip,
+                                        'ip': ip,
+                                        'access_ip': access_ip}
+            # Host/Argument starts with a letter, then we assume its a hostname
+            elif host[0].isalpha():
+                if ',' in host:
+                    try:
+                        hostname, ip, access_ip = host.split(',')
+                    except Exception:
+                        hostname, ip = host.split(',')
+                        access_ip = ip
+                if self.exists_hostname(all_hosts, host):
+                    self.debug("Skipping existing host {0}.".format(host))
+                    continue
+                elif self.exists_ip(all_hosts, ip):
+                    self.debug("Skipping existing host {0}.".format(ip))
+                    continue
+                all_hosts[hostname] = {'ansible_host': access_ip,
+                                       'ip': ip,
+                                       'access_ip': access_ip}
+        return all_hosts
+
+    # Expand IP ranges into individual addresses
+    def range2ips(self, hosts):
+        reworked_hosts = []
+
+        def ips(start_address, end_address):
+            try:
+                # Python 3.x
+                start = int(ip_address(start_address))
+                end = int(ip_address(end_address))
+            except Exception:
+                # Python 2.7
+                start = int(ip_address(str(start_address)))
+                end = int(ip_address(str(end_address)))
+            return [ip_address(ip).exploded for ip in range(start, end + 1)]
+
+        for host in hosts:
+            if '-' in host and not (host.startswith('-') or host[0].isalpha()):
+                start, end = host.strip().split('-')
+                try:
+                    reworked_hosts.extend(ips(start, end))
+                except ValueError:
+                    raise Exception("Range of ip_addresses isn't valid")
+            else:
+                reworked_hosts.append(host)
+        return reworked_hosts
+
+    def exists_hostname(self, existing_hosts, hostname):
+        return hostname in existing_hosts.keys()
+
+    def exists_ip(self, existing_hosts, ip):
+        for host_opts in existing_hosts.values():
+            if ip == self.get_ip_from_opts(host_opts):
+                return True
+        return False
+
+    def delete_host_by_ip(self, existing_hosts, ip):
+        for hostname, host_opts in existing_hosts.items():
+            if ip == self.get_ip_from_opts(host_opts):
+                del existing_hosts[hostname]
+                return
+        raise ValueError("Unable to find host by IP: {0}".format(ip))
+
+    def purge_invalid_hosts(self, hostnames, protected_names=[]):
+        for role in self.yaml_config['all']['children']:
+            if role != 'k8s_cluster' and self.yaml_config['all']['children'][role]['hosts']:  # noqa
+                all_hosts = self.yaml_config['all']['children'][role]['hosts'].copy()  # noqa
+                for host in all_hosts.keys():
+                    if host not in hostnames and host not in protected_names:
+                        self.debug(
+                            "Host {0} removed from role {1}".format(host, role))  # noqa
+                        del self.yaml_config['all']['children'][role]['hosts'][host]  # noqa
+        # purge from all
+        if self.yaml_config['all']['hosts']:
+            all_hosts = self.yaml_config['all']['hosts'].copy()
+            for host in all_hosts.keys():
+                if host not in hostnames and host not in protected_names:
+                    self.debug("Host {0} removed from role all".format(host))
+                    del self.yaml_config['all']['hosts'][host]
+
+    def add_host_to_group(self, group, host, opts=""):
+        self.debug("adding host {0} to group {1}".format(host, group))
+        if group == 'all':
+            if self.yaml_config['all']['hosts'] is None:
+                self.yaml_config['all']['hosts'] = {host: None}
+            self.yaml_config['all']['hosts'][host] = opts
+        elif group != 'k8s_cluster:children':
+            if self.yaml_config['all']['children'][group]['hosts'] is None:
+                self.yaml_config['all']['children'][group]['hosts'] = {
+                    host: None}
+            else:
+                self.yaml_config['all']['children'][group]['hosts'][host] = None  # noqa
+
+    def set_kube_control_plane(self, hosts):
+        for host in hosts:
+            self.add_host_to_group('kube_control_plane', host)
+
+    def set_all(self, hosts):
+        for host, opts in hosts.items():
+            self.add_host_to_group('all', host, opts)
+
+    def set_k8s_cluster(self):
+        k8s_cluster = {'children': {'kube_control_plane': None,
+                                    'kube_node': None}}
+        self.yaml_config['all']['children']['k8s_cluster'] = k8s_cluster
+
+    def set_calico_rr(self, hosts):
+        for host in hosts:
+            if host in self.yaml_config['all']['children']['kube_control_plane']: # noqa
+                self.debug("Not adding {0} to calico_rr group because it "
+                           "conflicts with kube_control_plane "
+                           "group".format(host))
+                continue
+            if host in self.yaml_config['all']['children']['kube_node']:
+                self.debug("Not adding {0} to calico_rr group because it "
+                           "conflicts with kube_node group".format(host))
+                continue
+            self.add_host_to_group('calico_rr', host)
+
+    def set_kube_node(self, hosts):
+        for host in hosts:
+            if len(self.yaml_config['all']['hosts']) >= SCALE_THRESHOLD:
+                if host in self.yaml_config['all']['children']['etcd']['hosts']:  # noqa
+                    self.debug("Not adding {0} to kube_node group because of "
+                               "scale deployment and host is in etcd "
+                               "group.".format(host))
+                    continue
+            if len(self.yaml_config['all']['hosts']) >= MASSIVE_SCALE_THRESHOLD:  # noqa
+                if host in self.yaml_config['all']['children']['kube_control_plane']['hosts']:  # noqa
+                    self.debug("Not adding {0} to kube_node group because of "
+                               "scale deployment and host is in "
+                               "kube_control_plane group.".format(host))
+                    continue
+            self.add_host_to_group('kube_node', host)
+
+    def set_etcd(self, hosts):
+        for host in hosts:
+            self.add_host_to_group('etcd', host)
+
+    def load_file(self, files=None):
+        '''Directly loads JSON to inventory.'''
+
+        if not files:
+            raise Exception("No input file specified.")
+
+        import json
+
+        for filename in list(files):
+            # Try JSON
+            try:
+                with open(filename, 'r') as f:
+                    data = json.load(f)
+            except ValueError:
+                raise Exception("Cannot read %s as JSON, or CSV", filename)
+
+            self.ensure_required_groups(ROLES)
+            self.set_k8s_cluster()
+            for group, hosts in data.items():
+                self.ensure_required_groups([group])
+                for host, opts in hosts.items():
+                    optstring = {'ansible_host': opts['ip'],
+                                 'ip': opts['ip'],
+                                 'access_ip': opts['ip']}
+                    self.add_host_to_group('all', host, optstring)
+                    self.add_host_to_group(group, host)
+            self.write_config(self.config_file)
+
+    def parse_command(self, command, args=None):
+        if command == 'help':
+            self.show_help()
+        elif command == 'print_cfg':
+            self.print_config()
+        elif command == 'print_ips':
+            self.print_ips()
+        elif command == 'print_hostnames':
+            self.print_hostnames()
+        elif command == 'load':
+            self.load_file(args)
+        else:
+            raise Exception("Invalid command specified.")
+
+    def show_help(self):
+        help_text = '''Usage: inventory.py ip1 [ip2 ...]
+Examples: inventory.py 10.10.1.3 10.10.1.4 10.10.1.5
+
+Available commands:
+help - Display this message
+print_cfg - Write inventory file to stdout
+print_ips - Write a space-delimited list of IPs from "all" group
+print_hostnames - Write a space-delimited list of Hostnames from "all" group
+add - Adds specified hosts into an already existing inventory
+
+Advanced usage:
+Create new or overwrite old inventory file: inventory.py 10.10.1.5
+Add another host after initial creation: inventory.py add 10.10.1.6
+Add range of hosts: inventory.py 10.10.1.3-10.10.1.5
+Add hosts with different ip and access ip: inventory.py 10.0.0.1,192.168.10.1 10.0.0.2,192.168.10.2 10.0.0.3,192.168.10.3
+Add hosts with a specific hostname, ip, and optional access ip: first,10.0.0.1,192.168.10.1 second,10.0.0.2 last,10.0.0.3
+Delete a host: inventory.py -10.10.1.3
+Delete a host by id: inventory.py -node1
+
+Configurable env vars:
+DEBUG                   Enable debug printing. Default: True
+CONFIG_FILE             File to write config to Default: ./inventory/sample/hosts.yaml
+HOST_PREFIX             Host prefix for generated hosts. Default: node
+KUBE_CONTROL_HOSTS      Set the number of kube-control-planes. Default: 2
+SCALE_THRESHOLD         Separate ETCD role if # of nodes >= 50
+MASSIVE_SCALE_THRESHOLD Separate K8s control-plane and ETCD if # of nodes >= 200
+'''  # noqa
+        print(help_text)
+
+    def print_config(self):
+        yaml.dump(self.yaml_config, sys.stdout)
+
+    def print_hostnames(self):
+        print(' '.join(self.yaml_config['all']['hosts'].keys()))
+
+    def print_ips(self):
+        ips = []
+        for host, opts in self.yaml_config['all']['hosts'].items():
+            ips.append(self.get_ip_from_opts(opts))
+        print(' '.join(ips))
+
+
+def main(argv=None):
+    if not argv:
+        argv = sys.argv[1:]
+    KubesprayInventory(argv, CONFIG_FILE)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

kubespray/contrib/inventory_builder/requirements.txt

@@ -0,0 +1,3 @@
+configparser>=3.3.0
+ruamel.yaml>=0.15.88
+ipaddress

kubespray/contrib/inventory_builder/setup.cfg

@@ -0,0 +1,3 @@
+[metadata]
+name = kubespray-inventory-builder
+version = 0.1

kubespray/contrib/inventory_builder/setup.py

@@ -0,0 +1,29 @@
+# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
+import setuptools
+
+# In python < 2.7.4, a lazy loading of package `pbr` will break
+# setuptools if some other modules registered functions in `atexit`.
+# solution from: http://bugs.python.org/issue15881#msg170215
+try:
+    import multiprocessing  # noqa
+except ImportError:
+    pass
+
+setuptools.setup(
+    setup_requires=[],
+    pbr=False)

kubespray/contrib/inventory_builder/test-requirements.txt

@@ -0,0 +1,3 @@
+hacking>=0.10.2
+pytest>=2.8.0
+mock>=1.3.0

kubespray/contrib/inventory_builder/tests/test_inventory.py

@@ -0,0 +1,595 @@
+# Copyright 2016 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import inventory
+from io import StringIO
+import unittest
+from unittest import mock
+
+from collections import OrderedDict
+import sys
+
+path = "./contrib/inventory_builder/"
+if path not in sys.path:
+    sys.path.append(path)
+
+import inventory  # noqa
+
+
+class TestInventoryPrintHostnames(unittest.TestCase):
+
+    @mock.patch('ruamel.yaml.YAML.load')
+    def test_print_hostnames(self, load_mock):
+        mock_io = mock.mock_open(read_data='')
+        load_mock.return_value = OrderedDict({'all': {'hosts': {
+            'node1': {'ansible_host': '10.90.0.2',
+                      'ip': '10.90.0.2',
+                      'access_ip': '10.90.0.2'},
+            'node2': {'ansible_host': '10.90.0.3',
+                      'ip': '10.90.0.3',
+                      'access_ip': '10.90.0.3'}}}})
+        with mock.patch('builtins.open', mock_io):
+            with self.assertRaises(SystemExit) as cm:
+                with mock.patch('sys.stdout', new_callable=StringIO) as stdout:
+                    inventory.KubesprayInventory(
+                        changed_hosts=["print_hostnames"],
+                        config_file="file")
+            self.assertEqual("node1 node2\n", stdout.getvalue())
+            self.assertEqual(cm.exception.code, 0)
+
+
+class TestInventory(unittest.TestCase):
+    @mock.patch('inventory.sys')
+    def setUp(self, sys_mock):
+        sys_mock.exit = mock.Mock()
+        super(TestInventory, self).setUp()
+        self.data = ['10.90.3.2', '10.90.3.3', '10.90.3.4']
+        self.inv = inventory.KubesprayInventory()
+
+    def test_get_ip_from_opts(self):
+        optstring = {'ansible_host': '10.90.3.2',
+                     'ip': '10.90.3.2',
+                     'access_ip': '10.90.3.2'}
+        expected = "10.90.3.2"
+        result = self.inv.get_ip_from_opts(optstring)
+        self.assertEqual(expected, result)
+
+    def test_get_ip_from_opts_invalid(self):
+        optstring = "notanaddr=value something random!chars:D"
+        self.assertRaisesRegex(ValueError, "IP parameter not found",
+                               self.inv.get_ip_from_opts, optstring)
+
+    def test_ensure_required_groups(self):
+        groups = ['group1', 'group2']
+        self.inv.ensure_required_groups(groups)
+        for group in groups:
+            self.assertIn(group, self.inv.yaml_config['all']['children'])
+
+    def test_get_host_id(self):
+        hostnames = ['node99', 'no99de01', '01node01', 'node1.domain',
+                     'node3.xyz123.aaa']
+        expected = [99, 1, 1, 1, 3]
+        for hostname, expected in zip(hostnames, expected):
+            result = self.inv.get_host_id(hostname)
+            self.assertEqual(expected, result)
+
+    def test_get_host_id_invalid(self):
+        bad_hostnames = ['node', 'no99de', '01node', 'node.111111']
+        for hostname in bad_hostnames:
+            self.assertRaisesRegex(ValueError, "Host name must end in an",
+                                   self.inv.get_host_id, hostname)
+
+    def test_build_hostnames_add_duplicate(self):
+        changed_hosts = ['10.90.0.2']
+        expected = OrderedDict([('node3',
+                                 {'ansible_host': '10.90.0.2',
+                                  'ip': '10.90.0.2',
+                                  'access_ip': '10.90.0.2'})])
+        self.inv.yaml_config['all']['hosts'] = expected
+        result = self.inv.build_hostnames(changed_hosts, True)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_add_two(self):
+        changed_hosts = ['10.90.0.2', '10.90.0.3']
+        expected = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        self.inv.yaml_config['all']['hosts'] = OrderedDict()
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_add_three(self):
+        changed_hosts = ['10.90.0.2', '10.90.0.3', '10.90.0.4']
+        expected = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'}),
+            ('node3', {'ansible_host': '10.90.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '10.90.0.4'})])
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_add_one(self):
+        changed_hosts = ['10.90.0.2']
+        expected = OrderedDict([('node1',
+                                 {'ansible_host': '10.90.0.2',
+                                  'ip': '10.90.0.2',
+                                  'access_ip': '10.90.0.2'})])
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_delete_first(self):
+        changed_hosts = ['-10.90.0.2']
+        existing_hosts = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        self.inv.yaml_config['all']['hosts'] = existing_hosts
+        expected = OrderedDict([
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        result = self.inv.build_hostnames(changed_hosts, True)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_delete_by_hostname(self):
+        changed_hosts = ['-node1']
+        existing_hosts = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        self.inv.yaml_config['all']['hosts'] = existing_hosts
+        expected = OrderedDict([
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        result = self.inv.build_hostnames(changed_hosts, True)
+        self.assertEqual(expected, result)
+
+    def test_exists_hostname_positive(self):
+        hostname = 'node1'
+        expected = True
+        existing_hosts = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        result = self.inv.exists_hostname(existing_hosts, hostname)
+        self.assertEqual(expected, result)
+
+    def test_exists_hostname_negative(self):
+        hostname = 'node99'
+        expected = False
+        existing_hosts = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        result = self.inv.exists_hostname(existing_hosts, hostname)
+        self.assertEqual(expected, result)
+
+    def test_exists_ip_positive(self):
+        ip = '10.90.0.2'
+        expected = True
+        existing_hosts = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        result = self.inv.exists_ip(existing_hosts, ip)
+        self.assertEqual(expected, result)
+
+    def test_exists_ip_negative(self):
+        ip = '10.90.0.200'
+        expected = False
+        existing_hosts = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        result = self.inv.exists_ip(existing_hosts, ip)
+        self.assertEqual(expected, result)
+
+    def test_delete_host_by_ip_positive(self):
+        ip = '10.90.0.2'
+        expected = OrderedDict([
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        existing_hosts = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        self.inv.delete_host_by_ip(existing_hosts, ip)
+        self.assertEqual(expected, existing_hosts)
+
+    def test_delete_host_by_ip_negative(self):
+        ip = '10.90.0.200'
+        existing_hosts = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        self.assertRaisesRegex(ValueError, "Unable to find host",
+                               self.inv.delete_host_by_ip, existing_hosts, ip)
+
+    def test_purge_invalid_hosts(self):
+        proper_hostnames = ['node1', 'node2']
+        bad_host = 'doesnotbelong2'
+        existing_hosts = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'}),
+            ('doesnotbelong2', {'whateveropts=ilike'})])
+        self.inv.yaml_config['all']['hosts'] = existing_hosts
+        self.inv.purge_invalid_hosts(proper_hostnames)
+        self.assertNotIn(
+            bad_host, self.inv.yaml_config['all']['hosts'].keys())
+
+    def test_add_host_to_group(self):
+        group = 'etcd'
+        host = 'node1'
+        opts = {'ip': '10.90.0.2'}
+
+        self.inv.add_host_to_group(group, host, opts)
+        self.assertEqual(
+            self.inv.yaml_config['all']['children'][group]['hosts'].get(host),
+            None)
+
+    def test_set_kube_control_plane(self):
+        group = 'kube_control_plane'
+        host = 'node1'
+
+        self.inv.set_kube_control_plane([host])
+        self.assertIn(
+            host, self.inv.yaml_config['all']['children'][group]['hosts'])
+
+    def test_set_all(self):
+        hosts = OrderedDict([
+            ('node1', 'opt1'),
+            ('node2', 'opt2')])
+
+        self.inv.set_all(hosts)
+        for host, opt in hosts.items():
+            self.assertEqual(
+                self.inv.yaml_config['all']['hosts'].get(host), opt)
+
+    def test_set_k8s_cluster(self):
+        group = 'k8s_cluster'
+        expected_hosts = ['kube_node', 'kube_control_plane']
+
+        self.inv.set_k8s_cluster()
+        for host in expected_hosts:
+            self.assertIn(
+                host,
+                self.inv.yaml_config['all']['children'][group]['children'])
+
+    def test_set_kube_node(self):
+        group = 'kube_node'
+        host = 'node1'
+
+        self.inv.set_kube_node([host])
+        self.assertIn(
+            host, self.inv.yaml_config['all']['children'][group]['hosts'])
+
+    def test_set_etcd(self):
+        group = 'etcd'
+        host = 'node1'
+
+        self.inv.set_etcd([host])
+        self.assertIn(
+            host, self.inv.yaml_config['all']['children'][group]['hosts'])
+
+    def test_scale_scenario_one(self):
+        num_nodes = 50
+        hosts = OrderedDict()
+
+        for hostid in range(1, num_nodes+1):
+            hosts["node" + str(hostid)] = ""
+
+        self.inv.set_all(hosts)
+        self.inv.set_etcd(list(hosts.keys())[0:3])
+        self.inv.set_kube_control_plane(list(hosts.keys())[0:2])
+        self.inv.set_kube_node(hosts.keys())
+        for h in range(3):
+            self.assertFalse(
+                list(hosts.keys())[h] in
+                self.inv.yaml_config['all']['children']['kube_node']['hosts'])
+
+    def test_scale_scenario_two(self):
+        num_nodes = 500
+        hosts = OrderedDict()
+
+        for hostid in range(1, num_nodes+1):
+            hosts["node" + str(hostid)] = ""
+
+        self.inv.set_all(hosts)
+        self.inv.set_etcd(list(hosts.keys())[0:3])
+        self.inv.set_kube_control_plane(list(hosts.keys())[3:5])
+        self.inv.set_kube_node(hosts.keys())
+        for h in range(5):
+            self.assertFalse(
+                list(hosts.keys())[h] in
+                self.inv.yaml_config['all']['children']['kube_node']['hosts'])
+
+    def test_range2ips_range(self):
+        changed_hosts = ['10.90.0.2', '10.90.0.4-10.90.0.6', '10.90.0.8']
+        expected = ['10.90.0.2',
+                    '10.90.0.4',
+                    '10.90.0.5',
+                    '10.90.0.6',
+                    '10.90.0.8']
+        result = self.inv.range2ips(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_range2ips_incorrect_range(self):
+        host_range = ['10.90.0.4-a.9b.c.e']
+        self.assertRaisesRegex(Exception, "Range of ip_addresses isn't valid",
+                               self.inv.range2ips, host_range)
+
+    def test_build_hostnames_create_with_one_different_ips(self):
+        changed_hosts = ['10.90.0.2,192.168.0.2']
+        expected = OrderedDict([('node1',
+                                 {'ansible_host': '192.168.0.2',
+                                  'ip': '10.90.0.2',
+                                  'access_ip': '192.168.0.2'})])
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_create_with_two_different_ips(self):
+        changed_hosts = ['10.90.0.2,192.168.0.2', '10.90.0.3,192.168.0.3']
+        expected = OrderedDict([
+            ('node1', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node2', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'})])
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_create_with_three_different_ips(self):
+        changed_hosts = ['10.90.0.2,192.168.0.2',
+                         '10.90.0.3,192.168.0.3',
+                         '10.90.0.4,192.168.0.4']
+        expected = OrderedDict([
+            ('node1', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node2', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node3', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'})])
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_overwrite_one_with_different_ips(self):
+        changed_hosts = ['10.90.0.2,192.168.0.2']
+        expected = OrderedDict([('node1',
+                                 {'ansible_host': '192.168.0.2',
+                                  'ip': '10.90.0.2',
+                                  'access_ip': '192.168.0.2'})])
+        existing = OrderedDict([('node5',
+                                 {'ansible_host': '192.168.0.5',
+                                  'ip': '10.90.0.5',
+                                  'access_ip': '192.168.0.5'})])
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_overwrite_three_with_different_ips(self):
+        changed_hosts = ['10.90.0.2,192.168.0.2']
+        expected = OrderedDict([('node1',
+                                 {'ansible_host': '192.168.0.2',
+                                  'ip': '10.90.0.2',
+                                  'access_ip': '192.168.0.2'})])
+        existing = OrderedDict([
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'}),
+            ('node5', {'ansible_host': '192.168.0.5',
+                       'ip': '10.90.0.5',
+                       'access_ip': '192.168.0.5'})])
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_different_ips_add_duplicate(self):
+        changed_hosts = ['10.90.0.2,192.168.0.2']
+        expected = OrderedDict([('node3',
+                                 {'ansible_host': '192.168.0.2',
+                                  'ip': '10.90.0.2',
+                                  'access_ip': '192.168.0.2'})])
+        existing = expected
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts, True)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_add_two_different_ips_into_one_existing(self):
+        changed_hosts = ['10.90.0.3,192.168.0.3', '10.90.0.4,192.168.0.4']
+        expected = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'})])
+
+        existing = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'})])
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts, True)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_add_two_different_ips_into_two_existing(self):
+        changed_hosts = ['10.90.0.4,192.168.0.4', '10.90.0.5,192.168.0.5']
+        expected = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'}),
+            ('node5', {'ansible_host': '192.168.0.5',
+                       'ip': '10.90.0.5',
+                       'access_ip': '192.168.0.5'})])
+
+        existing = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'})])
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts, True)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_add_two_different_ips_into_three_existing(self):
+        changed_hosts = ['10.90.0.5,192.168.0.5', '10.90.0.6,192.168.0.6']
+        expected = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'}),
+            ('node5', {'ansible_host': '192.168.0.5',
+                       'ip': '10.90.0.5',
+                       'access_ip': '192.168.0.5'}),
+            ('node6', {'ansible_host': '192.168.0.6',
+                       'ip': '10.90.0.6',
+                       'access_ip': '192.168.0.6'})])
+
+        existing = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'})])
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts, True)
+        self.assertEqual(expected, result)
+
+    # Add two IP addresses into a config that has
+    # three already defined IP addresses. One of the IP addresses
+    # is a duplicate.
+    def test_build_hostnames_add_two_duplicate_one_overlap(self):
+        changed_hosts = ['10.90.0.4,192.168.0.4', '10.90.0.5,192.168.0.5']
+        expected = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'}),
+            ('node5', {'ansible_host': '192.168.0.5',
+                       'ip': '10.90.0.5',
+                       'access_ip': '192.168.0.5'})])
+
+        existing = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'})])
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts, True)
+        self.assertEqual(expected, result)
+
+    # Add two duplicate IP addresses into a config that has
+    # three already defined IP addresses
+    def test_build_hostnames_add_two_duplicate_two_overlap(self):
+        changed_hosts = ['10.90.0.3,192.168.0.3', '10.90.0.4,192.168.0.4']
+        expected = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'})])
+
+        existing = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'})])
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts, True)
+        self.assertEqual(expected, result)

kubespray/contrib/inventory_builder/tox.ini

@@ -0,0 +1,28 @@
+[tox]
+minversion = 1.6
+skipsdist = True
+envlist = pep8, py33
+
+[testenv]
+whitelist_externals = py.test
+usedevelop = True
+deps =
+    -r{toxinidir}/requirements.txt
+    -r{toxinidir}/test-requirements.txt
+setenv = VIRTUAL_ENV={envdir}
+passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
+commands = pytest -vv #{posargs:./tests}
+
+[testenv:pep8]
+usedevelop = False
+whitelist_externals = bash
+commands =
+    bash -c "find {toxinidir}/* -type f -name '*.py' -print0 | xargs -0 flake8"
+
+[testenv:venv]
+commands = {posargs}
+
+[flake8]
+show-source = true
+builtins = _
+exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg

kubespray/contrib/kvm-setup/README.md

@@ -0,0 +1,11 @@
+# Kubespray on KVM Virtual Machines hypervisor preparation
+
+A simple playbook to ensure your system has the right settings to enable Kubespray
+deployment on VMs.
+
+This playbook does not create Virtual Machines, nor does it run Kubespray itself.
+
+## User creation
+
+If you want to create a user for running Kubespray deployment, you should specify
+both `k8s_deployment_user` and `k8s_deployment_user_pkey_path`.

kubespray/contrib/kvm-setup/group_vars/all

@@ -0,0 +1,3 @@
+#k8s_deployment_user: kubespray
+#k8s_deployment_user_pkey_path: /tmp/ssh_rsa
+

kubespray/contrib/kvm-setup/kvm-setup.yml

@@ -0,0 +1,8 @@
+---
+- hosts: localhost
+  gather_facts: False
+  become: yes
+  vars:
+    - bootstrap_os: none
+  roles:
+    - kvm-setup

kubespray/contrib/kvm-setup/roles/kvm-setup/tasks/main.yml

@@ -0,0 +1,30 @@
+---
+
+- name: Install required packages
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - bind-utils
+    - ntp
+  when: ansible_os_family == "RedHat"
+
+- name: Install required packages
+  apt:
+    upgrade: yes
+    update_cache: yes
+    cache_valid_time: 3600
+    name: "{{ item }}"
+    state: present
+    install_recommends: no
+  with_items:
+    - dnsutils
+    - ntp
+  when: ansible_os_family == "Debian"
+
+# Create deployment user if required
+- include: user.yml
+  when: k8s_deployment_user is defined
+
+# Set proper sysctl values
+- include: sysctl.yml

kubespray/contrib/kvm-setup/roles/kvm-setup/tasks/sysctl.yml

@@ -0,0 +1,46 @@
+---
+- name: Load br_netfilter module
+  modprobe:
+    name: br_netfilter
+    state: present
+  register: br_netfilter
+
+- name: Add br_netfilter into /etc/modules
+  lineinfile:
+    dest: /etc/modules
+    state: present
+    line: 'br_netfilter'
+  when: br_netfilter is defined and ansible_os_family == 'Debian'
+
+- name: Add br_netfilter into /etc/modules-load.d/kubespray.conf
+  copy:
+    dest: /etc/modules-load.d/kubespray.conf
+    content: |-
+      ### This file is managed by Ansible
+      br-netfilter
+    owner: root
+    group: root
+    mode: 0644
+  when: br_netfilter is defined
+
+
+- name: Enable net.ipv4.ip_forward in sysctl
+  sysctl:
+    name: net.ipv4.ip_forward
+    value: 1
+    sysctl_file: "{{ sysctl_file_path }}"
+    state: present
+    reload: yes
+
+- name: Set bridge-nf-call-{arptables,iptables} to 0
+  sysctl:
+    name: "{{ item }}"
+    state: present
+    value: 0
+    sysctl_file: "{{ sysctl_file_path }}"
+    reload: yes
+  with_items:
+    - net.bridge.bridge-nf-call-arptables
+    - net.bridge.bridge-nf-call-ip6tables
+    - net.bridge.bridge-nf-call-iptables
+  when: br_netfilter is defined

kubespray/contrib/kvm-setup/roles/kvm-setup/tasks/user.yml

@@ -0,0 +1,47 @@
+---
+- name: Create user {{ k8s_deployment_user }}
+  user:
+    name: "{{ k8s_deployment_user }}"
+    groups: adm
+    shell: /bin/bash
+
+- name: Ensure that .ssh exists
+  file:
+    path: "/home/{{ k8s_deployment_user }}/.ssh"
+    state: directory
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+    mode: 0700
+
+- name: Configure sudo for deployment user
+  copy:
+    content: |
+      %{{ k8s_deployment_user }} ALL=(ALL) NOPASSWD: ALL
+    dest: "/etc/sudoers.d/55-k8s-deployment"
+    owner: root
+    group: root
+    mode: 0644
+
+- name: Write private SSH key
+  copy:
+    src: "{{ k8s_deployment_user_pkey_path }}"
+    dest: "/home/{{ k8s_deployment_user }}/.ssh/id_rsa"
+    mode: 0400
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+  when: k8s_deployment_user_pkey_path is defined
+
+- name: Write public SSH key
+  shell: "ssh-keygen -y -f /home/{{ k8s_deployment_user }}/.ssh/id_rsa \
+            > /home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+  args:
+    creates: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+  when: k8s_deployment_user_pkey_path is defined
+
+- name: Fix ssh-pub-key permissions
+  file:
+    path: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+    mode: 0600
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+  when: k8s_deployment_user_pkey_path is defined

kubespray/contrib/misc/clusteradmin-rbac.yml

@@ -0,0 +1,15 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubernetes-dashboard
+  labels:
+    k8s-app: kubernetes-dashboard
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: kubernetes-dashboard
+  namespace: kube-system

kubespray/contrib/mitogen/mitogen.yml

@@ -0,0 +1,49 @@
+---
+- name: Check ansible version
+  import_playbook: ansible_version.yml
+
+- hosts: localhost
+  strategy: linear
+  vars:
+    mitogen_version: 0.3.2
+    mitogen_url: https://github.com/mitogen-hq/mitogen/archive/refs/tags/v{{ mitogen_version }}.tar.gz
+    ansible_connection: local
+  tasks:
+    - name: Create mitogen plugin dir
+      file:
+        path: "{{ item }}"
+        state: directory
+        mode: 0755
+      become: false
+      loop:
+        - "{{ playbook_dir }}/plugins/mitogen"
+        - "{{ playbook_dir }}/dist"
+
+    - name: download mitogen release
+      get_url:
+        url: "{{ mitogen_url }}"
+        dest: "{{ playbook_dir }}/dist/mitogen_{{ mitogen_version }}.tar.gz"
+        validate_certs: true
+
+    - name: extract archive
+      unarchive:
+        src: "{{ playbook_dir }}/dist/mitogen_{{ mitogen_version }}.tar.gz"
+        dest: "{{ playbook_dir }}/dist/"
+
+    - name: copy plugin
+      synchronize:
+        src: "{{ playbook_dir }}/dist/mitogen-{{ mitogen_version }}/"
+        dest: "{{ playbook_dir }}/plugins/mitogen"
+
+    - name: add strategy to ansible.cfg
+      ini_file:
+        path: ansible.cfg
+        mode: 0644
+        section: "{{ item.section | d('defaults') }}"
+        option: "{{ item.option }}"
+        value: "{{ item.value }}"
+      with_items:
+        - option: strategy
+          value: mitogen_linear
+        - option: strategy_plugins
+          value: plugins/mitogen/ansible_mitogen/plugins/strategy

kubespray/contrib/network-storage/glusterfs/README.md

@@ -0,0 +1,92 @@
+# Deploying a Kubespray Kubernetes Cluster with GlusterFS
+
+You can either deploy using Ansible on its own by supplying your own inventory file or by using Terraform to create the VMs and then providing a dynamic inventory to Ansible. The following two sections are self-contained, you don't need to go through one to use the other. So, if you want to provision with Terraform, you can skip the **Using an Ansible inventory** section, and if you want to provision with a pre-built ansible inventory, you can neglect the **Using Terraform and Ansible**  section.
+
+## Using an Ansible inventory
+
+In the same directory of this ReadMe file you should find a file named `inventory.example` which contains an example setup. Please note that, additionally to the Kubernetes nodes/masters, we define a set of machines for GlusterFS and we add them to the group `[gfs-cluster]`, which in turn is added to the larger `[network-storage]` group as a child group.
+
+Change that file to reflect your local setup (adding more machines or removing them and setting the adequate ip numbers), and save it to `inventory/sample/k8s_gfs_inventory`. Make sure that the settings on `inventory/sample/group_vars/all.yml` make sense with your deployment. Then execute change to the kubespray root folder, and execute (supposing that the machines are all using ubuntu):
+
+```shell
+ansible-playbook -b --become-user=root -i inventory/sample/k8s_gfs_inventory --user=ubuntu ./cluster.yml
+```
+
+This will provision your Kubernetes cluster. Then, to provision and configure the GlusterFS cluster, from the same directory execute:
+
+```shell
+ansible-playbook -b --become-user=root -i inventory/sample/k8s_gfs_inventory --user=ubuntu ./contrib/network-storage/glusterfs/glusterfs.yml
+```
+
+If your machines are not using Ubuntu, you need to change the `--user=ubuntu` to the correct user. Alternatively, if your Kubernetes machines are using one OS and your GlusterFS a different one, you can instead specify the `ansible_ssh_user=<correct-user>` variable in the inventory file that you just created, for each machine/VM:
+
+```shell
+k8s-master-1 ansible_ssh_host=192.168.0.147 ip=192.168.0.147 ansible_ssh_user=core
+k8s-master-node-1 ansible_ssh_host=192.168.0.148 ip=192.168.0.148 ansible_ssh_user=core
+k8s-master-node-2 ansible_ssh_host=192.168.0.146 ip=192.168.0.146 ansible_ssh_user=core
+```
+
+## Using Terraform and Ansible
+
+First step is to fill in a `my-kubespray-gluster-cluster.tfvars` file with the specification desired for your cluster. An example with all required variables would look like:
+
+```ini
+cluster_name = "cluster1"
+number_of_k8s_masters = "1"
+number_of_k8s_masters_no_floating_ip = "2"
+number_of_k8s_nodes_no_floating_ip = "0"
+number_of_k8s_nodes = "0"
+public_key_path = "~/.ssh/my-desired-key.pub"
+image = "Ubuntu 16.04"
+ssh_user = "ubuntu"
+flavor_k8s_node = "node-flavor-id-in-your-openstack"
+flavor_k8s_master = "master-flavor-id-in-your-openstack"
+network_name = "k8s-network"
+floatingip_pool = "net_external"
+
+# GlusterFS variables
+flavor_gfs_node = "gluster-flavor-id-in-your-openstack"
+image_gfs = "Ubuntu 16.04"
+number_of_gfs_nodes_no_floating_ip = "3"
+gfs_volume_size_in_gb = "50"
+ssh_user_gfs = "ubuntu"
+```
+
+As explained in the general terraform/openstack guide, you need to source your OpenStack credentials file, add your ssh-key to the ssh-agent and setup environment variables for terraform:
+
+```shell
+$ source ~/.stackrc
+$ eval $(ssh-agent -s)
+$ ssh-add ~/.ssh/my-desired-key
+$ echo Setting up Terraform creds && \
+  export TF_VAR_username=${OS_USERNAME} && \
+  export TF_VAR_password=${OS_PASSWORD} && \
+  export TF_VAR_tenant=${OS_TENANT_NAME} && \
+  export TF_VAR_auth_url=${OS_AUTH_URL}
+```
+
+Then, standing on the kubespray directory (root base of the Git checkout), issue the following terraform command to create the VMs for the cluster:
+
+```shell
+terraform apply -state=contrib/terraform/openstack/terraform.tfstate -var-file=my-kubespray-gluster-cluster.tfvars contrib/terraform/openstack
+```
+
+This will create both your Kubernetes and Gluster VMs. Make sure that the ansible file `contrib/terraform/openstack/group_vars/all.yml` includes any ansible variable that you want to setup (like, for instance, the type of machine for bootstrapping).
+
+Then, provision your Kubernetes (kubespray) cluster with the following ansible call:
+
+```shell
+ansible-playbook -b --become-user=root -i contrib/terraform/openstack/hosts ./cluster.yml
+```
+
+Finally, provision the glusterfs nodes and add the Persistent Volume setup for GlusterFS in Kubernetes through the following ansible call:
+
+```shell
+ansible-playbook -b --become-user=root -i contrib/terraform/openstack/hosts ./contrib/network-storage/glusterfs/glusterfs.yml
+```
+
+If you need to destroy the cluster, you can run:
+
+```shell
+terraform destroy -state=contrib/terraform/openstack/terraform.tfstate -var-file=my-kubespray-gluster-cluster.tfvars contrib/terraform/openstack
+```

kubespray/contrib/network-storage/glusterfs/glusterfs.yml

@@ -0,0 +1,24 @@
+---
+- hosts: gfs-cluster
+  gather_facts: false
+  vars:
+    ansible_ssh_pipelining: false
+  roles:
+    - { role: bootstrap-os, tags: bootstrap-os}
+
+- hosts: all
+  gather_facts: true
+
+- hosts: gfs-cluster
+  vars:
+    ansible_ssh_pipelining: true
+  roles:
+    - { role: glusterfs/server }
+
+- hosts: k8s_cluster
+  roles:
+    - { role: glusterfs/client }
+
+- hosts: kube_control_plane[0]
+  roles:
+    - { role: kubernetes-pv }

kubespray/contrib/network-storage/glusterfs/group_vars/all/all.yml

@@ -0,0 +1,140 @@
+---
+## Directory where the binaries will be installed
+bin_dir: /usr/local/bin
+
+## The access_ip variable is used to define how other nodes should access
+## the node.  This is used in flannel to allow other flannel nodes to see
+## this node for example.  The access_ip is really useful AWS and Google
+## environments where the nodes are accessed remotely by the "public" ip,
+## but don't know about that address themselves.
+# access_ip: 1.1.1.1
+
+
+## External LB example config
+## apiserver_loadbalancer_domain_name: "elb.some.domain"
+# loadbalancer_apiserver:
+#   address: 1.2.3.4
+#   port: 1234
+
+## Internal loadbalancers for apiservers
+# loadbalancer_apiserver_localhost: true
+# valid options are "nginx" or "haproxy"
+# loadbalancer_apiserver_type: nginx  # valid values "nginx" or "haproxy"
+
+## If the cilium is going to be used in strict mode, we can use the
+## localhost connection and not use the external LB. If this parameter is
+## not specified, the first node to connect to kubeapi will be used.
+# use_localhost_as_kubeapi_loadbalancer: true
+
+## Local loadbalancer should use this port
+## And must be set port 6443
+loadbalancer_apiserver_port: 6443
+
+## If loadbalancer_apiserver_healthcheck_port variable defined, enables proxy liveness check for nginx.
+loadbalancer_apiserver_healthcheck_port: 8081
+
+### OTHER OPTIONAL VARIABLES
+
+## By default, Kubespray collects nameservers on the host. It then adds the previously collected nameservers in nameserverentries.
+## If true, Kubespray does not include host nameservers in nameserverentries in dns_late stage. However, It uses the nameserver to make sure cluster installed safely in dns_early stage.
+## Use this option with caution, you may need to define your dns servers. Otherwise, the outbound queries such as www.google.com may fail.
+# disable_host_nameservers: false
+
+## Upstream dns servers
+# upstream_dns_servers:
+#   - 8.8.8.8
+#   - 8.8.4.4
+
+## There are some changes specific to the cloud providers
+## for instance we need to encapsulate packets with some network plugins
+## If set the possible values are either 'gce', 'aws', 'azure', 'openstack', 'vsphere', 'oci', or 'external'
+## When openstack is used make sure to source in the openstack credentials
+## like you would do when using openstack-client before starting the playbook.
+# cloud_provider:
+
+## When cloud_provider is set to 'external', you can set the cloud controller to deploy
+## Supported cloud controllers are: 'openstack', 'vsphere' and 'hcloud'
+## When openstack or vsphere are used make sure to source in the required fields
+# external_cloud_provider:
+
+## Set these proxy values in order to update package manager and docker daemon to use proxies
+# http_proxy: ""
+# https_proxy: ""
+
+## Refer to roles/kubespray-defaults/defaults/main.yml before modifying no_proxy
+# no_proxy: ""
+
+## Some problems may occur when downloading files over https proxy due to ansible bug
+## https://github.com/ansible/ansible/issues/32750. Set this variable to False to disable
+## SSL validation of get_url module. Note that kubespray will still be performing checksum validation.
+# download_validate_certs: False
+
+## If you need exclude all cluster nodes from proxy and other resources, add other resources here.
+# additional_no_proxy: ""
+
+## If you need to disable proxying of os package repositories but are still behind an http_proxy set
+## skip_http_proxy_on_os_packages to true
+## This will cause kubespray not to set proxy environment in /etc/yum.conf for centos and in /etc/apt/apt.conf for debian/ubuntu
+## Special information for debian/ubuntu - you have to set the no_proxy variable, then apt package will install from your source of wish
+# skip_http_proxy_on_os_packages: false
+
+## Since workers are included in the no_proxy variable by default, docker engine will be restarted on all nodes (all
+## pods will restart) when adding or removing workers.  To override this behaviour by only including master nodes in the
+## no_proxy variable, set below to true:
+no_proxy_exclude_workers: false
+
+## Certificate Management
+## This setting determines whether certs are generated via scripts.
+## Chose 'none' if you provide your own certificates.
+## Option is  "script", "none"
+# cert_management: script
+
+## Set to true to allow pre-checks to fail and continue deployment
+# ignore_assert_errors: false
+
+## The read-only port for the Kubelet to serve on with no authentication/authorization. Uncomment to enable.
+# kube_read_only_port: 10255
+
+## Set true to download and cache container
+# download_container: true
+
+## Deploy container engine
+# Set false if you want to deploy container engine manually.
+# deploy_container_engine: true
+
+## Red Hat Enterprise Linux subscription registration
+## Add either RHEL subscription Username/Password or Organization ID/Activation Key combination
+## Update RHEL subscription purpose usage, role and SLA if necessary
+# rh_subscription_username: ""
+# rh_subscription_password: ""
+# rh_subscription_org_id: ""
+# rh_subscription_activation_key: ""
+# rh_subscription_usage: "Development"
+# rh_subscription_role: "Red Hat Enterprise Server"
+# rh_subscription_sla: "Self-Support"
+
+## Check if access_ip responds to ping. Set false if your firewall blocks ICMP.
+# ping_access_ip: true
+
+# sysctl_file_path to add sysctl conf to
+# sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf"
+
+## Variables for webhook token auth https://kubernetes.io/docs/reference/access-authn-authz/authentication/#webhook-token-authentication
+kube_webhook_token_auth: false
+kube_webhook_token_auth_url_skip_tls_verify: false
+# kube_webhook_token_auth_url: https://...
+## base64-encoded string of the webhook's CA certificate
+# kube_webhook_token_auth_ca_data: "LS0t..."
+
+## NTP Settings
+# Start the ntpd or chrony service and enable it at system boot.
+ntp_enabled: false
+ntp_manage_config: false
+ntp_servers:
+  - "0.pool.ntp.org iburst"
+  - "1.pool.ntp.org iburst"
+  - "2.pool.ntp.org iburst"
+  - "3.pool.ntp.org iburst"
+
+## Used to control no_log attribute
+unsafe_show_logs: false

kubespray/contrib/network-storage/glusterfs/group_vars/all/aws.yml

@@ -0,0 +1,9 @@
+## To use AWS EBS CSI Driver to provision volumes, uncomment the first value
+## and configure the parameters below
+# aws_ebs_csi_enabled: true
+# aws_ebs_csi_enable_volume_scheduling: true
+# aws_ebs_csi_enable_volume_snapshot: false
+# aws_ebs_csi_enable_volume_resizing: false
+# aws_ebs_csi_controller_replicas: 1
+# aws_ebs_csi_plugin_image_tag: latest
+# aws_ebs_csi_extra_volume_tags: "Owner=owner,Team=team,Environment=environment'

kubespray/contrib/network-storage/glusterfs/group_vars/all/azure.yml

@@ -0,0 +1,40 @@
+## When azure is used, you need to also set the following variables.
+## see docs/azure.md for details on how to get these values
+
+# azure_cloud:
+# azure_tenant_id:
+# azure_subscription_id:
+# azure_aad_client_id:
+# azure_aad_client_secret:
+# azure_resource_group:
+# azure_location:
+# azure_subnet_name:
+# azure_security_group_name:
+# azure_security_group_resource_group:
+# azure_vnet_name:
+# azure_vnet_resource_group:
+# azure_route_table_name:
+# azure_route_table_resource_group:
+# supported values are 'standard' or 'vmss'
+# azure_vmtype: standard
+
+## Azure Disk CSI credentials and parameters
+## see docs/azure-csi.md for details on how to get these values
+
+# azure_csi_tenant_id:
+# azure_csi_subscription_id:
+# azure_csi_aad_client_id:
+# azure_csi_aad_client_secret:
+# azure_csi_location:
+# azure_csi_resource_group:
+# azure_csi_vnet_name:
+# azure_csi_vnet_resource_group:
+# azure_csi_subnet_name:
+# azure_csi_security_group_name:
+# azure_csi_use_instance_metadata:
+# azure_csi_tags: "Owner=owner,Team=team,Environment=environment'
+
+## To enable Azure Disk CSI, uncomment below
+# azure_csi_enabled: true
+# azure_csi_controller_replicas: 1
+# azure_csi_plugin_image_tag: latest

kubespray/contrib/network-storage/glusterfs/group_vars/all/containerd.yml

@@ -0,0 +1,50 @@
+---
+# Please see roles/container-engine/containerd/defaults/main.yml for more configuration options
+
+# containerd_storage_dir: "/var/lib/containerd"
+# containerd_state_dir: "/run/containerd"
+# containerd_oom_score: 0
+
+# containerd_default_runtime: "runc"
+# containerd_snapshotter: "native"
+
+# containerd_runc_runtime:
+#   name: runc
+#   type: "io.containerd.runc.v2"
+#   engine: ""
+#   root: ""
+
+# containerd_additional_runtimes:
+# Example for Kata Containers as additional runtime:
+#   - name: kata
+#     type: "io.containerd.kata.v2"
+#     engine: ""
+#     root: ""
+
+# containerd_grpc_max_recv_message_size: 16777216
+# containerd_grpc_max_send_message_size: 16777216
+
+# containerd_debug_level: "info"
+
+# containerd_metrics_address: ""
+
+# containerd_metrics_grpc_histogram: false
+
+## An obvious use case is allowing insecure-registry access to self hosted registries.
+## Can be ipaddress and domain_name.
+## example define mirror.registry.io or 172.19.16.11:5000
+## set "name": "url". insecure url must be started http://
+## Port number is also needed if the default HTTPS port is not used.
+# containerd_insecure_registries:
+#   "localhost": "http://127.0.0.1"
+#   "172.19.16.11:5000": "http://172.19.16.11:5000"
+
+# containerd_registries:
+#   "docker.io": "https://registry-1.docker.io"
+
+# containerd_max_container_log_line_size: -1
+
+# containerd_registry_auth:
+#   - registry: 10.0.0.2:5000
+#     username: user
+#     password: pass

kubespray/contrib/network-storage/glusterfs/group_vars/all/coreos.yml

@@ -0,0 +1,2 @@
+## Does coreos need auto upgrade, default is true
+# coreos_auto_upgrade: true

kubespray/contrib/network-storage/glusterfs/group_vars/all/cri-o.yml

@@ -0,0 +1,6 @@
+# crio_insecure_registries:
+#   - 10.0.0.2:5000
+# crio_registry_auth:
+#   - registry: 10.0.0.2:5000
+#     username: user
+#     password: pass

kubespray/contrib/network-storage/glusterfs/group_vars/all/docker.yml

@@ -0,0 +1,59 @@
+---
+## Uncomment this if you want to force overlay/overlay2 as docker storage driver
+## Please note that overlay2 is only supported on newer kernels
+# docker_storage_options: -s overlay2
+
+## Enable docker_container_storage_setup, it will configure devicemapper driver on Centos7 or RedHat7.
+docker_container_storage_setup: false
+
+## It must be define a disk path for docker_container_storage_setup_devs.
+## Otherwise docker-storage-setup will be executed incorrectly.
+# docker_container_storage_setup_devs: /dev/vdb
+
+## Uncomment this if you want to change the Docker Cgroup driver (native.cgroupdriver)
+## Valid options are systemd or cgroupfs, default is systemd
+# docker_cgroup_driver: systemd
+
+## Only set this if you have more than 3 nameservers:
+## If true Kubespray will only use the first 3, otherwise it will fail
+docker_dns_servers_strict: false
+
+# Path used to store Docker data
+docker_daemon_graph: "/var/lib/docker"
+
+## Used to set docker daemon iptables options to true
+docker_iptables_enabled: "false"
+
+# Docker log options
+# Rotate container stderr/stdout logs at 50m and keep last 5
+docker_log_opts: "--log-opt max-size=50m --log-opt max-file=5"
+
+# define docker bin_dir
+docker_bin_dir: "/usr/bin"
+
+# keep docker packages after installation; speeds up repeated ansible provisioning runs when '1'
+# kubespray deletes the docker package on each run, so caching the package makes sense
+docker_rpm_keepcache: 1
+
+## An obvious use case is allowing insecure-registry access to self hosted registries.
+## Can be ipaddress and domain_name.
+## example define 172.19.16.11 or mirror.registry.io
+# docker_insecure_registries:
+#   - mirror.registry.io
+#   - 172.19.16.11
+
+## Add other registry,example China registry mirror.
+# docker_registry_mirrors:
+#   - https://registry.docker-cn.com
+#   - https://mirror.aliyuncs.com
+
+## If non-empty will override default system MountFlags value.
+## This option takes a mount propagation flag: shared, slave
+## or private, which control whether mounts in the file system
+## namespace set up for docker will receive or propagate mounts
+## and unmounts. Leave empty for system default
+# docker_mount_flags:
+
+## A string of extra options to pass to the docker daemon.
+## This string should be exactly as you wish it to appear.
+# docker_options: ""

kubespray/contrib/network-storage/glusterfs/group_vars/all/etcd.yml

@@ -0,0 +1,16 @@
+---
+## Directory where etcd data stored
+etcd_data_dir: /var/lib/etcd
+
+## Container runtime
+## docker for docker, crio for cri-o and containerd for containerd.
+## Additionally you can set this to kubeadm if you want to install etcd using kubeadm
+## Kubeadm etcd deployment is experimental and only available for new deployments
+## If this is not set, container manager will be inherited from the Kubespray defaults
+## and not from k8s_cluster/k8s-cluster.yml, which might not be what you want.
+## Also this makes possible to use different container manager for etcd nodes.
+# container_manager: containerd
+
+## Settings for etcd deployment type
+# Set this to docker if you are using container_manager: docker
+etcd_deployment_type: host

kubespray/contrib/network-storage/glusterfs/group_vars/all/gcp.yml

@@ -0,0 +1,10 @@
+## GCP compute Persistent Disk CSI Driver credentials and parameters
+## See docs/gcp-pd-csi.md for information about the implementation
+
+## Specify the path to the file containing the service account credentials
+# gcp_pd_csi_sa_cred_file: "/my/safe/credentials/directory/cloud-sa.json"
+
+## To enable GCP Persistent Disk CSI driver, uncomment below
+# gcp_pd_csi_enabled: true
+# gcp_pd_csi_controller_replicas: 1
+# gcp_pd_csi_driver_image_tag: "v0.7.0-gke.0"

kubespray/contrib/network-storage/glusterfs/group_vars/all/hcloud.yml

@@ -0,0 +1,14 @@
+## Values for the external Hcloud Cloud Controller
+# external_hcloud_cloud:
+#   hcloud_api_token: ""
+#   token_secret_name: hcloud
+#   with_networks: false # Use the hcloud controller-manager with networks support https://github.com/hetznercloud/hcloud-cloud-controller-manager#networks-support
+#   service_account_name: cloud-controller-manager
+#
+#   controller_image_tag: "latest"
+#   ## A dictionary of extra arguments to add to the openstack cloud controller manager daemonset
+#   ## Format:
+#   ##  external_hcloud_cloud.controller_extra_args:
+#   ##    arg1: "value1"
+#   ##    arg2: "value2"
+#   controller_extra_args: {}

kubespray/contrib/network-storage/glusterfs/group_vars/all/oci.yml

@@ -0,0 +1,28 @@
+## When Oracle Cloud Infrastructure is used, set these variables
+# oci_private_key:
+# oci_region_id:
+# oci_tenancy_id:
+# oci_user_id:
+# oci_user_fingerprint:
+# oci_compartment_id:
+# oci_vnc_id:
+# oci_subnet1_id:
+# oci_subnet2_id:
+## Override these default/optional behaviors if you wish
+# oci_security_list_management: All
+## If you would like the controller to manage specific lists per subnet. This is a mapping of subnet ocids to security list ocids. Below are examples.
+# oci_security_lists:
+#   ocid1.subnet.oc1.phx.aaaaaaaasa53hlkzk6nzksqfccegk2qnkxmphkblst3riclzs4rhwg7rg57q: ocid1.securitylist.oc1.iad.aaaaaaaaqti5jsfvyw6ejahh7r4okb2xbtuiuguswhs746mtahn72r7adt7q
+#   ocid1.subnet.oc1.phx.aaaaaaaahuxrgvs65iwdz7ekwgg3l5gyah7ww5klkwjcso74u3e4i64hvtvq: ocid1.securitylist.oc1.iad.aaaaaaaaqti5jsfvyw6ejahh7r4okb2xbtuiuguswhs746mtahn72r7adt7q
+## If oci_use_instance_principals is true, you do not need to set the region, tenancy, user, key, passphrase, or fingerprint
+# oci_use_instance_principals: false
+# oci_cloud_controller_version: 0.6.0
+## If you would like to control OCI query rate limits for the controller
+# oci_rate_limit:
+#   rate_limit_qps_read:
+#   rate_limit_qps_write:
+#   rate_limit_bucket_read:
+#   rate_limit_bucket_write:
+## Other optional variables
+# oci_cloud_controller_pull_source: (default iad.ocir.io/oracle/cloud-provider-oci)
+# oci_cloud_controller_pull_secret: (name of pull secret to use if you define your own mirror above)

kubespray/contrib/network-storage/glusterfs/group_vars/all/offline.yml

@@ -0,0 +1,103 @@
+---
+## Global Offline settings
+### Private Container Image Registry
+# registry_host: "myprivateregisry.com"
+# files_repo: "http://myprivatehttpd"
+### If using CentOS, RedHat, AlmaLinux or Fedora
+# yum_repo: "http://myinternalyumrepo"
+### If using Debian
+# debian_repo: "http://myinternaldebianrepo"
+### If using Ubuntu
+# ubuntu_repo: "http://myinternalubunturepo"
+
+## Container Registry overrides
+# kube_image_repo: "{{ registry_host }}"
+# gcr_image_repo: "{{ registry_host }}"
+# github_image_repo: "{{ registry_host }}"
+# docker_image_repo: "{{ registry_host }}"
+# quay_image_repo: "{{ registry_host }}"
+
+## Kubernetes components
+# kubeadm_download_url: "{{ files_repo }}/storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/{{ image_arch }}/kubeadm"
+# kubectl_download_url: "{{ files_repo }}/storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubectl"
+# kubelet_download_url: "{{ files_repo }}/storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubelet"
+
+## CNI Plugins
+# cni_download_url: "{{ files_repo }}/github.com/containernetworking/plugins/releases/download/{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz"
+
+## cri-tools
+# crictl_download_url: "{{ files_repo }}/github.com/kubernetes-sigs/cri-tools/releases/download/{{ crictl_version }}/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
+
+## [Optional] etcd: only if you **DON'T** use etcd_deployment=host
+# etcd_download_url: "{{ files_repo }}/github.com/etcd-io/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
+
+# [Optional] Calico: If using Calico network plugin
+# calicoctl_download_url: "{{ files_repo }}/github.com/projectcalico/calico/releases/download/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
+# calicoctl_alternate_download_url: "{{ files_repo }}/github.com/projectcalico/calicoctl/releases/download/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
+# [Optional] Calico with kdd: If using Calico network plugin with kdd datastore
+# calico_crds_download_url: "{{ files_repo }}/github.com/projectcalico/calico/archive/{{ calico_version }}.tar.gz"
+
+# [Optional] Cilium: If using Cilium network plugin
+# ciliumcli_download_url: "{{ files_repo }}/github.com/cilium/cilium-cli/releases/download/{{ cilium_cli_version }}/cilium-linux-{{ image_arch }}.tar.gz"
+
+# [Optional] Flannel: If using Falnnel network plugin
+# flannel_cni_download_url: "{{ files_repo }}/kubernetes/flannel/{{ flannel_cni_version }}/flannel-{{ image_arch }}"
+
+# [Optional] helm: only if you set helm_enabled: true
+# helm_download_url: "{{ files_repo }}/get.helm.sh/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz"
+
+# [Optional] crun: only if you set crun_enabled: true
+# crun_download_url: "{{ files_repo }}/github.com/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}"
+
+# [Optional] kata: only if you set kata_containers_enabled: true
+# kata_containers_download_url: "{{ files_repo }}/github.com/kata-containers/kata-containers/releases/download/{{ kata_containers_version }}/kata-static-{{ kata_containers_version }}-{{ ansible_architecture }}.tar.xz"
+
+# [Optional] cri-dockerd: only if you set container_manager: docker
+# cri_dockerd_download_url: "{{ files_repo }}/github.com/Mirantis/cri-dockerd/releases/download/v{{ cri_dockerd_version }}/cri-dockerd-{{ cri_dockerd_version }}.{{ image_arch }}.tgz"
+
+# [Optional] cri-o: only if you set container_manager: crio
+# crio_download_base: "download.opensuse.org/repositories/devel:kubic:libcontainers:stable"
+# crio_download_crio: "http://{{ crio_download_base }}:/cri-o:/"
+
+# [Optional] runc,containerd: only if you set container_runtime: containerd
+# runc_download_url: "{{ files_repo }}/github.com/opencontainers/runc/releases/download/{{ runc_version }}/runc.{{ image_arch }}"
+# containerd_download_url: "{{ files_repo }}/github.com/containerd/containerd/releases/download/v{{ containerd_version }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz"
+# nerdctl_download_url: "{{ files_repo }}/github.com/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
+
+# [Optional] runsc,containerd-shim-runsc: only if you set gvisor_enabled: true
+# gvisor_runsc_download_url: "{{ files_repo }}/storage.googleapis.com/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}/runsc"
+# gvisor_containerd_shim_runsc_download_url: "{{ files_repo }}/storage.googleapis.com/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}/containerd-shim-runsc-v1"
+
+## CentOS/Redhat/AlmaLinux
+### For EL7, base and extras repo must be available, for EL8, baseos and appstream
+### By default we enable those repo automatically
+# rhel_enable_repos: false
+### Docker / Containerd
+# docker_rh_repo_base_url: "{{ yum_repo }}/docker-ce/$releasever/$basearch"
+# docker_rh_repo_gpgkey: "{{ yum_repo }}/docker-ce/gpg"
+
+## Fedora
+### Docker
+# docker_fedora_repo_base_url: "{{ yum_repo }}/docker-ce/{{ ansible_distribution_major_version }}/{{ ansible_architecture }}"
+# docker_fedora_repo_gpgkey: "{{ yum_repo }}/docker-ce/gpg"
+### Containerd
+# containerd_fedora_repo_base_url: "{{ yum_repo }}/containerd"
+# containerd_fedora_repo_gpgkey: "{{ yum_repo }}/docker-ce/gpg"
+
+## Debian
+### Docker
+# docker_debian_repo_base_url: "{{ debian_repo }}/docker-ce"
+# docker_debian_repo_gpgkey: "{{ debian_repo }}/docker-ce/gpg"
+### Containerd
+# containerd_debian_repo_base_url: "{{ debian_repo }}/containerd"
+# containerd_debian_repo_gpgkey: "{{ debian_repo }}/containerd/gpg"
+# containerd_debian_repo_repokey: 'YOURREPOKEY'
+
+## Ubuntu
+### Docker
+# docker_ubuntu_repo_base_url: "{{ ubuntu_repo }}/docker-ce"
+# docker_ubuntu_repo_gpgkey: "{{ ubuntu_repo }}/docker-ce/gpg"
+### Containerd
+# containerd_ubuntu_repo_base_url: "{{ ubuntu_repo }}/containerd"
+# containerd_ubuntu_repo_gpgkey: "{{ ubuntu_repo }}/containerd/gpg"
+# containerd_ubuntu_repo_repokey: 'YOURREPOKEY'

kubespray/contrib/network-storage/glusterfs/group_vars/all/openstack.yml

@@ -0,0 +1,49 @@
+## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (Fixed in 1.9: https://github.com/kubernetes/kubernetes/issues/50461)
+# openstack_blockstorage_version: "v1/v2/auto (default)"
+# openstack_blockstorage_ignore_volume_az: yes
+## When OpenStack is used, if LBaaSv2 is available you can enable it with the following 2 variables.
+# openstack_lbaas_enabled: True
+# openstack_lbaas_subnet_id: "Neutron subnet ID (not network ID) to create LBaaS VIP"
+## To enable automatic floating ip provisioning, specify a subnet.
+# openstack_lbaas_floating_network_id: "Neutron network ID (not subnet ID) to get floating IP from, disabled by default"
+## Override default LBaaS behavior
+# openstack_lbaas_use_octavia: False
+# openstack_lbaas_method: "ROUND_ROBIN"
+# openstack_lbaas_provider: "haproxy"
+# openstack_lbaas_create_monitor: "yes"
+# openstack_lbaas_monitor_delay: "1m"
+# openstack_lbaas_monitor_timeout: "30s"
+# openstack_lbaas_monitor_max_retries: "3"
+
+## Values for the external OpenStack Cloud Controller
+# external_openstack_lbaas_network_id: "Neutron network ID to create LBaaS VIP"
+# external_openstack_lbaas_subnet_id: "Neutron subnet ID to create LBaaS VIP"
+# external_openstack_lbaas_floating_network_id: "Neutron network ID to get floating IP from"
+# external_openstack_lbaas_floating_subnet_id: "Neutron subnet ID to get floating IP from"
+# external_openstack_lbaas_method: "ROUND_ROBIN"
+# external_openstack_lbaas_provider: "octavia"
+# external_openstack_lbaas_create_monitor: false
+# external_openstack_lbaas_monitor_delay: "1m"
+# external_openstack_lbaas_monitor_timeout: "30s"
+# external_openstack_lbaas_monitor_max_retries: "3"
+# external_openstack_lbaas_manage_security_groups: false
+# external_openstack_lbaas_internal_lb: false
+# external_openstack_network_ipv6_disabled: false
+# external_openstack_network_internal_networks: []
+# external_openstack_network_public_networks: []
+# external_openstack_metadata_search_order: "configDrive,metadataService"
+
+## Application credentials to authenticate against Keystone API
+## Those settings will take precedence over username and password that might be set your environment
+## All of them are required
+# external_openstack_application_credential_name:
+# external_openstack_application_credential_id:
+# external_openstack_application_credential_secret:
+
+## The tag of the external OpenStack Cloud Controller image
+# external_openstack_cloud_controller_image_tag: "latest"
+
+## To use Cinder CSI plugin to provision volumes set this value to true
+## Make sure to source in the openstack credentials
+# cinder_csi_enabled: true
+# cinder_csi_controller_replicas: 1

kubespray/contrib/network-storage/glusterfs/group_vars/all/upcloud.yml

@@ -0,0 +1,24 @@
+## Repo for UpClouds csi-driver: https://github.com/UpCloudLtd/upcloud-csi
+## To use UpClouds CSI plugin to provision volumes set this value to true
+## Remember to set UPCLOUD_USERNAME and UPCLOUD_PASSWORD
+# upcloud_csi_enabled: true
+# upcloud_csi_controller_replicas: 1
+## Override used image tags
+# upcloud_csi_provisioner_image_tag: "v3.1.0"
+# upcloud_csi_attacher_image_tag: "v3.4.0"
+# upcloud_csi_resizer_image_tag: "v1.4.0"
+# upcloud_csi_plugin_image_tag: "v0.3.3"
+# upcloud_csi_node_image_tag: "v2.5.0"
+# upcloud_tolerations: []
+## Storage class options
+# storage_classes:
+#   - name: standard
+#     is_default: true
+#     expand_persistent_volumes: true
+#     parameters:
+#       tier: maxiops
+#   - name: hdd
+#     is_default: false
+#     expand_persistent_volumes: true
+#     parameters:
+#       tier: hdd

kubespray/contrib/network-storage/glusterfs/group_vars/all/vsphere.yml

@@ -0,0 +1,32 @@
+## Values for the external vSphere Cloud Provider
+# external_vsphere_vcenter_ip: "myvcenter.domain.com"
+# external_vsphere_vcenter_port: "443"
+# external_vsphere_insecure: "true"
+# external_vsphere_user: "administrator@vsphere.local" # Can also be set via the `VSPHERE_USER` environment variable
+# external_vsphere_password: "K8s_admin" # Can also be set via the `VSPHERE_PASSWORD` environment variable
+# external_vsphere_datacenter: "DATACENTER_name"
+# external_vsphere_kubernetes_cluster_id: "kubernetes-cluster-id"
+
+## Vsphere version where located VMs
+# external_vsphere_version: "6.7u3"
+
+## Tags for the external vSphere Cloud Provider images
+## gcr.io/cloud-provider-vsphere/cpi/release/manager
+# external_vsphere_cloud_controller_image_tag: "latest"
+## gcr.io/cloud-provider-vsphere/csi/release/syncer
+# vsphere_syncer_image_tag: "v2.5.1"
+## registry.k8s.io/sig-storage/csi-attacher
+# vsphere_csi_attacher_image_tag: "v3.4.0"
+## gcr.io/cloud-provider-vsphere/csi/release/driver
+# vsphere_csi_controller: "v2.5.1"
+## registry.k8s.io/sig-storage/livenessprobe
+# vsphere_csi_liveness_probe_image_tag: "v2.6.0"
+## registry.k8s.io/sig-storage/csi-provisioner
+# vsphere_csi_provisioner_image_tag: "v3.1.0"
+## registry.k8s.io/sig-storage/csi-resizer
+## makes sense only for vSphere version >=7.0
+# vsphere_csi_resizer_tag: "v1.3.0"
+
+## To use vSphere CSI plugin to provision volumes set this value to true
+# vsphere_csi_enabled: true
+# vsphere_csi_controller_replicas: 1

kubespray/contrib/network-storage/glusterfs/group_vars/etcd.yml

@@ -0,0 +1,26 @@
+---
+## Etcd auto compaction retention for mvcc key value store in hour
+# etcd_compaction_retention: 0
+
+## Set level of detail for etcd exported metrics, specify 'extensive' to include histogram metrics.
+# etcd_metrics: basic
+
+## Etcd is restricted by default to 512M on systems under 4GB RAM, 512MB is not enough for much more than testing.
+## Set this if your etcd nodes have less than 4GB but you want more RAM for etcd. Set to 0 for unrestricted RAM.
+## This value is only relevant when deploying etcd with `etcd_deployment_type: docker`
+# etcd_memory_limit: "512M"
+
+## Etcd has a default of 2G for its space quota. If you put a value in etcd_memory_limit which is less than
+## etcd_quota_backend_bytes, you may encounter out of memory terminations of the etcd cluster. Please check
+## etcd documentation for more information.
+# 8G is a suggested maximum size for normal environments and etcd warns at startup if the configured value exceeds it.
+# etcd_quota_backend_bytes: "2147483648"
+
+# Maximum client request size in bytes the server will accept.
+# etcd is designed to handle small key value pairs typical for metadata.
+# Larger requests will work, but may increase the latency of other requests
+# etcd_max_request_bytes: "1572864"
+
+### ETCD: disable peer client cert authentication.
+# This affects ETCD_PEER_CLIENT_CERT_AUTH variable
+# etcd_peer_client_auth: true

kubespray/contrib/network-storage/glusterfs/group_vars/k8s_cluster/addons.yml

@@ -0,0 +1,228 @@
+---
+# Kubernetes dashboard
+# RBAC required. see docs/getting-started.md for access details.
+# dashboard_enabled: false
+
+# Helm deployment
+helm_enabled: false
+
+# Registry deployment
+registry_enabled: false
+# registry_namespace: kube-system
+# registry_storage_class: ""
+# registry_disk_size: "10Gi"
+
+# Metrics Server deployment
+metrics_server_enabled: false
+# metrics_server_container_port: 4443
+# metrics_server_kubelet_insecure_tls: true
+# metrics_server_metric_resolution: 15s
+# metrics_server_kubelet_preferred_address_types: "InternalIP,ExternalIP,Hostname"
+# metrics_server_host_network: false
+# metrics_server_replicas: 1
+
+# Rancher Local Path Provisioner
+local_path_provisioner_enabled: false
+# local_path_provisioner_namespace: "local-path-storage"
+# local_path_provisioner_storage_class: "local-path"
+# local_path_provisioner_reclaim_policy: Delete
+# local_path_provisioner_claim_root: /opt/local-path-provisioner/
+# local_path_provisioner_debug: false
+# local_path_provisioner_image_repo: "rancher/local-path-provisioner"
+# local_path_provisioner_image_tag: "v0.0.22"
+# local_path_provisioner_helper_image_repo: "busybox"
+# local_path_provisioner_helper_image_tag: "latest"
+
+# Local volume provisioner deployment
+local_volume_provisioner_enabled: false
+# local_volume_provisioner_namespace: kube-system
+# local_volume_provisioner_nodelabels:
+#   - kubernetes.io/hostname
+#   - topology.kubernetes.io/region
+#   - topology.kubernetes.io/zone
+# local_volume_provisioner_storage_classes:
+#   local-storage:
+#     host_dir: /mnt/disks
+#     mount_dir: /mnt/disks
+#     volume_mode: Filesystem
+#     fs_type: ext4
+#   fast-disks:
+#     host_dir: /mnt/fast-disks
+#     mount_dir: /mnt/fast-disks
+#     block_cleaner_command:
+#       - "/scripts/shred.sh"
+#       - "2"
+#     volume_mode: Filesystem
+#     fs_type: ext4
+# local_volume_provisioner_tolerations:
+#   - effect: NoSchedule
+#     operator: Exists
+
+# CSI Volume Snapshot Controller deployment, set this to true if your CSI is able to manage snapshots
+# currently, setting cinder_csi_enabled=true would automatically enable the snapshot controller
+# Longhorn is an extenal CSI that would also require setting this to true but it is not included in kubespray
+# csi_snapshot_controller_enabled: false
+# csi snapshot namespace
+# snapshot_controller_namespace: kube-system
+
+# CephFS provisioner deployment
+cephfs_provisioner_enabled: false
+# cephfs_provisioner_namespace: "cephfs-provisioner"
+# cephfs_provisioner_cluster: ceph
+# cephfs_provisioner_monitors: "172.24.0.1:6789,172.24.0.2:6789,172.24.0.3:6789"
+# cephfs_provisioner_admin_id: admin
+# cephfs_provisioner_secret: secret
+# cephfs_provisioner_storage_class: cephfs
+# cephfs_provisioner_reclaim_policy: Delete
+# cephfs_provisioner_claim_root: /volumes
+# cephfs_provisioner_deterministic_names: true
+
+# RBD provisioner deployment
+rbd_provisioner_enabled: false
+# rbd_provisioner_namespace: rbd-provisioner
+# rbd_provisioner_replicas: 2
+# rbd_provisioner_monitors: "172.24.0.1:6789,172.24.0.2:6789,172.24.0.3:6789"
+# rbd_provisioner_pool: kube
+# rbd_provisioner_admin_id: admin
+# rbd_provisioner_secret_name: ceph-secret-admin
+# rbd_provisioner_secret: ceph-key-admin
+# rbd_provisioner_user_id: kube
+# rbd_provisioner_user_secret_name: ceph-secret-user
+# rbd_provisioner_user_secret: ceph-key-user
+# rbd_provisioner_user_secret_namespace: rbd-provisioner
+# rbd_provisioner_fs_type: ext4
+# rbd_provisioner_image_format: "2"
+# rbd_provisioner_image_features: layering
+# rbd_provisioner_storage_class: rbd
+# rbd_provisioner_reclaim_policy: Delete
+
+# Nginx ingress controller deployment
+ingress_nginx_enabled: false
+# ingress_nginx_host_network: false
+ingress_publish_status_address: ""
+# ingress_nginx_nodeselector:
+#   kubernetes.io/os: "linux"
+# ingress_nginx_tolerations:
+#   - key: "node-role.kubernetes.io/master"
+#     operator: "Equal"
+#     value: ""
+#     effect: "NoSchedule"
+#   - key: "node-role.kubernetes.io/control-plane"
+#     operator: "Equal"
+#     value: ""
+#     effect: "NoSchedule"
+# ingress_nginx_namespace: "ingress-nginx"
+# ingress_nginx_insecure_port: 80
+# ingress_nginx_secure_port: 443
+# ingress_nginx_configmap:
+#   map-hash-bucket-size: "128"
+#   ssl-protocols: "TLSv1.2 TLSv1.3"
+# ingress_nginx_configmap_tcp_services:
+#   9000: "default/example-go:8080"
+# ingress_nginx_configmap_udp_services:
+#   53: "kube-system/coredns:53"
+# ingress_nginx_extra_args:
+#   - --default-ssl-certificate=default/foo-tls
+# ingress_nginx_termination_grace_period_seconds: 300
+# ingress_nginx_class: nginx
+
+# ALB ingress controller deployment
+ingress_alb_enabled: false
+# alb_ingress_aws_region: "us-east-1"
+# alb_ingress_restrict_scheme: "false"
+# Enables logging on all outbound requests sent to the AWS API.
+# If logging is desired, set to true.
+# alb_ingress_aws_debug: "false"
+
+# Cert manager deployment
+cert_manager_enabled: false
+# cert_manager_namespace: "cert-manager"
+# cert_manager_tolerations:
+#   - key: node-role.kubernetes.io/master
+#     effect: NoSchedule
+#   - key: node-role.kubernetes.io/control-plane
+#     effect: NoSchedule
+# cert_manager_affinity:
+#  nodeAffinity:
+#    preferredDuringSchedulingIgnoredDuringExecution:
+#    - weight: 100
+#      preference:
+#        matchExpressions:
+#        - key: node-role.kubernetes.io/control-plane
+#          operator: In
+#          values:
+#          - ""
+# cert_manager_nodeselector:
+#   kubernetes.io/os: "linux"
+
+# cert_manager_trusted_internal_ca: |
+#   -----BEGIN CERTIFICATE-----
+#   [REPLACE with your CA certificate]
+#   -----END CERTIFICATE-----
+# cert_manager_leader_election_namespace: kube-system
+
+# MetalLB deployment
+metallb_enabled: false
+metallb_speaker_enabled: "{{ metallb_enabled }}"
+# metallb_ip_range:
+#   - "10.5.0.50-10.5.0.99"
+# metallb_pool_name: "loadbalanced"
+# metallb_auto_assign: true
+# metallb_avoid_buggy_ips: false
+# metallb_speaker_nodeselector:
+#   kubernetes.io/os: "linux"
+# metallb_controller_nodeselector:
+#   kubernetes.io/os: "linux"
+# metallb_speaker_tolerations:
+#   - key: "node-role.kubernetes.io/master"
+#     operator: "Equal"
+#     value: ""
+#     effect: "NoSchedule"
+#   - key: "node-role.kubernetes.io/control-plane"
+#     operator: "Equal"
+#     value: ""
+#     effect: "NoSchedule"
+# metallb_controller_tolerations:
+#   - key: "node-role.kubernetes.io/master"
+#     operator: "Equal"
+#     value: ""
+#     effect: "NoSchedule"
+#   - key: "node-role.kubernetes.io/control-plane"
+#     operator: "Equal"
+#     value: ""
+#     effect: "NoSchedule"
+# metallb_version: v0.12.1
+# metallb_protocol: "layer2"
+# metallb_port: "7472"
+# metallb_memberlist_port: "7946"
+# metallb_additional_address_pools:
+#   kube_service_pool:
+#     ip_range:
+#       - "10.5.1.50-10.5.1.99"
+#     protocol: "layer2"
+#     auto_assign: false
+#     avoid_buggy_ips: false
+# metallb_protocol: "bgp"
+# metallb_peers:
+#   - peer_address: 192.0.2.1
+#     peer_asn: 64512
+#     my_asn: 4200000000
+#   - peer_address: 192.0.2.2
+#     peer_asn: 64513
+#     my_asn: 4200000000
+
+argocd_enabled: false
+# argocd_version: v2.5.5
+# argocd_namespace: argocd
+# Default password:
+#   - https://argo-cd.readthedocs.io/en/stable/getting_started/#4-login-using-the-cli
+#   ---
+#   The initial password is autogenerated to be the pod name of the Argo CD API server. This can be retrieved with the command:
+#   kubectl get pods -n argocd -l app.kubernetes.io/name=argocd-server -o name | cut -d'/' -f 2
+#   ---
+# Use the following var to set admin password
+# argocd_admin_password: "password"
+
+# The plugin manager for kubectl
+krew_enabled: false
+krew_root_dir: "/usr/local/krew"

kubespray/contrib/network-storage/glusterfs/group_vars/k8s_cluster/k8s-cluster.yml

@@ -0,0 +1,350 @@
+---
+# Kubernetes configuration dirs and system namespace.
+# Those are where all the additional config stuff goes
+# the kubernetes normally puts in /srv/kubernetes.
+# This puts them in a sane location and namespace.
+# Editing those values will almost surely break something.
+kube_config_dir: /etc/kubernetes
+kube_script_dir: "{{ bin_dir }}/kubernetes-scripts"
+kube_manifest_dir: "{{ kube_config_dir }}/manifests"
+
+# This is where all the cert scripts and certs will be located
+kube_cert_dir: "{{ kube_config_dir }}/ssl"
+
+# This is where all of the bearer tokens will be stored
+kube_token_dir: "{{ kube_config_dir }}/tokens"
+
+kube_api_anonymous_auth: true
+
+## Change this to use another Kubernetes version, e.g. a current beta release
+kube_version: v1.25.5
+
+# Where the binaries will be downloaded.
+# Note: ensure that you've enough disk space (about 1G)
+local_release_dir: "/tmp/releases"
+# Random shifts for retrying failed ops like pushing/downloading
+retry_stagger: 5
+
+# This is the user that owns tha cluster installation.
+kube_owner: kube
+
+# This is the group that the cert creation scripts chgrp the
+# cert files to. Not really changeable...
+kube_cert_group: kube-cert
+
+# Cluster Loglevel configuration
+kube_log_level: 2
+
+# Directory where credentials will be stored
+credentials_dir: "{{ inventory_dir }}/credentials"
+
+## It is possible to activate / deactivate selected authentication methods (oidc, static token auth)
+# kube_oidc_auth: false
+# kube_token_auth: false
+
+
+## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/
+## To use OpenID you have to deploy additional an OpenID Provider (e.g Dex, Keycloak, ...)
+
+# kube_oidc_url: https:// ...
+# kube_oidc_client_id: kubernetes
+## Optional settings for OIDC
+# kube_oidc_ca_file: "{{ kube_cert_dir }}/ca.pem"
+# kube_oidc_username_claim: sub
+# kube_oidc_username_prefix: 'oidc:'
+# kube_oidc_groups_claim: groups
+# kube_oidc_groups_prefix: 'oidc:'
+
+## Variables to control webhook authn/authz
+# kube_webhook_token_auth: false
+# kube_webhook_token_auth_url: https://...
+# kube_webhook_token_auth_url_skip_tls_verify: false
+
+## For webhook authorization, authorization_modes must include Webhook
+# kube_webhook_authorization: false
+# kube_webhook_authorization_url: https://...
+# kube_webhook_authorization_url_skip_tls_verify: false
+
+# Choose network plugin (cilium, calico, kube-ovn, weave or flannel. Use cni for generic cni plugin)
+# Can also be set to 'cloud', which lets the cloud provider setup appropriate routing
+kube_network_plugin: calico
+
+# Setting multi_networking to true will install Multus: https://github.com/intel/multus-cni
+kube_network_plugin_multus: false
+
+# Kubernetes internal network for services, unused block of space.
+kube_service_addresses: 10.233.0.0/18
+
+# internal network. When used, it will assign IP
+# addresses from this range to individual pods.
+# This network must be unused in your network infrastructure!
+kube_pods_subnet: 10.233.64.0/18
+
+# internal network node size allocation (optional). This is the size allocated
+# to each node for pod IP address allocation. Note that the number of pods per node is
+# also limited by the kubelet_max_pods variable which defaults to 110.
+#
+# Example:
+# Up to 64 nodes and up to 254 or kubelet_max_pods (the lowest of the two) pods per node:
+#  - kube_pods_subnet: 10.233.64.0/18
+#  - kube_network_node_prefix: 24
+#  - kubelet_max_pods: 110
+#
+# Example:
+# Up to 128 nodes and up to 126 or kubelet_max_pods (the lowest of the two) pods per node:
+#  - kube_pods_subnet: 10.233.64.0/18
+#  - kube_network_node_prefix: 25
+#  - kubelet_max_pods: 110
+kube_network_node_prefix: 24
+
+# Configure Dual Stack networking (i.e. both IPv4 and IPv6)
+enable_dual_stack_networks: false
+
+# Kubernetes internal network for IPv6 services, unused block of space.
+# This is only used if enable_dual_stack_networks is set to true
+# This provides 4096 IPv6 IPs
+kube_service_addresses_ipv6: fd85:ee78:d8a6:8607::1000/116
+
+# Internal network. When used, it will assign IPv6 addresses from this range to individual pods.
+# This network must not already be in your network infrastructure!
+# This is only used if enable_dual_stack_networks is set to true.
+# This provides room for 256 nodes with 254 pods per node.
+kube_pods_subnet_ipv6: fd85:ee78:d8a6:8607::1:0000/112
+
+# IPv6 subnet size allocated to each for pods.
+# This is only used if enable_dual_stack_networks is set to true
+# This provides room for 254 pods per node.
+kube_network_node_prefix_ipv6: 120
+
+# The port the API Server will be listening on.
+kube_apiserver_ip: "{{ kube_service_addresses|ipaddr('net')|ipaddr(1)|ipaddr('address') }}"
+kube_apiserver_port: 6443  # (https)
+
+# Kube-proxy proxyMode configuration.
+# Can be ipvs, iptables
+kube_proxy_mode: ipvs
+
+# configure arp_ignore and arp_announce to avoid answering ARP queries from kube-ipvs0 interface
+# must be set to true for MetalLB, kube-vip(ARP enabled) to work
+kube_proxy_strict_arp: false
+
+# A string slice of values which specify the addresses to use for NodePorts.
+# Values may be valid IP blocks (e.g. 1.2.3.0/24, 1.2.3.4/32).
+# The default empty string slice ([]) means to use all local addresses.
+# kube_proxy_nodeport_addresses_cidr is retained for legacy config
+kube_proxy_nodeport_addresses: >-
+  {%- if kube_proxy_nodeport_addresses_cidr is defined -%}
+  [{{ kube_proxy_nodeport_addresses_cidr }}]
+  {%- else -%}
+  []
+  {%- endif -%}
+
+# If non-empty, will use this string as identification instead of the actual hostname
+# kube_override_hostname: >-
+#   {%- if cloud_provider is defined and cloud_provider in [ 'aws' ] -%}
+#   {%- else -%}
+#   {{ inventory_hostname }}
+#   {%- endif -%}
+
+## Encrypting Secret Data at Rest
+kube_encrypt_secret_data: false
+
+# Graceful Node Shutdown (Kubernetes >= 1.21.0), see https://kubernetes.io/blog/2021/04/21/graceful-node-shutdown-beta/
+# kubelet_shutdown_grace_period had to be greater than kubelet_shutdown_grace_period_critical_pods to allow
+# non-critical podsa to also terminate gracefully
+# kubelet_shutdown_grace_period: 60s
+# kubelet_shutdown_grace_period_critical_pods: 20s
+
+# DNS configuration.
+# Kubernetes cluster name, also will be used as DNS domain
+cluster_name: cluster.local
+# Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods
+ndots: 2
+# dns_timeout: 2
+# dns_attempts: 2
+# Custom search domains to be added in addition to the default cluster search domains
+# searchdomains:
+#   - svc.{{ cluster_name }}
+#   - default.svc.{{ cluster_name  }}
+# Remove default cluster search domains (``default.svc.{{ dns_domain }}, svc.{{ dns_domain }}``).
+# remove_default_searchdomains: false
+# Can be coredns, coredns_dual, manual or none
+dns_mode: coredns
+# Set manual server if using a custom cluster DNS server
+# manual_dns_server: 10.x.x.x
+# Enable nodelocal dns cache
+enable_nodelocaldns: true
+enable_nodelocaldns_secondary: false
+nodelocaldns_ip: 169.254.25.10
+nodelocaldns_health_port: 9254
+nodelocaldns_second_health_port: 9256
+nodelocaldns_bind_metrics_host_ip: false
+nodelocaldns_secondary_skew_seconds: 5
+# nodelocaldns_external_zones:
+# - zones:
+#   - example.com
+#   - example.io:1053
+#   nameservers:
+#   - 1.1.1.1
+#   - 2.2.2.2
+#   cache: 5
+# - zones:
+#   - https://mycompany.local:4453
+#   nameservers:
+#   - 192.168.0.53
+#   cache: 0
+# - zones:
+#   - mydomain.tld
+#   nameservers:
+#   - 10.233.0.3
+#   cache: 5
+#   rewrite:
+#   - name website.tld website.namespace.svc.cluster.local
+# Enable k8s_external plugin for CoreDNS
+enable_coredns_k8s_external: false
+coredns_k8s_external_zone: k8s_external.local
+# Enable endpoint_pod_names option for kubernetes plugin
+enable_coredns_k8s_endpoint_pod_names: false
+# Set forward options for upstream DNS servers in coredns (and nodelocaldns) config
+# dns_upstream_forward_extra_opts:
+#   policy: sequential
+
+# Can be docker_dns, host_resolvconf or none
+resolvconf_mode: host_resolvconf
+# Deploy netchecker app to verify DNS resolve as an HTTP service
+deploy_netchecker: false
+# Ip address of the kubernetes skydns service
+skydns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('address') }}"
+skydns_server_secondary: "{{ kube_service_addresses|ipaddr('net')|ipaddr(4)|ipaddr('address') }}"
+dns_domain: "{{ cluster_name }}"
+
+## Container runtime
+## docker for docker, crio for cri-o and containerd for containerd.
+## Default: containerd
+container_manager: containerd
+
+# Additional container runtimes
+kata_containers_enabled: false
+
+kubeadm_certificate_key: "{{ lookup('password', credentials_dir + '/kubeadm_certificate_key.creds length=64 chars=hexdigits') | lower }}"
+
+# K8s image pull policy (imagePullPolicy)
+k8s_image_pull_policy: IfNotPresent
+
+# audit log for kubernetes
+kubernetes_audit: false
+
+# define kubelet config dir for dynamic kubelet
+# kubelet_config_dir:
+default_kubelet_config_dir: "{{ kube_config_dir }}/dynamic_kubelet_dir"
+
+# pod security policy (RBAC must be enabled either by having 'RBAC' in authorization_modes or kubeadm enabled)
+podsecuritypolicy_enabled: false
+
+# Custom PodSecurityPolicySpec for restricted policy
+# podsecuritypolicy_restricted_spec: {}
+
+# Custom PodSecurityPolicySpec for privileged policy
+# podsecuritypolicy_privileged_spec: {}
+
+# Make a copy of kubeconfig on the host that runs Ansible in {{ inventory_dir }}/artifacts
+# kubeconfig_localhost: false
+# Use ansible_host as external api ip when copying over kubeconfig.
+# kubeconfig_localhost_ansible_host: false
+# Download kubectl onto the host that runs Ansible in {{ bin_dir }}
+# kubectl_localhost: false
+
+# A comma separated list of levels of node allocatable enforcement to be enforced by kubelet.
+# Acceptable options are 'pods', 'system-reserved', 'kube-reserved' and ''. Default is "".
+# kubelet_enforce_node_allocatable: pods
+
+## Optionally reserve resources for OS system daemons.
+# system_reserved: true
+## Uncomment to override default values
+# system_memory_reserved: 512Mi
+# system_cpu_reserved: 500m
+# system_ephemeral_storage_reserved: 2Gi
+## Reservation for master hosts
+# system_master_memory_reserved: 256Mi
+# system_master_cpu_reserved: 250m
+# system_master_ephemeral_storage_reserved: 2Gi
+
+## Eviction Thresholds to avoid system OOMs
+# https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#eviction-thresholds
+# eviction_hard: {}
+# eviction_hard_control_plane: {}
+
+# An alternative flexvolume plugin directory
+# kubelet_flexvolumes_plugins_dir: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+
+## Supplementary addresses that can be added in kubernetes ssl keys.
+## That can be useful for example to setup a keepalived virtual IP
+# supplementary_addresses_in_ssl_keys: [10.0.0.1, 10.0.0.2, 10.0.0.3]
+
+## Running on top of openstack vms with cinder enabled may lead to unschedulable pods due to NoVolumeZoneConflict restriction in kube-scheduler.
+## See https://github.com/kubernetes-sigs/kubespray/issues/2141
+## Set this variable to true to get rid of this issue
+volume_cross_zone_attachment: false
+## Add Persistent Volumes Storage Class for corresponding cloud provider (supported: in-tree OpenStack, Cinder CSI,
+## AWS EBS CSI, Azure Disk CSI, GCP Persistent Disk CSI)
+persistent_volumes_enabled: false
+
+## Container Engine Acceleration
+## Enable container acceleration feature, for example use gpu acceleration in containers
+# nvidia_accelerator_enabled: true
+## Nvidia GPU driver install. Install will by done by a (init) pod running as a daemonset.
+## Important: if you use Ubuntu then you should set in all.yml 'docker_storage_options: -s overlay2'
+## Array with nvida_gpu_nodes, leave empty or comment if you don't want to install drivers.
+## Labels and taints won't be set to nodes if they are not in the array.
+# nvidia_gpu_nodes:
+#   - kube-gpu-001
+# nvidia_driver_version: "384.111"
+## flavor can be tesla or gtx
+# nvidia_gpu_flavor: gtx
+## NVIDIA driver installer images. Change them if you have trouble accessing gcr.io.
+# nvidia_driver_install_centos_container: atzedevries/nvidia-centos-driver-installer:2
+# nvidia_driver_install_ubuntu_container: gcr.io/google-containers/ubuntu-nvidia-driver-installer@sha256:7df76a0f0a17294e86f691c81de6bbb7c04a1b4b3d4ea4e7e2cccdc42e1f6d63
+## NVIDIA GPU device plugin image.
+# nvidia_gpu_device_plugin_container: "registry.k8s.io/nvidia-gpu-device-plugin@sha256:0842734032018be107fa2490c98156992911e3e1f2a21e059ff0105b07dd8e9e"
+
+## Support tls min version, Possible values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
+# tls_min_version: ""
+
+## Support tls cipher suites.
+# tls_cipher_suites: {}
+#   - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA
+#   - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
+#   - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
+#   - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA
+#   - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+#   - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
+#   - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA
+#   - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA
+#   - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA
+#   - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256
+#   - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
+#   - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA
+#   - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+#   - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305
+#   - TLS_ECDHE_RSA_WITH_RC4_128_SHA
+#   - TLS_RSA_WITH_3DES_EDE_CBC_SHA
+#   - TLS_RSA_WITH_AES_128_CBC_SHA
+#   - TLS_RSA_WITH_AES_128_CBC_SHA256
+#   - TLS_RSA_WITH_AES_128_GCM_SHA256
+#   - TLS_RSA_WITH_AES_256_CBC_SHA
+#   - TLS_RSA_WITH_AES_256_GCM_SHA384
+#   - TLS_RSA_WITH_RC4_128_SHA
+
+## Amount of time to retain events. (default 1h0m0s)
+event_ttl_duration: "1h0m0s"
+
+## Automatically renew K8S control plane certificates on first Monday of each month
+auto_renew_certificates: false
+# First Monday of each month
+# auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:{{ groups['kube_control_plane'].index(inventory_hostname) }}0:00"
+
+# kubeadm patches path
+kubeadm_patches:
+  enabled: false
+  source_dir: "{{ inventory_dir }}/patches"
+  dest_dir: "{{ kube_config_dir }}/patches"

kubespray/contrib/network-storage/glusterfs/group_vars/k8s_cluster/k8s-net-calico.yml

@@ -0,0 +1,131 @@
+---
+# see roles/network_plugin/calico/defaults/main.yml
+
+# the default value of name
+calico_cni_name: k8s-pod-network
+
+## With calico it is possible to distributed routes with border routers of the datacenter.
+## Warning : enabling router peering will disable calico's default behavior ('node mesh').
+## The subnets of each nodes will be distributed by the datacenter router
+# peer_with_router: false
+
+# Enables Internet connectivity from containers
+# nat_outgoing: true
+
+# Enables Calico CNI "host-local" IPAM plugin
+# calico_ipam_host_local: true
+
+# add default ippool name
+# calico_pool_name: "default-pool"
+
+# add default ippool blockSize (defaults kube_network_node_prefix)
+calico_pool_blocksize: 26
+
+# add default ippool CIDR (must be inside kube_pods_subnet, defaults to kube_pods_subnet otherwise)
+# calico_pool_cidr: 1.2.3.4/5
+
+# add default ippool CIDR to CNI config
+# calico_cni_pool: true
+
+# Add default IPV6 IPPool CIDR. Must be inside kube_pods_subnet_ipv6. Defaults to kube_pods_subnet_ipv6 if not set.
+# calico_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112
+
+# Add default IPV6 IPPool CIDR to CNI config
+# calico_cni_pool_ipv6: true
+
+# Global as_num (/calico/bgp/v1/global/as_num)
+# global_as_num: "64512"
+
+# If doing peering with node-assigned asn where the globas does not match your nodes, you want this
+# to be true.  All other cases, false.
+# calico_no_global_as_num: false
+
+# You can set MTU value here. If left undefined or empty, it will
+# not be specified in calico CNI config, so Calico will use built-in
+# defaults. The value should be a number, not a string.
+# calico_mtu: 1500
+
+# Configure the MTU to use for workload interfaces and tunnels.
+# - If Wireguard is enabled, subtract 60 from your network MTU (i.e 1500-60=1440)
+# - Otherwise, if VXLAN or BPF mode is enabled, subtract 50 from your network MTU (i.e. 1500-50=1450)
+# - Otherwise, if IPIP is enabled, subtract 20 from your network MTU (i.e. 1500-20=1480)
+# - Otherwise, if not using any encapsulation, set to your network MTU (i.e. 1500)
+# calico_veth_mtu: 1440
+
+# Advertise Cluster IPs
+# calico_advertise_cluster_ips: true
+
+# Advertise Service External IPs
+# calico_advertise_service_external_ips:
+# - x.x.x.x/24
+# - y.y.y.y/32
+
+# Advertise Service LoadBalancer IPs
+# calico_advertise_service_loadbalancer_ips:
+# - x.x.x.x/24
+# - y.y.y.y/16
+
+# Choose data store type for calico: "etcd" or "kdd" (kubernetes datastore)
+# calico_datastore: "kdd"
+
+# Choose Calico iptables backend: "Legacy", "Auto" or "NFT"
+# calico_iptables_backend: "Auto"
+
+# Use typha (only with kdd)
+# typha_enabled: false
+
+# Generate TLS certs for secure typha<->calico-node communication
+# typha_secure: false
+
+# Scaling typha: 1 replica per 100 nodes is adequate
+# Number of typha replicas
+# typha_replicas: 1
+
+# Set max typha connections
+# typha_max_connections_lower_limit: 300
+
+# Set calico network backend: "bird", "vxlan" or "none"
+# bird enable BGP routing, required for ipip and no encapsulation modes
+# calico_network_backend: vxlan
+
+# IP in IP and VXLAN is mutualy exclusive modes.
+# set IP in IP encapsulation mode: "Always", "CrossSubnet", "Never"
+# calico_ipip_mode: 'Never'
+
+# set VXLAN encapsulation mode: "Always", "CrossSubnet", "Never"
+# calico_vxlan_mode: 'Always'
+
+# set VXLAN port and VNI
+# calico_vxlan_vni: 4096
+# calico_vxlan_port: 4789
+
+# Enable eBPF mode
+# calico_bpf_enabled: false
+
+# If you want to use non default IP_AUTODETECTION_METHOD, IP6_AUTODETECTION_METHOD for calico node set this option to one of:
+# * can-reach=DESTINATION
+# * interface=INTERFACE-REGEX
+# see https://docs.projectcalico.org/reference/node/configuration
+# calico_ip_auto_method: "interface=eth.*"
+# calico_ip6_auto_method: "interface=eth.*"
+
+# Set FELIX_MTUIFACEPATTERN, Pattern used to discover the host’s interface for MTU auto-detection.
+# see https://projectcalico.docs.tigera.io/reference/felix/configuration
+# calico_felix_mtu_iface_pattern: "^((en|wl|ww|sl|ib)[opsx].*|(eth|wlan|wwan).*)"
+
+# Choose the iptables insert mode for Calico: "Insert" or "Append".
+# calico_felix_chaininsertmode: Insert
+
+# If you want use the default route interface when you use multiple interface with dynamique route (iproute2)
+# see https://docs.projectcalico.org/reference/node/configuration : FELIX_DEVICEROUTESOURCEADDRESS
+# calico_use_default_route_src_ipaddr: false
+
+# Enable calico traffic encryption with wireguard
+# calico_wireguard_enabled: false
+
+# Under certain situations liveness and readiness probes may need tunning
+# calico_node_livenessprobe_timeout: 10
+# calico_node_readinessprobe_timeout: 10
+
+# Calico apiserver (only with kdd)
+# calico_apiserver_enabled: false

kubespray/contrib/network-storage/glusterfs/group_vars/k8s_cluster/k8s-net-canal.yml

@@ -0,0 +1,10 @@
+# see roles/network_plugin/canal/defaults/main.yml
+
+# The interface used by canal for host <-> host communication.
+# If left blank, then the interface is choosing using the node's
+# default route.
+# canal_iface: ""
+
+# Whether or not to masquerade traffic to destinations not within
+# the pod network.
+# canal_masquerade: "true"

kubespray/contrib/network-storage/glusterfs/group_vars/k8s_cluster/k8s-net-cilium.yml

@@ -0,0 +1,245 @@
+---
+# cilium_version: "v1.12.1"
+
+# Log-level
+# cilium_debug: false
+
+# cilium_mtu: ""
+# cilium_enable_ipv4: true
+# cilium_enable_ipv6: false
+
+# Cilium agent health port
+# cilium_agent_health_port: "9879"
+
+# Identity allocation mode selects how identities are shared between cilium
+# nodes by setting how they are stored. The options are "crd" or "kvstore".
+# - "crd" stores identities in kubernetes as CRDs (custom resource definition).
+#   These can be queried with:
+#     `kubectl get ciliumid`
+# - "kvstore" stores identities in an etcd kvstore.
+# - In order to support External Workloads, "crd" is required
+#   - Ref: https://docs.cilium.io/en/stable/gettingstarted/external-workloads/#setting-up-support-for-external-workloads-beta
+# - KVStore operations are only required when cilium-operator is running with any of the below options:
+#   - --synchronize-k8s-services
+#   - --synchronize-k8s-nodes
+#   - --identity-allocation-mode=kvstore
+#   - Ref: https://docs.cilium.io/en/stable/internals/cilium_operator/#kvstore-operations
+# cilium_identity_allocation_mode: kvstore
+
+# Etcd SSL dirs
+# cilium_cert_dir: /etc/cilium/certs
+# kube_etcd_cacert_file: ca.pem
+# kube_etcd_cert_file: cert.pem
+# kube_etcd_key_file: cert-key.pem
+
+# Limits for apps
+# cilium_memory_limit: 500M
+# cilium_cpu_limit: 500m
+# cilium_memory_requests: 64M
+# cilium_cpu_requests: 100m
+
+# Overlay Network Mode
+# cilium_tunnel_mode: vxlan
+# Optional features
+# cilium_enable_prometheus: false
+# Enable if you want to make use of hostPort mappings
+# cilium_enable_portmap: false
+# Monitor aggregation level (none/low/medium/maximum)
+# cilium_monitor_aggregation: medium
+# The monitor aggregation flags determine which TCP flags which, upon the
+# first observation, cause monitor notifications to be generated.
+#
+# Only effective when monitor aggregation is set to "medium" or higher.
+# cilium_monitor_aggregation_flags: "all"
+# Kube Proxy Replacement mode (strict/probe/partial)
+# cilium_kube_proxy_replacement: probe
+
+# If upgrading from Cilium < 1.5, you may want to override some of these options
+# to prevent service disruptions. See also:
+# http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action
+# cilium_preallocate_bpf_maps: false
+
+# `cilium_tofqdns_enable_poller` is deprecated in 1.8, removed in 1.9
+# cilium_tofqdns_enable_poller: false
+
+# `cilium_enable_legacy_services` is deprecated in 1.6, removed in 1.9
+# cilium_enable_legacy_services: false
+
+# Unique ID of the cluster. Must be unique across all conneted clusters and
+# in the range of 1 and 255. Only relevant when building a mesh of clusters.
+# This value is not defined by default
+# cilium_cluster_id:
+
+# Deploy cilium even if kube_network_plugin is not cilium.
+# This enables to deploy cilium alongside another CNI to replace kube-proxy.
+# cilium_deploy_additionally: false
+
+# Auto direct nodes routes can be used to advertise pods routes in your cluster
+# without any tunelling (with `cilium_tunnel_mode` sets to `disabled`).
+# This works only if you have a L2 connectivity between all your nodes.
+# You wil also have to specify the variable `cilium_native_routing_cidr` to
+# make this work. Please refer to the cilium documentation for more
+# information about this kind of setups.
+# cilium_auto_direct_node_routes: false
+
+# Allows to explicitly specify the IPv4 CIDR for native routing.
+# When specified, Cilium assumes networking for this CIDR is preconfigured and
+# hands traffic destined for that range to the Linux network stack without
+# applying any SNAT.
+# Generally speaking, specifying a native routing CIDR implies that Cilium can
+# depend on the underlying networking stack to route packets to their
+# destination. To offer a concrete example, if Cilium is configured to use
+# direct routing and the Kubernetes CIDR is included in the native routing CIDR,
+# the user must configure the routes to reach pods, either manually or by
+# setting the auto-direct-node-routes flag.
+# cilium_native_routing_cidr: ""
+
+# Allows to explicitly specify the IPv6 CIDR for native routing.
+# cilium_native_routing_cidr_ipv6: ""
+
+# Enable transparent network encryption.
+# cilium_encryption_enabled: false
+
+# Encryption method. Can be either ipsec or wireguard.
+# Only effective when `cilium_encryption_enabled` is set to true.
+# cilium_encryption_type: "ipsec"
+
+# Enable encryption for pure node to node traffic.
+# This option is only effective when `cilium_encryption_type` is set to `ipsec`.
+# cilium_ipsec_node_encryption: false
+
+# If your kernel or distribution does not support WireGuard, Cilium agent can be configured to fall back on the user-space implementation.
+# When this flag is enabled and Cilium detects that the kernel has no native support for WireGuard,
+# it will fallback on the wireguard-go user-space implementation of WireGuard.
+# This option is only effective when `cilium_encryption_type` is set to `wireguard`.
+# cilium_wireguard_userspace_fallback: false
+
+# IP Masquerade Agent
+# https://docs.cilium.io/en/stable/concepts/networking/masquerading/
+# By default, all packets from a pod destined to an IP address outside of the cilium_native_routing_cidr range are masqueraded
+# cilium_ip_masq_agent_enable: false
+
+### A packet sent from a pod to a destination which belongs to any CIDR from the nonMasqueradeCIDRs is not going to be masqueraded
+# cilium_non_masquerade_cidrs:
+#   - 10.0.0.0/8
+#   - 172.16.0.0/12
+#   - 192.168.0.0/16
+#   - 100.64.0.0/10
+#   - 192.0.0.0/24
+#   - 192.0.2.0/24
+#   - 192.88.99.0/24
+#   - 198.18.0.0/15
+#   - 198.51.100.0/24
+#   - 203.0.113.0/24
+#   - 240.0.0.0/4
+### Indicates whether to masquerade traffic to the link local prefix.
+### If the masqLinkLocal is not set or set to false, then 169.254.0.0/16 is appended to the non-masquerade CIDRs list.
+# cilium_masq_link_local: false
+### A time interval at which the agent attempts to reload config from disk
+# cilium_ip_masq_resync_interval: 60s
+
+# Hubble
+### Enable Hubble without install
+# cilium_enable_hubble: false
+### Enable Hubble Metrics
+# cilium_enable_hubble_metrics: false
+### if cilium_enable_hubble_metrics: true
+# cilium_hubble_metrics: {}
+# - dns
+# - drop
+# - tcp
+# - flow
+# - icmp
+# - http
+### Enable Hubble install
+# cilium_hubble_install: false
+### Enable auto generate certs if cilium_hubble_install: true
+# cilium_hubble_tls_generate: false
+
+# IP address management mode for v1.9+.
+# https://docs.cilium.io/en/v1.9/concepts/networking/ipam/
+# cilium_ipam_mode: kubernetes
+
+# Extra arguments for the Cilium agent
+# cilium_agent_custom_args: []
+
+# For adding and mounting extra volumes to the cilium agent
+# cilium_agent_extra_volumes: []
+# cilium_agent_extra_volume_mounts: []
+
+# cilium_agent_extra_env_vars: []
+
+# cilium_operator_replicas: 2
+
+# The address at which the cillium operator bind health check api
+# cilium_operator_api_serve_addr: "127.0.0.1:9234"
+
+## A dictionary of extra config variables to add to cilium-config, formatted like:
+##  cilium_config_extra_vars:
+##    var1: "value1"
+##    var2: "value2"
+# cilium_config_extra_vars: {}
+
+# For adding and mounting extra volumes to the cilium operator
+# cilium_operator_extra_volumes: []
+# cilium_operator_extra_volume_mounts: []
+
+# Extra arguments for the Cilium Operator
+# cilium_operator_custom_args: []
+
+# Name of the cluster. Only relevant when building a mesh of clusters.
+# cilium_cluster_name: default
+
+# Make Cilium take ownership over the `/etc/cni/net.d` directory on the node, renaming all non-Cilium CNI configurations to `*.cilium_bak`.
+# This ensures no Pods can be scheduled using other CNI plugins during Cilium agent downtime.
+# Available for Cilium v1.10 and up.
+# cilium_cni_exclusive: true
+
+# Configure the log file for CNI logging with retention policy of 7 days.
+# Disable CNI file logging by setting this field to empty explicitly.
+# Available for Cilium v1.12 and up.
+# cilium_cni_log_file: "/var/run/cilium/cilium-cni.log"
+
+# -- Configure cgroup related configuration
+# -- Enable auto mount of cgroup2 filesystem.
+# When `cilium_cgroup_auto_mount` is enabled, cgroup2 filesystem is mounted at
+# `cilium_cgroup_host_root` path on the underlying host and inside the cilium agent pod.
+# If users disable `cilium_cgroup_auto_mount`, it's expected that users have mounted
+# cgroup2 filesystem at the specified `cilium_cgroup_auto_mount` volume, and then the
+# volume will be mounted inside the cilium agent pod at the same path.
+# Available for Cilium v1.11 and up
+# cilium_cgroup_auto_mount: true
+# -- Configure cgroup root where cgroup2 filesystem is mounted on the host
+# cilium_cgroup_host_root: "/run/cilium/cgroupv2"
+
+# Specifies the ratio (0.0-1.0) of total system memory to use for dynamic
+# sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.
+# cilium_bpf_map_dynamic_size_ratio: "0.0"
+
+# -- Enables masquerading of IPv4 traffic leaving the node from endpoints.
+# Available for Cilium v1.10 and up
+# cilium_enable_ipv4_masquerade: true
+# -- Enables masquerading of IPv6 traffic leaving the node from endpoints.
+# Available for Cilium v1.10 and up
+# cilium_enable_ipv6_masquerade: true
+
+# -- Enable native IP masquerade support in eBPF
+# cilium_enable_bpf_masquerade: false
+
+# -- Configure whether direct routing mode should route traffic via
+# host stack (true) or directly and more efficiently out of BPF (false) if
+# the kernel supports it. The latter has the implication that it will also
+# bypass netfilter in the host namespace.
+# cilium_enable_host_legacy_routing: true
+
+# -- Enable use of the remote node identity.
+# ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity
+# cilium_enable_remote_node_identity: true
+
+# -- Enable the use of well-known identities.
+# cilium_enable_well_known_identities: false
+
+# cilium_enable_bpf_clock_probe: true
+
+# -- Whether to enable CNP status updates.
+# cilium_disable_cnp_status_updates: true

kubespray/contrib/network-storage/glusterfs/group_vars/k8s_cluster/k8s-net-flannel.yml

@@ -0,0 +1,18 @@
+# see roles/network_plugin/flannel/defaults/main.yml
+
+## interface that should be used for flannel operations
+## This is actually an inventory cluster-level item
+# flannel_interface:
+
+## Select interface that should be used for flannel operations by regexp on Name or IP
+## This is actually an inventory cluster-level item
+## example: select interface with ip from net 10.0.0.0/23
+## single quote and escape backslashes
+# flannel_interface_regexp: '10\\.0\\.[0-2]\\.\\d{1,3}'
+
+# You can choose what type of flannel backend to use: 'vxlan',  'host-gw' or 'wireguard'
+# please refer to flannel's docs : https://github.com/coreos/flannel/blob/master/README.md
+# flannel_backend_type: "vxlan"
+# flannel_vxlan_vni: 1
+# flannel_vxlan_port: 8472
+# flannel_vxlan_direct_routing: false

kubespray/contrib/network-storage/glusterfs/group_vars/k8s_cluster/k8s-net-kube-ovn.yml

@@ -0,0 +1,57 @@
+---
+
+# geneve or vlan
+kube_ovn_network_type: geneve
+
+# geneve, vxlan or stt. ATTENTION: some networkpolicy cannot take effect when using vxlan and stt need custom compile ovs kernel module
+kube_ovn_tunnel_type: geneve
+
+## The nic to support container network can be a nic name or a group of regex separated by comma e.g: 'enp6s0f0,eth.*', if empty will use the nic that the default route use.
+# kube_ovn_iface: eth1
+## The MTU used by pod iface in overlay networks (default iface MTU - 100)
+# kube_ovn_mtu: 1333
+
+## Enable hw-offload, disable traffic mirror and set the iface to the physical port. Make sure that there is an IP address bind to the physical port.
+kube_ovn_hw_offload: false
+# traffic mirror
+kube_ovn_traffic_mirror: false
+
+# kube_ovn_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112
+# kube_ovn_default_interface_name: eth0
+
+kube_ovn_external_address: 8.8.8.8
+kube_ovn_external_address_ipv6: 2400:3200::1
+kube_ovn_external_dns: alauda.cn
+
+# kube_ovn_default_gateway: 10.233.64.1,fd85:ee78:d8a6:8607::1:0
+kube_ovn_default_gateway_check: true
+kube_ovn_default_logical_gateway: false
+# kube_ovn_default_exclude_ips: 10.16.0.1
+kube_ovn_node_switch_cidr: 100.64.0.0/16
+kube_ovn_node_switch_cidr_ipv6: fd00:100:64::/64
+
+## vlan config, set default interface name and vlan id
+# kube_ovn_default_interface_name: eth0
+kube_ovn_default_vlan_id: 100
+kube_ovn_vlan_name: product
+
+## pod nic type, support: veth-pair or internal-port
+kube_ovn_pod_nic_type: veth_pair
+
+## Enable load balancer
+kube_ovn_enable_lb: true
+
+## Enable network policy support
+kube_ovn_enable_np: true
+
+## Enable external vpc support
+kube_ovn_enable_external_vpc: true
+
+## Enable checksum
+kube_ovn_encap_checksum: true
+
+## enable ssl
+kube_ovn_enable_ssl: false
+
+## dpdk
+kube_ovn_dpdk_enabled: false

kubespray/contrib/network-storage/glusterfs/group_vars/k8s_cluster/k8s-net-kube-router.yml

@@ -0,0 +1,64 @@
+# See roles/network_plugin/kube-router//defaults/main.yml
+
+# Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP
+# kube_router_run_router: true
+
+# Enables Network Policy -- sets up iptables to provide ingress firewall for pods
+# kube_router_run_firewall: true
+
+# Enables Service Proxy -- sets up IPVS for Kubernetes Services
+# see docs/kube-router.md "Caveats" section
+# kube_router_run_service_proxy: false
+
+# Add Cluster IP of the service to the RIB so that it gets advertises to the BGP peers.
+# kube_router_advertise_cluster_ip: false
+
+# Add External IP of service to the RIB so that it gets advertised to the BGP peers.
+# kube_router_advertise_external_ip: false
+
+# Add LoadBalancer IP of service status as set by the LB provider to the RIB so that it gets advertised to the BGP peers.
+# kube_router_advertise_loadbalancer_ip: false
+
+# Adjust manifest of kube-router daemonset template with DSR needed changes
+# kube_router_enable_dsr: false
+
+# Array of arbitrary extra arguments to kube-router, see
+# https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md
+# kube_router_extra_args: []
+
+# ASN number of the cluster, used when communicating with external BGP routers
+# kube_router_cluster_asn: ~
+
+# ASN numbers of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr.
+# kube_router_peer_router_asns: ~
+
+# The ip address of the external router to which all nodes will peer and advertise the cluster ip and pod cidr's.
+# kube_router_peer_router_ips: ~
+
+# The remote port of the external BGP to which all nodes will peer. If not set, default BGP port (179) will be used.
+# kube_router_peer_router_ports: ~
+
+# Setups node CNI to allow hairpin mode, requires node reboots, see
+# https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md#hairpin-mode
+# kube_router_support_hairpin_mode: false
+
+# Select DNS Policy ClusterFirstWithHostNet, ClusterFirst, etc.
+# kube_router_dns_policy: ClusterFirstWithHostNet
+
+# Array of annotations for master
+# kube_router_annotations_master: []
+
+# Array of annotations for every node
+# kube_router_annotations_node: []
+
+# Array of common annotations for every node
+# kube_router_annotations_all: []
+
+# Enables scraping kube-router metrics with Prometheus
+# kube_router_enable_metrics: false
+
+# Path to serve Prometheus metrics on
+# kube_router_metrics_path: /metrics
+
+# Prometheus metrics port to use
+# kube_router_metrics_port: 9255

kubespray/contrib/network-storage/glusterfs/group_vars/k8s_cluster/k8s-net-macvlan.yml

@@ -0,0 +1,6 @@
+---
+# private interface, on a l2-network
+macvlan_interface: "eth1"
+
+# Enable nat in default gateway network interface
+enable_nat_default_gateway: true

kubespray/contrib/network-storage/glusterfs/group_vars/k8s_cluster/k8s-net-weave.yml

@@ -0,0 +1,64 @@
+# see roles/network_plugin/weave/defaults/main.yml
+
+# Weave's network password for encryption, if null then no network encryption.
+# weave_password: ~
+
+# If set to 1, disable checking for new Weave Net versions (default is blank,
+# i.e. check is enabled)
+# weave_checkpoint_disable: false
+
+# Soft limit on the number of connections between peers. Defaults to 100.
+# weave_conn_limit: 100
+
+# Weave Net defaults to enabling hairpin on the bridge side of the veth pair
+# for containers attached. If you need to disable hairpin, e.g. your kernel is
+# one of those that can panic if hairpin is enabled, then you can disable it by
+# setting `HAIRPIN_MODE=false`.
+# weave_hairpin_mode: true
+
+# The range of IP addresses used by Weave Net and the subnet they are placed in
+# (CIDR format; default 10.32.0.0/12)
+# weave_ipalloc_range: "{{ kube_pods_subnet }}"
+
+# Set to 0 to disable Network Policy Controller (default is on)
+# weave_expect_npc: "{{ enable_network_policy }}"
+
+# List of addresses of peers in the Kubernetes cluster (default is to fetch the
+# list from the api-server)
+# weave_kube_peers: ~
+
+# Set the initialization mode of the IP Address Manager (defaults to consensus
+# amongst the KUBE_PEERS)
+# weave_ipalloc_init: ~
+
+# Set the IP address used as a gateway from the Weave network to the host
+# network - this is useful if you are configuring the addon as a static pod.
+# weave_expose_ip: ~
+
+# Address and port that the Weave Net daemon will serve Prometheus-style
+# metrics on (defaults to 0.0.0.0:6782)
+# weave_metrics_addr: ~
+
+# Address and port that the Weave Net daemon will serve status requests on
+# (defaults to disabled)
+# weave_status_addr: ~
+
+# Weave Net defaults to 1376 bytes, but you can set a smaller size if your
+# underlying network has a tighter limit, or set a larger size for better
+# performance if your network supports jumbo frames (e.g. 8916)
+# weave_mtu: 1376
+
+# Set to 1 to preserve the client source IP address when accessing Service
+# annotated with `service.spec.externalTrafficPolicy=Local`. The feature works
+# only with Weave IPAM (default).
+# weave_no_masq_local: true
+
+# set to nft to use nftables backend for iptables (default is iptables)
+# weave_iptables_backend: iptables
+
+# Extra variables that passing to launch.sh, useful for enabling seed mode, see
+# https://www.weave.works/docs/net/latest/tasks/ipam/ipam/
+# weave_extra_args: ~
+
+# Extra variables for weave_npc that passing to launch.sh, useful for change log level, ex --log-level=error
+# weave_npc_extra_args: ~

kubespray/contrib/network-storage/glusterfs/inventory.example

@@ -0,0 +1,44 @@
+# ## Configure 'ip' variable to bind kubernetes services on a
+# ## different ip than the default iface
+# node1 ansible_ssh_host=95.54.0.12  # ip=10.3.0.1
+# node2 ansible_ssh_host=95.54.0.13  # ip=10.3.0.2
+# node3 ansible_ssh_host=95.54.0.14  # ip=10.3.0.3
+# node4 ansible_ssh_host=95.54.0.15  # ip=10.3.0.4
+# node5 ansible_ssh_host=95.54.0.16  # ip=10.3.0.5
+# node6 ansible_ssh_host=95.54.0.17  # ip=10.3.0.6
+#
+# ## GlusterFS nodes
+# ## Set disk_volume_device_1 to desired device for gluster brick, if different to /dev/vdb (default).
+# ## As in the previous case, you can set ip to give direct communication on internal IPs
+# gfs_node1 ansible_ssh_host=95.54.0.18 # disk_volume_device_1=/dev/vdc  ip=10.3.0.7
+# gfs_node2 ansible_ssh_host=95.54.0.19 # disk_volume_device_1=/dev/vdc  ip=10.3.0.8
+# gfs_node3 ansible_ssh_host=95.54.0.20 # disk_volume_device_1=/dev/vdc  ip=10.3.0.9
+
+# [kube_control_plane]
+# node1
+# node2
+
+# [etcd]
+# node1
+# node2
+# node3
+
+# [kube_node]
+# node2
+# node3
+# node4
+# node5
+# node6
+
+# [k8s_cluster:children]
+# kube_node
+# kube_control_plane
+
+# [gfs-cluster]
+# gfs_node1
+# gfs_node2
+# gfs_node3
+
+# [network-storage:children]
+# gfs-cluster
+

kubespray/contrib/network-storage/glusterfs/roles/bootstrap-os/defaults/main.yml

@@ -0,0 +1,32 @@
+---
+## CentOS/RHEL/AlmaLinux specific variables
+# Use the fastestmirror yum plugin
+centos_fastestmirror_enabled: false
+
+## Flatcar Container Linux specific variables
+# Disable locksmithd or leave it in its current state
+coreos_locksmithd_disable: false
+
+## Oracle Linux specific variables
+# Install public repo on Oracle Linux
+use_oracle_public_repo: true
+
+fedora_coreos_packages:
+  - python
+  - python3-libselinux
+  - ethtool                 # required in kubeadm preflight phase for verifying the environment
+  - ipset                   # required in kubeadm preflight phase for verifying the environment
+  - conntrack-tools         # required by kube-proxy
+
+## General
+# Set the hostname to inventory_hostname
+override_system_hostname: true
+
+is_fedora_coreos: false
+
+skip_http_proxy_on_os_packages: false
+
+# If this is true, debug information will be displayed but
+# may contain some private data, so it is recommended to set it to false
+# in the production environment.
+unsafe_show_logs: false

kubespray/contrib/network-storage/glusterfs/roles/bootstrap-os/files/bootstrap.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+set -e
+
+BINDIR="/opt/bin"
+if [[ -e $BINDIR/.bootstrapped ]]; then
+  exit 0
+fi
+
+ARCH=$(uname -m)
+case $ARCH in
+  "x86_64")
+    PYPY_ARCH=linux64
+    PYPI_HASH=46818cb3d74b96b34787548343d266e2562b531ddbaf330383ba930ff1930ed5
+    ;;
+  "aarch64")
+    PYPY_ARCH=aarch64
+    PYPI_HASH=2e1ae193d98bc51439642a7618d521ea019f45b8fb226940f7e334c548d2b4b9
+    ;;
+  *)
+    echo "Unsupported Architecture: ${ARCH}"
+    exit 1
+esac
+
+PYTHON_VERSION=3.9
+PYPY_VERSION=7.3.9
+PYPY_FILENAME="pypy${PYTHON_VERSION}-v${PYPY_VERSION}-${PYPY_ARCH}"
+PYPI_URL="https://downloads.python.org/pypy/${PYPY_FILENAME}.tar.bz2"
+
+mkdir -p $BINDIR
+
+cd $BINDIR
+
+TAR_FILE=pyp.tar.bz2
+wget -O "${TAR_FILE}" "${PYPI_URL}"
+echo "${PYPI_HASH} ${TAR_FILE}" | sha256sum -c -
+tar -xjf "${TAR_FILE}" && rm "${TAR_FILE}"
+mv -n "${PYPY_FILENAME}" pypy3
+
+ln -s ./pypy3/bin/pypy3 python
+$BINDIR/python --version
+
+touch $BINDIR/.bootstrapped

kubespray/contrib/network-storage/glusterfs/roles/bootstrap-os/handlers/main.yml

@@ -0,0 +1,4 @@
+---
+- name: RHEL auto-attach subscription
+  command: /sbin/subscription-manager attach --auto
+  become: true

kubespray/contrib/network-storage/glusterfs/roles/bootstrap-os/molecule/default/converge.yml

@@ -0,0 +1,6 @@
+---
+- name: Converge
+  hosts: all
+  gather_facts: no
+  roles:
+    - role: bootstrap-os

kubespray/contrib/network-storage/glusterfs/roles/bootstrap-os/molecule/default/molecule.yml

@@ -0,0 +1,57 @@
+---
+dependency:
+  name: galaxy
+lint: |
+  set -e
+  yamllint -c ../../.yamllint .
+driver:
+  name: vagrant
+  provider:
+    name: libvirt
+platforms:
+  - name: ubuntu16
+    box: generic/ubuntu1604
+    cpus: 1
+    memory: 512
+  - name: ubuntu18
+    box: generic/ubuntu1804
+    cpus: 1
+    memory: 512
+  - name: ubuntu20
+    box: generic/ubuntu2004
+    cpus: 1
+    memory: 512
+  - name: centos7
+    box: centos/7
+    cpus: 1
+    memory: 512
+  - name: almalinux8
+    box: almalinux/8
+    cpus: 1
+    memory: 512
+  - name: debian9
+    box: generic/debian9
+    cpus: 1
+    memory: 512
+  - name: debian10
+    box: generic/debian10
+    cpus: 1
+    memory: 512
+provisioner:
+  name: ansible
+  config_options:
+    defaults:
+      callbacks_enabled: profile_tasks
+      timeout: 120
+  lint:
+    name: ansible-lint
+  inventory:
+    group_vars:
+      all:
+        user:
+          name: foo
+          comment: My test comment
+verifier:
+  name: testinfra
+  lint:
+    name: flake8

kubespray/contrib/network-storage/glusterfs/roles/bootstrap-os/molecule/default/tests/test_default.py

@@ -0,0 +1,11 @@
+import os
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']
+).get_hosts('all')
+
+
+def test_python(host):
+    assert host.exists('python3') or host.exists('python')

kubespray/contrib/network-storage/glusterfs/roles/bootstrap-os/tasks/bootstrap-amazon.yml

@@ -0,0 +1,13 @@
+---
+- name: Enable EPEL repo for Amazon Linux
+  yum_repository:
+    name: epel
+    file: epel
+    description: Extra Packages for Enterprise Linux 7 - $basearch
+    baseurl: http://download.fedoraproject.org/pub/epel/7/$basearch
+    gpgcheck: yes
+    gpgkey: http://download.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7
+    skip_if_unavailable: yes
+    enabled: yes
+    repo_gpgcheck: no
+  when: epel_enabled

kubespray/contrib/network-storage/glusterfs/roles/bootstrap-os/tasks/bootstrap-centos.yml

@@ -0,0 +1,117 @@
+---
+- name: Gather host facts to get ansible_distribution_version ansible_distribution_major_version
+  setup:
+    gather_subset: '!all'
+    filter: ansible_distribution_*version
+
+- name: Add proxy to yum.conf or dnf.conf if http_proxy is defined
+  ini_file:
+    path: "{{ ( (ansible_distribution_major_version | int) < 8) | ternary('/etc/yum.conf','/etc/dnf/dnf.conf') }}"
+    section: main
+    option: proxy
+    value: "{{ http_proxy | default(omit) }}"
+    state: "{{ http_proxy | default(False) | ternary('present', 'absent') }}"
+    no_extra_spaces: true
+    mode: 0644
+  become: true
+  when: not skip_http_proxy_on_os_packages
+
+# For Oracle Linux install public repo
+- name: Download Oracle Linux public yum repo
+  get_url:
+    url: https://yum.oracle.com/public-yum-ol7.repo
+    dest: /etc/yum.repos.d/public-yum-ol7.repo
+  when:
+    - use_oracle_public_repo|default(true)
+    - '''ID="ol"'' in os_release.stdout_lines'
+    - (ansible_distribution_version | float) < 7.6
+  environment: "{{ proxy_env }}"
+
+- name: Enable Oracle Linux repo
+  ini_file:
+    dest: /etc/yum.repos.d/public-yum-ol7.repo
+    section: "{{ item }}"
+    option: enabled
+    value: "1"
+    mode: 0644
+  with_items:
+    - ol7_latest
+    - ol7_addons
+    - ol7_developer_EPEL
+  when:
+    - use_oracle_public_repo|default(true)
+    - '''ID="ol"'' in os_release.stdout_lines'
+    - (ansible_distribution_version | float) < 7.6
+
+- name: Install EPEL for Oracle Linux repo package
+  package:
+    name: "oracle-epel-release-el{{ ansible_distribution_major_version }}"
+    state: present
+  when:
+    - use_oracle_public_repo|default(true)
+    - '''ID="ol"'' in os_release.stdout_lines'
+    - (ansible_distribution_version | float) >= 7.6
+
+- name: Enable Oracle Linux repo
+  ini_file:
+    dest: "/etc/yum.repos.d/oracle-linux-ol{{ ansible_distribution_major_version }}.repo"
+    section: "ol{{ ansible_distribution_major_version }}_addons"
+    option: "{{ item.option }}"
+    value: "{{ item.value }}"
+    mode: 0644
+  with_items:
+    - { option: "name", value: "ol{{ ansible_distribution_major_version }}_addons" }
+    - { option: "enabled", value: "1" }
+    - { option: "baseurl", value: "http://yum.oracle.com/repo/OracleLinux/OL{{ ansible_distribution_major_version }}/addons/$basearch/" }
+  when:
+    - use_oracle_public_repo|default(true)
+    - '''ID="ol"'' in os_release.stdout_lines'
+    - (ansible_distribution_version | float) >= 7.6
+
+- name: Enable Centos extra repo for Oracle Linux
+  ini_file:
+    dest: "/etc/yum.repos.d/centos-extras.repo"
+    section: "extras"
+    option: "{{ item.option }}"
+    value: "{{ item.value }}"
+    mode: 0644
+  with_items:
+    - { option: "name", value: "CentOS-{{ ansible_distribution_major_version }} - Extras" }
+    - { option: "enabled", value: "1" }
+    - { option: "gpgcheck", value: "0" }
+    - { option: "baseurl", value: "http://mirror.centos.org/centos/{{ ansible_distribution_major_version }}/extras/$basearch/{% if ansible_distribution_major_version|int > 7 %}os/{% endif %}" }
+  when:
+    - use_oracle_public_repo|default(true)
+    - '''ID="ol"'' in os_release.stdout_lines'
+    - (ansible_distribution_version | float) >= 7.6
+    - (ansible_distribution_version | float) < 9
+
+# CentOS ships with python installed
+
+- name: Check presence of fastestmirror.conf
+  stat:
+    path: /etc/yum/pluginconf.d/fastestmirror.conf
+    get_attributes: no
+    get_checksum: no
+    get_mime: no
+  register: fastestmirror
+
+# the fastestmirror plugin can actually slow down Ansible deployments
+- name: Disable fastestmirror plugin if requested
+  lineinfile:
+    dest: /etc/yum/pluginconf.d/fastestmirror.conf
+    regexp: "^enabled=.*"
+    line: "enabled=0"
+    state: present
+  become: true
+  when:
+    - fastestmirror.stat.exists
+    - not centos_fastestmirror_enabled
+
+# libselinux-python is required on SELinux enabled hosts
+# See https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#managed-node-requirements
+- name: Install libselinux python package
+  package:
+    name: "{{ ( (ansible_distribution_major_version | int) < 8) | ternary('libselinux-python','python3-libselinux') }}"
+    state: present
+  become: true

kubespray/contrib/network-storage/glusterfs/roles/bootstrap-os/tasks/bootstrap-clearlinux.yml

@@ -0,0 +1,16 @@
+---
+# ClearLinux ships with Python installed
+
+- name: Install basic package to run containers
+  package:
+    name: containers-basic
+    state: present
+
+- name: Make sure docker service is enabled
+  systemd:
+    name: docker
+    masked: false
+    enabled: true
+    daemon_reload: true
+    state: started
+  become: true

kubespray/contrib/network-storage/glusterfs/roles/bootstrap-os/tasks/bootstrap-coreos.yml

@@ -0,0 +1,37 @@
+---
+# CoreOS ships without Python installed
+
+- name: Check if bootstrap is needed
+  raw: stat /opt/bin/.bootstrapped
+  register: need_bootstrap
+  failed_when: false
+  changed_when: false
+  tags:
+    - facts
+
+- name: Force binaries directory for Container Linux by CoreOS and Flatcar
+  set_fact:
+    bin_dir: "/opt/bin"
+  tags:
+    - facts
+
+- name: Run bootstrap.sh
+  script: bootstrap.sh
+  become: true
+  environment: "{{ proxy_env }}"
+  when:
+    - need_bootstrap.rc != 0
+
+- name: Set the ansible_python_interpreter fact
+  set_fact:
+    ansible_python_interpreter: "{{ bin_dir }}/python"
+  tags:
+    - facts
+
+- name: Disable auto-upgrade
+  systemd:
+    name: locksmithd.service
+    masked: true
+    state: stopped
+  when:
+    - coreos_locksmithd_disable

kubespray/contrib/network-storage/glusterfs/roles/bootstrap-os/tasks/bootstrap-debian.yml

@@ -0,0 +1,76 @@
+---
+# Some Debian based distros ship without Python installed
+
+- name: Check if bootstrap is needed
+  raw: which python3
+  register: need_bootstrap
+  failed_when: false
+  changed_when: false
+  # This command should always run, even in check mode
+  check_mode: false
+  tags:
+    - facts
+
+- name: Check http::proxy in apt configuration files
+  raw: apt-config dump | grep -qsi 'Acquire::http::proxy'
+  register: need_http_proxy
+  failed_when: false
+  changed_when: false
+  # This command should always run, even in check mode
+  check_mode: false
+
+- name: Add http_proxy to /etc/apt/apt.conf if http_proxy is defined
+  raw: echo 'Acquire::http::proxy "{{ http_proxy }}";' >> /etc/apt/apt.conf
+  become: true
+  when:
+    - http_proxy is defined
+    - need_http_proxy.rc != 0
+    - not skip_http_proxy_on_os_packages
+
+- name: Check https::proxy in apt configuration files
+  raw: apt-config dump | grep -qsi 'Acquire::https::proxy'
+  register: need_https_proxy
+  failed_when: false
+  changed_when: false
+  # This command should always run, even in check mode
+  check_mode: false
+
+- name: Add https_proxy to /etc/apt/apt.conf if https_proxy is defined
+  raw: echo 'Acquire::https::proxy "{{ https_proxy }}";' >> /etc/apt/apt.conf
+  become: true
+  when:
+    - https_proxy is defined
+    - need_https_proxy.rc != 0
+    - not skip_http_proxy_on_os_packages
+
+- name: Install python3
+  raw:
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y python3-minimal
+  become: true
+  when:
+    - need_bootstrap.rc != 0
+
+- name: Update Apt cache
+  raw: apt-get update --allow-releaseinfo-change
+  become: true
+  when:
+    - '''ID=debian'' in os_release.stdout_lines'
+    - '''VERSION_ID="10"'' in os_release.stdout_lines or ''VERSION_ID="11"'' in os_release.stdout_lines'
+  register: bootstrap_update_apt_result
+  changed_when:
+    - '"changed its" in bootstrap_update_apt_result.stdout'
+    - '"value from" in bootstrap_update_apt_result.stdout'
+  ignore_errors: true
+
+- name: Set the ansible_python_interpreter fact
+  set_fact:
+    ansible_python_interpreter: "/usr/bin/python3"
+
+# Workaround for https://github.com/ansible/ansible/issues/25543
+- name: Install dbus for the hostname module
+  package:
+    name: dbus
+    state: present
+    use: apt
+  become: true

kubespray/contrib/network-storage/glusterfs/roles/bootstrap-os/tasks/bootstrap-fedora-coreos.yml

@@ -0,0 +1,46 @@
+---
+
+- name: Check if bootstrap is needed
+  raw: which python
+  register: need_bootstrap
+  failed_when: false
+  changed_when: false
+  tags:
+    - facts
+
+- name: Remove podman network cni
+  raw: "podman network rm podman"
+  become: true
+  ignore_errors: true  # noqa ignore-errors
+  when: need_bootstrap.rc != 0
+
+- name: Clean up possible pending packages on fedora coreos
+  raw: "export http_proxy={{ http_proxy | default('') }};rpm-ostree cleanup -p }}"
+  become: true
+  when: need_bootstrap.rc != 0
+
+- name: Install required packages on fedora coreos
+  raw: "export http_proxy={{ http_proxy | default('') }};rpm-ostree install --allow-inactive {{ fedora_coreos_packages|join(' ') }}"
+  become: true
+  when: need_bootstrap.rc != 0
+
+- name: Reboot immediately for updated ostree
+  raw: "nohup bash -c 'sleep 5s && shutdown -r now'"
+  become: true
+  ignore_errors: true  # noqa ignore-errors
+  ignore_unreachable: yes
+  when: need_bootstrap.rc != 0
+
+- name: Wait for the reboot to complete
+  wait_for_connection:
+    timeout: 240
+    connect_timeout: 20
+    delay: 5
+    sleep: 5
+  when: need_bootstrap.rc != 0
+
+- name: Store the fact if this is an fedora core os host
+  set_fact:
+    is_fedora_coreos: True
+  tags:
+    - facts

kubespray/contrib/network-storage/glusterfs/roles/bootstrap-os/tasks/bootstrap-fedora.yml

@@ -0,0 +1,36 @@
+---
+# Some Fedora based distros ship without Python installed
+
+- name: Check if bootstrap is needed
+  raw: which python
+  register: need_bootstrap
+  failed_when: false
+  changed_when: false
+  tags:
+    - facts
+
+- name: Add proxy to dnf.conf if http_proxy is defined
+  ini_file:
+    path: "/etc/dnf/dnf.conf"
+    section: main
+    option: proxy
+    value: "{{ http_proxy | default(omit) }}"
+    state: "{{ http_proxy | default(False) | ternary('present', 'absent') }}"
+    no_extra_spaces: true
+    mode: 0644
+  become: true
+  when: not skip_http_proxy_on_os_packages
+
+- name: Install python3 on fedora
+  raw: "dnf install --assumeyes --quiet python3"
+  become: true
+  when:
+    - need_bootstrap.rc != 0
+
+# libselinux-python3 is required on SELinux enabled hosts
+# See https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#managed-node-requirements
+- name: Install libselinux-python3
+  package:
+    name: libselinux-python3
+    state: present
+  become: true