update

kubespray/roles/network_plugin/kube-router/OWNERS

@@ -0,0 +1,6 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+  - bozzo
+reviewers:
+  - bozzo

kubespray/roles/network_plugin/kube-router/defaults/main.yml

@@ -0,0 +1,66 @@
+---
+# Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP
+kube_router_run_router: true
+
+# Enables Network Policy -- sets up iptables to provide ingress firewall for pods
+kube_router_run_firewall: true
+
+# Enables Service Proxy -- sets up IPVS for Kubernetes Services
+# see docs/kube-router.md "Caveats" section
+kube_router_run_service_proxy: false
+
+# Add Cluster IP of the service to the RIB so that it gets advertises to the BGP peers.
+kube_router_advertise_cluster_ip: false
+
+# Add External IP of service to the RIB so that it gets advertised to the BGP peers.
+kube_router_advertise_external_ip: false
+
+# Add LoadBalancer IP of service status as set by the LB provider to the RIB so that it gets advertised to the BGP peers.
+kube_router_advertise_loadbalancer_ip: false
+
+# Adjust manifest of kube-router daemonset template with DSR needed changes
+kube_router_enable_dsr: false
+
+# Array of arbitrary extra arguments to kube-router, see
+# https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md
+kube_router_extra_args: []
+
+# ASN number of the cluster, used when communicating with external BGP routers
+kube_router_cluster_asn: ~
+
+# ASN numbers of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr.
+kube_router_peer_router_asns: ~
+
+# The ip address of the external router to which all nodes will peer and advertise the cluster ip and pod cidr's.
+kube_router_peer_router_ips: ~
+
+# The remote port of the external BGP to which all nodes will peer. If not set, default BGP port (179) will be used.
+kube_router_peer_router_ports: ~
+
+# Setups node CNI to allow hairpin mode, requires node reboots, see
+# https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md#hairpin-mode
+kube_router_support_hairpin_mode: false
+
+# Select DNS Policy ClusterFirstWithHostNet, ClusterFirst, etc.
+kube_router_dns_policy: ClusterFirstWithHostNet
+
+# Adds annotations to kubernetes nodes for advanced configuration of BGP Peers.
+# https://github.com/cloudnativelabs/kube-router/blob/master/docs/bgp.md
+
+# Array of annotations for master
+kube_router_annotations_master: []
+
+# Array of annotations for every node
+kube_router_annotations_node: []
+
+# Array of common annotations for every node
+kube_router_annotations_all: []
+
+# Enables scraping kube-router metrics with Prometheus
+kube_router_enable_metrics: false
+
+# Path to serve Prometheus metrics on
+kube_router_metrics_path: /metrics
+
+# Prometheus metrics port to use
+kube_router_metrics_port: 9255

kubespray/roles/network_plugin/kube-router/handlers/main.yml

@@ -0,0 +1,20 @@
+---
+- name: reset_kube_router
+  command: /bin/true
+  notify:
+    - Kube-router | delete kube-router docker containers
+    - Kube-router | delete kube-router crio/containerd containers
+
+- name: Kube-router | delete kube-router docker containers
+  shell: "{{ docker_bin_dir }}/docker ps -af name=k8s_POD_kube-router* -q | xargs --no-run-if-empty docker rm -f"
+  register: docker_kube_router_remove
+  until: docker_kube_router_remove is succeeded
+  retries: 5
+  when: container_manager in ["docker"]
+
+- name: Kube-router | delete kube-router crio/containerd containers
+  shell: '{{ bin_dir }}/crictl pods --name kube-router* -q | xargs -I% --no-run-if-empty bash -c "{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %"'
+  register: crictl_kube_router_remove
+  until: crictl_kube_router_remove is succeeded
+  retries: 5
+  when: container_manager in ["crio", "containerd"]

kubespray/roles/network_plugin/kube-router/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - role: network_plugin/cni

kubespray/roles/network_plugin/kube-router/tasks/annotate.yml

@@ -0,0 +1,21 @@
+---
+- name: kube-router | Add annotations on kube_control_plane
+  command: "{{ kubectl }} annotate --overwrite node {{ ansible_hostname }} {{ item }}"
+  with_items:
+  - "{{ kube_router_annotations_master }}"
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  when: kube_router_annotations_master is defined and inventory_hostname in groups['kube_control_plane']
+
+- name: kube-router | Add annotations on kube_node
+  command: "{{ kubectl }} annotate --overwrite node {{ ansible_hostname }} {{ item }}"
+  with_items:
+  - "{{ kube_router_annotations_node }}"
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  when: kube_router_annotations_node is defined and inventory_hostname in groups['kube_node']
+
+- name: kube-router | Add common annotations on all servers
+  command: "{{ kubectl }} annotate --overwrite node {{ ansible_hostname }} {{ item }}"
+  with_items:
+  - "{{ kube_router_annotations_all }}"
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  when: kube_router_annotations_all is defined and inventory_hostname in groups['k8s_cluster']

kubespray/roles/network_plugin/kube-router/tasks/main.yml

@@ -0,0 +1,62 @@
+---
+- name: kube-router | Create annotations
+  include: annotate.yml
+  tags: annotate
+
+- name: kube-router | Create config directory
+  file:
+    path: /var/lib/kube-router
+    state: directory
+    owner: "{{ kube_owner }}"
+    recurse: true
+    mode: 0755
+
+- name: kube-router | Create kubeconfig
+  template:
+    src: kubeconfig.yml.j2
+    dest: /var/lib/kube-router/kubeconfig
+    mode: 0644
+    owner: "{{ kube_owner }}"
+  notify:
+    - reset_kube_router
+
+- name: kube-router | Slurp cni config
+  slurp:
+    src: /etc/cni/net.d/10-kuberouter.conflist
+  register: cni_config_slurp
+  ignore_errors: true  # noqa ignore-errors
+
+- name: kube-router | Set cni_config variable
+  set_fact:
+    cni_config: "{{ cni_config_slurp.content | b64decode | from_json }}"
+  when:
+    - not cni_config_slurp.failed
+
+- name: kube-router | Set host_subnet variable
+  set_fact:
+    host_subnet: "{{ cni_config | json_query('plugins[?bridge==`kube-bridge`].ipam.subnet') | first }}"
+  when:
+    - cni_config is defined
+    - cni_config | json_query('plugins[?bridge==`kube-bridge`].ipam.subnet') | length > 0
+
+- name: kube-router | Create cni config
+  template:
+    src: cni-conf.json.j2
+    dest: /etc/cni/net.d/10-kuberouter.conflist
+    mode: 0644
+    owner: "{{ kube_owner }}"
+  notify:
+    - reset_kube_router
+
+- name: kube-router | Delete old configuration
+  file:
+    path: /etc/cni/net.d/10-kuberouter.conf
+    state: absent
+
+- name: kube-router | Create manifest
+  template:
+    src: kube-router.yml.j2
+    dest: "{{ kube_config_dir }}/kube-router.yml"
+    mode: 0644
+  delegate_to: "{{ groups['kube_control_plane'] | first }}"
+  run_once: true

kubespray/roles/network_plugin/kube-router/tasks/reset.yml

@@ -0,0 +1,28 @@
+---
+- name: reset | check kube-dummy-if network device
+  stat:
+    path: /sys/class/net/kube-dummy-if
+    get_attributes: no
+    get_checksum: no
+    get_mime: no
+  register: kube_dummy_if
+
+- name: reset | remove the network device created by kube-router
+  command: ip link del kube-dummy-if
+  when: kube_dummy_if.stat.exists
+
+- name: check kube-bridge exists
+  stat:
+    path: /sys/class/net/kube-bridge
+    get_attributes: no
+    get_checksum: no
+    get_mime: no
+  register: kube_bridge_if
+
+- name: reset | donw the network bridge create by kube-router
+  command: ip link set kube-bridge down
+  when: kube_bridge_if.stat.exists
+
+- name: reset | remove the network bridge create by kube-router
+  command: ip link del kube-bridge
+  when: kube_bridge_if.stat.exists

kubespray/roles/network_plugin/kube-router/templates/cni-conf.json.j2

@@ -0,0 +1,27 @@
+{
+   "cniVersion":"0.3.0",
+   "name":"kubernetes",
+   "plugins":[
+      {
+         "name":"kubernetes",
+         "type":"bridge",
+         "bridge":"kube-bridge",
+         "isDefaultGateway":true,
+{% if kube_router_support_hairpin_mode %}
+          "hairpinMode":true,
+{% endif %}
+         "ipam":{
+{% if host_subnet is defined %}
+            "subnet": "{{ host_subnet }}",
+{% endif %}
+            "type":"host-local"
+         }
+      },
+      {
+         "type":"portmap",
+         "capabilities":{
+            "portMappings":true
+         }
+      }
+   ]
+}

kubespray/roles/network_plugin/kube-router/templates/kube-router.yml.j2

@@ -0,0 +1,220 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    k8s-app: kube-router
+    tier: node
+  name: kube-router
+  namespace: kube-system
+spec:
+  minReadySeconds: 3
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      k8s-app: kube-router
+      tier: node
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-router
+        tier: node
+      annotations:
+{% if kube_router_enable_metrics %}
+        prometheus.io/path: {{ kube_router_metrics_path }}
+        prometheus.io/port: "{{ kube_router_metrics_port }}"
+        prometheus.io/scrape: "true"
+{% endif %}
+    spec:
+      priorityClassName: system-node-critical
+      serviceAccountName: kube-router
+      containers:
+      - name: kube-router
+        image: {{ kube_router_image_repo }}:{{ kube_router_image_tag }}
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        args:
+        - --run-router={{ kube_router_run_router | bool }}
+        - --run-firewall={{ kube_router_run_firewall | bool }}
+        - --run-service-proxy={{ kube_router_run_service_proxy | bool }}
+        - --kubeconfig=/var/lib/kube-router/kubeconfig
+        - --bgp-graceful-restart=true
+{% if kube_router_advertise_cluster_ip %}
+        - --advertise-cluster-ip
+{% endif %}
+{% if kube_router_advertise_external_ip %}
+        - --advertise-external-ip
+{% endif %}
+{% if kube_router_advertise_loadbalancer_ip %}
+        - --advertise-loadbalancer-ip
+{% endif %}
+{% if kube_router_cluster_asn %}
+        - --cluster-asn={{ kube_router_cluster_asn }}
+{% endif %}
+{% if kube_router_peer_router_asns %}
+        - --peer-router-asns={{ kube_router_peer_router_asns }}
+{% endif %}
+{% if kube_router_peer_router_ips %}
+        - --peer-router-ips={{ kube_router_peer_router_ips }}
+{% endif %}
+{% if kube_router_peer_router_ports %}
+        - --peer-router-ports={{ kube_router_peer_router_ports }}
+{% endif %}
+{% if kube_router_enable_metrics %}
+        - --metrics-path={{ kube_router_metrics_path }}
+        - --metrics-port={{ kube_router_metrics_port }}
+{% endif %}
+{% if kube_router_enable_dsr %}
+{% if container_manager == "docker" %}
+        - --runtime-endpoint=unix:///var/run/docker.sock
+{% endif %}
+{% if container_manager == "containerd" %}
+{% endif %}
+        - --runtime-endpoint=unix:///run/containerd/containerd.sock
+{% endif %}
+{% for arg in kube_router_extra_args %}
+        - "{{ arg }}"
+{% endfor %}
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: KUBE_ROUTER_CNI_CONF_FILE
+          value: /etc/cni/net.d/10-kuberouter.conflist
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 20244
+          initialDelaySeconds: 10
+          periodSeconds: 3
+        resources:
+          requests:
+            cpu: 250m
+            memory: 250Mi
+        securityContext:
+          privileged: true
+        volumeMounts:
+{% if kube_router_enable_dsr %}
+{% if container_manager == "docker" %}
+        - name: docker-socket
+          mountPath: /var/run/docker.sock
+          readOnly: true
+{% endif %}
+{% if container_manager == "containerd" %}
+        - name: containerd-socket
+          mountPath: /run/containerd/containerd.sock
+          readOnly: true
+{% endif %}
+{% endif %}
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: cni-conf-dir
+          mountPath: /etc/cni/net.d
+        - name: kubeconfig
+          mountPath: /var/lib/kube-router
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+          readOnly: false
+{% if kube_router_enable_metrics %}
+        ports:
+        - containerPort: {{ kube_router_metrics_port }}
+          hostPort: {{ kube_router_metrics_port }}
+          name: metrics
+          protocol: TCP
+{% endif %}
+      hostNetwork: true
+      dnsPolicy: {{ kube_router_dns_policy }}
+{% if kube_router_enable_dsr %}
+      hostIPC: true
+      hostPID: true
+{% endif %}
+      tolerations:
+      - operator: Exists
+      volumes:
+{% if kube_router_enable_dsr %}
+{% if container_manager == "docker" %}
+      - name: docker-socket
+        hostPath:
+          path: /var/run/docker.sock
+          type: Socket
+{% endif %}
+{% if container_manager == "containerd" %}
+      - name: containerd-socket
+        hostPath:
+          path: /run/containerd/containerd.sock
+          type: Socket
+{% endif %}
+{% endif %}
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: cni-conf-dir
+        hostPath:
+          path: /etc/cni/net.d
+      - name: kubeconfig
+        hostPath:
+          path: /var/lib/kube-router
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-router
+  namespace: kube-system
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kube-router
+  namespace: kube-system
+rules:
+  - apiGroups:
+    - ""
+    resources:
+      - namespaces
+      - pods
+      - services
+      - nodes
+      - endpoints
+    verbs:
+      - list
+      - get
+      - watch
+  - apiGroups:
+    - "networking.k8s.io"
+    resources:
+      - networkpolicies
+    verbs:
+      - list
+      - get
+      - watch
+  - apiGroups:
+    - extensions
+    resources:
+      - networkpolicies
+    verbs:
+      - get
+      - list
+      - watch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kube-router
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-router
+subjects:
+- kind: ServiceAccount
+  name: kube-router
+  namespace: kube-system

kubespray/roles/network_plugin/kube-router/templates/kubeconfig.yml.j2

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Config
+clusterCIDR: {{ kube_pods_subnet }}
+clusters:
+- name: cluster
+  cluster:
+    certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    server: {{ kube_apiserver_endpoint }}
+users:
+- name: kube-router
+  user:
+    tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+contexts:
+- context:
+    cluster: cluster
+    user: kube-router
+  name: kube-router-context
+current-context: kube-router-context