update

kubespray/scripts/collect-info.yaml

@@ -0,0 +1,146 @@
+---
+- hosts: all
+  become: true
+  gather_facts: no
+
+  vars:
+    docker_bin_dir: /usr/bin
+    bin_dir: /usr/local/bin
+    ansible_ssh_pipelining: true
+    etcd_cert_dir: /etc/ssl/etcd/ssl
+    kube_network_plugin: calico
+    archive_dirname: collect-info
+    commands:
+      - name: timedate_info
+        cmd: timedatectl status
+      - name: kernel_info
+        cmd: uname -r
+      - name: docker_info
+        cmd: "{{ docker_bin_dir }}/docker info"
+      - name: ip_info
+        cmd: ip -4 -o a
+      - name: route_info
+        cmd: ip ro
+      - name: proc_info
+        cmd: ps auxf | grep -v ]$
+      - name: systemctl_failed_info
+        cmd: systemctl --state=failed --no-pager
+      - name: k8s_info
+        cmd: "{{ bin_dir }}/kubectl get all --all-namespaces -o wide"
+      - name: errors_info
+        cmd: journalctl -p err --no-pager
+      - name: etcd_info
+        cmd: "{{ bin_dir }}/etcdctl endpoint --cluster health"
+      - name: calico_info
+        cmd: "{{ bin_dir }}/calicoctl node status"
+        when: '{{ kube_network_plugin == "calico" }}'
+      - name: calico_workload_info
+        cmd: "{{ bin_dir }}/calicoctl get workloadEndpoint -o wide"
+        when: '{{ kube_network_plugin == "calico" }}'
+      - name: calico_pool_info
+        cmd: "{{ bin_dir }}/calicoctl get ippool -o wide"
+        when: '{{ kube_network_plugin == "calico" }}'
+      - name: weave_info
+        cmd: weave report
+        when: '{{ kube_network_plugin == "weave" }}'
+      - name: weave_logs
+        cmd: "{{ docker_bin_dir }}/docker logs weave"
+        when: '{{ kube_network_plugin == "weave" }}'
+      - name: kube_describe_all
+        cmd: "{{ bin_dir }}/kubectl describe all --all-namespaces"
+      - name: kube_describe_nodes
+        cmd: "{{ bin_dir }}/kubectl describe nodes"
+      - name: kubelet_logs
+        cmd: journalctl -u kubelet --no-pager
+      - name: coredns_logs
+        cmd: "for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l k8s-app=coredns -o jsonpath={.items..metadata.name}`;
+          do {{ bin_dir }}/kubectl logs ${i} -n kube-system; done"
+      - name: apiserver_logs
+        cmd: "for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l component=kube-apiserver -o jsonpath={.items..metadata.name}`;
+          do {{ bin_dir }}/kubectl logs ${i} -n kube-system; done"
+      - name: controller_logs
+        cmd: "for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l component=kube-controller-manager -o jsonpath={.items..metadata.name}`;
+          do {{ bin_dir }}/kubectl logs ${i} -n kube-system; done"
+      - name: scheduler_logs
+        cmd: "for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l component=kube-scheduler -o jsonpath={.items..metadata.name}`;
+          do {{ bin_dir }}/kubectl logs ${i} -n kube-system; done"
+      - name: proxy_logs
+        cmd: "for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l k8s-app=kube-proxy -o jsonpath={.items..metadata.name}`;
+          do {{ bin_dir }}/kubectl logs ${i} -n kube-system; done"
+      - name: nginx_logs
+        cmd: "for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l k8s-app=kube-nginx -o jsonpath={.items..metadata.name}`;
+          do {{ bin_dir }}/kubectl logs ${i} -n kube-system; done"
+      - name: flannel_logs
+        cmd: "for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l app=flannel -o jsonpath={.items..metadata.name}`;
+          do {{ bin_dir }}/kubectl logs ${i} -n kube-system flannel-container; done"
+        when: '{{ kube_network_plugin == "flannel" }}'
+      - name: canal_logs
+        cmd: "for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l k8s-app=canal-node -o jsonpath={.items..metadata.name}`;
+          do {{ bin_dir }}/kubectl logs ${i} -n kube-system flannel; done"
+        when: '{{ kube_network_plugin == "canal" }}'
+      - name: calico_policy_logs
+        cmd: "for i in `{{ bin_dir }}/kubectl get pods -n kube-system -l k8s-app=calico-kube-controllers -o jsonpath={.items..metadata.name}`;
+          do {{ bin_dir }}/kubectl logs ${i} -n kube-system ; done"
+        when: '{{ kube_network_plugin in ["canal", "calico"] }}'
+      - name: helm_show_releases_history
+        cmd: "for i in `{{ bin_dir }}/helm list -q`; do {{ bin_dir }}/helm history ${i} --col-width=0; done"
+        when: "{{ helm_enabled|default(true) }}"
+
+    logs:
+      - /var/log/syslog
+      - /var/log/daemon.log
+      - /var/log/kern.log
+      - /var/log/dpkg.log
+      - /var/log/apt/history.log
+      - /var/log/yum.log
+      - /var/log/messages
+      - /var/log/dmesg
+
+  environment:
+    ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
+    ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
+
+  tasks:
+    - name: set etcd_access_addresses
+      set_fact:
+        etcd_access_addresses: |-
+          {% for item in groups['etcd'] -%}
+            https://{{ item }}:2379{% if not loop.last %},{% endif %}
+          {%- endfor %}
+      when: "'etcd' in groups"
+
+    - name: Storing commands output
+      shell: "{{ item.cmd }} &> {{ item.name }}"
+      failed_when: false
+      with_items: "{{ commands }}"
+      when: item.when | default(True)
+      no_log: True
+
+    - name: Fetch results
+      fetch: src={{ item.name }} dest=/tmp/{{ archive_dirname }}/commands
+      with_items: "{{ commands }}"
+      when: item.when | default(True)
+      failed_when: false
+
+    - name: Fetch logs
+      fetch: src={{ item }} dest=/tmp/{{ archive_dirname }}/logs
+      with_items: "{{ logs }}"
+      failed_when: false
+
+    - name: Pack results and logs
+      archive:
+        path: "/tmp/{{ archive_dirname }}"
+        dest: "{{ dir|default('.') }}/logs.tar.gz"
+        remove: true
+        mode: 0640
+      delegate_to: localhost
+      connection: local
+      become: false
+      run_once: true
+
+    - name: Clean up collected command outputs
+      file: path={{ item.name }} state=absent
+      with_items: "{{ commands }}"

kubespray/scripts/download_hash.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+
+# After a new version of Kubernetes has been released,
+# run this script to update roles/download/defaults/main.yml
+# with new hashes.
+
+import hashlib
+import sys
+
+import requests
+from ruamel.yaml import YAML
+
+MAIN_YML = "../roles/download/defaults/main.yml"
+
+def open_main_yaml():
+    yaml = YAML()
+    yaml.explicit_start = True
+    yaml.preserve_quotes = True
+    yaml.width = 4096
+
+    with open(MAIN_YML, "r") as main_yml:
+        data = yaml.load(main_yml)
+
+    return data, yaml
+
+
+def download_hash(versions):
+    architectures = ["arm", "arm64", "amd64", "ppc64le"]
+    downloads = ["kubelet", "kubectl", "kubeadm"]
+
+    data, yaml = open_main_yaml()
+
+    for download in downloads:
+        checksum_name = f"{download}_checksums"
+        for arch in architectures:
+            for version in versions:
+                if not version.startswith("v"):
+                    version = f"v{version}"
+                url = f"https://storage.googleapis.com/kubernetes-release/release/{version}/bin/linux/{arch}/{download}"
+                download_file = requests.get(url, allow_redirects=True)
+                download_file.raise_for_status()
+                sha256sum = hashlib.sha256(download_file.content).hexdigest()
+                data[checksum_name][arch][version] = sha256sum
+
+    with open(MAIN_YML, "w") as main_yml:
+        yaml.dump(data, main_yml)
+        print(f"\n\nUpdated {MAIN_YML}\n")
+
+
+def usage():
+    print(f"USAGE:\n    {sys.argv[0]} [k8s_version1] [[k8s_version2]....[k8s_versionN]]")
+
+
+def main(argv=None):
+    if not argv:
+        argv = sys.argv[1:]
+    if not argv:
+        usage()
+        return 1
+    download_hash(argv)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

kubespray/scripts/download_hash.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+set -eo pipefail
+
+VERSIONS="$@"
+ARCHITECTURES="arm arm64 amd64 ppc64le"
+DOWNLOADS="kubelet kubectl kubeadm"
+DOWNLOAD_DIR="tmp/kubeadm_hasher"
+
+if [ -z "$VERSIONS" ]; then
+  echo "USAGE: $0 <versions>"
+  exit 1
+fi
+
+mkdir -p ${DOWNLOAD_DIR}
+for download in ${DOWNLOADS}; do
+  echo -e "\n\n${download}_checksums:"
+  for arch in ${ARCHITECTURES}; do
+    echo -e "  ${arch}:"
+    for version in ${VERSIONS}; do
+      TARGET="${DOWNLOAD_DIR}/${download}-$version-$arch"
+      if [ ! -f ${TARGET} ]; then
+        curl -L -f -S -s -o ${TARGET} "https://storage.googleapis.com/kubernetes-release/release/${version}/bin/linux/${arch}/${download}"
+      fi
+      echo -e "    ${version}: $(sha256sum ${TARGET} | awk '{print $1}')"
+    done
+  done
+done
+echo -e "\n\nAdd these values to roles/download/defaults/main.yml"

kubespray/scripts/gen_tags.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+set -eo pipefail
+
+#Generate MD formatted tags from roles and cluster yaml files
+printf "|%25s |%9s\n" "Tag name" "Used for"
+echo "|--------------------------|---------"
+tags=$(grep -r tags: . | perl -ne '/tags:\s\[?(([\w\-_]+,?\s?)+)/ && printf "%s ", "$1"'|\
+  perl -ne 'print join "\n", split /\s|,/' | sort -u)
+for tag in $tags; do
+  match=$(cat docs/ansible.md | perl -ne "/^\|\s+${tag}\s\|\s+((\S+\s?)+)/ && printf \$1")
+  printf "|%25s |%s\n" "${tag}" " ${match}"
+done

kubespray/scripts/gitlab-branch-cleanup/.gitignore

@@ -0,0 +1,2 @@
+openrc
+venv

kubespray/scripts/gitlab-branch-cleanup/README.md

@@ -0,0 +1,24 @@
+# gitlab-branch-cleanup
+
+Cleanup old branches in a GitLab project
+
+## Installation
+
+```shell
+pip install -r requirements.txt
+python main.py --help
+```
+
+## Usage
+
+```console
+$ export GITLAB_API_TOKEN=foobar
+$ python main.py kargo-ci/kubernetes-sigs-kubespray
+Deleting branch pr-5220-containerd-systemd from 2020-02-17 ...
+Deleting branch pr-5561-feature/cinder_csi_fixes from 2020-02-17 ...
+Deleting branch pr-5607-add-flatcar from 2020-02-17 ...
+Deleting branch pr-5616-fix-typo from 2020-02-17 ...
+Deleting branch pr-5634-helm_310 from 2020-02-18 ...
+Deleting branch pr-5644-patch-1 from 2020-02-15 ...
+Deleting branch pr-5647-master from 2020-02-17 ...
+```

kubespray/scripts/gitlab-branch-cleanup/main.py

@@ -0,0 +1,38 @@
+import gitlab
+import argparse
+import os
+import sys
+from datetime import timedelta, datetime, timezone
+
+
+parser = argparse.ArgumentParser(
+    description='Cleanup old branches in a GitLab project')
+parser.add_argument('--api', default='https://gitlab.com/',
+    help='URL of GitLab API, defaults to gitlab.com')
+parser.add_argument('--age', type=int, default=30,
+    help='Delete branches older than this many days')
+parser.add_argument('--prefix', default='pr-',
+    help='Cleanup only branches with names matching this prefix')
+parser.add_argument('--dry-run', action='store_true',
+    help='Do not delete anything')
+parser.add_argument('project',
+    help='Path of the GitLab project')
+
+args = parser.parse_args()
+limit = datetime.now(timezone.utc) - timedelta(days=args.age)
+
+if os.getenv('GITLAB_API_TOKEN', '') == '':
+    print("Environment variable GITLAB_API_TOKEN is required.")
+    sys.exit(2)
+
+gl = gitlab.Gitlab(args.api, private_token=os.getenv('GITLAB_API_TOKEN'))
+gl.auth()
+
+p = gl.projects.get(args.project)
+for b in p.branches.list(all=True):
+    date = datetime.fromisoformat(b.commit['created_at'])
+    if date < limit and not b.protected and not b.default and b.name.startswith(args.prefix):
+        print("Deleting branch %s from %s ..." %
+            (b.name, date.date().isoformat()))
+        if not args.dry_run:
+            b.delete()

kubespray/scripts/gitlab-branch-cleanup/requirements.txt

@@ -0,0 +1 @@
+python-gitlab

kubespray/scripts/gitlab-runner.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+docker run -d --name gitlab-runner --restart always -v /srv/gitlab-runner/cache:/srv/gitlab-runner/cache -v /srv/gitlab-runner/config:/etc/gitlab-runner -v /var/run/docker.sock:/var/run/docker.sock gitlab/gitlab-runner:v1.10.0
+
+#
+#/srv/gitlab-runner/config# cat config.toml
+#concurrent = 10
+#check_interval = 1
+
+#[[runners]]
+#  name = "2edf3d71fe19"
+#  url = "https://gitlab.com"
+#  token = "THE TOKEN-CHANGEME"
+#  executor = "docker"
+#  [runners.docker]
+#    tls_verify = false
+#    image = "docker:latest"
+#    privileged = true
+#    disable_cache = false
+#    cache_dir = "/srv/gitlab-runner/cache"
+#    volumes = ["/var/run/docker.sock:/var/run/docker.sock", "/srv/gitlab-runner/cache:/cache:rw"]
+#  [runners.cache]

kubespray/scripts/openstack-cleanup/.gitignore

@@ -0,0 +1 @@
+openrc

kubespray/scripts/openstack-cleanup/README.md

@@ -0,0 +1,21 @@
+# openstack-cleanup
+
+Tool to deletes openstack servers older than a specific age (default 4h).
+
+Useful to cleanup orphan servers that are left behind when CI is manually cancelled or fails unexpectedly.
+
+## Installation
+
+```shell
+pip install -r requirements.txt
+python main.py --help
+```
+
+## Usage
+
+```console
+$ python main.py
+This will delete VMs... (ctrl+c to cancel)
+Will delete server example1
+Will delete server example2
+```

kubespray/scripts/openstack-cleanup/main.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+import argparse
+import openstack
+import logging
+import datetime
+import time
+
+DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
+PAUSE_SECONDS = 5
+
+log = logging.getLogger('openstack-cleanup')
+
+parser = argparse.ArgumentParser(description='Cleanup OpenStack resources')
+
+parser.add_argument('-v', '--verbose', action='store_true',
+                    help='Increase verbosity')
+parser.add_argument('--hours', type=int, default=4,
+                    help='Age (in hours) of VMs to cleanup (default: 4h)')
+parser.add_argument('--dry-run', action='store_true',
+                    help='Do not delete anything')
+
+args = parser.parse_args()
+
+oldest_allowed = datetime.datetime.now() - datetime.timedelta(hours=args.hours)
+
+
+def main():
+    if args.dry_run:
+        print('Running in dry-run mode')
+    else:
+        print('This will delete resources... (ctrl+c to cancel)')
+        time.sleep(PAUSE_SECONDS)
+
+    conn = openstack.connect()
+
+    print('Servers...')
+    map_if_old(conn.compute.delete_server,
+               conn.compute.servers())
+
+    print('Security groups...')
+    map_if_old(conn.network.delete_security_group,
+               conn.network.security_groups())
+
+    print('Ports...')
+    try:
+        map_if_old(conn.network.delete_port,
+                   conn.network.ports())
+    except openstack.exceptions.ConflictException as ex:
+        # Need to find subnet-id which should be removed from a router
+        for sn in conn.network.subnets():
+            try:
+                fn_if_old(conn.network.delete_subnet, sn)
+            except openstack.exceptions.ConflictException:
+                for r in conn.network.routers():
+                    print("Deleting subnet %s from router %s", sn, r)
+                    try:
+                        conn.network.remove_interface_from_router(
+                            r, subnet_id=sn.id)
+                    except Exception as ex:
+                        print("Failed to delete subnet from router as %s", ex)
+
+        # After removing unnecessary subnet from router, retry to delete ports
+        map_if_old(conn.network.delete_port,
+                   conn.network.ports())
+
+    print('Subnets...')
+    map_if_old(conn.network.delete_subnet,
+               conn.network.subnets())
+
+    print('Networks...')
+    for n in conn.network.networks():
+        if not n.is_router_external:
+            fn_if_old(conn.network.delete_network, n)
+
+
+# runs the given fn to all elements of the that are older than allowed
+def map_if_old(fn, items):
+    for item in items:
+        fn_if_old(fn, item)
+
+
+# run the given fn function only if the passed item is older than allowed
+def fn_if_old(fn, item):
+    created_at = datetime.datetime.strptime(item.created_at, DATE_FORMAT)
+    if item.name == "default":  # skip default security group
+        return
+    if created_at < oldest_allowed:
+        print('Will delete %(name)s (%(id)s)' % item)
+        if not args.dry_run:
+            fn(item)
+
+
+if __name__ == '__main__':
+    # execute only if run as a script
+    main()

kubespray/scripts/openstack-cleanup/requirements.txt

@@ -0,0 +1 @@
+openstacksdk>=0.43.0

kubespray/scripts/premoderator.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+# A naive premoderation script to allow Gitlab CI pipeline on a specific PRs' comment
+# Exits with 0, if the pipeline is good to go
+# Exits with 1, if the user is not allowed to start pipeline
+# Exits with 2, if script is unable to get issue id from CI_BUILD_REF_NAME variable
+# Exits with 3, if missing the magic comment in the pipeline to start the pipeline
+
+CURL_ARGS="-fs --retry 4 --retry-delay 5"
+MAGIC="${MAGIC:-ci check this}"
+exit_code=0
+
+# Get PR number from CI_BUILD_REF_NAME
+issue=$(echo ${CI_BUILD_REF_NAME} | perl -ne '/^pr-(\d+)-\S+$/ && print $1')
+
+if [ "$issue" = "" ]; then
+  echo "Unable to get issue id from: $CI_BUILD_REF_NAME"
+  exit 2
+fi
+
+echo "Fetching labels from PR $issue"
+labels=$(curl ${CURL_ARGS} "https://api.github.com/repos/kubernetes-sigs/kubespray/issues/${issue}?access_token=${GITHUB_TOKEN}" | jq '{labels: .labels}' | jq '.labels[].name' | jq -s '')
+labels_to_patch=$(echo -n $labels | jq '. + ["needs-ci-auth"]' | tr -d "\n")
+
+echo "Checking for '$MAGIC' comment in PR $issue"
+
+# Get the user name from the PR comments with the wanted magic incantation casted
+user=$(curl ${CURL_ARGS} "https://api.github.com/repos/kubernetes-sigs/kubespray/issues/${issue}/comments" | jq -M "map(select(.body | contains (\"$MAGIC\"))) | .[0] .user.login" | tr -d '"')
+
+# Check for the required user group membership to allow (exit 0) or decline (exit >0) the pipeline
+if [ "$user" = "" ] || [ "$user" = "null" ]; then
+  echo "Missing '$MAGIC' comment from one of the OWNERS"
+  exit_code=3
+else
+  echo "Found comment from user: $user"
+
+  curl ${CURL_ARGS} "https://api.github.com/orgs/kubernetes-sigs/members/${user}"
+
+  if [ $? -ne 0 ]; then
+    echo "User does not have permissions to start CI run"
+    exit_code=1
+  else
+    labels_to_patch=$(echo -n $labels | jq '. - ["needs-ci-auth"]' | tr -d "\n")
+    exit_code=0
+    echo "$user has allowed CI to start"
+  fi
+fi
+
+# Patch labels on PR
+curl ${CURL_ARGS} --request PATCH "https://api.github.com/repos/kubernetes-sigs/kubespray/issues/${issue}?access_token=${GITHUB_TOKEN}" -H "Content-Type: application/json" -d "{\"labels\": ${labels_to_patch}}"
+
+exit $exit_code