update

2023-05-30 14:44:26 +09:00
parent 9a3174deef
commit 4c32a7239d
2598 changed files with 164595 additions and 487 deletions
--- a/kubespray/roles/etcd/templates/etcd-docker.service.j2
+++ b/kubespray/roles/etcd/templates/etcd-docker.service.j2
@@ -0,0 +1,18 @@
+[Unit]
+Description=etcd docker wrapper
+Wants=docker.socket
+After=docker.service
+
+[Service]
+User=root
+PermissionsStartOnly=true
+EnvironmentFile=-/etc/etcd.env
+ExecStart={{ bin_dir }}/etcd
+ExecStartPre=-{{ docker_bin_dir }}/docker rm -f {{ etcd_member_name | default("etcd") }}
+ExecStop={{ docker_bin_dir }}/docker stop {{ etcd_member_name | default("etcd") }}
+Restart=always
+RestartSec=15s
+TimeoutStartSec=30s
+
+[Install]
+WantedBy=multi-user.target
--- a/kubespray/roles/etcd/templates/etcd-events-docker.service.j2
+++ b/kubespray/roles/etcd/templates/etcd-events-docker.service.j2
@@ -0,0 +1,18 @@
+[Unit]
+Description=etcd docker wrapper
+Wants=docker.socket
+After=docker.service
+
+[Service]
+User=root
+PermissionsStartOnly=true
+EnvironmentFile=-/etc/etcd-events.env
+ExecStart={{ bin_dir }}/etcd-events
+ExecStartPre=-{{ docker_bin_dir }}/docker rm -f {{ etcd_member_name }}-events
+ExecStop={{ docker_bin_dir }}/docker stop {{ etcd_member_name }}-events
+Restart=always
+RestartSec=15s
+TimeoutStartSec=30s
+
+[Install]
+WantedBy=multi-user.target
--- a/kubespray/roles/etcd/templates/etcd-events-host.service.j2
+++ b/kubespray/roles/etcd/templates/etcd-events-host.service.j2
@@ -0,0 +1,16 @@
+[Unit]
+Description=etcd
+After=network.target
+
+[Service]
+Type=notify
+User=root
+EnvironmentFile=/etc/etcd-events.env
+ExecStart={{ bin_dir }}/etcd
+NotifyAccess=all
+Restart=always
+RestartSec=10s
+LimitNOFILE=40000
+
+[Install]
+WantedBy=multi-user.target
--- a/kubespray/roles/etcd/templates/etcd-events.env.j2
+++ b/kubespray/roles/etcd/templates/etcd-events.env.j2
@@ -0,0 +1,43 @@
+ETCD_DATA_DIR={{ etcd_events_data_dir }}
+ETCD_ADVERTISE_CLIENT_URLS={{ etcd_events_client_url }}
+ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_events_peer_url }}
+ETCD_INITIAL_CLUSTER_STATE={% if etcd_events_cluster_is_healthy.rc == 0 | bool %}existing{% else %}new{% endif %}
+
+ETCD_METRICS={{ etcd_metrics }}
+ETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address }}:2383,https://127.0.0.1:2383
+ETCD_ELECTION_TIMEOUT={{ etcd_election_timeout }}
+ETCD_HEARTBEAT_INTERVAL={{ etcd_heartbeat_interval }}
+ETCD_INITIAL_CLUSTER_TOKEN=k8s_events_etcd
+ETCD_LISTEN_PEER_URLS=https://{{ etcd_address }}:2382
+ETCD_NAME={{ etcd_member_name }}-events
+ETCD_PROXY=off
+ETCD_INITIAL_CLUSTER={{ etcd_events_peer_addresses }}
+ETCD_AUTO_COMPACTION_RETENTION={{ etcd_compaction_retention }}
+{% if etcd_snapshot_count is defined %}
+ETCD_SNAPSHOT_COUNT={{ etcd_snapshot_count }}
+{% endif %}
+{% if etcd_quota_backend_bytes is defined %}
+ETCD_QUOTA_BACKEND_BYTES={{ etcd_quota_backend_bytes }}
+{% endif %}
+{% if etcd_max_request_bytes is defined %}
+ETCD_MAX_REQUEST_BYTES={{ etcd_max_request_bytes }}
+{% endif %}
+
+# TLS settings
+ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
+ETCD_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem
+ETCD_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem
+ETCD_CLIENT_CERT_AUTH={{ etcd_secure_client | lower}}
+
+ETCD_PEER_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
+ETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem
+ETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem
+ETCD_PEER_CLIENT_CERT_AUTH={{ etcd_peer_client_auth }}
+
+{% if etcd_tls_cipher_suites is defined %}
+ETCD_CIPHER_SUITES={% for tls in etcd_tls_cipher_suites %}{{ tls }}{{ "," if not loop.last else "" }}{% endfor %}
+{% endif %}
+
+{% for key, value in etcd_extra_vars.items() %}
+{{ key }}={{ value }}
+{% endfor %}
--- a/kubespray/roles/etcd/templates/etcd-events.j2
+++ b/kubespray/roles/etcd/templates/etcd-events.j2
@@ -0,0 +1,21 @@
+#!/bin/bash
+{{ docker_bin_dir }}/docker run \
+  --restart=on-failure:5 \
+  --env-file=/etc/etcd-events.env \
+  --net=host \
+  -v /etc/ssl/certs:/etc/ssl/certs:ro \
+  -v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \
+  -v {{ etcd_events_data_dir }}:{{ etcd_events_data_dir }}:rw \
+  {% if etcd_memory_limit is defined %}
+  --memory={{ etcd_memory_limit|regex_replace('Mi', 'M') }} \
+  {% endif %}
+  {% if etcd_cpu_limit is defined %}
+  --cpu-shares={{ etcd_cpu_limit|regex_replace('m', '') }} \
+  {% endif %}
+  {% if etcd_blkio_weight is defined %}
+  --blkio-weight={{ etcd_blkio_weight }} \
+  {% endif %}
+  --name={{ etcd_member_name }}-events \
+  {{ etcd_image_repo }}:{{ etcd_image_tag }} \
+  /usr/local/bin/etcd \
+  "$@"
--- a/kubespray/roles/etcd/templates/etcd-host.service.j2
+++ b/kubespray/roles/etcd/templates/etcd-host.service.j2
@@ -0,0 +1,16 @@
+[Unit]
+Description=etcd
+After=network.target
+
+[Service]
+Type=notify
+User=root
+EnvironmentFile=/etc/etcd.env
+ExecStart={{ bin_dir }}/etcd
+NotifyAccess=all
+Restart=always
+RestartSec=10s
+LimitNOFILE=40000
+
+[Install]
+WantedBy=multi-user.target
--- a/kubespray/roles/etcd/templates/etcd.env.j2
+++ b/kubespray/roles/etcd/templates/etcd.env.j2
@@ -0,0 +1,68 @@
+# Environment file for etcd {{ etcd_version }}
+ETCD_DATA_DIR={{ etcd_data_dir }}
+ETCD_ADVERTISE_CLIENT_URLS={{ etcd_client_url }}
+ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_peer_url }}
+ETCD_INITIAL_CLUSTER_STATE={% if etcd_cluster_is_healthy.rc == 0 | bool %}existing{% else %}new{% endif %}
+
+ETCD_METRICS={{ etcd_metrics }}
+{% if etcd_metrics_port is defined %}
+ETCD_LISTEN_METRICS_URLS=http://{{ etcd_address }}:{{ etcd_metrics_port }},http://127.0.0.1:{{ etcd_metrics_port }}
+{% endif %}
+ETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address }}:2379,https://127.0.0.1:2379
+ETCD_ELECTION_TIMEOUT={{ etcd_election_timeout }}
+ETCD_HEARTBEAT_INTERVAL={{ etcd_heartbeat_interval }}
+ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd
+ETCD_LISTEN_PEER_URLS=https://{{ etcd_address }}:2380
+ETCD_NAME={{ etcd_member_name }}
+ETCD_PROXY=off
+ETCD_INITIAL_CLUSTER={{ etcd_peer_addresses }}
+ETCD_AUTO_COMPACTION_RETENTION={{ etcd_compaction_retention }}
+{% if etcd_snapshot_count is defined %}
+ETCD_SNAPSHOT_COUNT={{ etcd_snapshot_count }}
+{% endif %}
+{% if etcd_quota_backend_bytes is defined %}
+ETCD_QUOTA_BACKEND_BYTES={{ etcd_quota_backend_bytes }}
+{% endif %}
+{% if etcd_max_request_bytes is defined %}
+ETCD_MAX_REQUEST_BYTES={{ etcd_max_request_bytes }}
+{% endif %}
+{% if etcd_log_level is defined %}
+ETCD_LOG_LEVEL={{ etcd_log_level }}
+{% endif %}
+{% if etcd_max_snapshots is defined %}
+ETCD_MAX_SNAPSHOTS={{ etcd_max_snapshots }}
+{% endif %}
+{% if etcd_max_wals is defined %}
+ETCD_MAX_WALS={{ etcd_max_wals }}
+{% endif %}
+# Flannel need etcd v2 API
+ETCD_ENABLE_V2=true
+
+# TLS settings
+ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
+ETCD_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem
+ETCD_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem
+ETCD_CLIENT_CERT_AUTH={{ etcd_secure_client | lower}}
+
+ETCD_PEER_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
+ETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem
+ETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem
+ETCD_PEER_CLIENT_CERT_AUTH={{ etcd_peer_client_auth }}
+
+{% if etcd_tls_cipher_suites is defined %}
+ETCD_CIPHER_SUITES={% for tls in etcd_tls_cipher_suites %}{{ tls }}{{ "," if not loop.last else "" }}{% endfor %}
+{% endif %}
+
+{% for key, value in etcd_extra_vars.items() %}
+{{ key }}={{ value }}
+{% endfor %}
+
+# CLI settings
+ETCDCTL_ENDPOINTS=https://127.0.0.1:2379
+ETCDCTL_CACERT={{ etcd_cert_dir }}/ca.pem
+ETCDCTL_KEY={{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem
+ETCDCTL_CERT={{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem
+
+# ETCD 3.5.x issue
+# https://groups.google.com/a/kubernetes.io/g/dev/c/B7gJs88XtQc/m/rSgNOzV2BwAJ?utm_medium=email&utm_source=footer
+ETCD_EXPERIMENTAL_INITIAL_CORRUPT_CHECK={{ etcd_experimental_initial_corrupt_check }}
--- a/kubespray/roles/etcd/templates/etcd.j2
+++ b/kubespray/roles/etcd/templates/etcd.j2
@@ -0,0 +1,21 @@
+#!/bin/bash
+{{ docker_bin_dir }}/docker run \
+  --restart=on-failure:5 \
+  --env-file=/etc/etcd.env \
+  --net=host \
+  -v /etc/ssl/certs:/etc/ssl/certs:ro \
+  -v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \
+  -v {{ etcd_data_dir }}:{{ etcd_data_dir }}:rw \
+{% if etcd_memory_limit is defined %}
+  --memory={{ etcd_memory_limit|regex_replace('Mi', 'M') }} \
+{% endif %}
+{% if etcd_cpu_limit is defined %}
+  --cpu-shares={{ etcd_cpu_limit|regex_replace('m', '') }} \
+{% endif %}
+{% if etcd_blkio_weight is defined %}
+  --blkio-weight={{ etcd_blkio_weight }} \
+{% endif %}
+  --name={{ etcd_member_name | default("etcd") }} \
+  {{ etcd_image_repo }}:{{ etcd_image_tag }} \
+  /usr/local/bin/etcd \
+  "$@"
--- a/kubespray/roles/etcd/templates/make-ssl-etcd.sh.j2
+++ b/kubespray/roles/etcd/templates/make-ssl-etcd.sh.j2
@@ -0,0 +1,103 @@
+#!/bin/bash
+
+# Author: Smana smainklh@gmail.com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o pipefail
+usage()
+{
+    cat << EOF
+Create self signed certificates
+
+Usage : $(basename $0) -f <config> [-d <ssldir>]
+      -h | --help         : Show this message
+      -f | --config       : Openssl configuration file
+      -d | --ssldir       : Directory where the certificates will be installed
+
+               ex :
+               $(basename $0) -f openssl.conf -d /srv/ssl
+EOF
+}
+
+# Options parsing
+while (($#)); do
+    case "$1" in
+        -h | --help)   usage;   exit 0;;
+        -f | --config) CONFIG=${2}; shift 2;;
+        -d | --ssldir) SSLDIR="${2}"; shift 2;;
+        *)
+            usage
+            echo "ERROR : Unknown option"
+            exit 3
+        ;;
+    esac
+done
+
+if [ -z ${CONFIG} ]; then
+    echo "ERROR: the openssl configuration file is missing. option -f"
+    exit 1
+fi
+if [ -z ${SSLDIR} ]; then
+    SSLDIR="/etc/ssl/etcd"
+fi
+
+tmpdir=$(mktemp -d /tmp/etcd_cacert.XXXXXX)
+trap 'rm -rf "${tmpdir}"' EXIT
+cd "${tmpdir}"
+
+mkdir -p "${SSLDIR}"
+
+# Root CA
+if [ -e "$SSLDIR/ca-key.pem" ]; then
+    # Reuse existing CA
+    cp $SSLDIR/{ca.pem,ca-key.pem} .
+else
+    openssl genrsa -out ca-key.pem {{certificates_key_size}} > /dev/null 2>&1
+    openssl req -x509 -new -nodes -key ca-key.pem -days {{certificates_duration}} -out ca.pem -subj "/CN=etcd-ca" > /dev/null 2>&1
+fi
+
+# ETCD member
+if [ -n "$MASTERS" ]; then
+    for host in $MASTERS; do
+        cn="${host%%.*}"
+        # Member key
+        openssl genrsa -out member-${host}-key.pem {{certificates_key_size}} > /dev/null 2>&1
+        openssl req -new -key member-${host}-key.pem -out member-${host}.csr -subj "/CN=etcd-member-${cn}" -config ${CONFIG} > /dev/null 2>&1
+        openssl x509 -req -in member-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member-${host}.pem -days {{certificates_duration}} -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
+
+        # Admin key
+        openssl genrsa -out admin-${host}-key.pem {{certificates_key_size}} > /dev/null 2>&1
+        openssl req -new -key admin-${host}-key.pem -out admin-${host}.csr -subj "/CN=etcd-admin-${cn}" > /dev/null 2>&1
+        openssl x509 -req -in admin-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out admin-${host}.pem -days {{certificates_duration}} -extensions ssl_client  -extfile ${CONFIG} > /dev/null 2>&1
+    done
+fi
+
+# Node keys
+if [ -n "$HOSTS" ]; then
+    for host in $HOSTS; do
+        cn="${host%%.*}"
+        openssl genrsa -out node-${host}-key.pem {{certificates_key_size}} > /dev/null 2>&1
+        openssl req -new -key node-${host}-key.pem -out node-${host}.csr -subj "/CN=etcd-node-${cn}" > /dev/null 2>&1
+        openssl x509 -req -in node-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out node-${host}.pem -days {{certificates_duration}} -extensions ssl_client  -extfile ${CONFIG} > /dev/null 2>&1
+    done
+fi
+
+# Install certs
+if [ -e "$SSLDIR/ca-key.pem" ]; then
+    # No pass existing CA
+    rm -f ca.pem ca-key.pem
+fi
+
+mv *.pem ${SSLDIR}/
--- a/kubespray/roles/etcd/templates/openssl.conf.j2
+++ b/kubespray/roles/etcd/templates/openssl.conf.j2
@@ -0,0 +1,45 @@
+{% set counter = {'dns': 2,'ip': 1,} %}{% macro increment(dct, key, inc=1)%}{% if dct.update({key: dct[key] + inc}) %} {% endif %}{% endmacro %}[req]
+req_extensions = v3_req
+distinguished_name = req_distinguished_name
+
+[req_distinguished_name]
+
+[ v3_req ]
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+subjectAltName = @alt_names
+
+[ ssl_client ]
+extendedKeyUsage = clientAuth, serverAuth
+basicConstraints = CA:FALSE
+subjectKeyIdentifier=hash
+authorityKeyIdentifier=keyid,issuer
+subjectAltName = @alt_names
+
+[ v3_ca ]
+basicConstraints = CA:TRUE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+subjectAltName = @alt_names
+authorityKeyIdentifier=keyid:always,issuer
+
+[alt_names]
+DNS.1 = localhost
+{% for host in groups['etcd'] %}
+DNS.{{ counter["dns"] }} = {{ host }}{{ increment(counter, 'dns') }}
+{% endfor %}
+{% if apiserver_loadbalancer_domain_name is defined %}
+DNS.{{ counter["dns"] }} = {{ apiserver_loadbalancer_domain_name }}{{ increment(counter, 'dns') }}
+{% endif %}
+{% for etcd_alt_name in etcd_cert_alt_names %}
+DNS.{{ counter["dns"] }} = {{ etcd_alt_name }}{{ increment(counter, 'dns') }}
+{% endfor %}
+{% for host in groups['etcd'] %}
+{% if hostvars[host]['access_ip'] is defined  %}
+IP.{{ counter["ip"] }} = {{ hostvars[host]['access_ip'] }}{{ increment(counter, 'ip') }}
+{% endif %}
+IP.{{ counter["ip"] }} = {{ hostvars[host]['ip'] | default(fallback_ips[host]) }}{{ increment(counter, 'ip') }}
+{% endfor %}
+{% for cert_alt_ip in etcd_cert_alt_ips %}
+IP.{{ counter["ip"] }} = {{ cert_alt_ip }}{{ increment(counter, 'ip') }}
+{% endfor %}
+IP.{{ counter["ip"] }} = 127.0.0.1