kubespray 2.24 추가

contrib/terraform/nifcloud/modules/kubernetes-cluster/main.tf

@@ -0,0 +1,301 @@
+#################################################
+##
+## Local variables
+##
+locals {
+  # e.g. east-11 is 11
+  az_num = reverse(split("-", var.availability_zone))[0]
+  # e.g. east-11 is e11
+  az_short_name = "${substr(reverse(split("-", var.availability_zone))[1], 0, 1)}${local.az_num}"
+
+  # Port used by the protocol
+  port_ssh     = 22
+  port_kubectl = 6443
+  port_kubelet = 10250
+
+  # calico: https://docs.tigera.io/calico/latest/getting-started/kubernetes/requirements#network-requirements
+  port_bgp   = 179
+  port_vxlan = 4789
+  port_etcd  = 2379
+}
+
+#################################################
+##
+## General
+##
+
+# data
+data "nifcloud_image" "this" {
+  image_name = var.image_name
+}
+
+# private lan
+resource "nifcloud_private_lan" "this" {
+  private_lan_name  = "${var.prefix}lan"
+  availability_zone = var.availability_zone
+  cidr_block        = var.private_network_cidr
+  accounting_type   = var.accounting_type
+}
+
+#################################################
+##
+## Bastion
+##
+resource "nifcloud_security_group" "bn" {
+  group_name        = "${var.prefix}bn"
+  description       = "${var.prefix} bastion"
+  availability_zone = var.availability_zone
+}
+
+resource "nifcloud_instance" "bn" {
+
+  instance_id    = "${local.az_short_name}${var.prefix}bn01"
+  security_group = nifcloud_security_group.bn.group_name
+  instance_type  = var.instance_type_bn
+
+  user_data = templatefile("${path.module}/templates/userdata.tftpl", {
+    private_ip_address = var.private_ip_bn
+    ssh_port           = local.port_ssh
+    hostname           = "${local.az_short_name}${var.prefix}bn01"
+  })
+
+  availability_zone = var.availability_zone
+  accounting_type   = var.accounting_type
+  image_id          = data.nifcloud_image.this.image_id
+  key_name          = var.instance_key_name
+
+  network_interface {
+    network_id = "net-COMMON_GLOBAL"
+  }
+  network_interface {
+    network_id = nifcloud_private_lan.this.network_id
+    ip_address = "static"
+  }
+
+  # The image_id changes when the OS image type is demoted from standard to public.
+  lifecycle {
+    ignore_changes = [
+      image_id,
+      user_data,
+    ]
+  }
+}
+
+#################################################
+##
+## Control Plane
+##
+resource "nifcloud_security_group" "cp" {
+  group_name        = "${var.prefix}cp"
+  description       = "${var.prefix} control plane"
+  availability_zone = var.availability_zone
+}
+
+resource "nifcloud_instance" "cp" {
+  for_each = var.instances_cp
+
+  instance_id    = "${local.az_short_name}${var.prefix}${each.key}"
+  security_group = nifcloud_security_group.cp.group_name
+  instance_type  = var.instance_type_cp
+  user_data = templatefile("${path.module}/templates/userdata.tftpl", {
+    private_ip_address = each.value.private_ip
+    ssh_port           = local.port_ssh
+    hostname           = "${local.az_short_name}${var.prefix}${each.key}"
+  })
+
+  availability_zone = var.availability_zone
+  accounting_type   = var.accounting_type
+  image_id          = data.nifcloud_image.this.image_id
+  key_name          = var.instance_key_name
+
+  network_interface {
+    network_id = "net-COMMON_GLOBAL"
+  }
+  network_interface {
+    network_id = nifcloud_private_lan.this.network_id
+    ip_address = "static"
+  }
+
+  # The image_id changes when the OS image type is demoted from standard to public.
+  lifecycle {
+    ignore_changes = [
+      image_id,
+      user_data,
+    ]
+  }
+}
+
+resource "nifcloud_load_balancer" "this" {
+  load_balancer_name = "${local.az_short_name}${var.prefix}cp"
+  accounting_type    = var.accounting_type
+  balancing_type     = 1 // Round-Robin
+  load_balancer_port = local.port_kubectl
+  instance_port      = local.port_kubectl
+  instances          = [for v in nifcloud_instance.cp : v.instance_id]
+  filter = concat(
+    [for k, v in nifcloud_instance.cp : v.public_ip],
+    [for k, v in nifcloud_instance.wk : v.public_ip],
+    var.additional_lb_filter,
+  )
+  filter_type = 1 // Allow
+}
+
+#################################################
+##
+## Worker
+##
+resource "nifcloud_security_group" "wk" {
+  group_name        = "${var.prefix}wk"
+  description       = "${var.prefix} worker"
+  availability_zone = var.availability_zone
+}
+
+resource "nifcloud_instance" "wk" {
+  for_each = var.instances_wk
+
+  instance_id    = "${local.az_short_name}${var.prefix}${each.key}"
+  security_group = nifcloud_security_group.wk.group_name
+  instance_type  = var.instance_type_wk
+  user_data = templatefile("${path.module}/templates/userdata.tftpl", {
+    private_ip_address = each.value.private_ip
+    ssh_port           = local.port_ssh
+    hostname           = "${local.az_short_name}${var.prefix}${each.key}"
+  })
+
+  availability_zone = var.availability_zone
+  accounting_type   = var.accounting_type
+  image_id          = data.nifcloud_image.this.image_id
+  key_name          = var.instance_key_name
+
+  network_interface {
+    network_id = "net-COMMON_GLOBAL"
+  }
+  network_interface {
+    network_id = nifcloud_private_lan.this.network_id
+    ip_address = "static"
+  }
+
+  # The image_id changes when the OS image type is demoted from standard to public.
+  lifecycle {
+    ignore_changes = [
+      image_id,
+      user_data,
+    ]
+  }
+}
+
+#################################################
+##
+## Security Group Rule: Kubernetes
+##
+
+# ssh
+resource "nifcloud_security_group_rule" "ssh_from_bastion" {
+  security_group_names = [
+    nifcloud_security_group.wk.group_name,
+    nifcloud_security_group.cp.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_ssh
+  to_port                    = local.port_ssh
+  protocol                   = "TCP"
+  source_security_group_name = nifcloud_security_group.bn.group_name
+}
+
+# kubectl
+resource "nifcloud_security_group_rule" "kubectl_from_worker" {
+  security_group_names = [
+    nifcloud_security_group.cp.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_kubectl
+  to_port                    = local.port_kubectl
+  protocol                   = "TCP"
+  source_security_group_name = nifcloud_security_group.wk.group_name
+}
+
+# kubelet
+resource "nifcloud_security_group_rule" "kubelet_from_worker" {
+  security_group_names = [
+    nifcloud_security_group.cp.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_kubelet
+  to_port                    = local.port_kubelet
+  protocol                   = "TCP"
+  source_security_group_name = nifcloud_security_group.wk.group_name
+}
+
+resource "nifcloud_security_group_rule" "kubelet_from_control_plane" {
+  security_group_names = [
+    nifcloud_security_group.wk.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_kubelet
+  to_port                    = local.port_kubelet
+  protocol                   = "TCP"
+  source_security_group_name = nifcloud_security_group.cp.group_name
+}
+
+#################################################
+##
+## Security Group Rule: calico
+##
+
+# vslan
+resource "nifcloud_security_group_rule" "vxlan_from_control_plane" {
+  security_group_names = [
+    nifcloud_security_group.wk.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_vxlan
+  to_port                    = local.port_vxlan
+  protocol                   = "UDP"
+  source_security_group_name = nifcloud_security_group.cp.group_name
+}
+
+resource "nifcloud_security_group_rule" "vxlan_from_worker" {
+  security_group_names = [
+    nifcloud_security_group.cp.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_vxlan
+  to_port                    = local.port_vxlan
+  protocol                   = "UDP"
+  source_security_group_name = nifcloud_security_group.wk.group_name
+}
+
+# bgp
+resource "nifcloud_security_group_rule" "bgp_from_control_plane" {
+  security_group_names = [
+    nifcloud_security_group.wk.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_bgp
+  to_port                    = local.port_bgp
+  protocol                   = "TCP"
+  source_security_group_name = nifcloud_security_group.cp.group_name
+}
+
+resource "nifcloud_security_group_rule" "bgp_from_worker" {
+  security_group_names = [
+    nifcloud_security_group.cp.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_bgp
+  to_port                    = local.port_bgp
+  protocol                   = "TCP"
+  source_security_group_name = nifcloud_security_group.wk.group_name
+}
+
+# etcd
+resource "nifcloud_security_group_rule" "etcd_from_worker" {
+  security_group_names = [
+    nifcloud_security_group.cp.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_etcd
+  to_port                    = local.port_etcd
+  protocol                   = "TCP"
+  source_security_group_name = nifcloud_security_group.wk.group_name
+}

contrib/terraform/nifcloud/modules/kubernetes-cluster/outputs.tf

@@ -0,0 +1,48 @@
+output "control_plane_lb" {
+  description = "The DNS name of LB for control plane"
+  value       = nifcloud_load_balancer.this.dns_name
+}
+
+output "security_group_name" {
+  description = "The security group used in the cluster"
+  value = {
+    bastion       = nifcloud_security_group.bn.group_name,
+    control_plane = nifcloud_security_group.cp.group_name,
+    worker        = nifcloud_security_group.wk.group_name,
+  }
+}
+
+output "private_network_id" {
+  description = "The private network used in the cluster"
+  value       = nifcloud_private_lan.this.id
+}
+
+output "bastion_info" {
+  description = "The basion information in cluster"
+  value = { (nifcloud_instance.bn.instance_id) : {
+    instance_id = nifcloud_instance.bn.instance_id,
+    unique_id   = nifcloud_instance.bn.unique_id,
+    private_ip  = nifcloud_instance.bn.private_ip,
+    public_ip   = nifcloud_instance.bn.public_ip,
+  } }
+}
+
+output "worker_info" {
+  description = "The worker information in cluster"
+  value = { for v in nifcloud_instance.wk : v.instance_id => {
+    instance_id = v.instance_id,
+    unique_id   = v.unique_id,
+    private_ip  = v.private_ip,
+    public_ip   = v.public_ip,
+  } }
+}
+
+output "control_plane_info" {
+  description = "The control plane information in cluster"
+  value = { for v in nifcloud_instance.cp : v.instance_id => {
+    instance_id = v.instance_id,
+    unique_id   = v.unique_id,
+    private_ip  = v.private_ip,
+    public_ip   = v.public_ip,
+  } }
+}

contrib/terraform/nifcloud/modules/kubernetes-cluster/templates/userdata.tftpl

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+#################################################
+##
+## IP Address
+##
+configure_private_ip_address () {
+  cat << EOS > /etc/netplan/01-netcfg.yaml
+network:
+    version: 2
+    renderer: networkd
+    ethernets:
+        ens192:
+            dhcp4: yes
+            dhcp6: yes
+            dhcp-identifier: mac
+        ens224:
+            dhcp4: no
+            dhcp6: no
+            addresses: [${private_ip_address}]
+EOS
+  netplan apply
+}
+configure_private_ip_address
+
+#################################################
+##
+## SSH
+##
+configure_ssh_port () {
+  sed -i 's/^#*Port [0-9]*/Port ${ssh_port}/' /etc/ssh/sshd_config
+}
+configure_ssh_port
+
+#################################################
+##
+## Hostname
+##
+hostnamectl set-hostname ${hostname}
+
+#################################################
+##
+## Disable swap files genereated by systemd-gpt-auto-generator
+##
+systemctl mask "dev-sda3.swap"

contrib/terraform/nifcloud/modules/kubernetes-cluster/terraform.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_version = ">=1.3.7"
+  required_providers {
+    nifcloud = {
+      source  = "nifcloud/nifcloud"
+      version = ">= 1.8.0, < 2.0.0"
+    }
+  }
+}

contrib/terraform/nifcloud/modules/kubernetes-cluster/variables.tf

@@ -0,0 +1,81 @@
+variable "availability_zone" {
+  description = "The availability zone"
+  type        = string
+}
+
+variable "prefix" {
+  description = "The prefix for the entire cluster"
+  type        = string
+  validation {
+    condition     = length(var.prefix) <= 5
+    error_message = "Must be a less than 5 character long."
+  }
+}
+
+variable "private_network_cidr" {
+  description = "The subnet of private network"
+  type        = string
+  validation {
+    condition     = can(cidrnetmask(var.private_network_cidr))
+    error_message = "Must be a valid IPv4 CIDR block address."
+  }
+}
+
+variable "private_ip_bn" {
+  description = "Private IP of bastion server"
+  type        = string
+}
+
+variable "instances_cp" {
+  type = map(object({
+    private_ip = string
+  }))
+}
+
+variable "instances_wk" {
+  type = map(object({
+    private_ip = string
+  }))
+}
+
+variable "instance_key_name" {
+  description = "The key name of the Key Pair to use for the instance"
+  type        = string
+}
+
+variable "instance_type_bn" {
+  description = "The instance type of bastion server"
+  type        = string
+}
+
+variable "instance_type_wk" {
+  description = "The instance type of worker"
+  type        = string
+}
+
+variable "instance_type_cp" {
+  description = "The instance type of control plane"
+  type        = string
+}
+
+variable "image_name" {
+  description = "The name of image"
+  type        = string
+}
+
+variable "additional_lb_filter" {
+  description = "Additional LB filter"
+  type        = list(string)
+}
+
+variable "accounting_type" {
+  type    = string
+  default = "1"
+  validation {
+    condition = anytrue([
+      var.accounting_type == "1", // Monthly
+      var.accounting_type == "2", // Pay per use
+    ])
+    error_message = "Must be a 1 or 2."
+  }
+}