From 2287d668ea1cff6f070a6e4b27bcab92631f6238 Mon Sep 17 00:00:00 2001 From: exhsgahm Date: Wed, 7 Sep 2022 14:49:58 +0900 Subject: [PATCH] add install doc --- doc/1_how_to_install_vpc.txt | 5 + doc/2_how_to_install_dev_cluster.txt | 421 +++++++++++++++++++++++++++ 2 files changed, 426 insertions(+) diff --git a/doc/1_how_to_install_vpc.txt b/doc/1_how_to_install_vpc.txt index e69de29..6d57bec 100644 --- a/doc/1_how_to_install_vpc.txt +++ b/doc/1_how_to_install_vpc.txt @@ -0,0 +1,5 @@ +cd terraform/tf-datasaker/ +terraform init +terraform plan +terraform apply +// terraform destroy diff --git a/doc/2_how_to_install_dev_cluster.txt b/doc/2_how_to_install_dev_cluster.txt index e69de29..d3a0f60 100644 --- a/doc/2_how_to_install_dev_cluster.txt +++ b/doc/2_how_to_install_dev_cluster.txt @@ -0,0 +1,421 @@ +export KOPS_STATE_STORE=s3://clusters.dev.datasaker.io + +kops create cluster \ + --name dev.datasaker.io \ + --vpc vpc-03cbb88e181ccb46e \ + --cloud aws \ + --state s3://clusters.dev.datasaker.io \ + --ssh-public-key /home/hsgahm/id_rsa_k8s.pub \ + --topology private --kubernetes-version "1.23.10" \ + --network-cidr "172.21.0.0/16" \ + --networking calico \ + --container-runtime containerd \ + --image ami-0ea5eb4b05645aa8a \ + --zones ap-northeast-2a,ap-northeast-2b,ap-northeast-2c \ + --master-count 3 \ + --master-size t3.small \ + --master-volume-size 50 \ + --node-count 3 \ + --node-size t3.small \ + --node-volume-size 100 \ + --utility-subnets "subnet-0d762a41fb41d63e5,subnet-0b4f418020349fb84,subnet-05b9f4f02955c3307" \ + --subnets "subnet-021536c4f12971c74,subnet-0c90842daa15aa7c7,subnet-0ae3ab7ae241fe761" \ + -v 10 + + +kops get instancegroups --name=dev.datasaker.io --state s3://clusters.dev.datasaker.io + + +kops --name=dev.datasaker.io delete instancegroup nodes-ap-northeast-2a +kops --name=dev.datasaker.io delete instancegroup nodes-ap-northeast-2b +kops --name=dev.datasaker.io delete instancegroup nodes-ap-northeast-2c + + +kops edit instancegroup --name=dev.datasaker.io master-ap-northeast-2a +kops edit instancegroup --name=dev.datasaker.io master-ap-northeast-2b +kops edit instancegroup --name=dev.datasaker.io master-ap-northeast-2c + rootVolumeSize: 50 + + +kops --name=dev.datasaker.io get ig + +kops get clusters +kops edit cluster dev.datasaker.io --state s3://clusters.dev.datasaker.io + +// subnet name 변경 +//ap-northeast-2a -> sbn-dev-a.datasaker +//ap-northeast-2b -> sbn-dev-b.datasaker +//ap-northeast-2c -> sbn-dev-c.datasaker + +//utility-ap-northeast-2a -> sbn-dmz-a.datasaker +//utility-ap-northeast-2b -> sbn-dmz-b.datasaker +//utility-ap-northeast-2c -> sbn-dmz-c.datasaker + + + +kops edit instancegroups --name=dev.datasaker.io master-ap-northeast-2a + +``` +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2022-09-06T05:44:09Z" + labels: + kops.k8s.io/cluster: dev.datasaker.io + name: master-ap-northeast-2a +spec: + image: ami-0ea5eb4b05645aa8a + instanceMetadata: + httpPutResponseHopLimit: 3 + httpTokens: required + machineType: t3.small + manager: CloudGroup + maxSize: 1 + minSize: 1 + nodeLabels: + kops.k8s.io/instancegroup: master-ap-northeast-2a + role: Master + rootVolumeSize: 50 + subnets: + - ap-northeast-2a + cloudLabels: + autoscale-off: "True" + autoscale-on: "True" + + +``` + + + +//kops create instancegroup --name=dev.datasaker.io dev-master-a --role master --subnet "ap-northeast-2a" +//kops create instancegroup --name=dev.datasaker.io dev-master-b --role node --subnet "ap-northeast-2b" +//kops create instancegroup --name=dev.datasaker.io dev-master-c --role node --subnet "ap-northeast-2c" + +// kops delete instancegroup --name=dev.datasaker.io dev-data-a +// kops delete instancegroup --name=dev.datasaker.io dev-data-b +// kops delete instancegroup --name=dev.datasaker.io dev-data-c + +kops create instancegroup --name=dev.datasaker.io dev-data-a --role node --subnet "ap-northeast-2a" +kops edit instancegroup --name=dev.datasaker.io dev-data-a + + ``` + apiVersion: kops.k8s.io/v1alpha2 + kind: InstanceGroup + metadata: + creationTimestamp: "2022-09-05T05:53:59Z" + labels: + kops.k8s.io/cluster: dev.datasaker.io + name: dev-data-a + spec: + image: ami-0ea5eb4b05645aa8a + machineType: m5.4xlarge + manager: CloudGroup + maxSize: 1 + minSize: 1 + nodeLabels: + kops.k8s.io/instancegroup: dev-data-a + datasaker/group: data + rootVolumeSize: 100 + role: Node + subnets: + - ap-northeast-2a + ``` + + ``` + cloudLabels: + autoscale-off: "True" + autoscale-on: "True" + ``` + +kops create instancegroup --name=dev.datasaker.io dev-data-b --role node --subnet "ap-northeast-2b" +kops edit instancegroup --name=dev.datasaker.io dev-data-b + + ``` + apiVersion: kops.k8s.io/v1alpha2 + kind: InstanceGroup + metadata: + creationTimestamp: "2022-09-05T06:00:56Z" + generation: 1 + labels: + kops.k8s.io/cluster: dev.datasaker.io + name: dev-data-b + spec: + image: ami-0ea5eb4b05645aa8a + machineType: m5.4xlarge + manager: CloudGroup + maxSize: 1 + minSize: 1 + nodeLabels: + datasaker/group: data + kops.k8s.io/instancegroup: dev-data-b + role: Node + rootVolumeSize: 100 + subnets: + - ap-northeast-2b + + ``` + +kops create instancegroup --name=dev.datasaker.io dev-data-c --role node --subnet "ap-northeast-2c" +kops edit instancegroup --name=dev.datasaker.io dev-data-c + + ``` + apiVersion: kops.k8s.io/v1alpha2 + kind: InstanceGroup + metadata: + creationTimestamp: null + name: dev-data-c + spec: + image: ami-0ea5eb4b05645aa8a + machineType: m5.4xlarge + manager: CloudGroup + maxSize: 1 + minSize: 1 + nodeLabels: + kops.k8s.io/instancegroup: dev-data-c + datasaker/group: data + rootVolumeSize: 100 + role: Node + subnets: + - ap-northeast-2c + + ``` + +// kops delete instancegroup --name=dev.datasaker.io dev-process-a +// kops delete instancegroup --name=dev.datasaker.io dev-process-b +// kops delete instancegroup --name=dev.datasaker.io dev-process-c +kops create instancegroup --name=dev.datasaker.io dev-process-a --role node --subnet "ap-northeast-2a" +kops edit instancegroup --name=dev.datasaker.io dev-process-a + + ``` + apiVersion: kops.k8s.io/v1alpha2 + kind: InstanceGroup + metadata: + creationTimestamp: null + name: dev-process-a + spec: + image: ami-0ea5eb4b05645aa8a + machineType: c5.xlarge + manager: CloudGroup + maxSize: 2 + minSize: 2 + nodeLabels: + kops.k8s.io/instancegroup: dev-process-a + datasaker/group: process + rootVolumeSize: 100 + role: Node + subnets: + - ap-northeast-2a + + ``` + +kops create instancegroup --name=dev.datasaker.io dev-process-b --role node --subnet "ap-northeast-2b" +kops edit instancegroup --name=dev.datasaker.io dev-process-b + ``` + apiVersion: kops.k8s.io/v1alpha2 + kind: InstanceGroup + metadata: + creationTimestamp: "2022-09-05T06:10:03Z" + labels: + kops.k8s.io/cluster: dev.datasaker.io + name: dev-process-b + spec: + image: ami-0ea5eb4b05645aa8a + machineType: c5.xlarge + manager: CloudGroup + maxSize: 2 + minSize: 2 + nodeLabels: + datasaker/group: process + kops.k8s.io/instancegroup: dev-process-b + role: Node + rootVolumeSize: 100 + subnets: + - ap-northeast-2b + + ``` + +kops create instancegroup --name=dev.datasaker.io dev-process-c --role node --subnet "ap-northeast-2c" +kops edit instancegroup --name=dev.datasaker.io dev-process-c + ``` + apiVersion: kops.k8s.io/v1alpha2 + kind: InstanceGroup + metadata: + creationTimestamp: "2022-09-05T06:10:59Z" + labels: + kops.k8s.io/cluster: dev.datasaker.io + name: dev-process-c + spec: + image: ami-0ea5eb4b05645aa8a + machineType: c5.xlarge + manager: CloudGroup + maxSize: 1 + minSize: 1 + nodeLabels: + datasaker/group: process + kops.k8s.io/instancegroup: dev-process-c + rootVolumeSize: 100 + role: Node + subnets: + - ap-northeast-2c + + ``` + +kops create instancegroup --name=dev.datasaker.io dev-mgmt-a --role node --subnet "ap-northeast-2a" +kops edit instancegroup --name=dev.datasaker.io dev-mgmt-a + ``` + apiVersion: kops.k8s.io/v1alpha2 + kind: InstanceGroup + metadata: + creationTimestamp: null + name: dev-mgmt-a + spec: + image: ami-0ea5eb4b05645aa8a + machineType: c5.xlarge + manager: CloudGroup + maxSize: 1 + minSize: 1 + nodeLabels: + kops.k8s.io/instancegroup: dev-mgmt-a + datasaker/group: mgmt + rootVolumeSize: 100 + role: Node + subnets: + - ap-northeast-2a + + ``` + +kops create instancegroup --name=dev.datasaker.io dev-mgmt-b --role node --subnet "ap-northeast-2b" +kops edit instancegroup --name=dev.datasaker.io dev-mgmt-b + + + +// cloudLabels: +// autoscale-off: "True" +// autoscale-on: "True" + +kops edit instancegroup --name=dev.datasaker.io dev-data-a +kops edit instancegroup --name=dev.datasaker.io dev-data-b +kops edit instancegroup --name=dev.datasaker.io dev-data-c + + + + + +kops get --state s3://clusters.dev.datasaker.io --name dev.datasaker.io -o yaml > dev.datasaker.io.yaml + +vi dev.datasaker.io.yaml + + subnets: + - cidr: 172.21.1.0/24 + name: ap-northeast-2a + type: Private + zone: ap-northeast-2a + - cidr: 172.21.2.0/24 + name: ap-northeast-2b + type: Private + zone: ap-northeast-2b + - cidr: 172.21.3.0/24 + name: ap-northeast-2c + type: Private + zone: ap-northeast-2c + - cidr: 172.21.0.48/28 + name: utility-ap-northeast-2a + type: Utility + zone: ap-northeast-2a + - cidr: 172.21.0.64/28 + name: utility-ap-northeast-2b + type: Utility + zone: ap-northeast-2b + - cidr: 172.21.0.80/28 + name: utility-ap-northeast-2c + type: Utility + zone: ap-northeast-2c + +export KOPS_STATE_STORE=s3://clusters.dev.datasaker.io + + +// kops delete cluster dev.datasaker.io --yes --state=s3://clusters.dev.datasaker.io +// kops delete -f=./dev.datasaker.io.yaml --yes +kops create -f=./dev.datasaker.io.yaml --state=s3://clusters.dev.datasaker.io +kops update cluster dev.datasaker.io --yes --admin --state=s3://clusters.dev.datasaker.io + + +kops export kubecfg --admin --state=s3://clusters.dev.datasaker.io +kops export kubecfg --admin --kubeconfig ~/workspace/kubeconfig --state=s3://clusters.dev.datasaker.io + +kops get secrets sshpublickey admin +kops get secrets sshpublickey admin -oplaintext +MgUKqpCUHLaEcYEuHXTM7ljlTpsnNYSs + +ssh admin@3.35.41.191 + +kops create instancegroup bastions --role Bastion --subnet utility-ap-northeast-2c + +kops create secret sshpublickey admin -i ~/.ssh/id_rsa.pub --state=s3://clusters.dev.datasaker.io + kops create secret sshpublickey admin -i id_rsa_k8s.pub --state=s3://clusters.dev.datasaker.io +kops update cluster --yes // to reconfigure the auto-scaling groups +kops update cluster --yes --state=s3://clusters.dev.datasaker.io +kops rolling-update cluster --name dev.datasaker.io --state=s3://clusters.dev.datasaker.io --yes + +kops rolling-update cluster --name --yes // to immediately roll all the machines so they have the new key (optional) + + +// Lambda 설정 변경. +get_names = ['ag-dmz-bastion-datasaker','master-ap-northeast-2a.masters.dev.datasaker.io','master-ap-northeast-2b.masters.dev.datasaker.io','master-ap-northeast-2c.masters.dev.datasaker.io','dev-process-a.dev.datasaker.io','dev-process-b.dev.datasaker.io','dev-process-c.dev.datasaker.io','dev-data-a.dev.datasaker.io','dev-data-b.dev.datasaker.io','dev-data-c.dev.datasaker.io','dev-mgmt-a.dev.datasaker.io','dev-mgmt-b.dev.datasaker.io'] + + +Suggestions: + * validate cluster: kops validate cluster --wait 10m + * list nodes: kubectl get nodes --show-labels + * ssh to the master: ssh -i ~/.ssh/id_rsa ubuntu@api.dev.datasaker.io + * the ubuntu user is specific to Ubuntu. If not using Ubuntu please use the appropriate user based on your OS. + * read about installing addons at: https://kops.sigs.k8s.io/addons. + + +// when kubecfg changed, due to master redeploy +kops export kubecfg --admin --state s3://clusters.dev.datasaker.io + +kops update cluster --name=dev.datasaker.io --state=s3://clusters.dev.datasaker.io --out=./tf-kops-dev-20200907-ip --target=terraform + + +115.178.73.2/32 exem router +115.178.73.91/32 proxy +3.35.247.45/32 bastion + +api-elb.dev.datasaker.io 에 115.178.73.2 만 적용되는 이슈가 있음. + + +kops edit cluster --name=dev.datasaker.io +from + kubernetesApiAccess: + - 0.0.0.0/0 + - ::/0 + + sshAccess: + - 0.0.0.0/0 + - ::/0 + +to + kubernetesApiAccess: + - 115.178.73.2/32 + - 115.178.73.91/32 + - 3.35.247.45/32 + + sshAccess: + - 115.178.73.2/32 + - 115.178.73.91/32 + - 3.35.247.45/32 + +kops update cluster --yes --state=s3://clusters.dev.datasaker.io + +kops rolling-update cluster --yes --state=s3://clusters.dev.datasaker.io + + +