export KOPS_STATE_STORE=s3://clusters.dev.datasaker.io

kops create cluster \
 --name dev.datasaker.io \
 --vpc vpc-03cbb88e181ccb46e \
 --cloud aws \
 --state s3://clusters.dev.datasaker.io \
 --ssh-public-key /home/hsgahm/id_rsa_k8s.pub \
 --topology private --kubernetes-version "1.23.10" \
 --network-cidr "172.21.0.0/16" \
 --networking calico \
 --container-runtime containerd \
 --image ami-0ea5eb4b05645aa8a \
 --zones ap-northeast-2a,ap-northeast-2b,ap-northeast-2c \
 --master-count 3 \
 --master-size t3.small \
 --master-volume-size 50 \
 --node-count 3 \
 --node-size t3.small \
 --node-volume-size 100 \
 --utility-subnets "subnet-0d762a41fb41d63e5,subnet-0b4f418020349fb84,subnet-05b9f4f02955c3307" \
 --subnets "subnet-021536c4f12971c74,subnet-0c90842daa15aa7c7,subnet-0ae3ab7ae241fe761" \
 -v 10


kops get instancegroups --name=dev.datasaker.io --state s3://clusters.dev.datasaker.io


kops --name=dev.datasaker.io delete instancegroup nodes-ap-northeast-2a
kops --name=dev.datasaker.io delete instancegroup nodes-ap-northeast-2b
kops --name=dev.datasaker.io delete instancegroup nodes-ap-northeast-2c


kops edit instancegroup --name=dev.datasaker.io master-ap-northeast-2a
kops edit instancegroup --name=dev.datasaker.io master-ap-northeast-2b
kops edit instancegroup --name=dev.datasaker.io master-ap-northeast-2c
  rootVolumeSize: 50


kops --name=dev.datasaker.io get ig 

kops get clusters
kops edit cluster dev.datasaker.io --state s3://clusters.dev.datasaker.io

// subnet name 변경
//ap-northeast-2a -> sbn-dev-a.datasaker
//ap-northeast-2b -> sbn-dev-b.datasaker
//ap-northeast-2c -> sbn-dev-c.datasaker

//utility-ap-northeast-2a -> sbn-dmz-a.datasaker
//utility-ap-northeast-2b -> sbn-dmz-b.datasaker
//utility-ap-northeast-2c -> sbn-dmz-c.datasaker



kops edit instancegroups --name=dev.datasaker.io master-ap-northeast-2a

```
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
  creationTimestamp: "2022-09-06T05:44:09Z"
  labels:
    kops.k8s.io/cluster: dev.datasaker.io
  name: master-ap-northeast-2a
spec:
  image: ami-0ea5eb4b05645aa8a
  instanceMetadata:
    httpPutResponseHopLimit: 3
    httpTokens: required
  machineType: t3.small
  manager: CloudGroup
  maxSize: 1
  minSize: 1
  nodeLabels:
    kops.k8s.io/instancegroup: master-ap-northeast-2a
  role: Master
  rootVolumeSize: 50
  subnets:
  - ap-northeast-2a
  cloudLabels:
    autoscale-off: "True"
    autoscale-on: "True"


```



//kops create instancegroup --name=dev.datasaker.io dev-master-a --role master --subnet "ap-northeast-2a"
//kops create instancegroup --name=dev.datasaker.io dev-master-b --role node --subnet "ap-northeast-2b"
//kops create instancegroup --name=dev.datasaker.io dev-master-c --role node --subnet "ap-northeast-2c"

// kops delete instancegroup --name=dev.datasaker.io dev-data-a
// kops delete instancegroup --name=dev.datasaker.io dev-data-b
// kops delete instancegroup --name=dev.datasaker.io dev-data-c

kops create instancegroup --name=dev.datasaker.io dev-data-a --role node --subnet "ap-northeast-2a"
kops edit instancegroup --name=dev.datasaker.io dev-data-a

  ```
  apiVersion: kops.k8s.io/v1alpha2
  kind: InstanceGroup
  metadata:
    creationTimestamp: "2022-09-05T05:53:59Z"
    labels:
      kops.k8s.io/cluster: dev.datasaker.io
    name: dev-data-a
  spec:
    image: ami-0ea5eb4b05645aa8a
    machineType: m5.4xlarge
    manager: CloudGroup
    maxSize: 1
    minSize: 1
    nodeLabels:
      kops.k8s.io/instancegroup: dev-data-a
      datasaker/group: data
    rootVolumeSize: 100
    role: Node
    subnets:
    - ap-northeast-2a
  ```

  ```  
  cloudLabels:
    autoscale-off: "True"
    autoscale-on: "True"
  ```

kops create instancegroup --name=dev.datasaker.io dev-data-b --role node --subnet "ap-northeast-2b"
kops edit instancegroup --name=dev.datasaker.io dev-data-b

  ```
  apiVersion: kops.k8s.io/v1alpha2
  kind: InstanceGroup
  metadata:
    creationTimestamp: "2022-09-05T06:00:56Z"
    generation: 1
    labels:
      kops.k8s.io/cluster: dev.datasaker.io
    name: dev-data-b
  spec:
    image: ami-0ea5eb4b05645aa8a
    machineType: m5.4xlarge
    manager: CloudGroup
    maxSize: 1
    minSize: 1
    nodeLabels:
      datasaker/group: data
      kops.k8s.io/instancegroup: dev-data-b
    role: Node
    rootVolumeSize: 100
    subnets:
    - ap-northeast-2b

  ```

kops create instancegroup --name=dev.datasaker.io dev-data-c --role node --subnet "ap-northeast-2c"
kops edit instancegroup --name=dev.datasaker.io dev-data-c

  ```
  apiVersion: kops.k8s.io/v1alpha2
  kind: InstanceGroup
  metadata:
    creationTimestamp: null
    name: dev-data-c
  spec:
    image: ami-0ea5eb4b05645aa8a
    machineType: m5.4xlarge
    manager: CloudGroup
    maxSize: 1
    minSize: 1
    nodeLabels:
      kops.k8s.io/instancegroup: dev-data-c
      datasaker/group: data
    rootVolumeSize: 100
    role: Node
    subnets:
    - ap-northeast-2c

  ```

// kops delete instancegroup --name=dev.datasaker.io dev-process-a
// kops delete instancegroup --name=dev.datasaker.io dev-process-b
// kops delete instancegroup --name=dev.datasaker.io dev-process-c
kops create instancegroup --name=dev.datasaker.io dev-process-a --role node --subnet "ap-northeast-2a"
kops edit instancegroup --name=dev.datasaker.io dev-process-a

  ```
  apiVersion: kops.k8s.io/v1alpha2
  kind: InstanceGroup
  metadata:
    creationTimestamp: null
    name: dev-process-a
  spec:
    image: ami-0ea5eb4b05645aa8a
    machineType: c5.xlarge
    manager: CloudGroup
    maxSize: 2
    minSize: 2
    nodeLabels:
      kops.k8s.io/instancegroup: dev-process-a
      datasaker/group: process
    rootVolumeSize: 100
    role: Node
    subnets:
    - ap-northeast-2a

  ```

kops create instancegroup --name=dev.datasaker.io dev-process-b --role node --subnet "ap-northeast-2b"
kops edit instancegroup --name=dev.datasaker.io dev-process-b
  ```
  apiVersion: kops.k8s.io/v1alpha2
  kind: InstanceGroup
  metadata:
    creationTimestamp: "2022-09-05T06:10:03Z"
    labels:
      kops.k8s.io/cluster: dev.datasaker.io
    name: dev-process-b
  spec:
    image: ami-0ea5eb4b05645aa8a
    machineType: c5.xlarge
    manager: CloudGroup
    maxSize: 2
    minSize: 2
    nodeLabels:
      datasaker/group: process
      kops.k8s.io/instancegroup: dev-process-b
    role: Node
    rootVolumeSize: 100
    subnets:
    - ap-northeast-2b

  ```

kops create instancegroup --name=dev.datasaker.io dev-process-c --role node --subnet "ap-northeast-2c"
kops edit instancegroup --name=dev.datasaker.io dev-process-c
  ```
  apiVersion: kops.k8s.io/v1alpha2
  kind: InstanceGroup
  metadata:
    creationTimestamp: "2022-09-05T06:10:59Z"
    labels:
      kops.k8s.io/cluster: dev.datasaker.io
    name: dev-process-c
  spec:
    image: ami-0ea5eb4b05645aa8a
    machineType: c5.xlarge
    manager: CloudGroup
    maxSize: 1
    minSize: 1
    nodeLabels:
      datasaker/group: process
      kops.k8s.io/instancegroup: dev-process-c
    rootVolumeSize: 100      
    role: Node
    subnets:
    - ap-northeast-2c

  ```

kops create instancegroup --name=dev.datasaker.io dev-mgmt-a --role node --subnet "ap-northeast-2a"
kops edit instancegroup --name=dev.datasaker.io dev-mgmt-a
  ```
  apiVersion: kops.k8s.io/v1alpha2
  kind: InstanceGroup
  metadata:
    creationTimestamp: null
    name: dev-mgmt-a
  spec:
    image: ami-0ea5eb4b05645aa8a
    machineType: c5.xlarge
    manager: CloudGroup
    maxSize: 1
    minSize: 1
    nodeLabels:
      kops.k8s.io/instancegroup: dev-mgmt-a
      datasaker/group: mgmt
    rootVolumeSize: 100  
    role: Node
    subnets:
    - ap-northeast-2a

  ```

kops create instancegroup --name=dev.datasaker.io dev-mgmt-b --role node --subnet "ap-northeast-2b"
kops edit instancegroup --name=dev.datasaker.io dev-mgmt-b

<!-- data instance group 변경
  t3.small
  maxSize: 2
  minSize: 1
  rootVolumeSize: 100
  nodeLabels:
    유지. kops.k8s.io/instancegroup: XXXX
    datasaker/group: data -->

//  cloudLabels:
//    autoscale-off: "True"
//    autoscale-on: "True"

kops edit instancegroup --name=dev.datasaker.io dev-data-a
kops edit instancegroup --name=dev.datasaker.io dev-data-b
kops edit instancegroup --name=dev.datasaker.io dev-data-c





kops get --state s3://clusters.dev.datasaker.io --name dev.datasaker.io -o yaml > dev.datasaker.io.yaml

vi dev.datasaker.io.yaml

  subnets:
  - cidr: 172.21.1.0/24
    name: ap-northeast-2a
    type: Private
    zone: ap-northeast-2a
  - cidr: 172.21.2.0/24
    name: ap-northeast-2b
    type: Private
    zone: ap-northeast-2b
  - cidr: 172.21.3.0/24
    name: ap-northeast-2c
    type: Private
    zone: ap-northeast-2c
  - cidr: 172.21.0.48/28
    name: utility-ap-northeast-2a
    type: Utility
    zone: ap-northeast-2a
  - cidr: 172.21.0.64/28
    name: utility-ap-northeast-2b
    type: Utility
    zone: ap-northeast-2b
  - cidr: 172.21.0.80/28
    name: utility-ap-northeast-2c
    type: Utility
    zone: ap-northeast-2c

export KOPS_STATE_STORE=s3://clusters.dev.datasaker.io


// kops delete cluster dev.datasaker.io --yes --state=s3://clusters.dev.datasaker.io
// kops delete -f=./dev.datasaker.io.yaml --yes
kops create -f=./dev.datasaker.io.yaml --state=s3://clusters.dev.datasaker.io
kops update cluster dev.datasaker.io --yes --admin --state=s3://clusters.dev.datasaker.io


kops export kubecfg --admin --state=s3://clusters.dev.datasaker.io
kops export kubecfg --admin --kubeconfig ~/workspace/kubeconfig --state=s3://clusters.dev.datasaker.io

kops get secrets sshpublickey admin 
kops get secrets sshpublickey admin -oplaintext
MgUKqpCUHLaEcYEuHXTM7ljlTpsnNYSs

ssh admin@3.35.41.191

kops create instancegroup bastions --role Bastion --subnet utility-ap-northeast-2c

kops create secret sshpublickey admin -i ~/.ssh/id_rsa.pub  --state=s3://clusters.dev.datasaker.io
 kops create secret sshpublickey admin -i id_rsa_k8s.pub  --state=s3://clusters.dev.datasaker.io
kops update cluster --yes // to reconfigure the auto-scaling groups
kops update cluster --yes --state=s3://clusters.dev.datasaker.io
kops rolling-update cluster --name dev.datasaker.io  --state=s3://clusters.dev.datasaker.io --yes

kops rolling-update cluster --name <clustername> --yes // to immediately roll all the machines so they have the new key (optional)


// Lambda 설정 변경.
get_names = ['ag-dmz-bastion-datasaker','master-ap-northeast-2a.masters.dev.datasaker.io','master-ap-northeast-2b.masters.dev.datasaker.io','master-ap-northeast-2c.masters.dev.datasaker.io','dev-process-a.dev.datasaker.io','dev-process-b.dev.datasaker.io','dev-process-c.dev.datasaker.io','dev-data-a.dev.datasaker.io','dev-data-b.dev.datasaker.io','dev-data-c.dev.datasaker.io','dev-mgmt-a.dev.datasaker.io','dev-mgmt-b.dev.datasaker.io']


Suggestions:
 * validate cluster: kops validate cluster --wait 10m
 * list nodes: kubectl get nodes --show-labels
 * ssh to the master: ssh -i ~/.ssh/id_rsa ubuntu@api.dev.datasaker.io
 * the ubuntu user is specific to Ubuntu. If not using Ubuntu please use the appropriate user based on your OS.
 * read about installing addons at: https://kops.sigs.k8s.io/addons.


// when kubecfg changed, due to master redeploy
// kops export kubecfg --admin  --state s3://clusters.dev.datasaker.io

kops update cluster --name=dev.datasaker.io --state=s3://clusters.dev.datasaker.io --out=./tf-kops-dev-20200907-ip --target=terraform


115.178.73.2/32	exem router
115.178.73.91/32	proxy
3.35.247.45/32  bastion

api-elb.dev.datasaker.io 에 115.178.73.2 만 적용되는 이슈가 있음.


kops edit cluster --name=dev.datasaker.io
from
  kubernetesApiAccess:
  - 0.0.0.0/0
  - ::/0

  sshAccess:
  - 0.0.0.0/0
  - ::/0

to
  kubernetesApiAccess:
  - 115.178.73.2/32
  - 115.178.73.91/32
  - 3.35.247.45/32

  sshAccess:
  - 115.178.73.2/32
  - 115.178.73.91/32
  - 3.35.247.45/32

kops update cluster --yes --state=s3://clusters.dev.datasaker.io

kops rolling-update cluster --yes --state=s3://clusters.dev.datasaker.io