








export KOPS_STATE_STORE=s3://clusters.dev.datasaker.io

kops update cluster --name dev.datasaker.io --state=s3://clusters.dev.datasaker.io
kops update cluster --name dev.datasaker.io --state=s3://clusters.dev.datasaker.io > changes-dev.datasaker.io-20221019.txt
kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io update cluster --yes --admin
kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io rolling-update cluster --yes --cloudonly

kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io -o yaml get > dev.datasaker.io-1.yaml
kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io -o yaml get > dev.datasaker.io-20221025.yaml
kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io edit cluster
kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io get ig
kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io edit ig master-ap-northeast-2a
kops export kubecfg --admin=8760h0m0s --name dev.datasaker.io --state s3://clusters.dev.datasaker.io --kubeconfig ~/.kube/config

kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io update cluster --out=./tf-kops-dev-20221025 --target=terraform


security-dev-bastion ami-0b6591f49cf24e237
security-dev-node ami-0abb33b73a78cae31


kops create cluster \
 --name dev.datasaker.io \
 --vpc vpc-0b6e0b906c678a22f \
 --cloud aws \
 --state s3://clusters.dev.datasaker.io \
 --ssh-public-key /home/hsgahm/.ssh/id_rsa_k8s.pub \
 --topology private --kubernetes-version "1.23.10" \
 --network-cidr "172.21.0.0/16" \
 --networking calico \
 --container-runtime containerd \
 --image ami-0ea5eb4b05645aa8a \
 --zones ap-northeast-2a,ap-northeast-2b,ap-northeast-2c \
 --master-count 3 \
 --master-size t3.small \
 --master-volume-size 50 \
 --node-count 3 \
 --node-size t3.small \
 --node-volume-size 100 \
 --utility-subnets "subnet-0de55619bee2411f8,subnet-0a5d787353f874684,subnet-0ee26ffc561efb292" \
 --subnets "subnet-0c875e254456809f7,subnet-05672a669943fc12f,subnet-0940fd78504acbbde" \
 -v 10

<!-- 
kops edit cluster  --name=dev.datasaker.io --state s3://clusters.dev.datasaker.io
```
spec:
  awsLoadBalancerController:
    enabled: true

```
 -->

kops --name dev.datasaker.io --state s3://clusters.dev.datasaker.io edit cluster
```
  containerd:
    configOverride: |
      version = 2
      imports = ["/etc/containerd/runtime_*.toml"]

      [plugins]
        [plugins."io.containerd.grpc.v1.cri"]
          sandbox_image = "registry.k8s.io/pause:3.6@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db"
          [plugins."io.containerd.grpc.v1.cri".containerd]
            [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
              [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
                runtime_type = "io.containerd.runc.v2"
                [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
                  SystemdCgroup = true
        [plugins."io.containerd.grpc.v1.cri".registry.configs."registry-1.docker.io".auth]
          username = "datasaker"
          password = "dckr_pat_kQP6vcHm_jMChWd_zvgH_G3kucc"

```

```

```


```
    enableWAF: true
    enableWAFv2: true
```

kops get instancegroups --name=dev.datasaker.io --state s3://clusters.dev.datasaker.io

kops --name=dev.datasaker.io delete instancegroup nodes-ap-northeast-2a
kops --name=dev.datasaker.io delete instancegroup nodes-ap-northeast-2b
kops --name=dev.datasaker.io delete instancegroup nodes-ap-northeast-2c

kops edit instancegroup --name=dev.datasaker.io master-ap-northeast-2a
kops edit instancegroup --name=dev.datasaker.io master-ap-northeast-2b
kops edit instancegroup --name=dev.datasaker.io master-ap-northeast-2c
  rootVolumeSize: 64 (default)


kops --name=dev.datasaker.io get ig 

kops get clusters
kops edit cluster dev.datasaker.io --state s3://clusters.dev.datasaker.io

// subnet name 변경
//ap-northeast-2a -> sbn-dev-a.datasaker
//ap-northeast-2b -> sbn-dev-b.datasaker
//ap-northeast-2c -> sbn-dev-c.datasaker

//utility-ap-northeast-2a -> sbn-dmz-a.datasaker
//utility-ap-northeast-2b -> sbn-dmz-b.datasaker
//utility-ap-northeast-2c -> sbn-dmz-c.datasaker



kops edit instancegroups --name=dev.datasaker.io master-ap-northeast-2a

```
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
  creationTimestamp: "2022-09-06T05:44:09Z"
  labels:
    kops.k8s.io/cluster: dev.datasaker.io
  name: master-ap-northeast-2a
spec:
  image: ami-0ea5eb4b05645aa8a
  instanceMetadata:
    httpPutResponseHopLimit: 3
    httpTokens: required
  machineType: t3.small
  manager: CloudGroup
  maxSize: 1
  minSize: 1
  nodeLabels:
    kops.k8s.io/instancegroup: master-ap-northeast-2a
  role: Master
  rootVolumeSize: 50
  subnets:
  - ap-northeast-2a
  cloudLabels:
    autoscale-off: "True"
    autoscale-on: "True"

```



//kops create instancegroup --name=dev.datasaker.io dev-master-a --role master --subnet "ap-northeast-2a"
//kops create instancegroup --name=dev.datasaker.io dev-master-b --role node --subnet "ap-northeast-2b"
//kops create instancegroup --name=dev.datasaker.io dev-master-c --role node --subnet "ap-northeast-2c"

// kops delete instancegroup --name=dev.datasaker.io dev-data-a
// kops delete instancegroup --name=dev.datasaker.io dev-data-b
// kops delete instancegroup --name=dev.datasaker.io dev-data-c


kops create instancegroup --name=dev.datasaker.io dev-data-druid-a --role node --subnet "ap-northeast-2a"
kops create instancegroup --name=dev.datasaker.io dev-data-druid-b --role node --subnet "ap-northeast-2b"
kops create instancegroup --name=dev.datasaker.io dev-data-druid-c --role node --subnet "ap-northeast-2c"
kops create instancegroup --name=dev.datasaker.io dev-data-kafka-a --role node --subnet "ap-northeast-2a"
kops create instancegroup --name=dev.datasaker.io dev-data-kafka-b --role node --subnet "ap-northeast-2b"
kops create instancegroup --name=dev.datasaker.io dev-data-kafka-c --role node --subnet "ap-northeast-2c"

```
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
  creationTimestamp: null
  name: dev-data-druid-a
spec:
  image: ami-0abb33b73a78cae31
  kubelet:
    anonymousAuth: false
    nodeLabels:
      node-role.kubernetes.io/node: ""
  machineType: m6i.2xlarge
  manager: CloudGroup
  maxSize: 1
  minSize: 1
  nodeLabels:
    kops.k8s.io/instancegroup: dev-data-druid-a
    datasaker/group: data-druid
  role: Node
  subnets:
  - ap-northeast-2a
  taints:
  - dev/data-druid:NoSchedule
```

```
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
  creationTimestamp: null
  name: dev-data-kafka-a
spec:
  image: ami-0abb33b73a78cae31
  kubelet:
    anonymousAuth: false
    nodeLabels:
      node-role.kubernetes.io/node: ""
  machineType: m6i.2xlarge
  manager: CloudGroup
  maxSize: 1
  minSize: 1
  nodeLabels:
    kops.k8s.io/instancegroup: dev-data-kafka-a
    datasaker/group: data-kafka
  role: Node
  subnets:
  - ap-northeast-2a
  taints:
  - dev/data-kafka:NoSchedule  

```


kops create instancegroup --name=dev.datasaker.io dev-data-a --role node --subnet "ap-northeast-2a"
kops edit instancegroup --name=dev.datasaker.io dev-data-a

  ```
  apiVersion: kops.k8s.io/v1alpha2
  kind: InstanceGroup
  metadata:
    creationTimestamp: "2022-09-05T05:53:59Z"
    labels:
      kops.k8s.io/cluster: dev.datasaker.io
    name: dev-data-a
  spec:
    image: ami-0ea5eb4b05645aa8a
    machineType: m5.4xlarge
    manager: CloudGroup
    maxSize: 1
    minSize: 1
    nodeLabels:
      kops.k8s.io/instancegroup: dev-data-a
      datasaker/group: data
    rootVolumeSize: 100
    role: Node
    subnets:
    - ap-northeast-2a
  ```

  ```  
  cloudLabels:
    autoscale-off: "True"
    autoscale-on: "True"
  ```

kops create instancegroup --name=dev.datasaker.io dev-data-b --role node --subnet "ap-northeast-2b"
kops edit instancegroup --name=dev.datasaker.io dev-data-b

  ```
  apiVersion: kops.k8s.io/v1alpha2
  kind: InstanceGroup
  metadata:
    creationTimestamp: "2022-09-05T06:00:56Z"
    generation: 1
    labels:
      kops.k8s.io/cluster: dev.datasaker.io
    name: dev-data-b
  spec:
    image: ami-0ea5eb4b05645aa8a
    machineType: m5.4xlarge
    manager: CloudGroup
    maxSize: 1
    minSize: 1
    nodeLabels:
      datasaker/group: data
      kops.k8s.io/instancegroup: dev-data-b
    role: Node
    rootVolumeSize: 100
    subnets:
    - ap-northeast-2b

  ```

kops create instancegroup --name=dev.datasaker.io dev-data-c --role node --subnet "ap-northeast-2c"
kops edit instancegroup --name=dev.datasaker.io dev-data-c

  ```
  apiVersion: kops.k8s.io/v1alpha2
  kind: InstanceGroup
  metadata:
    creationTimestamp: null
    name: dev-data-c
  spec:
    image: ami-0ea5eb4b05645aa8a
    machineType: m5.4xlarge
    manager: CloudGroup
    maxSize: 1
    minSize: 1
    nodeLabels:
      kops.k8s.io/instancegroup: dev-data-c
      datasaker/group: data
    rootVolumeSize: 100
    role: Node
    subnets:
    - ap-northeast-2c

  ```

// kops delete instancegroup --name=dev.datasaker.io dev-process-a
// kops delete instancegroup --name=dev.datasaker.io dev-process-b
// kops delete instancegroup --name=dev.datasaker.io dev-process-c
kops create instancegroup --name=dev.datasaker.io dev-process-a --role node --subnet "ap-northeast-2a"
kops edit instancegroup --name=dev.datasaker.io dev-process-a

  ```
  apiVersion: kops.k8s.io/v1alpha2
  kind: InstanceGroup
  metadata:
    creationTimestamp: null
    name: dev-process-a
  spec:
    image: ami-0ea5eb4b05645aa8a
    machineType: c5.xlarge
    manager: CloudGroup
    maxSize: 2
    minSize: 2
    nodeLabels:
      kops.k8s.io/instancegroup: dev-process-a
      datasaker/group: process
    rootVolumeSize: 100
    role: Node
    subnets:
    - ap-northeast-2a

  ```

kops create instancegroup --name=dev.datasaker.io dev-process-b --role node --subnet "ap-northeast-2b"
kops edit instancegroup --name=dev.datasaker.io dev-process-b
  ```
  apiVersion: kops.k8s.io/v1alpha2
  kind: InstanceGroup
  metadata:
    creationTimestamp: "2022-09-05T06:10:03Z"
    labels:
      kops.k8s.io/cluster: dev.datasaker.io
    name: dev-process-b
  spec:
    image: ami-0ea5eb4b05645aa8a
    machineType: c5.xlarge
    manager: CloudGroup
    maxSize: 2
    minSize: 2
    nodeLabels:
      datasaker/group: process
      kops.k8s.io/instancegroup: dev-process-b
    role: Node
    rootVolumeSize: 100
    subnets:
    - ap-northeast-2b

  ```

kops create instancegroup --name=dev.datasaker.io dev-process-c --role node --subnet "ap-northeast-2c"
kops edit instancegroup --name=dev.datasaker.io dev-process-c
  ```
  apiVersion: kops.k8s.io/v1alpha2
  kind: InstanceGroup
  metadata:
    creationTimestamp: "2022-09-05T06:10:59Z"
    labels:
      kops.k8s.io/cluster: dev.datasaker.io
    name: dev-process-c
  spec:
    image: ami-0ea5eb4b05645aa8a
    machineType: c5.xlarge
    manager: CloudGroup
    maxSize: 1
    minSize: 1
    nodeLabels:
      datasaker/group: process
      kops.k8s.io/instancegroup: dev-process-c
    rootVolumeSize: 100      
    role: Node
    subnets:
    - ap-northeast-2c
    taints:
    - dev/mgmt:NoSchedule
  ```

kops create instancegroup --name=dev.datasaker.io dev-mgmt-a --role node --subnet "ap-northeast-2a"
kops --state=s3://clusters.dev.datasaker.io --name=dev.datasaker.io edit instancegroup dev-mgmt-a
  ```
  apiVersion: kops.k8s.io/v1alpha2
  kind: InstanceGroup
  metadata:
    creationTimestamp: null
    name: dev-mgmt-a
  spec:
    image: ami-0ea5eb4b05645aa8a
    machineType: c5.xlarge
    manager: CloudGroup
    maxSize: 2
    minSize: 1
    nodeLabels:
      kops.k8s.io/instancegroup: dev-mgmt-a
      datasaker/group: mgmt
    rootVolumeSize: 100  
    role: Node
    subnets:
    - ap-northeast-2a
    taints:
    - dev/mgmt:NoSchedule

  ```

kops create instancegroup --name=dev.datasaker.io dev-mgmt-b --role node --subnet "ap-northeast-2b"
kops --state=s3://clusters.dev.datasaker.io --name=dev.datasaker.io edit instancegroup dev-mgmt-b

  ```
  apiVersion: kops.k8s.io/v1alpha2
  kind: InstanceGroup
  metadata:
    creationTimestamp: null
    name: dev-mgmt-b
  spec:
    image: ami-0abb33b73a78cae31
    machineType: c5.xlarge
    manager: CloudGroup
    maxSize: 1
    minSize: 1
    nodeLabels:
      kops.k8s.io/instancegroup: dev-mgmt-b
      datasaker/group: mgmt
    rootVolumeSize: 100  
    role: Node
    subnets:
    - ap-northeast-2b
    taints:
    - dev/mgmt:NoSchedule
    
  ```
  

kops create instancegroup --name=dev.datasaker.io dev-mgmt-c --role node --subnet "ap-northeast-2c"
kops --state=s3://clusters.dev.datasaker.io --name=dev.datasaker.io edit instancegroup dev-mgmt-c
  ```
  apiVersion: kops.k8s.io/v1alpha2
  kind: InstanceGroup
  metadata:
    creationTimestamp: null
    name: dev-mgmt-c
  spec:
    image: ami-0abb33b73a78cae31
    machineType: c5.xlarge
    manager: CloudGroup
    maxSize: 1
    minSize: 1
    nodeLabels:
      kops.k8s.io/instancegroup: dev-mgmt-a
      datasaker/group: mgmt
    rootVolumeSize: 100  
    role: Node
    subnets:
    - ap-northeast-2c
    taints:
    - dev/mgmt:NoSchedule

  ```
  


kops edit instancegroup --name=dev.datasaker.io dev-data-a
kops edit instancegroup --name=dev.datasaker.io dev-data-b
kops edit instancegroup --name=dev.datasaker.io dev-data-c





kops get --state s3://clusters.dev.datasaker.io --name dev.datasaker.io -o yaml > dev.datasaker.io.yaml

vi dev.datasaker.io.yaml

  subnets:
  - cidr: 172.21.1.0/24
    name: ap-northeast-2a
    type: Private
    zone: ap-northeast-2a
  - cidr: 172.21.2.0/24
    name: ap-northeast-2b
    type: Private
    zone: ap-northeast-2b
  - cidr: 172.21.3.0/24
    name: ap-northeast-2c
    type: Private
    zone: ap-northeast-2c
  - cidr: 172.21.0.48/28
    name: utility-ap-northeast-2a
    type: Utility
    zone: ap-northeast-2a
  - cidr: 172.21.0.64/28
    name: utility-ap-northeast-2b
    type: Utility
    zone: ap-northeast-2b
  - cidr: 172.21.0.80/28
    name: utility-ap-northeast-2c
    type: Utility
    zone: ap-northeast-2c

export KOPS_STATE_STORE=s3://clusters.dev.datasaker.io


// kops delete cluster dev.datasaker.io --yes --state=s3://clusters.dev.datasaker.io
// kops delete -f=./dev.datasaker.io.yaml --yes
kops create -f=./dev.datasaker.io.yaml --state=s3://clusters.dev.datasaker.io
kops update cluster dev.datasaker.io --yes --admin --state=s3://clusters.dev.datasaker.io


kops export kubecfg --admin --state=s3://clusters.dev.datasaker.io
kops export kubecfg --admin --kubeconfig ~/workspace/kubeconfig --state=s3://clusters.dev.datasaker.io

kops get secrets sshpublickey admin 
kops get secrets sshpublickey admin -oplaintext
MgUKqpCUHLaEcYEuHXTM7ljlTpsnNYSs

ssh ubuntu@3.37.243.25

//kops create instancegroup bastions --role Bastion --subnet utility-ap-northeast-2c

kops create secret sshpublickey admin -i ~/.ssh/id_rsa.pub  --state=s3://clusters.dev.datasaker.io
 kops create secret sshpublickey admin -i id_rsa_k8s.pub  --state=s3://clusters.dev.datasaker.io
kops update cluster --yes // to reconfigure the auto-scaling groups
kops update cluster --yes --state=s3://clusters.dev.datasaker.io
kops rolling-update cluster --name dev.datasaker.io  --state=s3://clusters.dev.datasaker.io --yes

kops rolling-update cluster --name <clustername> --yes // to immediately roll all the machines so they have the new key (optional)


// Lambda 설정 변경.
get_names = ['ag-dmz-bastion-datasaker','master-ap-northeast-2a.masters.dev.datasaker.io','master-ap-northeast-2b.masters.dev.datasaker.io','master-ap-northeast-2c.masters.dev.datasaker.io','dev-process-a.dev.datasaker.io','dev-process-b.dev.datasaker.io','dev-process-c.dev.datasaker.io','dev-data-a.dev.datasaker.io','dev-data-b.dev.datasaker.io','dev-data-c.dev.datasaker.io','dev-mgmt-a.dev.datasaker.io','dev-mgmt-b.dev.datasaker.io']


Suggestions:
 * validate cluster: kops validate cluster --wait 10m
 * list nodes: kubectl get nodes --show-labels
 * ssh to the master: ssh -i ~/.ssh/id_rsa ubuntu@api.dev.datasaker.io
 * the ubuntu user is specific to Ubuntu. If not using Ubuntu please use the appropriate user based on your OS.
 * read about installing addons at: https://kops.sigs.k8s.io/addons.


// when kubecfg changed, due to master redeploy
kops export kubecfg --admin --name dev.datasaker.io --state s3://clusters.dev.datasaker.io --kubeconfig ~/.kube/config
kops export kubecfg --admin=87600h0m0s --name dev.datasaker.io --state s3://clusters.dev.datasaker.io --kubeconfig ~/.kube/config
kops export kubecfg --admin=8760h0m0s --name dev.datasaker.io --state s3://clusters.dev.datasaker.io --kubeconfig ~/.kube/config


kops export kubecfg --admin=720h0m0s --name dev.datasaker.io --state s3://clusters.dev.datasaker.io --kubeconfig ~/.kube/config

kops update cluster --name=dev.datasaker.io --state=s3://clusters.dev.datasaker.io --out=./tf-kops-dev-20200916-ip --target=terraform


115.178.73.2/32	exem router
115.178.73.91/32	proxy
3.35.247.45/32  bastion

api-elb.dev.datasaker.io 에 115.178.73.2 만 적용되는 이슈가 있음.


kops edit cluster --name=dev.datasaker.io
from
  kubernetesApiAccess:
  - 0.0.0.0/0
  - ::/0

  sshAccess:
  - 0.0.0.0/0
  - ::/0

to
  kubernetesApiAccess:
  - 115.178.73.2/32
  - 115.178.73.91/32
  - 3.35.247.45/32

  sshAccess:
  - 115.178.73.2/32
  - 115.178.73.91/32
  - 3.35.247.45/32

kops update cluster --yes --state=s3://clusters.dev.datasaker.io

kops rolling-update cluster --yes --state=s3://clusters.dev.datasaker.io



##


## aws security group masters, nodes set for 30000, 30001 from bastion,elb
  secg-dmz-datasaker sg-07f27eba164d59dfa
  from-dev-bastion-to-ingress


1. elb 용 security group 생성
  service-elb.dev.datasaker.io  sg-08dd3bc6dac12a286	



1. 인증서 생성
// cert-static (IP 제한 없음)
am.dev.kr.datasaker.io (agent manager)
dgate-m.dev.kr.datasaker.io (datagate-metric)
dgate-j.dev.kr.datasaker.io (datagate-jaeger)
dgate-k.dev.kr.datasaker.io (datagate-menifest)
lgate.dev.kr.datasaker.io (loggate)
app.dev.kr.datasaker.io (app - ui)
auth.dev.kr.datasaker.io (keycloak)
api.dev.kr.datasaker.io (krakend)

1. master sg에서 elb sg로 부터 오는 입력 허용.
  TCP 30000 (HTTP), TCP 30001 (HTTPS)

  from-dev-alb-to-ingress

  elb sg에 80, 443 입력 허용.
  dev-from-all-80-to-ing
  dev-from-all-443-to-ing

1. target group 생성
  tg-dev-kr-30000-http-ingress
      http: 30000
      vpc: vpc-datasaker
      Protocol version http1
      healthcheck: http
      Health check path: /
      Advanced health check settings : 
        Success codes: 200,404
      AddTag: Name: tg-dev-kr-30000-http-ingress
      create

  tg-dev-kr-30001-https-ingress
      https: 30001
      vpc: dev.k8s.datasaker.io
      Protocol version http1  
      healthcheck: https
      Health check path: /
      Advanced health check settings : 
        Success codes: 200,404    
      AddTag: Name / tg-dev-kr-30001-https-ingress
      create

  tg-dev-kr-30001-http-ingress
      http: 30001
      vpc: dev.k8s.datasaker.io
      Protocol version http1  
      healthcheck: http
      Health check path: /
      Advanced health check settings : 
        Success codes: 200,404    
      AddTag: Name / tg-dev-kr-30001-https-ingress
      create


1. alb 생성

  alb-dev-kr-ingress
    Internet-facing
    IPv4
    vpc: vpc-datasaker
    Mappings: 
      - sbn-dmz-a.datasaker
      - sbn-dmz-b.datasaker
      - sbn-dmz-c.datasaker
    Security groups: 
      - service-elb.dev.datasaker.io
    Listeners and routing:
      - HTTP: 80 
        tg: tg-dev-kr-30000-http-ingress
      - HTTPS: 443 
        tg: tg-dev-kr-30001-https-ingress
    AddTag: Name / alb-dev-kr-ingress


1. 각 autoscaling group에 tg 연결

  nlb-dev-ingress
  internet-facing
  ipv4
  vpc: dev.k8s.datasaker.io
  mappings: subnet (utilityA,utilityB,utilityC)
  Listeners and routing: 
    TCP:80 -> targetGroup 지정. tg-dev-ingress-30000
    TCP:443 -> targetGroup 지정. tg-dev-ingress-30001



## mng
1. manage 인증서 생성
// manage-dev.kr.datasaker.io (특정 IP 제한)
argo.dev.kr.datasaker.io (argocd)
vlt.dev.kr.datasaker.io (vault)
jenkins.dev.kr.datasaker.io (jenkins)

1. managed SG 설정
  sg.dev.kr-managed-ingress
    VPC: vpc-datasaker
    AllTrafic -> AnyWhere
    AllTrafic -> AnyWhere
  Name: sg.dev.kr-managed-ingress

1. master sg에서 elb sg로 부터 오는 입력 허용.
  TCP 30000 (HTTP), TCP 30001 (HTTPS)

  from-dev-manage-elb-80-to-ingress
  from-dev-manage-elb-443-to-ingress

  elb sg에 80, 443 입력 허용.
  dev-from-all-80-to-ing
  dev-from-all-443-to-ing

1. target group 생성
  tg-dev-kr-30000-mng-http-ing
      http: 30000
      vpc: vpc-datasaker
      Protocol version http1
      healthcheck: http
      Health check path: /
      Advanced health check settings : 
        Success codes: 200,404,400
      AddTag: 
        Name: tg-dev-kr-30000-mng-http-ing
      create

  tg-dev-kr-30001-mng-https-ing
      https: 30001
      vpc: dev.k8s.datasaker.io
      Protocol version http1  
      healthcheck: https
      Health check path: /
      Advanced health check settings : 
        Success codes: 200,404,400
      AddTag: 
        Name / tg-dev-kr-30001-mng-https-ing
      create

1. alb 생성

  alb-dev-kr-mng-ing
    Internet-facing
    IPv4
    vpc: vpc-datasaker
    Mappings: 
      - sbn-dmz-a.datasaker
      - sbn-dmz-b.datasaker
      - sbn-dmz-c.datasaker
    Security groups: 
      - service-elb.dev.datasaker.io
    Listeners and routing:
      - HTTP: 80 
        tg: tg-dev-kr-30000-http-ingress
      - HTTPS: 443 
        tg: tg-dev-kr-30001-https-ingress
    AddTag: Name / alb-dev-kr-mng-ing



1. 각 autoscaling group에 tg 연결
  master asg에
    TCP:80 -> targetGroup 지정. tg-dev-ingress-30000
    TCP:443 -> targetGroup 지정. tg-dev-ingress-30001



## nodeport 연결
1. nlb 용 security group 생성
  sg.nlb.dev.kr.datasaker.io
    Custom TCP: 30010 - 32768 : 0.0.0.0/0
    Custom TCP: 30010 - 32768 : ::/0

1. master sg에서 nlb sg로 부터 오는 입력 허용.
  TCP 30000-32768
  from-nlb-30000-32768-to-nodeport

// 1개의 nodeport 지정 (범위로 지정 가능한지 검토 필요) // 
1. target group 생성 (ingress로 health check로만 사용) // 수정 필요.
  31428

  tg-dev-kr-tcp-np
      tcp: 31428
      vpc: vpc-datasaker
      Protocol version http1
      healthcheck: http
      Health check path: /
      Advanced health check settings : 
        Success codes: 200,404,400
      AddTag: Name: tg-dev-kr-tcp-np
      create

  
1. nlb 생성 및 tg 연결

  nlb-dev-kr-mng-np
    Internet-facing
    IPv4
    vpc: vpc-datasaker
    Mappings: 
      - sbn-dmz-a.datasaker
      - sbn-dmz-b.datasaker
      - sbn-dmz-c.datasaker
    Security groups: 
      - service-elb.dev.datasaker.io
    Listeners and routing:
      - HTTP: 80 
        tg: tg-dev-kr-30000-http-ingress
      - HTTPS: 443 
        tg: tg-dev-kr-30001-https-ingress
    AddTag: Name / nlb-dev-kr-mng-np


1. 각 autoscaling group에 tg 연결
  master asg에
    tg-dev-kr-tcp-np 
    // arn:aws:elasticloadbalancing:ap-northeast-2:508259851457:targetgroup/tg-dev-kr-tcp-np/e86a5f0c14928131


##



1. autoscaling group에서 lb를 이용한 target group 등록
  nodes-ap-northeast-2a.dev.k8s.datasaker.io, nodes-ap-northeast-2b.dev.k8s.datasaker.io, nodes-ap-northeast-2c.dev.k8s.datasaker.io
  Load balancing: tg-dev-ingress-30000, tg-dev-ingress-30001

1. target group에서 instance 가 보이는지 확인

1. nlb A Record 주소 확인
  nlb-dev-ingress-f266e4f0bead8225.elb.ap-northeast-2.amazonaws.com

1. route53에 등록 
  ex) g2048.dev.kr.datasaker.io
  g2048.dev.kr A Alias nlb-dev-ingress-f266e4f0bead8225.elb.ap-northeast-2.amazonaws.com. 300 Simple routing

  argo.dev.datasaker.io A Alias nlb-dev-ingress-f266e4f0bead8225.elb.ap-northeast-2.amazonaws.com. 300 Simple routing

  simple-app.dev.datasaker.io

1. test app
https://blog.leiwang.info/simple-app
// git clone https://github.com/tendant/simple-app.git
// helm install simple-app simple-app -n simple-app --create-namespace


ssh ubuntu@bastion.dev.k8s.datasaker.io
curl -v 172.20.68.243:30000/healthz

masters.dev.k8s.datasaker.io
from-bastion-30000-30001
sgr-0d891ac3623e03e7b	–	사용자 지정 TCP	TCP	30000 - 30001	sg-0fadf3368999e9eaf / bastion.dev.k8s.datasaker.io	–

nodes.dev.k8s.datasaker.io
from-bastion-30000-30001
sgr-0d891ac3623e03e7b	–	사용자 지정 TCP	TCP	30000 - 30001	sg-0fadf3368999e9eaf / bastion.dev.k8s.datasaker.io	–


nc -z -v 172.20.68.243 30000-30001
nc -z -v 172.20.68.243 32679


simple-app.dev.datasaker.io


tg-dev-kr-30000-http-ingress  
  // arn:aws:elasticloadbalancing:ap-northeast-2:508259851457:targetgroup/tg-dev-kr-30000-http-ingress/c722b2d641bcfc87
tg-dev-kr-30001-https-ingress
  // arn:aws:elasticloadbalancing:ap-northeast-2:508259851457:targetgroup/tg-dev-kr-30001-https-ingress/d41767571f8a7bb8

tg-dev-kr-30000-mng-http-ing
  // arn:aws:elasticloadbalancing:ap-northeast-2:508259851457:targetgroup/tg-dev-kr-30000-mng-http-ing/474dc8d6f6ad2106
tg-dev-kr-30001-mng-https-ing
  // arn:aws:elasticloadbalancing:ap-northeast-2:508259851457:targetgroup/tg-dev-kr-30001-mng-https-ing/960e93df1bb9a326

##




<!-- 

--utility-subnets "subnet-0c298ebbccf528cc1,subnet-0dcda9a0e47e17998,subnet-0b805200e89d9095c" \
--subnets "subnet-066a9c7883bac8665,subnet-03eeee967799ec024,subnet-01d16f4fb3bc70413" \

--master-zones ap-northeast-2a,ap-northeast-2b,ap-northeast-2c \
--utility-subnets "subnet-0779691e403086418,subnet-075fd7be078c73b72" \
  Error: [spec.subnets[3].id: Forbidden: cannot mix subnets with specified ID and unspecified ID, spec.subnets[4].id: Forbidden: cannot mix subnets with specified ID and unspecified ID]
  모든 서브넷을 다 지정하면 해결될 듯.
--node-security-groups "sg-0682386228f9859d9" \
--master-security-groups "sg-0682386228f9859d9" \

--bastion \
  // private subnet에 생성됨.

--admin-access 115.178.73.2
--ssh-access 115.178.73.2
--cloud-labels 

 --image ami-054a058b04f721571 \ (x64)

-->

## ing 생성
krakend-dev     krakend-develop                          NodePort    100.65.7.164     <none>        80:32701/TCP                                                                                    7d19h
saas-dev        sam-agentapi-develop                     NodePort    100.71.124.134   <none>        8080:32199/TCP                                                                                  3d17h
saas-dev        sam-app-sender-develop                   NodePort    100.65.88.171    <none>        8000:31514/TCP                                                                                  5d17h
saas-dev        sam-dashboardapi-develop                 NodePort    100.66.2.234     <none>        8080:30696/TCP                                                                                  5d17h
saas-dev        sam-infrastructureapi-develop            NodePort    100.68.69.163    <none>        8000:31648/TCP                                                                                  3d23h
saas-dev        sam-jaeger-agent-develop                 NodePort    100.67.217.7     <none>        14271:30772/TCP,5778:30835/TCP,6831:30834/UDP                                                   4d23h
saas-dev        sam-ui-develop                           NodePort    100.70.74.238    <none>        80:30208/TCP                                                                                    5d20h
saas-dev        sam-usergate-develop                     NodePort    100.66.175.184   <none>        8080:31085/TCP                                                                                  3d22h
saas-dev        sample-app-develop-sample-app-deploy     ClusterIP   100.65.249.165   <none>        80/TCP  

k -n argocd edit ing argocd-server

```

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    kubernetes.io/ingress.class: nginx
    meta.helm.sh/release-name: argocd
    meta.helm.sh/release-namespace: argocd
    nginx.ingress.kubernetes.io/backend-protocol: HTTPS
    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
  creationTimestamp: "2022-09-26T04:31:11Z"
  generation: 4
  labels:
    app.kubernetes.io/component: server
    app.kubernetes.io/instance: argocd
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: argocd-server
    app.kubernetes.io/part-of: argocd
    helm.sh/chart: argo-cd-4.9.11
  name: argocd-server
  namespace: argocd
  resourceVersion: "1567505"
  uid: 567190bd-a080-4628-9e21-5f6b56ffd5e1
spec:
  rules:
  - host: argo.dev.kr.datasaker.io
    http:
      paths:
      - backend:
          service:
            name: argocd-server
            port:
              number: 80
        path: /
        pathType: Prefix
status:
  loadBalancer:
    ingress:
    - ip: 100.71.12.82

```


#### sam-ui

saas-dev        sam-ui-develop                           NodePort    100.70.74.238    <none>        80:30208/TCP     

```
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: sam-ui-develop
  annotations:
    kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
  namespace: saas-dev
spec:
  rules:
  - host: app.dev.kr.datasaker.io
    http:
      paths:
      - backend:
          service:
            name: sam-ui-develop
            port:
              number: 80
        path: /
        pathType: Prefix

```


## keycloak ingress
keycloak                        NodePort    100.67.217.51    <none>        80:30100/TCP,443:30101/TCP

```

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/backend-protocol: HTTPS
    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
  name: keycloak-dev
  namespace: infra-dev
spec:
  rules:
  - host: auth.dev.kr.datasaker.io
    http:
      paths:
      - backend:
          service:
            name: keycloak
            port:
              number: 80
        path: /
        pathType: Prefix

```

```
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    kubernetes.io/ingress.class: nginx
    meta.helm.sh/release-name: keycloak
    meta.helm.sh/release-namespace: infra-dev
  creationTimestamp: "2022-10-04T04:46:18Z"
  generation: 2
  labels:
    app.kubernetes.io/component: keycloak
    app.kubernetes.io/instance: keycloak
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: keycloak
    helm.sh/chart: keycloak-7.1.17
  name: keycloak
  namespace: infra-dev
  resourceVersion: "2184553"
  uid: 1ca8be2a-6580-4537-8488-c825839a7512
spec:
  rules:
  - host: auth.dev.kr.datasaker.io
    http:
      paths:
      - backend:
          service:
            name: keycloak
            port:
              name: https
        path: /
        pathType: ImplementationSpecific
status:
  loadBalancer: {}

```

https://community.gooddata.com/administration-61/how-to-properly-terminate-ssl-using-aws-alb-with-acm-391
  annotations:
    kubernetes.io/ingress.class: "nginx"
    nginx.ingress.kubernetes.io/configuration-snippet: |
      add_header 'Access-Control-Allow-Origin' '*';
      add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
      add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
      add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';



  add_header X-Frame-Options "sameorigin";
  add_header X-Content-Type-Options nosniff;
  add_header Referrer-Policy 'same-origin';

  add_header 'Access-Control-Allow-Origin' '*';
  add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
  add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
  add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';



#### am.dev.kr.datasaker.io (agent manager)

```
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
  name: agentmanager-develop
  namespace: saas-dev
spec:
  ingressClassName: nginx
  rules:
  - host: am.dev.kr.datasaker.io
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: sam-agentmanager-cloud-4534
            port:
              number: 8080



<!-- apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: agentmanager-develop
  annotations:
    kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
  namespace: saas-dev
spec:
  rules:
  - host: am.dev.kr.datasaker.io
    http:
      paths:
      - backend:
          service:
            name: sam-agentmanager-cloud-4534
            port:
              number: 8080
        path: /
        pathType: Prefix -->

```
####

https://aws.amazon.com/blogs/aws/new-application-load-balancer-support-for-end-to-end-http-2-and-grpc/

#### dgate-j.dev.kr.datasaker.io (datagate-jaeger)

```
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
  name: datagate-jaeger-develop
  namespace: saas-dev
spec:
  ingressClassName: nginx
  rules:
  - host: dgate-j.dev.kr.datasaker.io
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: jaeger-sam-datagate-cloud-test
            port:
              number: 8080

<!-- apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: datagate-jaeger-develop
  annotations:
    kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
  namespace: saas-dev
spec:
  rules:
  - host: dgate-j.dev.kr.datasaker.io
    http:
      paths:
      - backend:
          service:
            name: jaeger-sam-datagate-cloud-test
            port:
              number: 8080
        path: /
        pathType: Prefix -->

```


#### dgate-m.dev.kr.datasaker.io (datagate-metric)
```
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
  name: datagate-metric-develop
  namespace: saas-dev
spec:
  ingressClassName: nginx
  rules:
  - host: dgate-m.dev.kr.datasaker.io
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: metric-sam-datagate-cloud-test
            port:
              number: 8080


<!-- apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: datagate-metric-develop
  annotations:
    kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
  namespace: saas-dev
spec:
  rules:
  - host: dgate-m.dev.kr.datasaker.io
    http:
      paths:
      - backend:
          service:
            name: metric-sam-datagate-cloud-test
            port:
              number: 8080
        path: /
        pathType: Prefix -->

```

#### dgate-k.dev.kr.datasaker.io (datagate-menifest)
```

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
  name: datagate-menifest-develop
  namespace: saas-dev
spec:
  ingressClassName: nginx
  rules:
  - host: dgate-k.dev.kr.datasaker.io
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: manifest-sam-datagate-cloud-test
            port:
              number: 8080

<!-- apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: datagate-menifest-develop
  annotations:
    kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
  namespace: saas-dev
spec:
  rules:
  - host: dgate-k.dev.kr.datasaker.io
    http:
      paths:
      - backend:
          service:
            name: manifest-sam-datagate-cloud-test
            port:
              number: 8080
        path: /
        pathType: Prefix -->

```


#### postgresql.dev.kr.datasaker.io
postgresql                      NodePort    100.69.229.168   <none>        5432:32713/TCP 

```
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: postgresql-develop
  annotations:
    kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
  namespace: infra-dev
spec:
  rules:
  - host: postgresql.dev.kr.datasaker.io
    http:
      paths:
      - backend:
          service:
            name: postgresql
            port:
              number: 5432
        path: /
        pathType: Prefix

```




