export KOPS_STATE_STORE=s3://clusters.dev.datasaker.io kops update cluster --name dev.datasaker.io --state=s3://clusters.dev.datasaker.io kops update cluster --name dev.datasaker.io --state=s3://clusters.dev.datasaker.io > changes-dev.datasaker.io-20221019.txt kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io update cluster --yes --admin kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io rolling-update cluster --yes --cloudonly kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io -o yaml get > dev.datasaker.io-1.yaml kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io -o yaml get > dev.datasaker.io-20221025.yaml kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io edit cluster kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io get ig kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io edit ig master-ap-northeast-2a kops export kubecfg --admin=8760h0m0s --name dev.datasaker.io --state s3://clusters.dev.datasaker.io --kubeconfig ~/.kube/config kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io update cluster --out=./tf-kops-dev-20221025 --target=terraform security-dev-bastion ami-0b6591f49cf24e237 security-dev-node ami-0abb33b73a78cae31 kops create cluster \ --name dev.datasaker.io \ --vpc vpc-0b6e0b906c678a22f \ --cloud aws \ --state s3://clusters.dev.datasaker.io \ --ssh-public-key /home/hsgahm/.ssh/id_rsa_k8s.pub \ --topology private --kubernetes-version "1.23.10" \ --network-cidr "172.21.0.0/16" \ --networking calico \ --container-runtime containerd \ --image ami-0ea5eb4b05645aa8a \ --zones ap-northeast-2a,ap-northeast-2b,ap-northeast-2c \ --master-count 3 \ --master-size t3.small \ --master-volume-size 50 \ --node-count 3 \ --node-size t3.small \ --node-volume-size 100 \ --utility-subnets "subnet-0de55619bee2411f8,subnet-0a5d787353f874684,subnet-0ee26ffc561efb292" \ --subnets "subnet-0c875e254456809f7,subnet-05672a669943fc12f,subnet-0940fd78504acbbde" \ -v 10 kops --name dev.datasaker.io --state s3://clusters.dev.datasaker.io edit cluster ``` containerd: configOverride: | version = 2 imports = ["/etc/containerd/runtime_*.toml"] [plugins] [plugins."io.containerd.grpc.v1.cri"] sandbox_image = "registry.k8s.io/pause:3.6@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db" [plugins."io.containerd.grpc.v1.cri".containerd] [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] runtime_type = "io.containerd.runc.v2" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] SystemdCgroup = true [plugins."io.containerd.grpc.v1.cri".registry.configs."registry-1.docker.io".auth] username = "datasaker" password = "dckr_pat_kQP6vcHm_jMChWd_zvgH_G3kucc" ``` ``` ``` ``` enableWAF: true enableWAFv2: true ``` kops get instancegroups --name=dev.datasaker.io --state s3://clusters.dev.datasaker.io kops --name=dev.datasaker.io delete instancegroup nodes-ap-northeast-2a kops --name=dev.datasaker.io delete instancegroup nodes-ap-northeast-2b kops --name=dev.datasaker.io delete instancegroup nodes-ap-northeast-2c kops edit instancegroup --name=dev.datasaker.io master-ap-northeast-2a kops edit instancegroup --name=dev.datasaker.io master-ap-northeast-2b kops edit instancegroup --name=dev.datasaker.io master-ap-northeast-2c rootVolumeSize: 64 (default) kops --name=dev.datasaker.io get ig kops get clusters kops edit cluster dev.datasaker.io --state s3://clusters.dev.datasaker.io // subnet name 변경 //ap-northeast-2a -> sbn-dev-a.datasaker //ap-northeast-2b -> sbn-dev-b.datasaker //ap-northeast-2c -> sbn-dev-c.datasaker //utility-ap-northeast-2a -> sbn-dmz-a.datasaker //utility-ap-northeast-2b -> sbn-dmz-b.datasaker //utility-ap-northeast-2c -> sbn-dmz-c.datasaker kops edit instancegroups --name=dev.datasaker.io master-ap-northeast-2a ``` apiVersion: kops.k8s.io/v1alpha2 kind: InstanceGroup metadata: creationTimestamp: "2022-09-06T05:44:09Z" labels: kops.k8s.io/cluster: dev.datasaker.io name: master-ap-northeast-2a spec: image: ami-0ea5eb4b05645aa8a instanceMetadata: httpPutResponseHopLimit: 3 httpTokens: required machineType: t3.small manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: kops.k8s.io/instancegroup: master-ap-northeast-2a role: Master rootVolumeSize: 50 subnets: - ap-northeast-2a cloudLabels: autoscale-off: "True" autoscale-on: "True" ``` //kops create instancegroup --name=dev.datasaker.io dev-master-a --role master --subnet "ap-northeast-2a" //kops create instancegroup --name=dev.datasaker.io dev-master-b --role node --subnet "ap-northeast-2b" //kops create instancegroup --name=dev.datasaker.io dev-master-c --role node --subnet "ap-northeast-2c" // kops delete instancegroup --name=dev.datasaker.io dev-data-a // kops delete instancegroup --name=dev.datasaker.io dev-data-b // kops delete instancegroup --name=dev.datasaker.io dev-data-c kops create instancegroup --name=dev.datasaker.io dev-data-druid-a --role node --subnet "ap-northeast-2a" kops create instancegroup --name=dev.datasaker.io dev-data-druid-b --role node --subnet "ap-northeast-2b" kops create instancegroup --name=dev.datasaker.io dev-data-druid-c --role node --subnet "ap-northeast-2c" kops create instancegroup --name=dev.datasaker.io dev-data-kafka-a --role node --subnet "ap-northeast-2a" kops create instancegroup --name=dev.datasaker.io dev-data-kafka-b --role node --subnet "ap-northeast-2b" kops create instancegroup --name=dev.datasaker.io dev-data-kafka-c --role node --subnet "ap-northeast-2c" ``` apiVersion: kops.k8s.io/v1alpha2 kind: InstanceGroup metadata: creationTimestamp: null name: dev-data-druid-a spec: image: ami-0abb33b73a78cae31 kubelet: anonymousAuth: false nodeLabels: node-role.kubernetes.io/node: "" machineType: m6i.2xlarge manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: kops.k8s.io/instancegroup: dev-data-druid-a datasaker/group: data-druid role: Node subnets: - ap-northeast-2a taints: - dev/data-druid:NoSchedule ``` ``` apiVersion: kops.k8s.io/v1alpha2 kind: InstanceGroup metadata: creationTimestamp: null name: dev-data-kafka-a spec: image: ami-0abb33b73a78cae31 kubelet: anonymousAuth: false nodeLabels: node-role.kubernetes.io/node: "" machineType: m6i.2xlarge manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: kops.k8s.io/instancegroup: dev-data-kafka-a datasaker/group: data-kafka role: Node subnets: - ap-northeast-2a taints: - dev/data-kafka:NoSchedule ``` kops create instancegroup --name=dev.datasaker.io dev-data-a --role node --subnet "ap-northeast-2a" kops edit instancegroup --name=dev.datasaker.io dev-data-a ``` apiVersion: kops.k8s.io/v1alpha2 kind: InstanceGroup metadata: creationTimestamp: "2022-09-05T05:53:59Z" labels: kops.k8s.io/cluster: dev.datasaker.io name: dev-data-a spec: image: ami-0ea5eb4b05645aa8a machineType: m5.4xlarge manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: kops.k8s.io/instancegroup: dev-data-a datasaker/group: data rootVolumeSize: 100 role: Node subnets: - ap-northeast-2a ``` ``` cloudLabels: autoscale-off: "True" autoscale-on: "True" ``` kops create instancegroup --name=dev.datasaker.io dev-data-b --role node --subnet "ap-northeast-2b" kops edit instancegroup --name=dev.datasaker.io dev-data-b ``` apiVersion: kops.k8s.io/v1alpha2 kind: InstanceGroup metadata: creationTimestamp: "2022-09-05T06:00:56Z" generation: 1 labels: kops.k8s.io/cluster: dev.datasaker.io name: dev-data-b spec: image: ami-0ea5eb4b05645aa8a machineType: m5.4xlarge manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: datasaker/group: data kops.k8s.io/instancegroup: dev-data-b role: Node rootVolumeSize: 100 subnets: - ap-northeast-2b ``` kops create instancegroup --name=dev.datasaker.io dev-data-c --role node --subnet "ap-northeast-2c" kops edit instancegroup --name=dev.datasaker.io dev-data-c ``` apiVersion: kops.k8s.io/v1alpha2 kind: InstanceGroup metadata: creationTimestamp: null name: dev-data-c spec: image: ami-0ea5eb4b05645aa8a machineType: m5.4xlarge manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: kops.k8s.io/instancegroup: dev-data-c datasaker/group: data rootVolumeSize: 100 role: Node subnets: - ap-northeast-2c ``` // kops delete instancegroup --name=dev.datasaker.io dev-process-a // kops delete instancegroup --name=dev.datasaker.io dev-process-b // kops delete instancegroup --name=dev.datasaker.io dev-process-c kops create instancegroup --name=dev.datasaker.io dev-process-a --role node --subnet "ap-northeast-2a" kops edit instancegroup --name=dev.datasaker.io dev-process-a ``` apiVersion: kops.k8s.io/v1alpha2 kind: InstanceGroup metadata: creationTimestamp: null name: dev-process-a spec: image: ami-0ea5eb4b05645aa8a machineType: c5.xlarge manager: CloudGroup maxSize: 2 minSize: 2 nodeLabels: kops.k8s.io/instancegroup: dev-process-a datasaker/group: process rootVolumeSize: 100 role: Node subnets: - ap-northeast-2a ``` kops create instancegroup --name=dev.datasaker.io dev-process-b --role node --subnet "ap-northeast-2b" kops edit instancegroup --name=dev.datasaker.io dev-process-b ``` apiVersion: kops.k8s.io/v1alpha2 kind: InstanceGroup metadata: creationTimestamp: "2022-09-05T06:10:03Z" labels: kops.k8s.io/cluster: dev.datasaker.io name: dev-process-b spec: image: ami-0ea5eb4b05645aa8a machineType: c5.xlarge manager: CloudGroup maxSize: 2 minSize: 2 nodeLabels: datasaker/group: process kops.k8s.io/instancegroup: dev-process-b role: Node rootVolumeSize: 100 subnets: - ap-northeast-2b ``` kops create instancegroup --name=dev.datasaker.io dev-process-c --role node --subnet "ap-northeast-2c" kops edit instancegroup --name=dev.datasaker.io dev-process-c ``` apiVersion: kops.k8s.io/v1alpha2 kind: InstanceGroup metadata: creationTimestamp: "2022-09-05T06:10:59Z" labels: kops.k8s.io/cluster: dev.datasaker.io name: dev-process-c spec: image: ami-0ea5eb4b05645aa8a machineType: c5.xlarge manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: datasaker/group: process kops.k8s.io/instancegroup: dev-process-c rootVolumeSize: 100 role: Node subnets: - ap-northeast-2c taints: - dev/mgmt:NoSchedule ``` kops create instancegroup --name=dev.datasaker.io dev-mgmt-a --role node --subnet "ap-northeast-2a" kops --state=s3://clusters.dev.datasaker.io --name=dev.datasaker.io edit instancegroup dev-mgmt-a ``` apiVersion: kops.k8s.io/v1alpha2 kind: InstanceGroup metadata: creationTimestamp: null name: dev-mgmt-a spec: image: ami-0ea5eb4b05645aa8a machineType: c5.xlarge manager: CloudGroup maxSize: 2 minSize: 1 nodeLabels: kops.k8s.io/instancegroup: dev-mgmt-a datasaker/group: mgmt rootVolumeSize: 100 role: Node subnets: - ap-northeast-2a taints: - dev/mgmt:NoSchedule ``` kops create instancegroup --name=dev.datasaker.io dev-mgmt-b --role node --subnet "ap-northeast-2b" kops --state=s3://clusters.dev.datasaker.io --name=dev.datasaker.io edit instancegroup dev-mgmt-b ``` apiVersion: kops.k8s.io/v1alpha2 kind: InstanceGroup metadata: creationTimestamp: null name: dev-mgmt-b spec: image: ami-0abb33b73a78cae31 machineType: c5.xlarge manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: kops.k8s.io/instancegroup: dev-mgmt-b datasaker/group: mgmt rootVolumeSize: 100 role: Node subnets: - ap-northeast-2b taints: - dev/mgmt:NoSchedule ``` kops create instancegroup --name=dev.datasaker.io dev-mgmt-c --role node --subnet "ap-northeast-2c" kops --state=s3://clusters.dev.datasaker.io --name=dev.datasaker.io edit instancegroup dev-mgmt-c ``` apiVersion: kops.k8s.io/v1alpha2 kind: InstanceGroup metadata: creationTimestamp: null name: dev-mgmt-c spec: image: ami-0abb33b73a78cae31 machineType: c5.xlarge manager: CloudGroup maxSize: 1 minSize: 1 nodeLabels: kops.k8s.io/instancegroup: dev-mgmt-a datasaker/group: mgmt rootVolumeSize: 100 role: Node subnets: - ap-northeast-2c taints: - dev/mgmt:NoSchedule ``` kops edit instancegroup --name=dev.datasaker.io dev-data-a kops edit instancegroup --name=dev.datasaker.io dev-data-b kops edit instancegroup --name=dev.datasaker.io dev-data-c kops get --state s3://clusters.dev.datasaker.io --name dev.datasaker.io -o yaml > dev.datasaker.io.yaml vi dev.datasaker.io.yaml subnets: - cidr: 172.21.1.0/24 name: ap-northeast-2a type: Private zone: ap-northeast-2a - cidr: 172.21.2.0/24 name: ap-northeast-2b type: Private zone: ap-northeast-2b - cidr: 172.21.3.0/24 name: ap-northeast-2c type: Private zone: ap-northeast-2c - cidr: 172.21.0.48/28 name: utility-ap-northeast-2a type: Utility zone: ap-northeast-2a - cidr: 172.21.0.64/28 name: utility-ap-northeast-2b type: Utility zone: ap-northeast-2b - cidr: 172.21.0.80/28 name: utility-ap-northeast-2c type: Utility zone: ap-northeast-2c export KOPS_STATE_STORE=s3://clusters.dev.datasaker.io // kops delete cluster dev.datasaker.io --yes --state=s3://clusters.dev.datasaker.io // kops delete -f=./dev.datasaker.io.yaml --yes kops create -f=./dev.datasaker.io.yaml --state=s3://clusters.dev.datasaker.io kops update cluster dev.datasaker.io --yes --admin --state=s3://clusters.dev.datasaker.io kops export kubecfg --admin --state=s3://clusters.dev.datasaker.io kops export kubecfg --admin --kubeconfig ~/workspace/kubeconfig --state=s3://clusters.dev.datasaker.io kops get secrets sshpublickey admin kops get secrets sshpublickey admin -oplaintext MgUKqpCUHLaEcYEuHXTM7ljlTpsnNYSs ssh ubuntu@3.37.243.25 //kops create instancegroup bastions --role Bastion --subnet utility-ap-northeast-2c kops create secret sshpublickey admin -i ~/.ssh/id_rsa.pub --state=s3://clusters.dev.datasaker.io kops create secret sshpublickey admin -i id_rsa_k8s.pub --state=s3://clusters.dev.datasaker.io kops update cluster --yes // to reconfigure the auto-scaling groups kops update cluster --yes --state=s3://clusters.dev.datasaker.io kops rolling-update cluster --name dev.datasaker.io --state=s3://clusters.dev.datasaker.io --yes kops rolling-update cluster --name --yes // to immediately roll all the machines so they have the new key (optional) // Lambda 설정 변경. get_names = ['ag-dmz-bastion-datasaker','master-ap-northeast-2a.masters.dev.datasaker.io','master-ap-northeast-2b.masters.dev.datasaker.io','master-ap-northeast-2c.masters.dev.datasaker.io','dev-process-a.dev.datasaker.io','dev-process-b.dev.datasaker.io','dev-process-c.dev.datasaker.io','dev-data-a.dev.datasaker.io','dev-data-b.dev.datasaker.io','dev-data-c.dev.datasaker.io','dev-mgmt-a.dev.datasaker.io','dev-mgmt-b.dev.datasaker.io'] Suggestions: * validate cluster: kops validate cluster --wait 10m * list nodes: kubectl get nodes --show-labels * ssh to the master: ssh -i ~/.ssh/id_rsa ubuntu@api.dev.datasaker.io * the ubuntu user is specific to Ubuntu. If not using Ubuntu please use the appropriate user based on your OS. * read about installing addons at: https://kops.sigs.k8s.io/addons. // when kubecfg changed, due to master redeploy kops export kubecfg --admin --name dev.datasaker.io --state s3://clusters.dev.datasaker.io --kubeconfig ~/.kube/config kops export kubecfg --admin=87600h0m0s --name dev.datasaker.io --state s3://clusters.dev.datasaker.io --kubeconfig ~/.kube/config kops export kubecfg --admin=8760h0m0s --name dev.datasaker.io --state s3://clusters.dev.datasaker.io --kubeconfig ~/.kube/config kops export kubecfg --admin=720h0m0s --name dev.datasaker.io --state s3://clusters.dev.datasaker.io --kubeconfig ~/.kube/config kops update cluster --name=dev.datasaker.io --state=s3://clusters.dev.datasaker.io --out=./tf-kops-dev-20200916-ip --target=terraform 115.178.73.2/32 exem router 115.178.73.91/32 proxy 3.35.247.45/32 bastion api-elb.dev.datasaker.io 에 115.178.73.2 만 적용되는 이슈가 있음. kops edit cluster --name=dev.datasaker.io from kubernetesApiAccess: - 0.0.0.0/0 - ::/0 sshAccess: - 0.0.0.0/0 - ::/0 to kubernetesApiAccess: - 115.178.73.2/32 - 115.178.73.91/32 - 3.35.247.45/32 sshAccess: - 115.178.73.2/32 - 115.178.73.91/32 - 3.35.247.45/32 kops update cluster --yes --state=s3://clusters.dev.datasaker.io kops rolling-update cluster --yes --state=s3://clusters.dev.datasaker.io ## ## aws security group masters, nodes set for 30000, 30001 from bastion,elb secg-dmz-datasaker sg-07f27eba164d59dfa from-dev-bastion-to-ingress 1. elb 용 security group 생성 service-elb.dev.datasaker.io sg-08dd3bc6dac12a286 1. 인증서 생성 // cert-static (IP 제한 없음) am.dev.kr.datasaker.io (agent manager) dgate-m.dev.kr.datasaker.io (datagate-metric) dgate-j.dev.kr.datasaker.io (datagate-jaeger) dgate-k.dev.kr.datasaker.io (datagate-menifest) lgate.dev.kr.datasaker.io (loggate) app.dev.kr.datasaker.io (app - ui) auth.dev.kr.datasaker.io (keycloak) api.dev.kr.datasaker.io (krakend) 1. master sg에서 elb sg로 부터 오는 입력 허용. TCP 30000 (HTTP), TCP 30001 (HTTPS) from-dev-alb-to-ingress elb sg에 80, 443 입력 허용. dev-from-all-80-to-ing dev-from-all-443-to-ing 1. target group 생성 tg-dev-kr-30000-http-ingress http: 30000 vpc: vpc-datasaker Protocol version http1 healthcheck: http Health check path: / Advanced health check settings : Success codes: 200,404 AddTag: Name: tg-dev-kr-30000-http-ingress create tg-dev-kr-30001-https-ingress https: 30001 vpc: dev.k8s.datasaker.io Protocol version http1 healthcheck: https Health check path: / Advanced health check settings : Success codes: 200,404 AddTag: Name / tg-dev-kr-30001-https-ingress create tg-dev-kr-30001-http-ingress http: 30001 vpc: dev.k8s.datasaker.io Protocol version http1 healthcheck: http Health check path: / Advanced health check settings : Success codes: 200,404 AddTag: Name / tg-dev-kr-30001-https-ingress create 1. alb 생성 alb-dev-kr-ingress Internet-facing IPv4 vpc: vpc-datasaker Mappings: - sbn-dmz-a.datasaker - sbn-dmz-b.datasaker - sbn-dmz-c.datasaker Security groups: - service-elb.dev.datasaker.io Listeners and routing: - HTTP: 80 tg: tg-dev-kr-30000-http-ingress - HTTPS: 443 tg: tg-dev-kr-30001-https-ingress AddTag: Name / alb-dev-kr-ingress 1. 각 autoscaling group에 tg 연결 nlb-dev-ingress internet-facing ipv4 vpc: dev.k8s.datasaker.io mappings: subnet (utilityA,utilityB,utilityC) Listeners and routing: TCP:80 -> targetGroup 지정. tg-dev-ingress-30000 TCP:443 -> targetGroup 지정. tg-dev-ingress-30001 ## mng 1. manage 인증서 생성 // manage-dev.kr.datasaker.io (특정 IP 제한) argo.dev.kr.datasaker.io (argocd) vlt.dev.kr.datasaker.io (vault) jenkins.dev.kr.datasaker.io (jenkins) 1. managed SG 설정 sg.dev.kr-managed-ingress VPC: vpc-datasaker AllTrafic -> AnyWhere AllTrafic -> AnyWhere Name: sg.dev.kr-managed-ingress 1. master sg에서 elb sg로 부터 오는 입력 허용. TCP 30000 (HTTP), TCP 30001 (HTTPS) from-dev-manage-elb-80-to-ingress from-dev-manage-elb-443-to-ingress elb sg에 80, 443 입력 허용. dev-from-all-80-to-ing dev-from-all-443-to-ing 1. target group 생성 tg-dev-kr-30000-mng-http-ing http: 30000 vpc: vpc-datasaker Protocol version http1 healthcheck: http Health check path: / Advanced health check settings : Success codes: 200,404,400 AddTag: Name: tg-dev-kr-30000-mng-http-ing create tg-dev-kr-30001-mng-https-ing https: 30001 vpc: dev.k8s.datasaker.io Protocol version http1 healthcheck: https Health check path: / Advanced health check settings : Success codes: 200,404,400 AddTag: Name / tg-dev-kr-30001-mng-https-ing create 1. alb 생성 alb-dev-kr-mng-ing Internet-facing IPv4 vpc: vpc-datasaker Mappings: - sbn-dmz-a.datasaker - sbn-dmz-b.datasaker - sbn-dmz-c.datasaker Security groups: - service-elb.dev.datasaker.io Listeners and routing: - HTTP: 80 tg: tg-dev-kr-30000-http-ingress - HTTPS: 443 tg: tg-dev-kr-30001-https-ingress AddTag: Name / alb-dev-kr-mng-ing 1. 각 autoscaling group에 tg 연결 master asg에 TCP:80 -> targetGroup 지정. tg-dev-ingress-30000 TCP:443 -> targetGroup 지정. tg-dev-ingress-30001 ## nodeport 연결 1. nlb 용 security group 생성 sg.nlb.dev.kr.datasaker.io Custom TCP: 30010 - 32768 : 0.0.0.0/0 Custom TCP: 30010 - 32768 : ::/0 1. master sg에서 nlb sg로 부터 오는 입력 허용. TCP 30000-32768 from-nlb-30000-32768-to-nodeport // 1개의 nodeport 지정 (범위로 지정 가능한지 검토 필요) // 1. target group 생성 (ingress로 health check로만 사용) // 수정 필요. 31428 tg-dev-kr-tcp-np tcp: 31428 vpc: vpc-datasaker Protocol version http1 healthcheck: http Health check path: / Advanced health check settings : Success codes: 200,404,400 AddTag: Name: tg-dev-kr-tcp-np create 1. nlb 생성 및 tg 연결 nlb-dev-kr-mng-np Internet-facing IPv4 vpc: vpc-datasaker Mappings: - sbn-dmz-a.datasaker - sbn-dmz-b.datasaker - sbn-dmz-c.datasaker Security groups: - service-elb.dev.datasaker.io Listeners and routing: - HTTP: 80 tg: tg-dev-kr-30000-http-ingress - HTTPS: 443 tg: tg-dev-kr-30001-https-ingress AddTag: Name / nlb-dev-kr-mng-np 1. 각 autoscaling group에 tg 연결 master asg에 tg-dev-kr-tcp-np // arn:aws:elasticloadbalancing:ap-northeast-2:508259851457:targetgroup/tg-dev-kr-tcp-np/e86a5f0c14928131 ## 1. autoscaling group에서 lb를 이용한 target group 등록 nodes-ap-northeast-2a.dev.k8s.datasaker.io, nodes-ap-northeast-2b.dev.k8s.datasaker.io, nodes-ap-northeast-2c.dev.k8s.datasaker.io Load balancing: tg-dev-ingress-30000, tg-dev-ingress-30001 1. target group에서 instance 가 보이는지 확인 1. nlb A Record 주소 확인 nlb-dev-ingress-f266e4f0bead8225.elb.ap-northeast-2.amazonaws.com 1. route53에 등록 ex) g2048.dev.kr.datasaker.io g2048.dev.kr A Alias nlb-dev-ingress-f266e4f0bead8225.elb.ap-northeast-2.amazonaws.com. 300 Simple routing argo.dev.datasaker.io A Alias nlb-dev-ingress-f266e4f0bead8225.elb.ap-northeast-2.amazonaws.com. 300 Simple routing simple-app.dev.datasaker.io 1. test app https://blog.leiwang.info/simple-app // git clone https://github.com/tendant/simple-app.git // helm install simple-app simple-app -n simple-app --create-namespace ssh ubuntu@bastion.dev.k8s.datasaker.io curl -v 172.20.68.243:30000/healthz masters.dev.k8s.datasaker.io from-bastion-30000-30001 sgr-0d891ac3623e03e7b – 사용자 지정 TCP TCP 30000 - 30001 sg-0fadf3368999e9eaf / bastion.dev.k8s.datasaker.io – nodes.dev.k8s.datasaker.io from-bastion-30000-30001 sgr-0d891ac3623e03e7b – 사용자 지정 TCP TCP 30000 - 30001 sg-0fadf3368999e9eaf / bastion.dev.k8s.datasaker.io – nc -z -v 172.20.68.243 30000-30001 nc -z -v 172.20.68.243 32679 simple-app.dev.datasaker.io tg-dev-kr-30000-http-ingress // arn:aws:elasticloadbalancing:ap-northeast-2:508259851457:targetgroup/tg-dev-kr-30000-http-ingress/c722b2d641bcfc87 tg-dev-kr-30001-https-ingress // arn:aws:elasticloadbalancing:ap-northeast-2:508259851457:targetgroup/tg-dev-kr-30001-https-ingress/d41767571f8a7bb8 tg-dev-kr-30000-mng-http-ing // arn:aws:elasticloadbalancing:ap-northeast-2:508259851457:targetgroup/tg-dev-kr-30000-mng-http-ing/474dc8d6f6ad2106 tg-dev-kr-30001-mng-https-ing // arn:aws:elasticloadbalancing:ap-northeast-2:508259851457:targetgroup/tg-dev-kr-30001-mng-https-ing/960e93df1bb9a326 ## ## ing 생성 krakend-dev krakend-develop NodePort 100.65.7.164 80:32701/TCP 7d19h saas-dev sam-agentapi-develop NodePort 100.71.124.134 8080:32199/TCP 3d17h saas-dev sam-app-sender-develop NodePort 100.65.88.171 8000:31514/TCP 5d17h saas-dev sam-dashboardapi-develop NodePort 100.66.2.234 8080:30696/TCP 5d17h saas-dev sam-infrastructureapi-develop NodePort 100.68.69.163 8000:31648/TCP 3d23h saas-dev sam-jaeger-agent-develop NodePort 100.67.217.7 14271:30772/TCP,5778:30835/TCP,6831:30834/UDP 4d23h saas-dev sam-ui-develop NodePort 100.70.74.238 80:30208/TCP 5d20h saas-dev sam-usergate-develop NodePort 100.66.175.184 8080:31085/TCP 3d22h saas-dev sample-app-develop-sample-app-deploy ClusterIP 100.65.249.165 80/TCP k -n argocd edit ing argocd-server ``` apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: kubernetes.io/ingress.class: nginx meta.helm.sh/release-name: argocd meta.helm.sh/release-namespace: argocd nginx.ingress.kubernetes.io/backend-protocol: HTTPS nginx.ingress.kubernetes.io/force-ssl-redirect: "true" nginx.ingress.kubernetes.io/ssl-passthrough: "true" creationTimestamp: "2022-09-26T04:31:11Z" generation: 4 labels: app.kubernetes.io/component: server app.kubernetes.io/instance: argocd app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: argocd-server app.kubernetes.io/part-of: argocd helm.sh/chart: argo-cd-4.9.11 name: argocd-server namespace: argocd resourceVersion: "1567505" uid: 567190bd-a080-4628-9e21-5f6b56ffd5e1 spec: rules: - host: argo.dev.kr.datasaker.io http: paths: - backend: service: name: argocd-server port: number: 80 path: / pathType: Prefix status: loadBalancer: ingress: - ip: 100.71.12.82 ``` #### sam-ui saas-dev sam-ui-develop NodePort 100.70.74.238 80:30208/TCP ``` apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: sam-ui-develop annotations: kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/force-ssl-redirect: "true" nginx.ingress.kubernetes.io/ssl-passthrough: "true" namespace: saas-dev spec: rules: - host: app.dev.kr.datasaker.io http: paths: - backend: service: name: sam-ui-develop port: number: 80 path: / pathType: Prefix ``` ## keycloak ingress keycloak NodePort 100.67.217.51 80:30100/TCP,443:30101/TCP ``` apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/backend-protocol: HTTPS nginx.ingress.kubernetes.io/force-ssl-redirect: "true" nginx.ingress.kubernetes.io/ssl-passthrough: "true" name: keycloak-dev namespace: infra-dev spec: rules: - host: auth.dev.kr.datasaker.io http: paths: - backend: service: name: keycloak port: number: 80 path: / pathType: Prefix ``` ``` apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: kubernetes.io/ingress.class: nginx meta.helm.sh/release-name: keycloak meta.helm.sh/release-namespace: infra-dev creationTimestamp: "2022-10-04T04:46:18Z" generation: 2 labels: app.kubernetes.io/component: keycloak app.kubernetes.io/instance: keycloak app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: keycloak helm.sh/chart: keycloak-7.1.17 name: keycloak namespace: infra-dev resourceVersion: "2184553" uid: 1ca8be2a-6580-4537-8488-c825839a7512 spec: rules: - host: auth.dev.kr.datasaker.io http: paths: - backend: service: name: keycloak port: name: https path: / pathType: ImplementationSpecific status: loadBalancer: {} ``` https://community.gooddata.com/administration-61/how-to-properly-terminate-ssl-using-aws-alb-with-acm-391 annotations: kubernetes.io/ingress.class: "nginx" nginx.ingress.kubernetes.io/configuration-snippet: | add_header 'Access-Control-Allow-Origin' '*'; add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; add_header X-Frame-Options "sameorigin"; add_header X-Content-Type-Options nosniff; add_header Referrer-Policy 'same-origin'; add_header 'Access-Control-Allow-Origin' '*'; add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; #### am.dev.kr.datasaker.io (agent manager) ``` apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: nginx.ingress.kubernetes.io/force-ssl-redirect: "true" nginx.ingress.kubernetes.io/ssl-passthrough: "true" name: agentmanager-develop namespace: saas-dev spec: ingressClassName: nginx rules: - host: am.dev.kr.datasaker.io http: paths: - path: / pathType: Prefix backend: service: name: sam-agentmanager-cloud-4534 port: number: 8080 ``` #### https://aws.amazon.com/blogs/aws/new-application-load-balancer-support-for-end-to-end-http-2-and-grpc/ #### dgate-j.dev.kr.datasaker.io (datagate-jaeger) ``` apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: nginx.ingress.kubernetes.io/ssl-redirect: "true" nginx.ingress.kubernetes.io/backend-protocol: "GRPC" name: datagate-jaeger-develop namespace: saas-dev spec: ingressClassName: nginx rules: - host: dgate-j.dev.kr.datasaker.io http: paths: - path: / pathType: Prefix backend: service: name: jaeger-sam-datagate-cloud-test port: number: 8080 ``` #### dgate-m.dev.kr.datasaker.io (datagate-metric) ``` apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: nginx.ingress.kubernetes.io/ssl-redirect: "true" nginx.ingress.kubernetes.io/backend-protocol: "GRPC" name: datagate-metric-develop namespace: saas-dev spec: ingressClassName: nginx rules: - host: dgate-m.dev.kr.datasaker.io http: paths: - path: / pathType: Prefix backend: service: name: metric-sam-datagate-cloud-test port: number: 8080 ``` #### dgate-k.dev.kr.datasaker.io (datagate-menifest) ``` apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: nginx.ingress.kubernetes.io/ssl-redirect: "true" nginx.ingress.kubernetes.io/backend-protocol: "GRPC" name: datagate-menifest-develop namespace: saas-dev spec: ingressClassName: nginx rules: - host: dgate-k.dev.kr.datasaker.io http: paths: - path: / pathType: Prefix backend: service: name: manifest-sam-datagate-cloud-test port: number: 8080 ``` #### postgresql.dev.kr.datasaker.io postgresql NodePort 100.69.229.168 5432:32713/TCP ``` apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: postgresql-develop annotations: kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/force-ssl-redirect: "true" nginx.ingress.kubernetes.io/ssl-passthrough: "true" namespace: infra-dev spec: rules: - host: postgresql.dev.kr.datasaker.io http: paths: - backend: service: name: postgresql port: number: 5432 path: / pathType: Prefix ```