diff --git a/doc/2_how_to_install_dev_cluster_20221026.txt b/doc/2_how_to_install_dev_cluster_20221026.txt new file mode 100644 index 0000000..7103c47 --- /dev/null +++ b/doc/2_how_to_install_dev_cluster_20221026.txt @@ -0,0 +1,1322 @@ + + + + + + + + + +export KOPS_STATE_STORE=s3://clusters.dev.datasaker.io + +kops update cluster --name dev.datasaker.io --state=s3://clusters.dev.datasaker.io +kops update cluster --name dev.datasaker.io --state=s3://clusters.dev.datasaker.io > changes-dev.datasaker.io-20221019.txt +kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io update cluster --yes --admin +kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io rolling-update cluster --yes --cloudonly + +kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io -o yaml get > dev.datasaker.io-1.yaml +kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io -o yaml get > dev.datasaker.io-20221025.yaml +kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io edit cluster +kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io get ig +kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io edit ig master-ap-northeast-2a +kops export kubecfg --admin=8760h0m0s --name dev.datasaker.io --state s3://clusters.dev.datasaker.io --kubeconfig ~/.kube/config + +kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io update cluster --out=./tf-kops-dev-20221025 --target=terraform + + +security-dev-bastion ami-0b6591f49cf24e237 +security-dev-node ami-0abb33b73a78cae31 + + +kops create cluster \ + --name dev.datasaker.io \ + --vpc vpc-0b6e0b906c678a22f \ + --cloud aws \ + --state s3://clusters.dev.datasaker.io \ + --ssh-public-key /home/hsgahm/.ssh/id_rsa_k8s.pub \ + --topology private --kubernetes-version "1.23.10" \ + --network-cidr "172.21.0.0/16" \ + --networking calico \ + --container-runtime containerd \ + --image ami-0ea5eb4b05645aa8a \ + --zones ap-northeast-2a,ap-northeast-2b,ap-northeast-2c \ + --master-count 3 \ + --master-size t3.small \ + --master-volume-size 50 \ + --node-count 3 \ + --node-size t3.small \ + --node-volume-size 100 \ + --utility-subnets "subnet-0de55619bee2411f8,subnet-0a5d787353f874684,subnet-0ee26ffc561efb292" \ + --subnets "subnet-0c875e254456809f7,subnet-05672a669943fc12f,subnet-0940fd78504acbbde" \ + -v 10 + + + +kops --name dev.datasaker.io --state s3://clusters.dev.datasaker.io edit cluster +``` + containerd: + configOverride: | + version = 2 + imports = ["/etc/containerd/runtime_*.toml"] + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + sandbox_image = "registry.k8s.io/pause:3.6@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db" + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry.configs."registry-1.docker.io".auth] + username = "datasaker" + password = "dckr_pat_kQP6vcHm_jMChWd_zvgH_G3kucc" + +``` + +``` + +``` + + +``` + enableWAF: true + enableWAFv2: true +``` + +kops get instancegroups --name=dev.datasaker.io --state s3://clusters.dev.datasaker.io + +kops --name=dev.datasaker.io delete instancegroup nodes-ap-northeast-2a +kops --name=dev.datasaker.io delete instancegroup nodes-ap-northeast-2b +kops --name=dev.datasaker.io delete instancegroup nodes-ap-northeast-2c + +kops edit instancegroup --name=dev.datasaker.io master-ap-northeast-2a +kops edit instancegroup --name=dev.datasaker.io master-ap-northeast-2b +kops edit instancegroup --name=dev.datasaker.io master-ap-northeast-2c + rootVolumeSize: 64 (default) + + +kops --name=dev.datasaker.io get ig + +kops get clusters +kops edit cluster dev.datasaker.io --state s3://clusters.dev.datasaker.io + +// subnet name 변경 +//ap-northeast-2a -> sbn-dev-a.datasaker +//ap-northeast-2b -> sbn-dev-b.datasaker +//ap-northeast-2c -> sbn-dev-c.datasaker + +//utility-ap-northeast-2a -> sbn-dmz-a.datasaker +//utility-ap-northeast-2b -> sbn-dmz-b.datasaker +//utility-ap-northeast-2c -> sbn-dmz-c.datasaker + + + +kops edit instancegroups --name=dev.datasaker.io master-ap-northeast-2a + +``` +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2022-09-06T05:44:09Z" + labels: + kops.k8s.io/cluster: dev.datasaker.io + name: master-ap-northeast-2a +spec: + image: ami-0ea5eb4b05645aa8a + instanceMetadata: + httpPutResponseHopLimit: 3 + httpTokens: required + machineType: t3.small + manager: CloudGroup + maxSize: 1 + minSize: 1 + nodeLabels: + kops.k8s.io/instancegroup: master-ap-northeast-2a + role: Master + rootVolumeSize: 50 + subnets: + - ap-northeast-2a + cloudLabels: + autoscale-off: "True" + autoscale-on: "True" + +``` + + + +//kops create instancegroup --name=dev.datasaker.io dev-master-a --role master --subnet "ap-northeast-2a" +//kops create instancegroup --name=dev.datasaker.io dev-master-b --role node --subnet "ap-northeast-2b" +//kops create instancegroup --name=dev.datasaker.io dev-master-c --role node --subnet "ap-northeast-2c" + +// kops delete instancegroup --name=dev.datasaker.io dev-data-a +// kops delete instancegroup --name=dev.datasaker.io dev-data-b +// kops delete instancegroup --name=dev.datasaker.io dev-data-c + + +kops create instancegroup --name=dev.datasaker.io dev-data-druid-a --role node --subnet "ap-northeast-2a" +kops create instancegroup --name=dev.datasaker.io dev-data-druid-b --role node --subnet "ap-northeast-2b" +kops create instancegroup --name=dev.datasaker.io dev-data-druid-c --role node --subnet "ap-northeast-2c" +kops create instancegroup --name=dev.datasaker.io dev-data-kafka-a --role node --subnet "ap-northeast-2a" +kops create instancegroup --name=dev.datasaker.io dev-data-kafka-b --role node --subnet "ap-northeast-2b" +kops create instancegroup --name=dev.datasaker.io dev-data-kafka-c --role node --subnet "ap-northeast-2c" + +``` +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: null + name: dev-data-druid-a +spec: + image: ami-0abb33b73a78cae31 + kubelet: + anonymousAuth: false + nodeLabels: + node-role.kubernetes.io/node: "" + machineType: m6i.2xlarge + manager: CloudGroup + maxSize: 1 + minSize: 1 + nodeLabels: + kops.k8s.io/instancegroup: dev-data-druid-a + datasaker/group: data-druid + role: Node + subnets: + - ap-northeast-2a + taints: + - dev/data-druid:NoSchedule +``` + +``` +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: null + name: dev-data-kafka-a +spec: + image: ami-0abb33b73a78cae31 + kubelet: + anonymousAuth: false + nodeLabels: + node-role.kubernetes.io/node: "" + machineType: m6i.2xlarge + manager: CloudGroup + maxSize: 1 + minSize: 1 + nodeLabels: + kops.k8s.io/instancegroup: dev-data-kafka-a + datasaker/group: data-kafka + role: Node + subnets: + - ap-northeast-2a + taints: + - dev/data-kafka:NoSchedule + +``` + + +kops create instancegroup --name=dev.datasaker.io dev-data-a --role node --subnet "ap-northeast-2a" +kops edit instancegroup --name=dev.datasaker.io dev-data-a + + ``` + apiVersion: kops.k8s.io/v1alpha2 + kind: InstanceGroup + metadata: + creationTimestamp: "2022-09-05T05:53:59Z" + labels: + kops.k8s.io/cluster: dev.datasaker.io + name: dev-data-a + spec: + image: ami-0ea5eb4b05645aa8a + machineType: m5.4xlarge + manager: CloudGroup + maxSize: 1 + minSize: 1 + nodeLabels: + kops.k8s.io/instancegroup: dev-data-a + datasaker/group: data + rootVolumeSize: 100 + role: Node + subnets: + - ap-northeast-2a + ``` + + ``` + cloudLabels: + autoscale-off: "True" + autoscale-on: "True" + ``` + +kops create instancegroup --name=dev.datasaker.io dev-data-b --role node --subnet "ap-northeast-2b" +kops edit instancegroup --name=dev.datasaker.io dev-data-b + + ``` + apiVersion: kops.k8s.io/v1alpha2 + kind: InstanceGroup + metadata: + creationTimestamp: "2022-09-05T06:00:56Z" + generation: 1 + labels: + kops.k8s.io/cluster: dev.datasaker.io + name: dev-data-b + spec: + image: ami-0ea5eb4b05645aa8a + machineType: m5.4xlarge + manager: CloudGroup + maxSize: 1 + minSize: 1 + nodeLabels: + datasaker/group: data + kops.k8s.io/instancegroup: dev-data-b + role: Node + rootVolumeSize: 100 + subnets: + - ap-northeast-2b + + ``` + +kops create instancegroup --name=dev.datasaker.io dev-data-c --role node --subnet "ap-northeast-2c" +kops edit instancegroup --name=dev.datasaker.io dev-data-c + + ``` + apiVersion: kops.k8s.io/v1alpha2 + kind: InstanceGroup + metadata: + creationTimestamp: null + name: dev-data-c + spec: + image: ami-0ea5eb4b05645aa8a + machineType: m5.4xlarge + manager: CloudGroup + maxSize: 1 + minSize: 1 + nodeLabels: + kops.k8s.io/instancegroup: dev-data-c + datasaker/group: data + rootVolumeSize: 100 + role: Node + subnets: + - ap-northeast-2c + + ``` + +// kops delete instancegroup --name=dev.datasaker.io dev-process-a +// kops delete instancegroup --name=dev.datasaker.io dev-process-b +// kops delete instancegroup --name=dev.datasaker.io dev-process-c +kops create instancegroup --name=dev.datasaker.io dev-process-a --role node --subnet "ap-northeast-2a" +kops edit instancegroup --name=dev.datasaker.io dev-process-a + + ``` + apiVersion: kops.k8s.io/v1alpha2 + kind: InstanceGroup + metadata: + creationTimestamp: null + name: dev-process-a + spec: + image: ami-0ea5eb4b05645aa8a + machineType: c5.xlarge + manager: CloudGroup + maxSize: 2 + minSize: 2 + nodeLabels: + kops.k8s.io/instancegroup: dev-process-a + datasaker/group: process + rootVolumeSize: 100 + role: Node + subnets: + - ap-northeast-2a + + ``` + +kops create instancegroup --name=dev.datasaker.io dev-process-b --role node --subnet "ap-northeast-2b" +kops edit instancegroup --name=dev.datasaker.io dev-process-b + ``` + apiVersion: kops.k8s.io/v1alpha2 + kind: InstanceGroup + metadata: + creationTimestamp: "2022-09-05T06:10:03Z" + labels: + kops.k8s.io/cluster: dev.datasaker.io + name: dev-process-b + spec: + image: ami-0ea5eb4b05645aa8a + machineType: c5.xlarge + manager: CloudGroup + maxSize: 2 + minSize: 2 + nodeLabels: + datasaker/group: process + kops.k8s.io/instancegroup: dev-process-b + role: Node + rootVolumeSize: 100 + subnets: + - ap-northeast-2b + + ``` + +kops create instancegroup --name=dev.datasaker.io dev-process-c --role node --subnet "ap-northeast-2c" +kops edit instancegroup --name=dev.datasaker.io dev-process-c + ``` + apiVersion: kops.k8s.io/v1alpha2 + kind: InstanceGroup + metadata: + creationTimestamp: "2022-09-05T06:10:59Z" + labels: + kops.k8s.io/cluster: dev.datasaker.io + name: dev-process-c + spec: + image: ami-0ea5eb4b05645aa8a + machineType: c5.xlarge + manager: CloudGroup + maxSize: 1 + minSize: 1 + nodeLabels: + datasaker/group: process + kops.k8s.io/instancegroup: dev-process-c + rootVolumeSize: 100 + role: Node + subnets: + - ap-northeast-2c + taints: + - dev/mgmt:NoSchedule + ``` + +kops create instancegroup --name=dev.datasaker.io dev-mgmt-a --role node --subnet "ap-northeast-2a" +kops --state=s3://clusters.dev.datasaker.io --name=dev.datasaker.io edit instancegroup dev-mgmt-a + ``` + apiVersion: kops.k8s.io/v1alpha2 + kind: InstanceGroup + metadata: + creationTimestamp: null + name: dev-mgmt-a + spec: + image: ami-0ea5eb4b05645aa8a + machineType: c5.xlarge + manager: CloudGroup + maxSize: 2 + minSize: 1 + nodeLabels: + kops.k8s.io/instancegroup: dev-mgmt-a + datasaker/group: mgmt + rootVolumeSize: 100 + role: Node + subnets: + - ap-northeast-2a + taints: + - dev/mgmt:NoSchedule + + ``` + +kops create instancegroup --name=dev.datasaker.io dev-mgmt-b --role node --subnet "ap-northeast-2b" +kops --state=s3://clusters.dev.datasaker.io --name=dev.datasaker.io edit instancegroup dev-mgmt-b + + ``` + apiVersion: kops.k8s.io/v1alpha2 + kind: InstanceGroup + metadata: + creationTimestamp: null + name: dev-mgmt-b + spec: + image: ami-0abb33b73a78cae31 + machineType: c5.xlarge + manager: CloudGroup + maxSize: 1 + minSize: 1 + nodeLabels: + kops.k8s.io/instancegroup: dev-mgmt-b + datasaker/group: mgmt + rootVolumeSize: 100 + role: Node + subnets: + - ap-northeast-2b + taints: + - dev/mgmt:NoSchedule + + ``` + + +kops create instancegroup --name=dev.datasaker.io dev-mgmt-c --role node --subnet "ap-northeast-2c" +kops --state=s3://clusters.dev.datasaker.io --name=dev.datasaker.io edit instancegroup dev-mgmt-c + ``` + apiVersion: kops.k8s.io/v1alpha2 + kind: InstanceGroup + metadata: + creationTimestamp: null + name: dev-mgmt-c + spec: + image: ami-0abb33b73a78cae31 + machineType: c5.xlarge + manager: CloudGroup + maxSize: 1 + minSize: 1 + nodeLabels: + kops.k8s.io/instancegroup: dev-mgmt-a + datasaker/group: mgmt + rootVolumeSize: 100 + role: Node + subnets: + - ap-northeast-2c + taints: + - dev/mgmt:NoSchedule + + ``` + + + +kops edit instancegroup --name=dev.datasaker.io dev-data-a +kops edit instancegroup --name=dev.datasaker.io dev-data-b +kops edit instancegroup --name=dev.datasaker.io dev-data-c + + + + + +kops get --state s3://clusters.dev.datasaker.io --name dev.datasaker.io -o yaml > dev.datasaker.io.yaml + +vi dev.datasaker.io.yaml + + subnets: + - cidr: 172.21.1.0/24 + name: ap-northeast-2a + type: Private + zone: ap-northeast-2a + - cidr: 172.21.2.0/24 + name: ap-northeast-2b + type: Private + zone: ap-northeast-2b + - cidr: 172.21.3.0/24 + name: ap-northeast-2c + type: Private + zone: ap-northeast-2c + - cidr: 172.21.0.48/28 + name: utility-ap-northeast-2a + type: Utility + zone: ap-northeast-2a + - cidr: 172.21.0.64/28 + name: utility-ap-northeast-2b + type: Utility + zone: ap-northeast-2b + - cidr: 172.21.0.80/28 + name: utility-ap-northeast-2c + type: Utility + zone: ap-northeast-2c + +export KOPS_STATE_STORE=s3://clusters.dev.datasaker.io + + +// kops delete cluster dev.datasaker.io --yes --state=s3://clusters.dev.datasaker.io +// kops delete -f=./dev.datasaker.io.yaml --yes +kops create -f=./dev.datasaker.io.yaml --state=s3://clusters.dev.datasaker.io +kops update cluster dev.datasaker.io --yes --admin --state=s3://clusters.dev.datasaker.io + + +kops export kubecfg --admin --state=s3://clusters.dev.datasaker.io +kops export kubecfg --admin --kubeconfig ~/workspace/kubeconfig --state=s3://clusters.dev.datasaker.io + +kops get secrets sshpublickey admin +kops get secrets sshpublickey admin -oplaintext +MgUKqpCUHLaEcYEuHXTM7ljlTpsnNYSs + +ssh ubuntu@3.37.243.25 + +//kops create instancegroup bastions --role Bastion --subnet utility-ap-northeast-2c + +kops create secret sshpublickey admin -i ~/.ssh/id_rsa.pub --state=s3://clusters.dev.datasaker.io + kops create secret sshpublickey admin -i id_rsa_k8s.pub --state=s3://clusters.dev.datasaker.io +kops update cluster --yes // to reconfigure the auto-scaling groups +kops update cluster --yes --state=s3://clusters.dev.datasaker.io +kops rolling-update cluster --name dev.datasaker.io --state=s3://clusters.dev.datasaker.io --yes + +kops rolling-update cluster --name --yes // to immediately roll all the machines so they have the new key (optional) + + +// Lambda 설정 변경. +get_names = ['ag-dmz-bastion-datasaker','master-ap-northeast-2a.masters.dev.datasaker.io','master-ap-northeast-2b.masters.dev.datasaker.io','master-ap-northeast-2c.masters.dev.datasaker.io','dev-process-a.dev.datasaker.io','dev-process-b.dev.datasaker.io','dev-process-c.dev.datasaker.io','dev-data-a.dev.datasaker.io','dev-data-b.dev.datasaker.io','dev-data-c.dev.datasaker.io','dev-mgmt-a.dev.datasaker.io','dev-mgmt-b.dev.datasaker.io'] + + +Suggestions: + * validate cluster: kops validate cluster --wait 10m + * list nodes: kubectl get nodes --show-labels + * ssh to the master: ssh -i ~/.ssh/id_rsa ubuntu@api.dev.datasaker.io + * the ubuntu user is specific to Ubuntu. If not using Ubuntu please use the appropriate user based on your OS. + * read about installing addons at: https://kops.sigs.k8s.io/addons. + + +// when kubecfg changed, due to master redeploy +kops export kubecfg --admin --name dev.datasaker.io --state s3://clusters.dev.datasaker.io --kubeconfig ~/.kube/config +kops export kubecfg --admin=87600h0m0s --name dev.datasaker.io --state s3://clusters.dev.datasaker.io --kubeconfig ~/.kube/config +kops export kubecfg --admin=8760h0m0s --name dev.datasaker.io --state s3://clusters.dev.datasaker.io --kubeconfig ~/.kube/config + + +kops export kubecfg --admin=720h0m0s --name dev.datasaker.io --state s3://clusters.dev.datasaker.io --kubeconfig ~/.kube/config + +kops update cluster --name=dev.datasaker.io --state=s3://clusters.dev.datasaker.io --out=./tf-kops-dev-20200916-ip --target=terraform + + +115.178.73.2/32 exem router +115.178.73.91/32 proxy +3.35.247.45/32 bastion + +api-elb.dev.datasaker.io 에 115.178.73.2 만 적용되는 이슈가 있음. + + +kops edit cluster --name=dev.datasaker.io +from + kubernetesApiAccess: + - 0.0.0.0/0 + - ::/0 + + sshAccess: + - 0.0.0.0/0 + - ::/0 + +to + kubernetesApiAccess: + - 115.178.73.2/32 + - 115.178.73.91/32 + - 3.35.247.45/32 + + sshAccess: + - 115.178.73.2/32 + - 115.178.73.91/32 + - 3.35.247.45/32 + +kops update cluster --yes --state=s3://clusters.dev.datasaker.io + +kops rolling-update cluster --yes --state=s3://clusters.dev.datasaker.io + + + +## + + +## aws security group masters, nodes set for 30000, 30001 from bastion,elb + secg-dmz-datasaker sg-07f27eba164d59dfa + from-dev-bastion-to-ingress + + +1. elb 용 security group 생성 + service-elb.dev.datasaker.io sg-08dd3bc6dac12a286 + + + +1. 인증서 생성 +// cert-static (IP 제한 없음) +am.dev.kr.datasaker.io (agent manager) +dgate-m.dev.kr.datasaker.io (datagate-metric) +dgate-j.dev.kr.datasaker.io (datagate-jaeger) +dgate-k.dev.kr.datasaker.io (datagate-menifest) +lgate.dev.kr.datasaker.io (loggate) +app.dev.kr.datasaker.io (app - ui) +auth.dev.kr.datasaker.io (keycloak) +api.dev.kr.datasaker.io (krakend) + +1. master sg에서 elb sg로 부터 오는 입력 허용. + TCP 30000 (HTTP), TCP 30001 (HTTPS) + + from-dev-alb-to-ingress + + elb sg에 80, 443 입력 허용. + dev-from-all-80-to-ing + dev-from-all-443-to-ing + +1. target group 생성 + tg-dev-kr-30000-http-ingress + http: 30000 + vpc: vpc-datasaker + Protocol version http1 + healthcheck: http + Health check path: / + Advanced health check settings : + Success codes: 200,404 + AddTag: Name: tg-dev-kr-30000-http-ingress + create + + tg-dev-kr-30001-https-ingress + https: 30001 + vpc: dev.k8s.datasaker.io + Protocol version http1 + healthcheck: https + Health check path: / + Advanced health check settings : + Success codes: 200,404 + AddTag: Name / tg-dev-kr-30001-https-ingress + create + + tg-dev-kr-30001-http-ingress + http: 30001 + vpc: dev.k8s.datasaker.io + Protocol version http1 + healthcheck: http + Health check path: / + Advanced health check settings : + Success codes: 200,404 + AddTag: Name / tg-dev-kr-30001-https-ingress + create + + +1. alb 생성 + + alb-dev-kr-ingress + Internet-facing + IPv4 + vpc: vpc-datasaker + Mappings: + - sbn-dmz-a.datasaker + - sbn-dmz-b.datasaker + - sbn-dmz-c.datasaker + Security groups: + - service-elb.dev.datasaker.io + Listeners and routing: + - HTTP: 80 + tg: tg-dev-kr-30000-http-ingress + - HTTPS: 443 + tg: tg-dev-kr-30001-https-ingress + AddTag: Name / alb-dev-kr-ingress + + +1. 각 autoscaling group에 tg 연결 + + nlb-dev-ingress + internet-facing + ipv4 + vpc: dev.k8s.datasaker.io + mappings: subnet (utilityA,utilityB,utilityC) + Listeners and routing: + TCP:80 -> targetGroup 지정. tg-dev-ingress-30000 + TCP:443 -> targetGroup 지정. tg-dev-ingress-30001 + + + +## mng +1. manage 인증서 생성 +// manage-dev.kr.datasaker.io (특정 IP 제한) +argo.dev.kr.datasaker.io (argocd) +vlt.dev.kr.datasaker.io (vault) +jenkins.dev.kr.datasaker.io (jenkins) + +1. managed SG 설정 + sg.dev.kr-managed-ingress + VPC: vpc-datasaker + AllTrafic -> AnyWhere + AllTrafic -> AnyWhere + Name: sg.dev.kr-managed-ingress + +1. master sg에서 elb sg로 부터 오는 입력 허용. + TCP 30000 (HTTP), TCP 30001 (HTTPS) + + from-dev-manage-elb-80-to-ingress + from-dev-manage-elb-443-to-ingress + + elb sg에 80, 443 입력 허용. + dev-from-all-80-to-ing + dev-from-all-443-to-ing + +1. target group 생성 + tg-dev-kr-30000-mng-http-ing + http: 30000 + vpc: vpc-datasaker + Protocol version http1 + healthcheck: http + Health check path: / + Advanced health check settings : + Success codes: 200,404,400 + AddTag: + Name: tg-dev-kr-30000-mng-http-ing + create + + tg-dev-kr-30001-mng-https-ing + https: 30001 + vpc: dev.k8s.datasaker.io + Protocol version http1 + healthcheck: https + Health check path: / + Advanced health check settings : + Success codes: 200,404,400 + AddTag: + Name / tg-dev-kr-30001-mng-https-ing + create + +1. alb 생성 + + alb-dev-kr-mng-ing + Internet-facing + IPv4 + vpc: vpc-datasaker + Mappings: + - sbn-dmz-a.datasaker + - sbn-dmz-b.datasaker + - sbn-dmz-c.datasaker + Security groups: + - service-elb.dev.datasaker.io + Listeners and routing: + - HTTP: 80 + tg: tg-dev-kr-30000-http-ingress + - HTTPS: 443 + tg: tg-dev-kr-30001-https-ingress + AddTag: Name / alb-dev-kr-mng-ing + + + +1. 각 autoscaling group에 tg 연결 + master asg에 + TCP:80 -> targetGroup 지정. tg-dev-ingress-30000 + TCP:443 -> targetGroup 지정. tg-dev-ingress-30001 + + + +## nodeport 연결 +1. nlb 용 security group 생성 + sg.nlb.dev.kr.datasaker.io + Custom TCP: 30010 - 32768 : 0.0.0.0/0 + Custom TCP: 30010 - 32768 : ::/0 + +1. master sg에서 nlb sg로 부터 오는 입력 허용. + TCP 30000-32768 + from-nlb-30000-32768-to-nodeport + +// 1개의 nodeport 지정 (범위로 지정 가능한지 검토 필요) // +1. target group 생성 (ingress로 health check로만 사용) // 수정 필요. + 31428 + + tg-dev-kr-tcp-np + tcp: 31428 + vpc: vpc-datasaker + Protocol version http1 + healthcheck: http + Health check path: / + Advanced health check settings : + Success codes: 200,404,400 + AddTag: Name: tg-dev-kr-tcp-np + create + + +1. nlb 생성 및 tg 연결 + + nlb-dev-kr-mng-np + Internet-facing + IPv4 + vpc: vpc-datasaker + Mappings: + - sbn-dmz-a.datasaker + - sbn-dmz-b.datasaker + - sbn-dmz-c.datasaker + Security groups: + - service-elb.dev.datasaker.io + Listeners and routing: + - HTTP: 80 + tg: tg-dev-kr-30000-http-ingress + - HTTPS: 443 + tg: tg-dev-kr-30001-https-ingress + AddTag: Name / nlb-dev-kr-mng-np + + +1. 각 autoscaling group에 tg 연결 + master asg에 + tg-dev-kr-tcp-np + // arn:aws:elasticloadbalancing:ap-northeast-2:508259851457:targetgroup/tg-dev-kr-tcp-np/e86a5f0c14928131 + + +## + + + +1. autoscaling group에서 lb를 이용한 target group 등록 + nodes-ap-northeast-2a.dev.k8s.datasaker.io, nodes-ap-northeast-2b.dev.k8s.datasaker.io, nodes-ap-northeast-2c.dev.k8s.datasaker.io + Load balancing: tg-dev-ingress-30000, tg-dev-ingress-30001 + +1. target group에서 instance 가 보이는지 확인 + +1. nlb A Record 주소 확인 + nlb-dev-ingress-f266e4f0bead8225.elb.ap-northeast-2.amazonaws.com + +1. route53에 등록 + ex) g2048.dev.kr.datasaker.io + g2048.dev.kr A Alias nlb-dev-ingress-f266e4f0bead8225.elb.ap-northeast-2.amazonaws.com. 300 Simple routing + + argo.dev.datasaker.io A Alias nlb-dev-ingress-f266e4f0bead8225.elb.ap-northeast-2.amazonaws.com. 300 Simple routing + + simple-app.dev.datasaker.io + +1. test app +https://blog.leiwang.info/simple-app +// git clone https://github.com/tendant/simple-app.git +// helm install simple-app simple-app -n simple-app --create-namespace + + +ssh ubuntu@bastion.dev.k8s.datasaker.io +curl -v 172.20.68.243:30000/healthz + +masters.dev.k8s.datasaker.io +from-bastion-30000-30001 +sgr-0d891ac3623e03e7b – 사용자 지정 TCP TCP 30000 - 30001 sg-0fadf3368999e9eaf / bastion.dev.k8s.datasaker.io – + +nodes.dev.k8s.datasaker.io +from-bastion-30000-30001 +sgr-0d891ac3623e03e7b – 사용자 지정 TCP TCP 30000 - 30001 sg-0fadf3368999e9eaf / bastion.dev.k8s.datasaker.io – + + +nc -z -v 172.20.68.243 30000-30001 +nc -z -v 172.20.68.243 32679 + + +simple-app.dev.datasaker.io + + +tg-dev-kr-30000-http-ingress + // arn:aws:elasticloadbalancing:ap-northeast-2:508259851457:targetgroup/tg-dev-kr-30000-http-ingress/c722b2d641bcfc87 +tg-dev-kr-30001-https-ingress + // arn:aws:elasticloadbalancing:ap-northeast-2:508259851457:targetgroup/tg-dev-kr-30001-https-ingress/d41767571f8a7bb8 + +tg-dev-kr-30000-mng-http-ing + // arn:aws:elasticloadbalancing:ap-northeast-2:508259851457:targetgroup/tg-dev-kr-30000-mng-http-ing/474dc8d6f6ad2106 +tg-dev-kr-30001-mng-https-ing + // arn:aws:elasticloadbalancing:ap-northeast-2:508259851457:targetgroup/tg-dev-kr-30001-mng-https-ing/960e93df1bb9a326 + +## + + + + + + +## ing 생성 +krakend-dev krakend-develop NodePort 100.65.7.164 80:32701/TCP 7d19h +saas-dev sam-agentapi-develop NodePort 100.71.124.134 8080:32199/TCP 3d17h +saas-dev sam-app-sender-develop NodePort 100.65.88.171 8000:31514/TCP 5d17h +saas-dev sam-dashboardapi-develop NodePort 100.66.2.234 8080:30696/TCP 5d17h +saas-dev sam-infrastructureapi-develop NodePort 100.68.69.163 8000:31648/TCP 3d23h +saas-dev sam-jaeger-agent-develop NodePort 100.67.217.7 14271:30772/TCP,5778:30835/TCP,6831:30834/UDP 4d23h +saas-dev sam-ui-develop NodePort 100.70.74.238 80:30208/TCP 5d20h +saas-dev sam-usergate-develop NodePort 100.66.175.184 8080:31085/TCP 3d22h +saas-dev sample-app-develop-sample-app-deploy ClusterIP 100.65.249.165 80/TCP + +k -n argocd edit ing argocd-server + +``` + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + kubernetes.io/ingress.class: nginx + meta.helm.sh/release-name: argocd + meta.helm.sh/release-namespace: argocd + nginx.ingress.kubernetes.io/backend-protocol: HTTPS + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + creationTimestamp: "2022-09-26T04:31:11Z" + generation: 4 + labels: + app.kubernetes.io/component: server + app.kubernetes.io/instance: argocd + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: argocd-server + app.kubernetes.io/part-of: argocd + helm.sh/chart: argo-cd-4.9.11 + name: argocd-server + namespace: argocd + resourceVersion: "1567505" + uid: 567190bd-a080-4628-9e21-5f6b56ffd5e1 +spec: + rules: + - host: argo.dev.kr.datasaker.io + http: + paths: + - backend: + service: + name: argocd-server + port: + number: 80 + path: / + pathType: Prefix +status: + loadBalancer: + ingress: + - ip: 100.71.12.82 + +``` + + +#### sam-ui + +saas-dev sam-ui-develop NodePort 100.70.74.238 80:30208/TCP + +``` +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: sam-ui-develop + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + namespace: saas-dev +spec: + rules: + - host: app.dev.kr.datasaker.io + http: + paths: + - backend: + service: + name: sam-ui-develop + port: + number: 80 + path: / + pathType: Prefix + +``` + + +## keycloak ingress +keycloak NodePort 100.67.217.51 80:30100/TCP,443:30101/TCP + +``` + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/backend-protocol: HTTPS + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + name: keycloak-dev + namespace: infra-dev +spec: + rules: + - host: auth.dev.kr.datasaker.io + http: + paths: + - backend: + service: + name: keycloak + port: + number: 80 + path: / + pathType: Prefix + +``` + +``` +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + kubernetes.io/ingress.class: nginx + meta.helm.sh/release-name: keycloak + meta.helm.sh/release-namespace: infra-dev + creationTimestamp: "2022-10-04T04:46:18Z" + generation: 2 + labels: + app.kubernetes.io/component: keycloak + app.kubernetes.io/instance: keycloak + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: keycloak + helm.sh/chart: keycloak-7.1.17 + name: keycloak + namespace: infra-dev + resourceVersion: "2184553" + uid: 1ca8be2a-6580-4537-8488-c825839a7512 +spec: + rules: + - host: auth.dev.kr.datasaker.io + http: + paths: + - backend: + service: + name: keycloak + port: + name: https + path: / + pathType: ImplementationSpecific +status: + loadBalancer: {} + +``` + +https://community.gooddata.com/administration-61/how-to-properly-terminate-ssl-using-aws-alb-with-acm-391 + annotations: + kubernetes.io/ingress.class: "nginx" + nginx.ingress.kubernetes.io/configuration-snippet: | + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; + + + + add_header X-Frame-Options "sameorigin"; + add_header X-Content-Type-Options nosniff; + add_header Referrer-Policy 'same-origin'; + + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; + + + +#### am.dev.kr.datasaker.io (agent manager) + +``` +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + name: agentmanager-develop + namespace: saas-dev +spec: + ingressClassName: nginx + rules: + - host: am.dev.kr.datasaker.io + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: sam-agentmanager-cloud-4534 + port: + number: 8080 + + + + + +``` +#### + +https://aws.amazon.com/blogs/aws/new-application-load-balancer-support-for-end-to-end-http-2-and-grpc/ + +#### dgate-j.dev.kr.datasaker.io (datagate-jaeger) + +``` +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/backend-protocol: "GRPC" + name: datagate-jaeger-develop + namespace: saas-dev +spec: + ingressClassName: nginx + rules: + - host: dgate-j.dev.kr.datasaker.io + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: jaeger-sam-datagate-cloud-test + port: + number: 8080 + + + +``` + + +#### dgate-m.dev.kr.datasaker.io (datagate-metric) +``` +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/backend-protocol: "GRPC" + name: datagate-metric-develop + namespace: saas-dev +spec: + ingressClassName: nginx + rules: + - host: dgate-m.dev.kr.datasaker.io + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: metric-sam-datagate-cloud-test + port: + number: 8080 + + + + +``` + +#### dgate-k.dev.kr.datasaker.io (datagate-menifest) +``` + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "true" + nginx.ingress.kubernetes.io/backend-protocol: "GRPC" + name: datagate-menifest-develop + namespace: saas-dev +spec: + ingressClassName: nginx + rules: + - host: dgate-k.dev.kr.datasaker.io + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: manifest-sam-datagate-cloud-test + port: + number: 8080 + + + +``` + + +#### postgresql.dev.kr.datasaker.io +postgresql NodePort 100.69.229.168 5432:32713/TCP + +``` +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: postgresql-develop + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + namespace: infra-dev +spec: + rules: + - host: postgresql.dev.kr.datasaker.io + http: + paths: + - backend: + service: + name: postgresql + port: + number: 5432 + path: / + pathType: Prefix + +``` + + + +