sa_8001/dsk-iac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
2224a6f6e7fb066f8c1be7dfea23804edd022552
dsk-iac/doc/2_how_to_install_dev_cluster_20221026.txt
exhsgahm 4236a8aee5 update kops dev howto
2022-10-26 10:07:49 +09:00

1323 lines
36 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

export KOPS_STATE_STORE=s3://clusters.dev.datasaker.io
kops update cluster --name dev.datasaker.io --state=s3://clusters.dev.datasaker.io
kops update cluster --name dev.datasaker.io --state=s3://clusters.dev.datasaker.io > changes-dev.datasaker.io-20221019.txt
kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io update cluster --yes --admin
kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io rolling-update cluster --yes --cloudonly
kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io -o yaml get > dev.datasaker.io-1.yaml
kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io -o yaml get > dev.datasaker.io-20221025.yaml
kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io edit cluster
kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io get ig
kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io edit ig master-ap-northeast-2a
kops export kubecfg --admin=8760h0m0s --name dev.datasaker.io --state s3://clusters.dev.datasaker.io --kubeconfig ~/.kube/config
kops --state=s3://clusters.dev.datasaker.io --name dev.datasaker.io update cluster --out=./tf-kops-dev-20221025 --target=terraform
security-dev-bastion ami-0b6591f49cf24e237
security-dev-node ami-0abb33b73a78cae31
kops create cluster \
--name dev.datasaker.io \
--vpc vpc-0b6e0b906c678a22f \
--cloud aws \
--state s3://clusters.dev.datasaker.io \
--ssh-public-key /home/hsgahm/.ssh/id_rsa_k8s.pub \
--topology private --kubernetes-version "1.23.10" \
--network-cidr "172.21.0.0/16" \
--networking calico \
--container-runtime containerd \
--image ami-0ea5eb4b05645aa8a \
--zones ap-northeast-2a,ap-northeast-2b,ap-northeast-2c \
--master-count 3 \
--master-size t3.small \
--master-volume-size 50 \
--node-count 3 \
--node-size t3.small \
--node-volume-size 100 \
--utility-subnets "subnet-0de55619bee2411f8,subnet-0a5d787353f874684,subnet-0ee26ffc561efb292" \
--subnets "subnet-0c875e254456809f7,subnet-05672a669943fc12f,subnet-0940fd78504acbbde" \
-v 10
<!--
kops edit cluster --name=dev.datasaker.io --state s3://clusters.dev.datasaker.io
```
spec:
awsLoadBalancerController:
enabled: true
```
-->
kops --name dev.datasaker.io --state s3://clusters.dev.datasaker.io edit cluster
```
containerd:
configOverride: |
version = 2
imports = ["/etc/containerd/runtime_*.toml"]
[plugins]
[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = "registry.k8s.io/pause:3.6@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db"
[plugins."io.containerd.grpc.v1.cri".containerd]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
SystemdCgroup = true
[plugins."io.containerd.grpc.v1.cri".registry.configs."registry-1.docker.io".auth]
username = "datasaker"
password = "dckr_pat_kQP6vcHm_jMChWd_zvgH_G3kucc"
```
```
```
```
enableWAF: true
enableWAFv2: true
```
kops get instancegroups --name=dev.datasaker.io --state s3://clusters.dev.datasaker.io
kops --name=dev.datasaker.io delete instancegroup nodes-ap-northeast-2a
kops --name=dev.datasaker.io delete instancegroup nodes-ap-northeast-2b
kops --name=dev.datasaker.io delete instancegroup nodes-ap-northeast-2c
kops edit instancegroup --name=dev.datasaker.io master-ap-northeast-2a
kops edit instancegroup --name=dev.datasaker.io master-ap-northeast-2b
kops edit instancegroup --name=dev.datasaker.io master-ap-northeast-2c
rootVolumeSize: 64 (default)
kops --name=dev.datasaker.io get ig
kops get clusters
kops edit cluster dev.datasaker.io --state s3://clusters.dev.datasaker.io
// subnet name 변경
//ap-northeast-2a -> sbn-dev-a.datasaker
//ap-northeast-2b -> sbn-dev-b.datasaker
//ap-northeast-2c -> sbn-dev-c.datasaker
//utility-ap-northeast-2a -> sbn-dmz-a.datasaker
//utility-ap-northeast-2b -> sbn-dmz-b.datasaker
//utility-ap-northeast-2c -> sbn-dmz-c.datasaker
kops edit instancegroups --name=dev.datasaker.io master-ap-northeast-2a
```
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: "2022-09-06T05:44:09Z"
labels:
kops.k8s.io/cluster: dev.datasaker.io
name: master-ap-northeast-2a
spec:
image: ami-0ea5eb4b05645aa8a
instanceMetadata:
httpPutResponseHopLimit: 3
httpTokens: required
machineType: t3.small
manager: CloudGroup
maxSize: 1
minSize: 1
nodeLabels:
kops.k8s.io/instancegroup: master-ap-northeast-2a
role: Master
rootVolumeSize: 50
subnets:
- ap-northeast-2a
cloudLabels:
autoscale-off: "True"
autoscale-on: "True"
```
//kops create instancegroup --name=dev.datasaker.io dev-master-a --role master --subnet "ap-northeast-2a"
//kops create instancegroup --name=dev.datasaker.io dev-master-b --role node --subnet "ap-northeast-2b"
//kops create instancegroup --name=dev.datasaker.io dev-master-c --role node --subnet "ap-northeast-2c"
// kops delete instancegroup --name=dev.datasaker.io dev-data-a
// kops delete instancegroup --name=dev.datasaker.io dev-data-b
// kops delete instancegroup --name=dev.datasaker.io dev-data-c
kops create instancegroup --name=dev.datasaker.io dev-data-druid-a --role node --subnet "ap-northeast-2a"
kops create instancegroup --name=dev.datasaker.io dev-data-druid-b --role node --subnet "ap-northeast-2b"
kops create instancegroup --name=dev.datasaker.io dev-data-druid-c --role node --subnet "ap-northeast-2c"
kops create instancegroup --name=dev.datasaker.io dev-data-kafka-a --role node --subnet "ap-northeast-2a"
kops create instancegroup --name=dev.datasaker.io dev-data-kafka-b --role node --subnet "ap-northeast-2b"
kops create instancegroup --name=dev.datasaker.io dev-data-kafka-c --role node --subnet "ap-northeast-2c"
```
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: null
name: dev-data-druid-a
spec:
image: ami-0abb33b73a78cae31
kubelet:
anonymousAuth: false
nodeLabels:
node-role.kubernetes.io/node: ""
machineType: m6i.2xlarge
manager: CloudGroup
maxSize: 1
minSize: 1
nodeLabels:
kops.k8s.io/instancegroup: dev-data-druid-a
datasaker/group: data-druid
role: Node
subnets:
- ap-northeast-2a
taints:
- dev/data-druid:NoSchedule
```
```
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: null
name: dev-data-kafka-a
spec:
image: ami-0abb33b73a78cae31
kubelet:
anonymousAuth: false
nodeLabels:
node-role.kubernetes.io/node: ""
machineType: m6i.2xlarge
manager: CloudGroup
maxSize: 1
minSize: 1
nodeLabels:
kops.k8s.io/instancegroup: dev-data-kafka-a
datasaker/group: data-kafka
role: Node
subnets:
- ap-northeast-2a
taints:
- dev/data-kafka:NoSchedule
```
kops create instancegroup --name=dev.datasaker.io dev-data-a --role node --subnet "ap-northeast-2a"
kops edit instancegroup --name=dev.datasaker.io dev-data-a
```
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: "2022-09-05T05:53:59Z"
labels:
kops.k8s.io/cluster: dev.datasaker.io
name: dev-data-a
spec:
image: ami-0ea5eb4b05645aa8a
machineType: m5.4xlarge
manager: CloudGroup
maxSize: 1
minSize: 1
nodeLabels:
kops.k8s.io/instancegroup: dev-data-a
datasaker/group: data
rootVolumeSize: 100
role: Node
subnets:
- ap-northeast-2a
```
```
cloudLabels:
autoscale-off: "True"
autoscale-on: "True"
```
kops create instancegroup --name=dev.datasaker.io dev-data-b --role node --subnet "ap-northeast-2b"
kops edit instancegroup --name=dev.datasaker.io dev-data-b
```
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: "2022-09-05T06:00:56Z"
generation: 1
labels:
kops.k8s.io/cluster: dev.datasaker.io
name: dev-data-b
spec:
image: ami-0ea5eb4b05645aa8a
machineType: m5.4xlarge
manager: CloudGroup
maxSize: 1
minSize: 1
nodeLabels:
datasaker/group: data
kops.k8s.io/instancegroup: dev-data-b
role: Node
rootVolumeSize: 100
subnets:
- ap-northeast-2b
```
kops create instancegroup --name=dev.datasaker.io dev-data-c --role node --subnet "ap-northeast-2c"
kops edit instancegroup --name=dev.datasaker.io dev-data-c
```
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: null
name: dev-data-c
spec:
image: ami-0ea5eb4b05645aa8a
machineType: m5.4xlarge
manager: CloudGroup
maxSize: 1
minSize: 1
nodeLabels:
kops.k8s.io/instancegroup: dev-data-c
datasaker/group: data
rootVolumeSize: 100
role: Node
subnets:
- ap-northeast-2c
```
// kops delete instancegroup --name=dev.datasaker.io dev-process-a
// kops delete instancegroup --name=dev.datasaker.io dev-process-b
// kops delete instancegroup --name=dev.datasaker.io dev-process-c
kops create instancegroup --name=dev.datasaker.io dev-process-a --role node --subnet "ap-northeast-2a"
kops edit instancegroup --name=dev.datasaker.io dev-process-a
```
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: null
name: dev-process-a
spec:
image: ami-0ea5eb4b05645aa8a
machineType: c5.xlarge
manager: CloudGroup
maxSize: 2
minSize: 2
nodeLabels:
kops.k8s.io/instancegroup: dev-process-a
datasaker/group: process
rootVolumeSize: 100
role: Node
subnets:
- ap-northeast-2a
```
kops create instancegroup --name=dev.datasaker.io dev-process-b --role node --subnet "ap-northeast-2b"
kops edit instancegroup --name=dev.datasaker.io dev-process-b
```
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: "2022-09-05T06:10:03Z"
labels:
kops.k8s.io/cluster: dev.datasaker.io
name: dev-process-b
spec:
image: ami-0ea5eb4b05645aa8a
machineType: c5.xlarge
manager: CloudGroup
maxSize: 2
minSize: 2
nodeLabels:
datasaker/group: process
kops.k8s.io/instancegroup: dev-process-b
role: Node
rootVolumeSize: 100
subnets:
- ap-northeast-2b
```
kops create instancegroup --name=dev.datasaker.io dev-process-c --role node --subnet "ap-northeast-2c"
kops edit instancegroup --name=dev.datasaker.io dev-process-c
```
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: "2022-09-05T06:10:59Z"
labels:
kops.k8s.io/cluster: dev.datasaker.io
name: dev-process-c
spec:
image: ami-0ea5eb4b05645aa8a
machineType: c5.xlarge
manager: CloudGroup
maxSize: 1
minSize: 1
nodeLabels:
datasaker/group: process
kops.k8s.io/instancegroup: dev-process-c
rootVolumeSize: 100
role: Node
subnets:
- ap-northeast-2c
taints:
- dev/mgmt:NoSchedule
```
kops create instancegroup --name=dev.datasaker.io dev-mgmt-a --role node --subnet "ap-northeast-2a"
kops --state=s3://clusters.dev.datasaker.io --name=dev.datasaker.io edit instancegroup dev-mgmt-a
```
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: null
name: dev-mgmt-a
spec:
image: ami-0ea5eb4b05645aa8a
machineType: c5.xlarge
manager: CloudGroup
maxSize: 2
minSize: 1
nodeLabels:
kops.k8s.io/instancegroup: dev-mgmt-a
datasaker/group: mgmt
rootVolumeSize: 100
role: Node
subnets:
- ap-northeast-2a
taints:
- dev/mgmt:NoSchedule
```
kops create instancegroup --name=dev.datasaker.io dev-mgmt-b --role node --subnet "ap-northeast-2b"
kops --state=s3://clusters.dev.datasaker.io --name=dev.datasaker.io edit instancegroup dev-mgmt-b
```
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: null
name: dev-mgmt-b
spec:
image: ami-0abb33b73a78cae31
machineType: c5.xlarge
manager: CloudGroup
maxSize: 1
minSize: 1
nodeLabels:
kops.k8s.io/instancegroup: dev-mgmt-b
datasaker/group: mgmt
rootVolumeSize: 100
role: Node
subnets:
- ap-northeast-2b
taints:
- dev/mgmt:NoSchedule
```
kops create instancegroup --name=dev.datasaker.io dev-mgmt-c --role node --subnet "ap-northeast-2c"
kops --state=s3://clusters.dev.datasaker.io --name=dev.datasaker.io edit instancegroup dev-mgmt-c
```
apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
creationTimestamp: null
name: dev-mgmt-c
spec:
image: ami-0abb33b73a78cae31
machineType: c5.xlarge
manager: CloudGroup
maxSize: 1
minSize: 1
nodeLabels:
kops.k8s.io/instancegroup: dev-mgmt-a
datasaker/group: mgmt
rootVolumeSize: 100
role: Node
subnets:
- ap-northeast-2c
taints:
- dev/mgmt:NoSchedule
```
kops edit instancegroup --name=dev.datasaker.io dev-data-a
kops edit instancegroup --name=dev.datasaker.io dev-data-b
kops edit instancegroup --name=dev.datasaker.io dev-data-c
kops get --state s3://clusters.dev.datasaker.io --name dev.datasaker.io -o yaml > dev.datasaker.io.yaml
vi dev.datasaker.io.yaml
subnets:
- cidr: 172.21.1.0/24
name: ap-northeast-2a
type: Private
zone: ap-northeast-2a
- cidr: 172.21.2.0/24
name: ap-northeast-2b
type: Private
zone: ap-northeast-2b
- cidr: 172.21.3.0/24
name: ap-northeast-2c
type: Private
zone: ap-northeast-2c
- cidr: 172.21.0.48/28
name: utility-ap-northeast-2a
type: Utility
zone: ap-northeast-2a
- cidr: 172.21.0.64/28
name: utility-ap-northeast-2b
type: Utility
zone: ap-northeast-2b
- cidr: 172.21.0.80/28
name: utility-ap-northeast-2c
type: Utility
zone: ap-northeast-2c
export KOPS_STATE_STORE=s3://clusters.dev.datasaker.io
// kops delete cluster dev.datasaker.io --yes --state=s3://clusters.dev.datasaker.io
// kops delete -f=./dev.datasaker.io.yaml --yes
kops create -f=./dev.datasaker.io.yaml --state=s3://clusters.dev.datasaker.io
kops update cluster dev.datasaker.io --yes --admin --state=s3://clusters.dev.datasaker.io
kops export kubecfg --admin --state=s3://clusters.dev.datasaker.io
kops export kubecfg --admin --kubeconfig ~/workspace/kubeconfig --state=s3://clusters.dev.datasaker.io
kops get secrets sshpublickey admin
kops get secrets sshpublickey admin -oplaintext
MgUKqpCUHLaEcYEuHXTM7ljlTpsnNYSs
ssh ubuntu@3.37.243.25
//kops create instancegroup bastions --role Bastion --subnet utility-ap-northeast-2c
kops create secret sshpublickey admin -i ~/.ssh/id_rsa.pub --state=s3://clusters.dev.datasaker.io
kops create secret sshpublickey admin -i id_rsa_k8s.pub --state=s3://clusters.dev.datasaker.io
kops update cluster --yes // to reconfigure the auto-scaling groups
kops update cluster --yes --state=s3://clusters.dev.datasaker.io
kops rolling-update cluster --name dev.datasaker.io --state=s3://clusters.dev.datasaker.io --yes
kops rolling-update cluster --name <clustername> --yes // to immediately roll all the machines so they have the new key (optional)
// Lambda 설정 변경.
get_names = ['ag-dmz-bastion-datasaker','master-ap-northeast-2a.masters.dev.datasaker.io','master-ap-northeast-2b.masters.dev.datasaker.io','master-ap-northeast-2c.masters.dev.datasaker.io','dev-process-a.dev.datasaker.io','dev-process-b.dev.datasaker.io','dev-process-c.dev.datasaker.io','dev-data-a.dev.datasaker.io','dev-data-b.dev.datasaker.io','dev-data-c.dev.datasaker.io','dev-mgmt-a.dev.datasaker.io','dev-mgmt-b.dev.datasaker.io']
Suggestions:
* validate cluster: kops validate cluster --wait 10m
* list nodes: kubectl get nodes --show-labels
* ssh to the master: ssh -i ~/.ssh/id_rsa ubuntu@api.dev.datasaker.io
* the ubuntu user is specific to Ubuntu. If not using Ubuntu please use the appropriate user based on your OS.
* read about installing addons at: https://kops.sigs.k8s.io/addons.
// when kubecfg changed, due to master redeploy
kops export kubecfg --admin --name dev.datasaker.io --state s3://clusters.dev.datasaker.io --kubeconfig ~/.kube/config
kops export kubecfg --admin=87600h0m0s --name dev.datasaker.io --state s3://clusters.dev.datasaker.io --kubeconfig ~/.kube/config
kops export kubecfg --admin=8760h0m0s --name dev.datasaker.io --state s3://clusters.dev.datasaker.io --kubeconfig ~/.kube/config
kops export kubecfg --admin=720h0m0s --name dev.datasaker.io --state s3://clusters.dev.datasaker.io --kubeconfig ~/.kube/config
kops update cluster --name=dev.datasaker.io --state=s3://clusters.dev.datasaker.io --out=./tf-kops-dev-20200916-ip --target=terraform
115.178.73.2/32 exem router
115.178.73.91/32 proxy
3.35.247.45/32 bastion
api-elb.dev.datasaker.io 에 115.178.73.2 만 적용되는 이슈가 있음.
kops edit cluster --name=dev.datasaker.io
from
kubernetesApiAccess:
- 0.0.0.0/0
- ::/0
sshAccess:
- 0.0.0.0/0
- ::/0
to
kubernetesApiAccess:
- 115.178.73.2/32
- 115.178.73.91/32
- 3.35.247.45/32
sshAccess:
- 115.178.73.2/32
- 115.178.73.91/32
- 3.35.247.45/32
kops update cluster --yes --state=s3://clusters.dev.datasaker.io
kops rolling-update cluster --yes --state=s3://clusters.dev.datasaker.io
##
## aws security group masters, nodes set for 30000, 30001 from bastion,elb
secg-dmz-datasaker sg-07f27eba164d59dfa
from-dev-bastion-to-ingress
1. elb 용 security group 생성
service-elb.dev.datasaker.io sg-08dd3bc6dac12a286
1. 인증서 생성
// cert-static (IP 제한 없음)
am.dev.kr.datasaker.io (agent manager)
dgate-m.dev.kr.datasaker.io (datagate-metric)
dgate-j.dev.kr.datasaker.io (datagate-jaeger)
dgate-k.dev.kr.datasaker.io (datagate-menifest)
lgate.dev.kr.datasaker.io (loggate)
app.dev.kr.datasaker.io (app - ui)
auth.dev.kr.datasaker.io (keycloak)
api.dev.kr.datasaker.io (krakend)
1. master sg에서 elb sg로 부터 오는 입력 허용.
TCP 30000 (HTTP), TCP 30001 (HTTPS)
from-dev-alb-to-ingress
elb sg에 80, 443 입력 허용.
dev-from-all-80-to-ing
dev-from-all-443-to-ing
1. target group 생성
tg-dev-kr-30000-http-ingress
http: 30000
vpc: vpc-datasaker
Protocol version http1
healthcheck: http
Health check path: /
Advanced health check settings :
Success codes: 200,404
AddTag: Name: tg-dev-kr-30000-http-ingress
create
tg-dev-kr-30001-https-ingress
https: 30001
vpc: dev.k8s.datasaker.io
Protocol version http1
healthcheck: https
Health check path: /
Advanced health check settings :
Success codes: 200,404
AddTag: Name / tg-dev-kr-30001-https-ingress
create
tg-dev-kr-30001-http-ingress
http: 30001
vpc: dev.k8s.datasaker.io
Protocol version http1
healthcheck: http
Health check path: /
Advanced health check settings :
Success codes: 200,404
AddTag: Name / tg-dev-kr-30001-https-ingress
create
1. alb 생성
alb-dev-kr-ingress
Internet-facing
IPv4
vpc: vpc-datasaker
Mappings:
- sbn-dmz-a.datasaker
- sbn-dmz-b.datasaker
- sbn-dmz-c.datasaker
Security groups:
- service-elb.dev.datasaker.io
Listeners and routing:
- HTTP: 80
tg: tg-dev-kr-30000-http-ingress
- HTTPS: 443
tg: tg-dev-kr-30001-https-ingress
AddTag: Name / alb-dev-kr-ingress
1. 각 autoscaling group에 tg 연결
nlb-dev-ingress
internet-facing
ipv4
vpc: dev.k8s.datasaker.io
mappings: subnet (utilityA,utilityB,utilityC)
Listeners and routing:
TCP:80 -> targetGroup 지정. tg-dev-ingress-30000
TCP:443 -> targetGroup 지정. tg-dev-ingress-30001
## mng
1. manage 인증서 생성
// manage-dev.kr.datasaker.io (특정 IP 제한)
argo.dev.kr.datasaker.io (argocd)
vlt.dev.kr.datasaker.io (vault)
jenkins.dev.kr.datasaker.io (jenkins)
1. managed SG 설정
sg.dev.kr-managed-ingress
VPC: vpc-datasaker
AllTrafic -> AnyWhere
AllTrafic -> AnyWhere
Name: sg.dev.kr-managed-ingress
1. master sg에서 elb sg로 부터 오는 입력 허용.
TCP 30000 (HTTP), TCP 30001 (HTTPS)
from-dev-manage-elb-80-to-ingress
from-dev-manage-elb-443-to-ingress
elb sg에 80, 443 입력 허용.
dev-from-all-80-to-ing
dev-from-all-443-to-ing
1. target group 생성
tg-dev-kr-30000-mng-http-ing
http: 30000
vpc: vpc-datasaker
Protocol version http1
healthcheck: http
Health check path: /
Advanced health check settings :
Success codes: 200,404,400
AddTag:
Name: tg-dev-kr-30000-mng-http-ing
create
tg-dev-kr-30001-mng-https-ing
https: 30001
vpc: dev.k8s.datasaker.io
Protocol version http1
healthcheck: https
Health check path: /
Advanced health check settings :
Success codes: 200,404,400
AddTag:
Name / tg-dev-kr-30001-mng-https-ing
create
1. alb 생성
alb-dev-kr-mng-ing
Internet-facing
IPv4
vpc: vpc-datasaker
Mappings:
- sbn-dmz-a.datasaker
- sbn-dmz-b.datasaker
- sbn-dmz-c.datasaker
Security groups:
- service-elb.dev.datasaker.io
Listeners and routing:
- HTTP: 80
tg: tg-dev-kr-30000-http-ingress
- HTTPS: 443
tg: tg-dev-kr-30001-https-ingress
AddTag: Name / alb-dev-kr-mng-ing
1. 각 autoscaling group에 tg 연결
master asg에
TCP:80 -> targetGroup 지정. tg-dev-ingress-30000
TCP:443 -> targetGroup 지정. tg-dev-ingress-30001
## nodeport 연결
1. nlb 용 security group 생성
sg.nlb.dev.kr.datasaker.io
Custom TCP: 30010 - 32768 : 0.0.0.0/0
Custom TCP: 30010 - 32768 : ::/0
1. master sg에서 nlb sg로 부터 오는 입력 허용.
TCP 30000-32768
from-nlb-30000-32768-to-nodeport
// 1개의 nodeport 지정 (범위로 지정 가능한지 검토 필요) //
1. target group 생성 (ingress로 health check로만 사용) // 수정 필요.
31428
tg-dev-kr-tcp-np
tcp: 31428
vpc: vpc-datasaker
Protocol version http1
healthcheck: http
Health check path: /
Advanced health check settings :
Success codes: 200,404,400
AddTag: Name: tg-dev-kr-tcp-np
create
1. nlb 생성 및 tg 연결
nlb-dev-kr-mng-np
Internet-facing
IPv4
vpc: vpc-datasaker
Mappings:
- sbn-dmz-a.datasaker
- sbn-dmz-b.datasaker
- sbn-dmz-c.datasaker
Security groups:
- service-elb.dev.datasaker.io
Listeners and routing:
- HTTP: 80
tg: tg-dev-kr-30000-http-ingress
- HTTPS: 443
tg: tg-dev-kr-30001-https-ingress
AddTag: Name / nlb-dev-kr-mng-np
1. 각 autoscaling group에 tg 연결
master asg에
tg-dev-kr-tcp-np
// arn:aws:elasticloadbalancing:ap-northeast-2:508259851457:targetgroup/tg-dev-kr-tcp-np/e86a5f0c14928131
##
1. autoscaling group에서 lb를 이용한 target group 등록
nodes-ap-northeast-2a.dev.k8s.datasaker.io, nodes-ap-northeast-2b.dev.k8s.datasaker.io, nodes-ap-northeast-2c.dev.k8s.datasaker.io
Load balancing: tg-dev-ingress-30000, tg-dev-ingress-30001
1. target group에서 instance 가 보이는지 확인
1. nlb A Record 주소 확인
nlb-dev-ingress-f266e4f0bead8225.elb.ap-northeast-2.amazonaws.com
1. route53에 등록
ex) g2048.dev.kr.datasaker.io
g2048.dev.kr A Alias nlb-dev-ingress-f266e4f0bead8225.elb.ap-northeast-2.amazonaws.com. 300 Simple routing
argo.dev.datasaker.io A Alias nlb-dev-ingress-f266e4f0bead8225.elb.ap-northeast-2.amazonaws.com. 300 Simple routing
simple-app.dev.datasaker.io
1. test app
https://blog.leiwang.info/simple-app
// git clone https://github.com/tendant/simple-app.git
// helm install simple-app simple-app -n simple-app --create-namespace
ssh ubuntu@bastion.dev.k8s.datasaker.io
curl -v 172.20.68.243:30000/healthz
masters.dev.k8s.datasaker.io
from-bastion-30000-30001
sgr-0d891ac3623e03e7b – 사용자 지정 TCP TCP 30000 - 30001 sg-0fadf3368999e9eaf / bastion.dev.k8s.datasaker.io –
nodes.dev.k8s.datasaker.io
from-bastion-30000-30001
sgr-0d891ac3623e03e7b – 사용자 지정 TCP TCP 30000 - 30001 sg-0fadf3368999e9eaf / bastion.dev.k8s.datasaker.io –
nc -z -v 172.20.68.243 30000-30001
nc -z -v 172.20.68.243 32679
simple-app.dev.datasaker.io
tg-dev-kr-30000-http-ingress
// arn:aws:elasticloadbalancing:ap-northeast-2:508259851457:targetgroup/tg-dev-kr-30000-http-ingress/c722b2d641bcfc87
tg-dev-kr-30001-https-ingress
// arn:aws:elasticloadbalancing:ap-northeast-2:508259851457:targetgroup/tg-dev-kr-30001-https-ingress/d41767571f8a7bb8
tg-dev-kr-30000-mng-http-ing
// arn:aws:elasticloadbalancing:ap-northeast-2:508259851457:targetgroup/tg-dev-kr-30000-mng-http-ing/474dc8d6f6ad2106
tg-dev-kr-30001-mng-https-ing
// arn:aws:elasticloadbalancing:ap-northeast-2:508259851457:targetgroup/tg-dev-kr-30001-mng-https-ing/960e93df1bb9a326
##
<!--
--utility-subnets "subnet-0c298ebbccf528cc1,subnet-0dcda9a0e47e17998,subnet-0b805200e89d9095c" \
--subnets "subnet-066a9c7883bac8665,subnet-03eeee967799ec024,subnet-01d16f4fb3bc70413" \
--master-zones ap-northeast-2a,ap-northeast-2b,ap-northeast-2c \
--utility-subnets "subnet-0779691e403086418,subnet-075fd7be078c73b72" \
Error: [spec.subnets[3].id: Forbidden: cannot mix subnets with specified ID and unspecified ID, spec.subnets[4].id: Forbidden: cannot mix subnets with specified ID and unspecified ID]
모든 서브넷을 다 지정하면 해결될 듯.
--node-security-groups "sg-0682386228f9859d9" \
--master-security-groups "sg-0682386228f9859d9" \
--bastion \
// private subnet에 생성됨.
--admin-access 115.178.73.2
--ssh-access 115.178.73.2
--cloud-labels
--image ami-054a058b04f721571 \ (x64)
-->
## ing 생성
krakend-dev krakend-develop NodePort 100.65.7.164 <none> 80:32701/TCP 7d19h
saas-dev sam-agentapi-develop NodePort 100.71.124.134 <none> 8080:32199/TCP 3d17h
saas-dev sam-app-sender-develop NodePort 100.65.88.171 <none> 8000:31514/TCP 5d17h
saas-dev sam-dashboardapi-develop NodePort 100.66.2.234 <none> 8080:30696/TCP 5d17h
saas-dev sam-infrastructureapi-develop NodePort 100.68.69.163 <none> 8000:31648/TCP 3d23h
saas-dev sam-jaeger-agent-develop NodePort 100.67.217.7 <none> 14271:30772/TCP,5778:30835/TCP,6831:30834/UDP 4d23h
saas-dev sam-ui-develop NodePort 100.70.74.238 <none> 80:30208/TCP 5d20h
saas-dev sam-usergate-develop NodePort 100.66.175.184 <none> 8080:31085/TCP 3d22h
saas-dev sample-app-develop-sample-app-deploy ClusterIP 100.65.249.165 <none> 80/TCP
k -n argocd edit ing argocd-server
```
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
kubernetes.io/ingress.class: nginx
meta.helm.sh/release-name: argocd
meta.helm.sh/release-namespace: argocd
nginx.ingress.kubernetes.io/backend-protocol: HTTPS
nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
nginx.ingress.kubernetes.io/ssl-passthrough: "true"
creationTimestamp: "2022-09-26T04:31:11Z"
generation: 4
labels:
app.kubernetes.io/component: server
app.kubernetes.io/instance: argocd
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/name: argocd-server
app.kubernetes.io/part-of: argocd
helm.sh/chart: argo-cd-4.9.11
name: argocd-server
namespace: argocd
resourceVersion: "1567505"
uid: 567190bd-a080-4628-9e21-5f6b56ffd5e1
spec:
rules:
- host: argo.dev.kr.datasaker.io
http:
paths:
- backend:
service:
name: argocd-server
port:
number: 80
path: /
pathType: Prefix
status:
loadBalancer:
ingress:
- ip: 100.71.12.82
```
#### sam-ui
saas-dev sam-ui-develop NodePort 100.70.74.238 <none> 80:30208/TCP
```
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: sam-ui-develop
annotations:
kubernetes.io/ingress.class: nginx
nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
nginx.ingress.kubernetes.io/ssl-passthrough: "true"
namespace: saas-dev
spec:
rules:
- host: app.dev.kr.datasaker.io
http:
paths:
- backend:
service:
name: sam-ui-develop
port:
number: 80
path: /
pathType: Prefix
```
## keycloak ingress
keycloak NodePort 100.67.217.51 <none> 80:30100/TCP,443:30101/TCP
```
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
kubernetes.io/ingress.class: nginx
nginx.ingress.kubernetes.io/backend-protocol: HTTPS
nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
nginx.ingress.kubernetes.io/ssl-passthrough: "true"
name: keycloak-dev
namespace: infra-dev
spec:
rules:
- host: auth.dev.kr.datasaker.io
http:
paths:
- backend:
service:
name: keycloak
port:
number: 80
path: /
pathType: Prefix
```
```
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
kubernetes.io/ingress.class: nginx
meta.helm.sh/release-name: keycloak
meta.helm.sh/release-namespace: infra-dev
creationTimestamp: "2022-10-04T04:46:18Z"
generation: 2
labels:
app.kubernetes.io/component: keycloak
app.kubernetes.io/instance: keycloak
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/name: keycloak
helm.sh/chart: keycloak-7.1.17
name: keycloak
namespace: infra-dev
resourceVersion: "2184553"
uid: 1ca8be2a-6580-4537-8488-c825839a7512
spec:
rules:
- host: auth.dev.kr.datasaker.io
http:
paths:
- backend:
service:
name: keycloak
port:
name: https
path: /
pathType: ImplementationSpecific
status:
loadBalancer: {}
```
https://community.gooddata.com/administration-61/how-to-properly-terminate-ssl-using-aws-alb-with-acm-391
annotations:
kubernetes.io/ingress.class: "nginx"
nginx.ingress.kubernetes.io/configuration-snippet: |
add_header 'Access-Control-Allow-Origin' '*';
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';
add_header X-Frame-Options "sameorigin";
add_header X-Content-Type-Options nosniff;
add_header Referrer-Policy 'same-origin';
add_header 'Access-Control-Allow-Origin' '*';
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range';
#### am.dev.kr.datasaker.io (agent manager)
```
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
nginx.ingress.kubernetes.io/ssl-passthrough: "true"
name: agentmanager-develop
namespace: saas-dev
spec:
ingressClassName: nginx
rules:
- host: am.dev.kr.datasaker.io
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: sam-agentmanager-cloud-4534
port:
number: 8080
<!-- apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: agentmanager-develop
annotations:
kubernetes.io/ingress.class: nginx
nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
nginx.ingress.kubernetes.io/ssl-passthrough: "true"
namespace: saas-dev
spec:
rules:
- host: am.dev.kr.datasaker.io
http:
paths:
- backend:
service:
name: sam-agentmanager-cloud-4534
port:
number: 8080
path: /
pathType: Prefix -->
```
####
https://aws.amazon.com/blogs/aws/new-application-load-balancer-support-for-end-to-end-http-2-and-grpc/
#### dgate-j.dev.kr.datasaker.io (datagate-jaeger)
```
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/ssl-redirect: "true"
nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
name: datagate-jaeger-develop
namespace: saas-dev
spec:
ingressClassName: nginx
rules:
- host: dgate-j.dev.kr.datasaker.io
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: jaeger-sam-datagate-cloud-test
port:
number: 8080
<!-- apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: datagate-jaeger-develop
annotations:
kubernetes.io/ingress.class: nginx
nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
nginx.ingress.kubernetes.io/ssl-passthrough: "true"
namespace: saas-dev
spec:
rules:
- host: dgate-j.dev.kr.datasaker.io
http:
paths:
- backend:
service:
name: jaeger-sam-datagate-cloud-test
port:
number: 8080
path: /
pathType: Prefix -->
```
#### dgate-m.dev.kr.datasaker.io (datagate-metric)
```
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/ssl-redirect: "true"
nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
name: datagate-metric-develop
namespace: saas-dev
spec:
ingressClassName: nginx
rules:
- host: dgate-m.dev.kr.datasaker.io
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: metric-sam-datagate-cloud-test
port:
number: 8080
<!-- apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: datagate-metric-develop
annotations:
kubernetes.io/ingress.class: nginx
nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
nginx.ingress.kubernetes.io/ssl-passthrough: "true"
namespace: saas-dev
spec:
rules:
- host: dgate-m.dev.kr.datasaker.io
http:
paths:
- backend:
service:
name: metric-sam-datagate-cloud-test
port:
number: 8080
path: /
pathType: Prefix -->
```
#### dgate-k.dev.kr.datasaker.io (datagate-menifest)
```
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/ssl-redirect: "true"
nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
name: datagate-menifest-develop
namespace: saas-dev
spec:
ingressClassName: nginx
rules:
- host: dgate-k.dev.kr.datasaker.io
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: manifest-sam-datagate-cloud-test
port:
number: 8080
<!-- apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: datagate-menifest-develop
annotations:
kubernetes.io/ingress.class: nginx
nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
nginx.ingress.kubernetes.io/ssl-passthrough: "true"
namespace: saas-dev
spec:
rules:
- host: dgate-k.dev.kr.datasaker.io
http:
paths:
- backend:
service:
name: manifest-sam-datagate-cloud-test
port:
number: 8080
path: /
pathType: Prefix -->
```
#### postgresql.dev.kr.datasaker.io
postgresql NodePort 100.69.229.168 <none> 5432:32713/TCP
```
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: postgresql-develop
annotations:
kubernetes.io/ingress.class: nginx
nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
nginx.ingress.kubernetes.io/ssl-passthrough: "true"
namespace: infra-dev
spec:
rules:
- host: postgresql.dev.kr.datasaker.io
http:
paths:
- backend:
service:
name: postgresql
port:
number: 5432
path: /
pathType: Prefix
```
Reference in New Issue View Git Blame Copy Permalink