From fa9a7cff544a2f6c3a98b90dc47083e5c7bf6fbf Mon Sep 17 00:00:00 2001 From: minchulahn Date: Thu, 29 Dec 2022 12:28:14 +0900 Subject: [PATCH] Update spot scripts --- .../terraform/.terraform.lock.hcl | 37 ------- .../aws_kops_prod/terraform/99_variables.tf | 6 +- scripts/spot/{ => lambda}/lambda_restart.sh | 6 +- scripts/spot/lambda/lambda_start.sh | 100 ++++++++++++++++++ scripts/spot/{ => lambda}/lambda_stop.sh | 4 +- scripts/spot/spot.sh | 69 ++++++++++++ 6 files changed, 177 insertions(+), 45 deletions(-) rename scripts/spot/{ => lambda}/lambda_restart.sh (98%) create mode 100755 scripts/spot/lambda/lambda_start.sh rename scripts/spot/{ => lambda}/lambda_stop.sh (99%) create mode 100755 scripts/spot/spot.sh diff --git a/aws_kubernetes/aws_kops_prod/terraform/.terraform.lock.hcl b/aws_kubernetes/aws_kops_prod/terraform/.terraform.lock.hcl index edcfb3a..717e06d 100644 --- a/aws_kubernetes/aws_kops_prod/terraform/.terraform.lock.hcl +++ b/aws_kubernetes/aws_kops_prod/terraform/.terraform.lock.hcl @@ -1,24 +1,6 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. -provider "registry.terraform.io/hashicorp/archive" { - version = "2.2.0" - hashes = [ - "h1:CIWi5G6ob7p2wWoThRQbOB8AbmFlCzp7Ka81hR3cVp0=", - "zh:06bd875932288f235c16e2237142b493c2c2b6aba0e82e8c85068332a8d2a29e", - "zh:0c681b481372afcaefddacc7ccdf1d3bb3a0c0d4678a526bc8b02d0c331479bc", - "zh:100fc5b3fc01ea463533d7bbfb01cb7113947a969a4ec12e27f5b2be49884d6c", - "zh:55c0d7ddddbd0a46d57c51fcfa9b91f14eed081a45101dbfc7fd9d2278aa1403", - "zh:73a5dd68379119167934c48afa1101b09abad2deb436cd5c446733e705869d6b", - "zh:841fc4ac6dc3479981330974d44ad2341deada8a5ff9e3b1b4510702dfbdbed9", - "zh:91be62c9b41edb137f7f835491183628d484e9d6efa82fcb75cfa538c92791c5", - "zh:acd5f442bd88d67eb948b18dc2ed421c6c3faee62d3a12200e442bfff0aa7d8b", - "zh:ad5720da5524641ad718a565694821be5f61f68f1c3c5d2cfa24426b8e774bef", - "zh:e63f12ea938520b3f83634fc29da28d92eed5cfbc5cc8ca08281a6a9c36cca65", - "zh:f6542918faa115df46474a36aabb4c3899650bea036b5f8a5e296be6f8f25767", - ] -} - provider "registry.terraform.io/hashicorp/aws" { version = "4.45.0" constraints = ">= 4.0.0" @@ -41,22 +23,3 @@ provider "registry.terraform.io/hashicorp/aws" { "zh:f6d355a2fb3bcebb597f68bbca4fa2aaa364efd29240236c582375e219d77656", ] } - -provider "registry.terraform.io/hashicorp/null" { - version = "3.2.1" - hashes = [ - "h1:FbGfc+muBsC17Ohy5g806iuI1hQc4SIexpYCrQHQd8w=", - "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840", - "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb", - "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5", - "zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238", - "zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc", - "zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970", - "zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2", - "zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5", - "zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f", - "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694", - ] -} diff --git a/aws_kubernetes/aws_kops_prod/terraform/99_variables.tf b/aws_kubernetes/aws_kops_prod/terraform/99_variables.tf index 10dc1ed..29904af 100644 --- a/aws_kubernetes/aws_kops_prod/terraform/99_variables.tf +++ b/aws_kubernetes/aws_kops_prod/terraform/99_variables.tf @@ -8,15 +8,15 @@ variable "port_list" { #---------------------------------------------------------------# # LB 구성 후 target group 매칭 시 필요한 인스턴스 ID variable "master-2a" { - default = "i-0e740951af7f1af18" + default = "i-080276c1ee1237696" } variable "master-2b" { - default = "i-04699e92fe5063435" + default = "i-023274b3691fb2c07" } variable "master-2c" { - default = "i-05258725ca60459fe" + default = "i-033d278c81542933e" } #---------------------------------------------------------------# diff --git a/scripts/spot/lambda_restart.sh b/scripts/spot/lambda/lambda_restart.sh similarity index 98% rename from scripts/spot/lambda_restart.sh rename to scripts/spot/lambda/lambda_restart.sh index 2e4b800..953a110 100755 --- a/scripts/spot/lambda_restart.sh +++ b/scripts/spot/lambda/lambda_restart.sh @@ -3,16 +3,16 @@ if [ -z "$BASH_VERSION" ]; then exec bash "$0" "$@"; exit; fi #---------------------------------------------------------------------------------------------------------------- # 변수 선언 echo_line="==========================================================================" -search_tag="spot" -stop_function="spot_stop" +search_tag="prod" start_function="spot_start" +stop_function="spot_stop" #---------------------------------------------------------------------------------------------------------------- # aws 인스턴스 조회 _get_aws_status (){ aws_query="Reservations[].Instances[].[ InstanceId, Tags[?Key=='Name'].Value|[0] ]" aws_filter="Name=instance-state-name,Values=running" - current_ec2_num=`aws ec2 describe-instances --query "${aws_query}" --filter ${aws_filter} --output text | grep ${search_tag} | wc -l` + current_ec2_num=`aws ec2 describe-instances --query "${aws_query}" --filter ${aws_filter} --output text | grep ${search_tag} | egrep -iv '(bastion|wireguard)' |wc -l` } #---------------------------------------------------------------------------------------------------------------- diff --git a/scripts/spot/lambda/lambda_start.sh b/scripts/spot/lambda/lambda_start.sh new file mode 100755 index 0000000..84d1656 --- /dev/null +++ b/scripts/spot/lambda/lambda_start.sh @@ -0,0 +1,100 @@ +if [ -z "$BASH_VERSION" ]; then exec bash "$0" "$@"; exit; fi + +#---------------------------------------------------------------------------------------------------------------- +# 변수 선언 +echo_line="==========================================================================" +search_tag="prod" +start_function="spot_start" +stop_function="spot_stop" + +#---------------------------------------------------------------------------------------------------------------- +# aws 인스턴스 조회 +_get_aws_status (){ + aws_query="Reservations[].Instances[].[ InstanceId, Tags[?Key=='Name'].Value|[0] ]" + aws_filter="Name=instance-state-name,Values=running" + current_ec2_num=`aws ec2 describe-instances --query "${aws_query}" --filter ${aws_filter} --output text | grep ${search_tag} | egrep -iv '(bastion|wireguard)' |wc -l` +} + +#---------------------------------------------------------------------------------------------------------------- +# log 출력 +_get_time_log (){ + datetime=`date +'%Y-%m-%d %H:%M:%S'` + string=$1 + echo "[${datetime}] ${string}" +} + +#---------------------------------------------------------------------------------------------------------------- +# aws lambda 실행 후 인스턴스 조회하여 성공/실패 여부 체크 + +_main (){ + lambda_function=$1 + echo ${echo_line} + if [[ ${lambda_fuction} == *"stop"* ]]; then + what_func='stop' + else + what_func='start' + fi + + _get_time_log "[INFO] aws lambda ${lambda_function} start" + + aws lambda invoke --function-name ${lambda_function} --cli-binary-format raw-in-base64-out --payload '{ "key": "value" }' response.json > /tmp/aws_func_result + #echo "aws lambda invoke --function-name ${lambda_function} --cli-binary-format raw-in-base64-out --payload '{ "key": "value" }' response.json" + + succ_flag=0 + while read line + do + if [[ $line == *"200"* ]]; then + succ_flag=1 + _get_time_log "[INFO] aws lambda ${lambda_function} success!" + break + else + succ_flag=0 + fi + done < /tmp/aws_func_result + + if [[ ${succ_flag} == 1 ]]; then + aws lambda invoke --function-name ${lambda_function} out --log-type Tail --query 'LogResult' --output text | base64 -d > /tmp/aws_func_output + if [[ ${what_func} == 'stop' ]]; then + final_ec2_num=0 + else + final_ec2_num=`grep "Starting" /tmp/aws_func_output | wc -l` + fi + + num=0 + while true + do + _get_aws_status + num=`echo $(( $num + 1 ))` + if [[ ${current_ec2_num} == ${final_ec2_num} ]]; then + _get_time_log "[INFO] aws ${search_tag} instance number: ${final_ec2_num} check success!" + break + elif [[ ${num} == 30 ]]; then + _get_time_log "[Error] time out!" + break + else + _get_time_log "[Retry: ${num}] aws ${search_tag} instance number: ${final_ec2_num} checking..." + fi + sleep 5 + done + else + echo ${echo_line} + _get_time_log "[Error] aws lambda ${lambda_function} failed!" + exit + fi + echo ${echo_line} + #--------------- + +} +#---------------------------------------------------------------------------------------------------------------- +# 실행 + +_main ${start_function} + +#---------------------------------------------------------------------------------------------------------------- +# 임시 파일 삭제 + +unlink out +unlink response.json +unlink /tmp/aws_func_output +unlink /tmp/aws_func_result +#---------------------------------------------------------------------------------------------------------------- diff --git a/scripts/spot/lambda_stop.sh b/scripts/spot/lambda/lambda_stop.sh similarity index 99% rename from scripts/spot/lambda_stop.sh rename to scripts/spot/lambda/lambda_stop.sh index b8be4fd..ae99f98 100755 --- a/scripts/spot/lambda_stop.sh +++ b/scripts/spot/lambda/lambda_stop.sh @@ -3,9 +3,9 @@ if [ -z "$BASH_VERSION" ]; then exec bash "$0" "$@"; exit; fi #---------------------------------------------------------------------------------------------------------------- # 변수 선언 echo_line="==========================================================================" -search_tag="spot" -stop_function="spot_stop" +search_tag="prod" start_function="spot_start" +stop_function="spot_stop" #---------------------------------------------------------------------------------------------------------------- # aws 인스턴스 조회 diff --git a/scripts/spot/spot.sh b/scripts/spot/spot.sh new file mode 100755 index 0000000..8da4b1f --- /dev/null +++ b/scripts/spot/spot.sh @@ -0,0 +1,69 @@ +#!/bin/bash +if [ -z "$BASH_VERSION" ]; then exec bash "$0" "$@"; exit; fi + +#---------------------------------------------------------------------------------------------------------------- +# 변수 선언 +current_path=`pwd` +line="*-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-*" + +#---------------------------------------------------------------------------------------------------------------- +# 파일 및 폴더 경로 확인 +__find_path (){ + script_file="00_terraform_var_change.sh" + cd ../../ + tmp_path=`find -name ${script_file} | grep -vi old` + tmp_num=`grep -c "/" ${tmp_path}` + path_num=`echo $(( $tmp_num - 1 ))` + script_path=`echo ${tmp_path}| cut -d "/" -f1-${path_num}` + _bash=`which bash` + _terraform=`which terraform` +} + +#---------------------------------------------------------------------------------------------------------------- +# lambda 실행 +__lambda_execute (){ + lambda_file="lambda_${1}.sh" + cd "${current_path}/lambda" + #./${lambda_file} + echo ${line} + echo "lambda file : $(pwd)/${lambda_file}" #DEBUG + ${_bash} ${lambda_file} + echo ${line} +} + +#---------------------------------------------------------------------------------------------------------------- +# 변수 변경 스크립트 실행 뒤 terraform apply 실행 +__terraform_target (){ + cd ${current_path} + cd ../../ + cd ${script_path} + echo "script_file : ${script_file}" #DEBUG + ${_bash} ${script_file} + echo ${line} + ${_terraform} init + echo ${line} + sleep 1 + ${_terraform} apply +} + +#---------------------------------------------------------------------------------------------------------------- + +if [[ ${1} == '' ]];then + echo "[Usage] ${0} { start | stop | restart } " + exit +else + script_func=${1} + __find_path + if [[ ${script_func} == 'start' ]];then + __lambda_execute start + __terraform_target + elif [[ ${script_func} == 'stop' ]];then + __lambda_execute stop + elif [[ ${script_func} == 'restart' ]];then + __lambda_execute restart + __terraform_target + else + echo "[Usage] ${0} { start | stop | restart } " + exit + fi +fi