Update spot scripts

2022-12-29 12:28:14 +09:00
parent 781f0ebd65
commit fa9a7cff54
6 changed files with 177 additions and 45 deletions
--- a/aws_kubernetes/aws_kops_prod/terraform/.terraform.lock.hcl
+++ b/aws_kubernetes/aws_kops_prod/terraform/.terraform.lock.hcl
@@ -1,24 +1,6 @@
 # This file is maintained automatically by "terraform init".
 # Manual edits may be lost in future updates.

-provider "registry.terraform.io/hashicorp/archive" {
-  version = "2.2.0"
-  hashes = [
-    "h1:CIWi5G6ob7p2wWoThRQbOB8AbmFlCzp7Ka81hR3cVp0=",
-    "zh:06bd875932288f235c16e2237142b493c2c2b6aba0e82e8c85068332a8d2a29e",
-    "zh:0c681b481372afcaefddacc7ccdf1d3bb3a0c0d4678a526bc8b02d0c331479bc",
-    "zh:100fc5b3fc01ea463533d7bbfb01cb7113947a969a4ec12e27f5b2be49884d6c",
-    "zh:55c0d7ddddbd0a46d57c51fcfa9b91f14eed081a45101dbfc7fd9d2278aa1403",
-    "zh:73a5dd68379119167934c48afa1101b09abad2deb436cd5c446733e705869d6b",
-    "zh:841fc4ac6dc3479981330974d44ad2341deada8a5ff9e3b1b4510702dfbdbed9",
-    "zh:91be62c9b41edb137f7f835491183628d484e9d6efa82fcb75cfa538c92791c5",
-    "zh:acd5f442bd88d67eb948b18dc2ed421c6c3faee62d3a12200e442bfff0aa7d8b",
-    "zh:ad5720da5524641ad718a565694821be5f61f68f1c3c5d2cfa24426b8e774bef",
-    "zh:e63f12ea938520b3f83634fc29da28d92eed5cfbc5cc8ca08281a6a9c36cca65",
-    "zh:f6542918faa115df46474a36aabb4c3899650bea036b5f8a5e296be6f8f25767",
-  ]
-}
-
 provider "registry.terraform.io/hashicorp/aws" {
  version     = "4.45.0"
  constraints = ">= 4.0.0"
@@ -41,22 +23,3 @@ provider "registry.terraform.io/hashicorp/aws" {
    "zh:f6d355a2fb3bcebb597f68bbca4fa2aaa364efd29240236c582375e219d77656",
  ]
 }
-
-provider "registry.terraform.io/hashicorp/null" {
-  version = "3.2.1"
-  hashes = [
-    "h1:FbGfc+muBsC17Ohy5g806iuI1hQc4SIexpYCrQHQd8w=",
-    "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840",
-    "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb",
-    "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5",
-    "zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3",
-    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
-    "zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238",
-    "zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc",
-    "zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970",
-    "zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2",
-    "zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5",
-    "zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f",
-    "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694",
-  ]
-}
--- a/aws_kubernetes/aws_kops_prod/terraform/99_variables.tf
+++ b/aws_kubernetes/aws_kops_prod/terraform/99_variables.tf
@@ -8,15 +8,15 @@ variable "port_list" {
 #---------------------------------------------------------------#
 # LB 구성 후 target group 매칭 시 필요한 인스턴스 ID
 variable "master-2a" {
-  default = "i-0e740951af7f1af18"
+  default = "i-080276c1ee1237696"
 }

 variable "master-2b" {
-  default = "i-04699e92fe5063435"
+  default = "i-023274b3691fb2c07"
 }

 variable "master-2c" {
-  default = "i-05258725ca60459fe"
+  default = "i-033d278c81542933e"
 }

 #---------------------------------------------------------------#
--- a/scripts/spot/lambda/lambda_restart.sh
+++ b/scripts/spot/lambda/lambda_restart.sh
@@ -3,16 +3,16 @@ if [ -z "$BASH_VERSION" ]; then exec bash "$0" "$@"; exit; fi
 #----------------------------------------------------------------------------------------------------------------
 # 변수 선언
 echo_line="=========================================================================="
-search_tag="spot"
-stop_function="spot_stop"
+search_tag="prod"
 start_function="spot_start"
+stop_function="spot_stop"

 #----------------------------------------------------------------------------------------------------------------
 # aws 인스턴스 조회
 _get_aws_status (){
    aws_query="Reservations[].Instances[].[ InstanceId, Tags[?Key=='Name'].Value|[0] ]"
    aws_filter="Name=instance-state-name,Values=running"
-    current_ec2_num=`aws ec2 describe-instances --query "${aws_query}" --filter ${aws_filter} --output text | grep ${search_tag} | wc -l`
+    current_ec2_num=`aws ec2 describe-instances --query "${aws_query}" --filter ${aws_filter} --output text | grep ${search_tag} | egrep -iv '(bastion|wireguard)' |wc -l`
 }

 #----------------------------------------------------------------------------------------------------------------
--- a/scripts/spot/lambda/lambda_start.sh
+++ b/scripts/spot/lambda/lambda_start.sh
@@ -0,0 +1,100 @@
+if [ -z "$BASH_VERSION" ]; then exec bash "$0" "$@"; exit; fi
+
+#----------------------------------------------------------------------------------------------------------------
+# 변수 선언
+echo_line="=========================================================================="
+search_tag="prod"
+start_function="spot_start"
+stop_function="spot_stop"
+
+#----------------------------------------------------------------------------------------------------------------
+# aws 인스턴스 조회
+_get_aws_status (){
+    aws_query="Reservations[].Instances[].[ InstanceId, Tags[?Key=='Name'].Value|[0] ]"
+    aws_filter="Name=instance-state-name,Values=running"
+    current_ec2_num=`aws ec2 describe-instances --query "${aws_query}" --filter ${aws_filter} --output text | grep ${search_tag} | egrep -iv '(bastion|wireguard)' |wc -l`
+}
+
+#----------------------------------------------------------------------------------------------------------------
+# log 출력
+_get_time_log (){
+    datetime=`date +'%Y-%m-%d %H:%M:%S'`
+    string=$1
+    echo "[${datetime}] ${string}"
+}
+
+#----------------------------------------------------------------------------------------------------------------
+# aws lambda 실행 후 인스턴스 조회하여 성공/실패 여부 체크
+
+_main (){
+    lambda_function=$1
+    echo ${echo_line}
+    if [[ ${lambda_fuction} == *"stop"*  ]]; then
+        what_func='stop'
+    else
+        what_func='start'
+    fi
+
+    _get_time_log "[INFO] aws lambda ${lambda_function} start"
+
+    aws lambda invoke --function-name ${lambda_function} --cli-binary-format raw-in-base64-out --payload '{ "key": "value" }' response.json > /tmp/aws_func_result
+    #echo "aws lambda invoke --function-name ${lambda_function} --cli-binary-format raw-in-base64-out --payload '{ "key": "value" }' response.json"
+
+    succ_flag=0
+    while read line
+    do
+        if [[ $line == *"200"* ]]; then
+          succ_flag=1
+          _get_time_log "[INFO] aws lambda ${lambda_function} success!"
+          break
+        else
+          succ_flag=0
+        fi
+    done < /tmp/aws_func_result
+
+    if [[ ${succ_flag} == 1 ]]; then
+        aws lambda invoke --function-name ${lambda_function} out --log-type Tail --query 'LogResult' --output text |  base64 -d > /tmp/aws_func_output
+        if [[ ${what_func} == 'stop' ]]; then
+            final_ec2_num=0
+        else
+            final_ec2_num=`grep "Starting" /tmp/aws_func_output | wc -l`
+        fi
+
+        num=0
+        while true
+        do
+            _get_aws_status
+            num=`echo $(( $num + 1 ))`
+            if [[ ${current_ec2_num} == ${final_ec2_num} ]]; then
+                _get_time_log "[INFO] aws ${search_tag} instance number: ${final_ec2_num} check success!"
+                break
+            elif [[ ${num} == 30 ]]; then
+                _get_time_log "[Error] time out!"
+                break
+            else
+                _get_time_log "[Retry: ${num}] aws ${search_tag} instance number: ${final_ec2_num} checking..."
+            fi
+            sleep 5
+        done
+    else
+        echo ${echo_line}
+        _get_time_log "[Error] aws lambda ${lambda_function} failed!"
+        exit
+    fi
+    echo ${echo_line}
+    #---------------
+
+}
+#----------------------------------------------------------------------------------------------------------------
+# 실행
+
+_main ${start_function}
+
+#----------------------------------------------------------------------------------------------------------------
+# 임시 파일 삭제
+
+unlink out
+unlink response.json
+unlink /tmp/aws_func_output
+unlink /tmp/aws_func_result
+#----------------------------------------------------------------------------------------------------------------
--- a/scripts/spot/lambda/lambda_stop.sh
+++ b/scripts/spot/lambda/lambda_stop.sh
@@ -3,9 +3,9 @@ if [ -z "$BASH_VERSION" ]; then exec bash "$0" "$@"; exit; fi
 #----------------------------------------------------------------------------------------------------------------
 # 변수 선언
 echo_line="=========================================================================="
-search_tag="spot"
-stop_function="spot_stop"
+search_tag="prod"
 start_function="spot_start"
+stop_function="spot_stop"

 #----------------------------------------------------------------------------------------------------------------
 # aws 인스턴스 조회
--- a/scripts/spot/spot.sh
+++ b/scripts/spot/spot.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+if [ -z "$BASH_VERSION" ]; then exec bash "$0" "$@"; exit; fi
+
+#----------------------------------------------------------------------------------------------------------------
+# 변수 선언
+current_path=`pwd`
+line="*-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-*"
+
+#----------------------------------------------------------------------------------------------------------------
+# 파일 및 폴더 경로 확인
+__find_path (){
+    script_file="00_terraform_var_change.sh"
+    cd ../../
+    tmp_path=`find -name ${script_file} | grep -vi old`
+    tmp_num=`grep -c "/" ${tmp_path}`
+    path_num=`echo $(( $tmp_num - 1 ))`
+    script_path=`echo ${tmp_path}| cut -d "/" -f1-${path_num}`
+    _bash=`which bash`
+    _terraform=`which terraform`
+}
+
+#----------------------------------------------------------------------------------------------------------------
+# lambda 실행
+__lambda_execute (){
+    lambda_file="lambda_${1}.sh"
+    cd "${current_path}/lambda"
+    #./${lambda_file}
+    echo ${line}
+    echo "lambda file : $(pwd)/${lambda_file}" #DEBUG
+    ${_bash} ${lambda_file}
+    echo ${line}
+}
+
+#----------------------------------------------------------------------------------------------------------------
+# 변수 변경 스크립트 실행 뒤 terraform apply 실행
+__terraform_target (){
+    cd ${current_path}
+    cd ../../
+    cd ${script_path}
+    echo "script_file : ${script_file}" #DEBUG
+    ${_bash} ${script_file}
+    echo ${line}
+    ${_terraform} init
+    echo ${line}
+    sleep 1
+    ${_terraform} apply
+}
+
+#----------------------------------------------------------------------------------------------------------------
+
+if [[ ${1} == ''  ]];then
+    echo "[Usage] ${0} { start | stop | restart } "
+    exit
+else
+    script_func=${1}
+    __find_path
+    if [[ ${script_func} == 'start' ]];then
+        __lambda_execute start
+        __terraform_target
+    elif [[ ${script_func} == 'stop'  ]];then
+        __lambda_execute stop
+    elif [[ ${script_func} == 'restart'  ]];then
+        __lambda_execute restart
+        __terraform_target
+    else
+        echo "[Usage] ${0} { start | stop | restart } "
+        exit
+    fi
+fi