디렉토리 구조 및 각 서비스 추가
This commit is contained in:
dsk-minchulahn
197
helm/airflow/override_values.yaml
Normal file
197
helm/airflow/override_values.yaml
Normal file
@@ -0,0 +1,197 @@
|
||||
## Prerequirements
|
||||
|
||||
## Airflow Namespace 생성
|
||||
# kubectl create namespace airflow
|
||||
|
||||
## Web Server Secret Key 생성
|
||||
# kubectl create secret -n airflow generic webserver-secret --from-literal="webserver-secret-key=$(python3 -c 'import secrets; print(secrets.token_hex(16))')"
|
||||
|
||||
## GitHub 연동을 위한 Secret Key 생성
|
||||
# kubectl create secret generic airflow-git-ssh-secret \
|
||||
# --from-file=gitSshKey=[사용할 SSH Private key] \
|
||||
# -n airflow
|
||||
|
||||
## keda 활용 시 설치
|
||||
# helm repo add kedacore https://kedacore.github.io/charts
|
||||
# helm repo update
|
||||
# kubectl create namespace keda
|
||||
# helm install keda kedacore/keda --namespace keda --version "v2.0.0"
|
||||
|
||||
---
|
||||
|
||||
webserverSecretKeySecretName: webserver-secret
|
||||
|
||||
executor: "CeleryKubernetesExecutor"
|
||||
|
||||
workers:
|
||||
replicas: 1
|
||||
resources:
|
||||
limits:
|
||||
cpu: 1
|
||||
memory: 3Gi
|
||||
requests:
|
||||
cpu: 1
|
||||
memory: 2Gi
|
||||
keda:
|
||||
enabled: true
|
||||
minReplicaCount: 1
|
||||
maxReplicaCount: 3
|
||||
# advanced:
|
||||
# horizontalPodAutoscalerConfig:
|
||||
# behavior:
|
||||
# scaleDown:
|
||||
# stabilizationWindowSeconds: 600
|
||||
# policies:
|
||||
# - type: Percent
|
||||
# value: 100
|
||||
# periodSeconds: 15
|
||||
|
||||
scheduler:
|
||||
replicas: 1
|
||||
resources:
|
||||
limits:
|
||||
cpu: 1
|
||||
memory: 1Gi
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
|
||||
webserver:
|
||||
replicas: 1
|
||||
resources:
|
||||
limits:
|
||||
cpu: 1
|
||||
memory: 2Gi
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
service:
|
||||
type: NodePort
|
||||
ports:
|
||||
- name: airflow-ui
|
||||
port: 8080
|
||||
targetPort: 8080
|
||||
nodePort: 30180
|
||||
defaultUser:
|
||||
email: minchulahn@ex-em.com
|
||||
|
||||
triggerer:
|
||||
replicas: 1
|
||||
resources:
|
||||
limits:
|
||||
cpu: 1
|
||||
memory: 1Gi
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
|
||||
statsd:
|
||||
resources:
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
|
||||
redis:
|
||||
resources:
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
|
||||
dags:
|
||||
gitSync:
|
||||
enabled: true
|
||||
repo: git@github.com:cloudmoa/dsk-airflow-pipeline.git
|
||||
branch: main
|
||||
subPath: dags
|
||||
sshKeySecret: airflow-git-ssh-secret
|
||||
|
||||
postgresql:
|
||||
primary:
|
||||
service:
|
||||
type: NodePort
|
||||
nodePorts:
|
||||
postgresql: 30185
|
||||
|
||||
logs:
|
||||
persistence:
|
||||
enabled: true
|
||||
|
||||
config:
|
||||
core:
|
||||
dags_folder: '{{ include "airflow_dags" . }}'
|
||||
# This is ignored when used with the official Docker image
|
||||
load_examples: 'False'
|
||||
executor: '{{ .Values.executor }}'
|
||||
# For Airflow 1.10, backward compatibility; moved to [logging] in 2.0
|
||||
colored_console_log: 'False'
|
||||
remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}'
|
||||
logging:
|
||||
remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}'
|
||||
colored_console_log: 'False'
|
||||
metrics:
|
||||
statsd_on: '{{ ternary "True" "False" .Values.statsd.enabled }}'
|
||||
statsd_port: 9125
|
||||
statsd_prefix: airflow
|
||||
statsd_host: '{{ printf "%s-statsd" .Release.Name }}'
|
||||
webserver:
|
||||
enable_proxy_fix: 'True'
|
||||
# For Airflow 1.10
|
||||
rbac: 'True'
|
||||
celery:
|
||||
flower_url_prefix: '{{ .Values.ingress.flower.path }}'
|
||||
worker_concurrency: 16
|
||||
scheduler:
|
||||
standalone_dag_processor: '{{ ternary "True" "False" .Values.dagProcessor.enabled }}'
|
||||
# statsd params included for Airflow 1.10 backward compatibility; moved to [metrics] in 2.0
|
||||
statsd_on: '{{ ternary "True" "False" .Values.statsd.enabled }}'
|
||||
statsd_port: 9125
|
||||
statsd_prefix: airflow
|
||||
statsd_host: '{{ printf "%s-statsd" .Release.Name }}'
|
||||
# `run_duration` included for Airflow 1.10 backward compatibility; removed in 2.0.
|
||||
run_duration: 41460
|
||||
elasticsearch:
|
||||
json_format: 'True'
|
||||
log_id_template: "{dag_id}_{task_id}_{execution_date}_{try_number}"
|
||||
elasticsearch_configs:
|
||||
max_retries: 3
|
||||
timeout: 30
|
||||
retry_timeout: 'True'
|
||||
kerberos:
|
||||
keytab: '{{ .Values.kerberos.keytabPath }}'
|
||||
reinit_frequency: '{{ .Values.kerberos.reinitFrequency }}'
|
||||
principal: '{{ .Values.kerberos.principal }}'
|
||||
ccache: '{{ .Values.kerberos.ccacheMountPath }}/{{ .Values.kerberos.ccacheFileName }}'
|
||||
celery_kubernetes_executor:
|
||||
kubernetes_queue: 'kubernetes'
|
||||
# The `kubernetes` section is deprecated in Airflow >= 2.5.0 due to an airflow.cfg schema change.
|
||||
# The `kubernetes` section can be removed once the helm chart no longer supports Airflow < 2.5.0.
|
||||
kubernetes:
|
||||
namespace: '{{ .Release.Namespace }}'
|
||||
# The following `airflow_` entries are for Airflow 1, and can be removed when it is no longer supported.
|
||||
airflow_configmap: '{{ include "airflow_config" . }}'
|
||||
airflow_local_settings_configmap: '{{ include "airflow_config" . }}'
|
||||
pod_template_file: '{{ include "airflow_pod_template_file" . }}/pod_template_file.yaml'
|
||||
worker_container_repository: '{{ .Values.images.airflow.repository | default .Values.defaultAirflowRepository }}'
|
||||
worker_container_tag: '{{ .Values.images.airflow.tag | default .Values.defaultAirflowTag }}'
|
||||
multi_namespace_mode: '{{ ternary "True" "False" .Values.multiNamespaceMode }}'
|
||||
# The `kubernetes_executor` section duplicates the `kubernetes` section in Airflow >= 2.5.0 due to an airflow.cfg schema change.
|
||||
kubernetes_executor:
|
||||
namespace: '{{ .Release.Namespace }}'
|
||||
pod_template_file: '{{ include "airflow_pod_template_file" . }}/pod_template_file.yaml'
|
||||
worker_container_repository: '{{ .Values.images.airflow.repository | default .Values.defaultAirflowRepository }}'
|
||||
worker_container_tag: '{{ .Values.images.airflow.tag | default .Values.defaultAirflowTag }}'
|
||||
multi_namespace_mode: '{{ ternary "True" "False" .Values.multiNamespaceMode }}'
|
||||
email:
|
||||
email_backend: airflow.providers.amazon.aws.utils.emailer.send_email
|
||||
email_conn_id: aws_ses
|
||||
default_email_on_retry: True
|
||||
default_email_on_failure: True
|
||||
from_email: DataSaker <notice@datasaker.io>
|
||||
api:
|
||||
auth_backends: airflow.api.auth.backend.basic_auth
|
||||
Reference in New Issue
Block a user