197 lines
5.8 KiB
YAML
197 lines
5.8 KiB
YAML
## Prerequirements
|
|
|
|
## Airflow Namespace 생성
|
|
# kubectl create namespace airflow
|
|
|
|
## Web Server Secret Key 생성
|
|
# kubectl create secret -n airflow generic webserver-secret --from-literal="webserver-secret-key=$(python3 -c 'import secrets; print(secrets.token_hex(16))')"
|
|
|
|
## GitHub 연동을 위한 Secret Key 생성
|
|
# kubectl create secret generic airflow-git-ssh-secret \
|
|
# --from-file=gitSshKey=[사용할 SSH Private key] \
|
|
# -n airflow
|
|
|
|
## keda 활용 시 설치
|
|
# helm repo add kedacore https://kedacore.github.io/charts
|
|
# helm repo update
|
|
# kubectl create namespace keda
|
|
# helm install keda kedacore/keda --namespace keda --version "v2.0.0"
|
|
|
|
---
|
|
|
|
webserverSecretKeySecretName: webserver-secret
|
|
|
|
executor: "CeleryKubernetesExecutor"
|
|
|
|
workers:
|
|
replicas: 1
|
|
resources:
|
|
limits:
|
|
cpu: 1
|
|
memory: 3Gi
|
|
requests:
|
|
cpu: 1
|
|
memory: 2Gi
|
|
keda:
|
|
enabled: true
|
|
minReplicaCount: 1
|
|
maxReplicaCount: 3
|
|
# advanced:
|
|
# horizontalPodAutoscalerConfig:
|
|
# behavior:
|
|
# scaleDown:
|
|
# stabilizationWindowSeconds: 600
|
|
# policies:
|
|
# - type: Percent
|
|
# value: 100
|
|
# periodSeconds: 15
|
|
|
|
scheduler:
|
|
replicas: 1
|
|
resources:
|
|
limits:
|
|
cpu: 1
|
|
memory: 1Gi
|
|
requests:
|
|
cpu: 500m
|
|
memory: 1Gi
|
|
|
|
webserver:
|
|
replicas: 1
|
|
resources:
|
|
limits:
|
|
cpu: 1
|
|
memory: 2Gi
|
|
requests:
|
|
cpu: 500m
|
|
memory: 1Gi
|
|
service:
|
|
type: NodePort
|
|
ports:
|
|
- name: airflow-ui
|
|
port: 8080
|
|
targetPort: 8080
|
|
nodePort: 30180
|
|
defaultUser:
|
|
email: minchulahn@ex-em.com
|
|
|
|
triggerer:
|
|
replicas: 1
|
|
resources:
|
|
limits:
|
|
cpu: 1
|
|
memory: 1Gi
|
|
requests:
|
|
cpu: 500m
|
|
memory: 1Gi
|
|
|
|
statsd:
|
|
resources:
|
|
limits:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
requests:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
|
|
redis:
|
|
resources:
|
|
limits:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
requests:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
|
|
dags:
|
|
gitSync:
|
|
enabled: true
|
|
repo: git@github.com:cloudmoa/dsk-airflow-pipeline.git
|
|
branch: main
|
|
subPath: dags
|
|
sshKeySecret: airflow-git-ssh-secret
|
|
|
|
postgresql:
|
|
primary:
|
|
service:
|
|
type: NodePort
|
|
nodePorts:
|
|
postgresql: 30185
|
|
|
|
logs:
|
|
persistence:
|
|
enabled: true
|
|
|
|
config:
|
|
core:
|
|
dags_folder: '{{ include "airflow_dags" . }}'
|
|
# This is ignored when used with the official Docker image
|
|
load_examples: 'False'
|
|
executor: '{{ .Values.executor }}'
|
|
# For Airflow 1.10, backward compatibility; moved to [logging] in 2.0
|
|
colored_console_log: 'False'
|
|
remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}'
|
|
logging:
|
|
remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}'
|
|
colored_console_log: 'False'
|
|
metrics:
|
|
statsd_on: '{{ ternary "True" "False" .Values.statsd.enabled }}'
|
|
statsd_port: 9125
|
|
statsd_prefix: airflow
|
|
statsd_host: '{{ printf "%s-statsd" .Release.Name }}'
|
|
webserver:
|
|
enable_proxy_fix: 'True'
|
|
# For Airflow 1.10
|
|
rbac: 'True'
|
|
celery:
|
|
flower_url_prefix: '{{ .Values.ingress.flower.path }}'
|
|
worker_concurrency: 16
|
|
scheduler:
|
|
standalone_dag_processor: '{{ ternary "True" "False" .Values.dagProcessor.enabled }}'
|
|
# statsd params included for Airflow 1.10 backward compatibility; moved to [metrics] in 2.0
|
|
statsd_on: '{{ ternary "True" "False" .Values.statsd.enabled }}'
|
|
statsd_port: 9125
|
|
statsd_prefix: airflow
|
|
statsd_host: '{{ printf "%s-statsd" .Release.Name }}'
|
|
# `run_duration` included for Airflow 1.10 backward compatibility; removed in 2.0.
|
|
run_duration: 41460
|
|
elasticsearch:
|
|
json_format: 'True'
|
|
log_id_template: "{dag_id}_{task_id}_{execution_date}_{try_number}"
|
|
elasticsearch_configs:
|
|
max_retries: 3
|
|
timeout: 30
|
|
retry_timeout: 'True'
|
|
kerberos:
|
|
keytab: '{{ .Values.kerberos.keytabPath }}'
|
|
reinit_frequency: '{{ .Values.kerberos.reinitFrequency }}'
|
|
principal: '{{ .Values.kerberos.principal }}'
|
|
ccache: '{{ .Values.kerberos.ccacheMountPath }}/{{ .Values.kerberos.ccacheFileName }}'
|
|
celery_kubernetes_executor:
|
|
kubernetes_queue: 'kubernetes'
|
|
# The `kubernetes` section is deprecated in Airflow >= 2.5.0 due to an airflow.cfg schema change.
|
|
# The `kubernetes` section can be removed once the helm chart no longer supports Airflow < 2.5.0.
|
|
kubernetes:
|
|
namespace: '{{ .Release.Namespace }}'
|
|
# The following `airflow_` entries are for Airflow 1, and can be removed when it is no longer supported.
|
|
airflow_configmap: '{{ include "airflow_config" . }}'
|
|
airflow_local_settings_configmap: '{{ include "airflow_config" . }}'
|
|
pod_template_file: '{{ include "airflow_pod_template_file" . }}/pod_template_file.yaml'
|
|
worker_container_repository: '{{ .Values.images.airflow.repository | default .Values.defaultAirflowRepository }}'
|
|
worker_container_tag: '{{ .Values.images.airflow.tag | default .Values.defaultAirflowTag }}'
|
|
multi_namespace_mode: '{{ ternary "True" "False" .Values.multiNamespaceMode }}'
|
|
# The `kubernetes_executor` section duplicates the `kubernetes` section in Airflow >= 2.5.0 due to an airflow.cfg schema change.
|
|
kubernetes_executor:
|
|
namespace: '{{ .Release.Namespace }}'
|
|
pod_template_file: '{{ include "airflow_pod_template_file" . }}/pod_template_file.yaml'
|
|
worker_container_repository: '{{ .Values.images.airflow.repository | default .Values.defaultAirflowRepository }}'
|
|
worker_container_tag: '{{ .Values.images.airflow.tag | default .Values.defaultAirflowTag }}'
|
|
multi_namespace_mode: '{{ ternary "True" "False" .Values.multiNamespaceMode }}'
|
|
email:
|
|
email_backend: airflow.providers.amazon.aws.utils.emailer.send_email
|
|
email_conn_id: aws_ses
|
|
default_email_on_retry: True
|
|
default_email_on_failure: True
|
|
from_email: DataSaker <notice@datasaker.io>
|
|
api:
|
|
auth_backends: airflow.api.auth.backend.basic_auth |