디렉토리 구조 및 각 서비스 추가
This commit is contained in:
dsk-minchulahn
206
helm/airflow/templates/NOTES.txt
Normal file
206
helm/airflow/templates/NOTES.txt
Normal file
@@ -0,0 +1,206 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
|
||||
Thank you for installing Apache {{ title .Chart.Name }} {{ .Values.airflowVersion }}!
|
||||
|
||||
Your release is named {{ .Release.Name }}.
|
||||
|
||||
{{- if or .Values.ingress.web.enabled .Values.ingress.flower.enabled .Values.ingress.enabled }}
|
||||
You can now access your service(s) by following defined Ingress urls:
|
||||
|
||||
{{- if .Values.ingress.web.host }}
|
||||
|
||||
DEPRECATION WARNING:
|
||||
`ingress.web.host` has been renamed to `ingress.web.hosts` and is now an array.
|
||||
Please change your values as support for the old name will be dropped in a future release.
|
||||
|
||||
{{- end }}
|
||||
|
||||
{{- if .Values.ingress.web.tls }}
|
||||
|
||||
DEPRECATION WARNING:
|
||||
`ingress.web.tls` has been renamed to `ingress.web.hosts[*].tls` and can be set per host.
|
||||
Please change your values as support for the old name will be dropped in a future release.
|
||||
|
||||
{{- end }}
|
||||
|
||||
{{- if .Values.ingress.flower.host }}
|
||||
|
||||
DEPRECATION WARNING:
|
||||
`ingress.flower.host` has been renamed to `ingress.flower.hosts` and is now an array.
|
||||
Please change your values as support for the old name will be dropped in a future release.
|
||||
|
||||
{{- end }}
|
||||
|
||||
|
||||
{{- if .Values.ingress.flower.tls }}
|
||||
|
||||
DEPRECATION WARNING:
|
||||
`ingress.flower.tls` has been renamed to `ingress.flower.hosts[*].tls` and can be set per host.
|
||||
Please change your values as support for the old name will be dropped in a future release.
|
||||
|
||||
{{- end }}
|
||||
|
||||
{{- if .Values.ingress.enabled }}
|
||||
|
||||
DEPRECATION WARNING:
|
||||
`ingress.enabled` has been deprecated. There are now separate flags to control the webserver and
|
||||
flower individually, ``ingress.web.enabled`` and ``ingress.flower.enabled``.
|
||||
Please change your values as support for the old name will be dropped in a future release.
|
||||
|
||||
{{- end }}
|
||||
|
||||
{{- if or .Values.ingress.web.enabled .Values.ingress.enabled }}
|
||||
Airflow Webserver:
|
||||
{{- range .Values.ingress.web.hosts | default (list .Values.ingress.web.host) }}
|
||||
{{- $tlsEnabled := $.Values.ingress.web.tls.enabled -}}
|
||||
{{- $hostname := $.Values.ingress.web.host -}}
|
||||
{{- if . | kindIs "string" | not }}
|
||||
{{- if .tls }}
|
||||
{{- $tlsEnabled = .tls.enabled -}}
|
||||
{{- $hostname = .name -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
http{{ if $tlsEnabled }}s{{ end }}://{{ $hostname }}{{ $.Values.ingress.web.path }}/
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if and (or .Values.ingress.flower.enabled .Values.ingress.enabled) (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")) }}
|
||||
Flower dashboard:
|
||||
{{- range .Values.ingress.flower.hosts | default (list .Values.ingress.flower.host) }}
|
||||
{{- $tlsEnabled := $.Values.ingress.flower.tls.enabled -}}
|
||||
{{- $hostname := $.Values.ingress.flower.host -}}
|
||||
{{- if . | kindIs "string" | not }}
|
||||
{{- if .tls }}
|
||||
{{- $tlsEnabled = .tls.enabled -}}
|
||||
{{- $hostname = .name -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
http{{ if $tlsEnabled }}s{{ end }}://{{ $hostname }}{{ $.Values.ingress.flower.path }}/
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- else }}
|
||||
You can now access your dashboard(s) by executing the following command(s) and visiting the corresponding port at localhost in your browser:
|
||||
|
||||
Airflow Webserver: kubectl port-forward svc/{{ include "airflow.fullname" . }}-webserver {{ .Values.ports.airflowUI }}:{{ .Values.ports.airflowUI }} --namespace {{ .Release.Namespace }}
|
||||
|
||||
{{- if .Values.flower.enabled }}
|
||||
{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")}}
|
||||
Flower dashboard: kubectl port-forward svc/{{ include "airflow.fullname" . }}-flower {{ .Values.ports.flowerUI }}:{{ .Values.ports.flowerUI }} --namespace {{ .Release.Namespace }}
|
||||
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
|
||||
{{- if .Values.webserver.defaultUser.enabled}}
|
||||
Default Webserver (Airflow UI) Login credentials:
|
||||
username: {{ .Values.webserver.defaultUser.username }}
|
||||
password: {{ .Values.webserver.defaultUser.password }}
|
||||
{{- end }}
|
||||
|
||||
{{- if .Values.postgresql.enabled }}
|
||||
Default Postgres connection credentials:
|
||||
username: {{ .Values.data.metadataConnection.user }}
|
||||
password: {{ .Values.data.metadataConnection.pass }}
|
||||
port: {{ .Values.data.metadataConnection.port }}
|
||||
|
||||
{{- end }}
|
||||
|
||||
{{- if not .Values.fernetKeySecretName }}
|
||||
|
||||
You can get Fernet Key value by running the following:
|
||||
|
||||
echo Fernet Key: $(kubectl get secret --namespace {{ .Release.Namespace }} {{ .Release.Name }}-fernet-key -o jsonpath="{.data.fernet-key}" | base64 --decode)
|
||||
|
||||
{{- end }}
|
||||
|
||||
{{- if or (eq .Values.executor "KubernetesExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
|
||||
{{- if and (not .Values.logs.persistence.enabled) (eq (lower (tpl .Values.config.logging.remote_logging .)) "false") }}
|
||||
|
||||
WARNING:
|
||||
Kubernetes workers task logs may not persist unless you configure log persistence or remote logging!
|
||||
Logging options can be found at: https://airflow.apache.org/docs/helm-chart/stable/manage-logs.html
|
||||
(This warning can be ignored if logging is configured with environment variables or secrets backend)
|
||||
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.sshKeySecret (not .Values.dags.gitSync.knownHosts)}}
|
||||
|
||||
#####################################################
|
||||
# WARNING: You should set dags.gitSync.knownHosts #
|
||||
#####################################################
|
||||
|
||||
You are using ssh authentication for your gitsync repo, however you currently have SSH known_hosts verification disabled,
|
||||
making you susceptible to man-in-the-middle attacks!
|
||||
|
||||
Information on how to set knownHosts can be found here:
|
||||
https://airflow.apache.org/docs/helm-chart/stable/production-guide.html#knownhosts
|
||||
|
||||
{{- end }}
|
||||
|
||||
{{- if .Values.flower.extraNetworkPolicies }}
|
||||
|
||||
DEPRECATION WARNING:
|
||||
`flower.extraNetworkPolicies` has been renamed to `flower.networkPolicy.peers`.
|
||||
Please change your values as support for the old name will be dropped in a future release.
|
||||
|
||||
{{- end }}
|
||||
|
||||
|
||||
{{- if .Values.webserver.extraNetworkPolicies }}
|
||||
|
||||
DEPRECATION WARNING:
|
||||
`webserver.extraNetworkPolicies` has been renamed to `webserver.networkPolicy.peers`.
|
||||
Please change your values as support for the old name will be dropped in a future release.
|
||||
|
||||
{{- end }}
|
||||
|
||||
{{- if not (or .Values.webserverSecretKey .Values.webserverSecretKeySecretName) }}
|
||||
|
||||
{{- if .Values.securityContext }}
|
||||
|
||||
DEPRECATION WARNING:
|
||||
`securityContext` has been renamed to `securityContexts`, to be enabled on container and pod level.
|
||||
Please change your values as support for the old name will be dropped in a future release.
|
||||
|
||||
{{- end }}
|
||||
|
||||
###########################################################
|
||||
# WARNING: You should set a static webserver secret key #
|
||||
###########################################################
|
||||
|
||||
You are using a dynamically generated webserver secret key, which can lead to
|
||||
unnecessary restarts of your Airflow components.
|
||||
|
||||
Information on how to set a static webserver secret key can be found here:
|
||||
https://airflow.apache.org/docs/helm-chart/stable/production-guide.html#webserver-secret-key
|
||||
|
||||
{{- end }}
|
||||
|
||||
{{- if or .Values.postgresql.postgresqlUsername .Values.postgresql.postgresqlPassword }}
|
||||
|
||||
{{ fail "postgresql.postgresqlUsername and postgresql.postgresqlPassword are no longer supported. If you wish to use the 'postgres' user, set its password with postgresql.auth.postgresPassword. If you wish to create a different user, do so with postgresql.auth.username and postgresql.auth.password." }}
|
||||
|
||||
{{- end }}
|
||||
|
||||
{{- if ne .Values.executor (tpl .Values.config.core.executor $) }}
|
||||
{{ fail "Please configure the executor with `executor`, not `config.core.executor`." }}
|
||||
{{- end }}
|
||||
985
helm/airflow/templates/_helpers.yaml
Normal file
985
helm/airflow/templates/_helpers.yaml
Normal file
@@ -0,0 +1,985 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
{{/*
|
||||
Create a default fully qualified app name.
|
||||
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
|
||||
If release name contains chart name it will be used as a full name.
|
||||
*/}}
|
||||
{{- define "airflow.fullname" -}}
|
||||
{{- if not .Values.useStandardNaming }}
|
||||
{{- .Release.Name }}
|
||||
{{- else if .Values.fullnameOverride }}
|
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
|
||||
{{- else }}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride }}
|
||||
{{- if contains $name .Release.Name }}
|
||||
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
|
||||
{{- else }}
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "airflow.serviceAccountName" -}}
|
||||
{{ if .Values.fullnameOverride }}
|
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
|
||||
{{- else }}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride }}
|
||||
{{- if contains $name .Release.Name }}
|
||||
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
|
||||
{{- else }}
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/* Standard Airflow environment variables */}}
|
||||
{{- define "standard_airflow_environment" }}
|
||||
# Hard Coded Airflow Envs
|
||||
{{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__CORE__FERNET_KEY }}
|
||||
- name: AIRFLOW__CORE__FERNET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ template "fernet_key_secret" . }}
|
||||
key: fernet-key
|
||||
{{- end }}
|
||||
# For Airflow <2.3, backward compatibility; moved to [database] in 2.3
|
||||
{{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__CORE__SQL_ALCHEMY_CONN }}
|
||||
- name: AIRFLOW__CORE__SQL_ALCHEMY_CONN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ template "airflow_metadata_secret" . }}
|
||||
key: connection
|
||||
{{- end }}
|
||||
{{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__DATABASE__SQL_ALCHEMY_CONN }}
|
||||
- name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ template "airflow_metadata_secret" . }}
|
||||
key: connection
|
||||
{{- end }}
|
||||
{{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW_CONN_AIRFLOW_DB }}
|
||||
- name: AIRFLOW_CONN_AIRFLOW_DB
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ template "airflow_metadata_secret" . }}
|
||||
key: connection
|
||||
{{- end }}
|
||||
{{- if and .Values.workers.keda.enabled .Values.pgbouncer.enabled (not .Values.workers.keda.usePgbouncer) }}
|
||||
- name: KEDA_DB_CONN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ template "airflow_metadata_secret" . }}
|
||||
key: kedaConnection
|
||||
{{- end }}
|
||||
{{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__WEBSERVER__SECRET_KEY }}
|
||||
- name: AIRFLOW__WEBSERVER__SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ template "webserver_secret_key_secret" . }}
|
||||
key: webserver-secret-key
|
||||
{{- end }}
|
||||
{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
|
||||
{{- if or (semverCompare "<2.4.0" .Values.airflowVersion) (.Values.data.resultBackendSecretName) (.Values.data.resultBackendConnection) }}
|
||||
{{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__CELERY__CELERY_RESULT_BACKEND }}
|
||||
# (Airflow 1.10.* variant)
|
||||
- name: AIRFLOW__CELERY__CELERY_RESULT_BACKEND
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ template "airflow_result_backend_secret" . }}
|
||||
key: connection
|
||||
{{- end }}
|
||||
{{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__CELERY__RESULT_BACKEND }}
|
||||
- name: AIRFLOW__CELERY__RESULT_BACKEND
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ template "airflow_result_backend_secret" . }}
|
||||
key: connection
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__CELERY__BROKER_URL }}
|
||||
- name: AIRFLOW__CELERY__BROKER_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ default (printf "%s-broker-url" .Release.Name) .Values.data.brokerUrlSecretName }}
|
||||
key: connection
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.elasticsearch.enabled }}
|
||||
# The elasticsearch variables were updated to the shorter names in v1.10.4
|
||||
{{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__ELASTICSEARCH__HOST }}
|
||||
- name: AIRFLOW__ELASTICSEARCH__HOST
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ template "elasticsearch_secret" . }}
|
||||
key: connection
|
||||
{{- end }}
|
||||
{{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__ELASTICSEARCH__ELASTICSEARCH_HOST }}
|
||||
# This is the older format for these variable names, kept here for backward compatibility
|
||||
- name: AIRFLOW__ELASTICSEARCH__ELASTICSEARCH_HOST
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ template "elasticsearch_secret" . }}
|
||||
key: connection
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/* User defined Airflow environment variables */}}
|
||||
{{- define "custom_airflow_environment" }}
|
||||
# Dynamically created environment variables
|
||||
{{- range $i, $config := .Values.env }}
|
||||
- name: {{ $config.name }}
|
||||
value: {{ $config.value | quote }}
|
||||
{{- if or (eq $.Values.executor "KubernetesExecutor") (eq $.Values.executor "LocalKubernetesExecutor") (eq $.Values.executor "CeleryKubernetesExecutor") }}
|
||||
- name: AIRFLOW__KUBERNETES_ENVIRONMENT_VARIABLES__{{ $config.name }}
|
||||
value: {{ $config.value | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
# Dynamically created secret envs
|
||||
{{- range $i, $config := .Values.secret }}
|
||||
- name: {{ $config.envName }}
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ $config.secretName }}
|
||||
key: {{ default "value" $config.secretKey }}
|
||||
{{- end }}
|
||||
{{- if or (eq $.Values.executor "LocalKubernetesExecutor") (eq $.Values.executor "KubernetesExecutor") (eq $.Values.executor "CeleryKubernetesExecutor") }}
|
||||
{{- range $i, $config := .Values.secret }}
|
||||
- name: AIRFLOW__KUBERNETES_SECRETS__{{ $config.envName }}
|
||||
value: {{ printf "%s=%s" $config.secretName $config.secretKey }}
|
||||
{{- end }}
|
||||
{{ end }}
|
||||
# Extra env
|
||||
{{- $Global := . }}
|
||||
{{- with .Values.extraEnv }}
|
||||
{{- tpl . $Global | nindent 2 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/* User defined Airflow environment from */}}
|
||||
{{- define "custom_airflow_environment_from" }}
|
||||
{{- $Global := . }}
|
||||
{{- with .Values.extraEnvFrom }}
|
||||
{{- tpl . $Global | nindent 2 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/* Git ssh key volume */}}
|
||||
{{- define "git_sync_ssh_key_volume" }}
|
||||
- name: git-sync-ssh-key
|
||||
secret:
|
||||
secretName: {{ .Values.dags.gitSync.sshKeySecret }}
|
||||
defaultMode: 288
|
||||
{{- end }}
|
||||
|
||||
{{/* Git sync container */}}
|
||||
{{- define "git_sync_container" }}
|
||||
- name: {{ .Values.dags.gitSync.containerName }}{{ if .is_init }}-init{{ end }}
|
||||
image: {{ template "git_sync_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.gitSync.pullPolicy }}
|
||||
securityContext: {{- include "localContainerSecurityContext" .Values.dags.gitSync | nindent 4 }}
|
||||
env:
|
||||
{{- if .Values.dags.gitSync.sshKeySecret }}
|
||||
- name: GIT_SSH_KEY_FILE
|
||||
value: "/etc/git-secret/ssh"
|
||||
- name: GIT_SYNC_SSH
|
||||
value: "true"
|
||||
{{- if .Values.dags.gitSync.knownHosts }}
|
||||
- name: GIT_KNOWN_HOSTS
|
||||
value: "true"
|
||||
- name: GIT_SSH_KNOWN_HOSTS_FILE
|
||||
value: "/etc/git-secret/known_hosts"
|
||||
{{- else }}
|
||||
- name: GIT_KNOWN_HOSTS
|
||||
value: "false"
|
||||
{{- end }}
|
||||
{{ else if .Values.dags.gitSync.credentialsSecret }}
|
||||
- name: GIT_SYNC_USERNAME
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Values.dags.gitSync.credentialsSecret | quote }}
|
||||
key: GIT_SYNC_USERNAME
|
||||
- name: GIT_SYNC_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Values.dags.gitSync.credentialsSecret | quote }}
|
||||
key: GIT_SYNC_PASSWORD
|
||||
{{- end }}
|
||||
- name: GIT_SYNC_REV
|
||||
value: {{ .Values.dags.gitSync.rev | quote }}
|
||||
- name: GIT_SYNC_BRANCH
|
||||
value: {{ .Values.dags.gitSync.branch | quote }}
|
||||
- name: GIT_SYNC_REPO
|
||||
value: {{ .Values.dags.gitSync.repo | quote }}
|
||||
- name: GIT_SYNC_DEPTH
|
||||
value: {{ .Values.dags.gitSync.depth | quote }}
|
||||
- name: GIT_SYNC_ROOT
|
||||
value: "/git"
|
||||
- name: GIT_SYNC_DEST
|
||||
value: "repo"
|
||||
- name: GIT_SYNC_ADD_USER
|
||||
value: "true"
|
||||
- name: GIT_SYNC_WAIT
|
||||
value: {{ .Values.dags.gitSync.wait | quote }}
|
||||
- name: GIT_SYNC_MAX_SYNC_FAILURES
|
||||
value: {{ .Values.dags.gitSync.maxFailures | quote }}
|
||||
{{- if .is_init }}
|
||||
- name: GIT_SYNC_ONE_TIME
|
||||
value: "true"
|
||||
{{- end }}
|
||||
{{- with .Values.dags.gitSync.env }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
resources: {{ toYaml .Values.dags.gitSync.resources | nindent 6 }}
|
||||
volumeMounts:
|
||||
- name: dags
|
||||
mountPath: /git
|
||||
{{- if .Values.dags.gitSync.sshKeySecret }}
|
||||
- name: git-sync-ssh-key
|
||||
mountPath: /etc/git-secret/ssh
|
||||
readOnly: true
|
||||
subPath: gitSshKey
|
||||
{{- if .Values.dags.gitSync.knownHosts }}
|
||||
- name: config
|
||||
mountPath: /etc/git-secret/known_hosts
|
||||
readOnly: true
|
||||
subPath: known_hosts
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.dags.gitSync.extraVolumeMounts }}
|
||||
{{- tpl (toYaml .Values.dags.gitSync.extraVolumeMounts) . | nindent 2 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/* This helper will change when customers deploy a new image */}}
|
||||
{{- define "airflow_image" -}}
|
||||
{{- $repository := .Values.images.airflow.repository | default .Values.defaultAirflowRepository -}}
|
||||
{{- $tag := .Values.images.airflow.tag | default .Values.defaultAirflowTag -}}
|
||||
{{- $digest := .Values.images.airflow.digest | default .Values.defaultAirflowDigest -}}
|
||||
{{- if $digest }}
|
||||
{{- printf "%s@%s" $repository $digest -}}
|
||||
{{- else }}
|
||||
{{- printf "%s:%s" $repository $tag -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "pod_template_image" -}}
|
||||
{{- printf "%s:%s" (.Values.images.pod_template.repository | default .Values.defaultAirflowRepository) (.Values.images.pod_template.tag | default .Values.defaultAirflowTag) }}
|
||||
{{- end }}
|
||||
|
||||
{{/* This helper is used for airflow containers that do not need the users code */}}
|
||||
{{ define "default_airflow_image" -}}
|
||||
{{- $repository := .Values.defaultAirflowRepository -}}
|
||||
{{- $tag := .Values.defaultAirflowTag -}}
|
||||
{{- $digest := .Values.defaultAirflowDigest -}}
|
||||
{{- if $digest }}
|
||||
{{- printf "%s@%s" $repository $digest -}}
|
||||
{{- else }}
|
||||
{{- printf "%s:%s" $repository $tag -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{ define "airflow_image_for_migrations" -}}
|
||||
{{- if .Values.images.useDefaultImageForMigration }}
|
||||
{{- template "default_airflow_image" . }}
|
||||
{{- else }}
|
||||
{{- template "airflow_image" . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "flower_image" -}}
|
||||
{{- printf "%s:%s" (.Values.images.flower.repository | default .Values.defaultAirflowRepository) (.Values.images.flower.tag | default .Values.defaultAirflowTag) }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "statsd_image" -}}
|
||||
{{- printf "%s:%s" .Values.images.statsd.repository .Values.images.statsd.tag }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "redis_image" -}}
|
||||
{{- printf "%s:%s" .Values.images.redis.repository .Values.images.redis.tag }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "pgbouncer_image" -}}
|
||||
{{- printf "%s:%s" .Values.images.pgbouncer.repository .Values.images.pgbouncer.tag }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "pgbouncer_exporter_image" -}}
|
||||
{{- printf "%s:%s" .Values.images.pgbouncerExporter.repository .Values.images.pgbouncerExporter.tag }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "git_sync_image" -}}
|
||||
{{- printf "%s:%s" .Values.images.gitSync.repository .Values.images.gitSync.tag }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "fernet_key_secret" -}}
|
||||
{{- default (printf "%s-fernet-key" .Release.Name) .Values.fernetKeySecretName }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "webserver_secret_key_secret" -}}
|
||||
{{- default (printf "%s-webserver-secret-key" (include "airflow.fullname" .)) .Values.webserverSecretKeySecretName }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "redis_password_secret" -}}
|
||||
{{- default (printf "%s-redis-password" .Release.Name) .Values.redis.passwordSecretName }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "airflow_metadata_secret" -}}
|
||||
{{- default (printf "%s-metadata" (include "airflow.fullname" .)) .Values.data.metadataSecretName }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "airflow_result_backend_secret" -}}
|
||||
{{- default (printf "%s-result-backend" (include "airflow.fullname" .)) .Values.data.resultBackendSecretName }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "airflow_pod_template_file" -}}
|
||||
{{- printf "%s/pod_templates" .Values.airflowHome }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "pgbouncer_config_secret" -}}
|
||||
{{- default (printf "%s-pgbouncer-config" (include "airflow.fullname" .)) .Values.pgbouncer.configSecretName }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "pgbouncer_certificates_secret" -}}
|
||||
{{- printf "%s-pgbouncer-certificates" (include "airflow.fullname" .) }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "pgbouncer_stats_secret" -}}
|
||||
{{- default (printf "%s-pgbouncer-stats" (include "airflow.fullname" .)) .Values.pgbouncer.metricsExporterSidecar.statsSecretName }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "registry_secret" -}}
|
||||
{{- default (printf "%s-registry" (include "airflow.fullname" .)) .Values.registry.secretName }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "elasticsearch_secret" -}}
|
||||
{{- default (printf "%s-elasticsearch" (include "airflow.fullname" .)) .Values.elasticsearch.secretName }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "flower_secret" -}}
|
||||
{{- default (printf "%s-flower" (include "airflow.fullname" .)) .Values.flower.secretName }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "kerberos_keytab_secret" -}}
|
||||
{{- printf "%s-kerberos-keytab" (include "airflow.fullname" .) }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "kerberos_ccache_path" -}}
|
||||
{{- printf "%s/%s" .Values.kerberos.ccacheMountPath .Values.kerberos.ccacheFileName }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "celery_executor_namespace" -}}
|
||||
{{- if semverCompare ">=2.7.0" .Values.airflowVersion }}
|
||||
{{- print "airflow.providers.celery.executors.celery_executor.app" -}}
|
||||
{{- else }}
|
||||
{{- print "airflow.executors.celery_executor.app" -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "pgbouncer_config" -}}
|
||||
{{ $resultBackendConnection := .Values.data.resultBackendConnection | default .Values.data.metadataConnection }}
|
||||
{{ $pgMetadataHost := .Values.data.metadataConnection.host | default (printf "%s-%s.%s" .Release.Name "postgresql" .Release.Namespace) }}
|
||||
{{ $pgResultBackendHost := $resultBackendConnection.host | default (printf "%s-%s.%s" .Release.Name "postgresql" .Release.Namespace) }}
|
||||
[databases]
|
||||
{{ .Release.Name }}-metadata = host={{ $pgMetadataHost }} dbname={{ .Values.data.metadataConnection.db }} port={{ .Values.data.metadataConnection.port }} pool_size={{ .Values.pgbouncer.metadataPoolSize }} {{ .Values.pgbouncer.extraIniMetadata | default "" }}
|
||||
{{ .Release.Name }}-result-backend = host={{ $pgResultBackendHost }} dbname={{ $resultBackendConnection.db }} port={{ $resultBackendConnection.port }} pool_size={{ .Values.pgbouncer.resultBackendPoolSize }} {{ .Values.pgbouncer.extraIniResultBackend | default "" }}
|
||||
|
||||
[pgbouncer]
|
||||
pool_mode = transaction
|
||||
listen_port = {{ .Values.ports.pgbouncer }}
|
||||
listen_addr = *
|
||||
auth_type = {{ .Values.pgbouncer.auth_type }}
|
||||
auth_file = {{ .Values.pgbouncer.auth_file }}
|
||||
stats_users = {{ .Values.data.metadataConnection.user }}
|
||||
ignore_startup_parameters = extra_float_digits
|
||||
max_client_conn = {{ .Values.pgbouncer.maxClientConn }}
|
||||
verbose = {{ .Values.pgbouncer.verbose }}
|
||||
log_disconnections = {{ .Values.pgbouncer.logDisconnections }}
|
||||
log_connections = {{ .Values.pgbouncer.logConnections }}
|
||||
|
||||
server_tls_sslmode = {{ .Values.pgbouncer.sslmode }}
|
||||
server_tls_ciphers = {{ .Values.pgbouncer.ciphers }}
|
||||
|
||||
{{- if .Values.pgbouncer.ssl.ca }}
|
||||
server_tls_ca_file = /etc/pgbouncer/root.crt
|
||||
{{- end }}
|
||||
{{- if .Values.pgbouncer.ssl.cert }}
|
||||
server_tls_cert_file = /etc/pgbouncer/server.crt
|
||||
{{- end }}
|
||||
{{- if .Values.pgbouncer.ssl.key }}
|
||||
server_tls_key_file = /etc/pgbouncer/server.key
|
||||
{{- end }}
|
||||
|
||||
{{- if .Values.pgbouncer.extraIni }}
|
||||
{{ .Values.pgbouncer.extraIni }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{ define "pgbouncer_users" }}
|
||||
{{- $resultBackendConnection := .Values.data.resultBackendConnection | default .Values.data.metadataConnection }}
|
||||
{{ .Values.data.metadataConnection.user | quote }} {{ .Values.data.metadataConnection.pass | quote }}
|
||||
{{ $resultBackendConnection.user | quote }} {{ $resultBackendConnection.pass | quote }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "airflow_logs" -}}
|
||||
{{- printf "%s/logs" .Values.airflowHome | quote }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "airflow_logs_no_quote" -}}
|
||||
{{- printf "%s/logs" .Values.airflowHome }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "airflow_logs_volume_claim" -}}
|
||||
{{- if .Values.logs.persistence.existingClaim }}
|
||||
{{- .Values.logs.persistence.existingClaim }}
|
||||
{{- else }}
|
||||
{{- printf "%s-logs" .Release.Name }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "airflow_dags" -}}
|
||||
{{- if .Values.dags.gitSync.enabled }}
|
||||
{{- printf "%s/dags/repo/%s" .Values.airflowHome .Values.dags.gitSync.subPath }}
|
||||
{{- else }}
|
||||
{{- printf "%s/dags" .Values.airflowHome }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "airflow_dags_volume_claim" -}}
|
||||
{{- if .Values.dags.persistence.existingClaim }}
|
||||
{{- .Values.dags.persistence.existingClaim }}
|
||||
{{- else }}
|
||||
{{- printf "%s-dags" .Release.Name }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "airflow_dags_mount" -}}
|
||||
- name: dags
|
||||
mountPath: {{ printf "%s/dags" .Values.airflowHome }}
|
||||
{{- if .Values.dags.persistence.subPath }}
|
||||
subPath: {{ .Values.dags.persistence.subPath }}
|
||||
{{- end }}
|
||||
readOnly: {{ .Values.dags.gitSync.enabled | ternary "True" "False" }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "airflow_config_path" -}}
|
||||
{{- printf "%s/airflow.cfg" .Values.airflowHome | quote }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "airflow_webserver_config_path" -}}
|
||||
{{- printf "%s/webserver_config.py" .Values.airflowHome | quote }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "airflow_webserver_config_configmap_name" -}}
|
||||
{{- default (printf "%s-webserver-config" .Release.Name) .Values.webserver.webserverConfigConfigMapName }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "airflow_webserver_config_mount" -}}
|
||||
- name: webserver-config
|
||||
mountPath: {{ template "airflow_webserver_config_path" . }}
|
||||
subPath: webserver_config.py
|
||||
readOnly: True
|
||||
{{- end }}
|
||||
|
||||
{{- define "airflow_local_setting_path" -}}
|
||||
{{- printf "%s/config/airflow_local_settings.py" .Values.airflowHome | quote }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "airflow_config" -}}
|
||||
{{- printf "%s-config" (include "airflow.fullname" .) }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "airflow_config_mount" -}}
|
||||
- name: config
|
||||
mountPath: {{ template "airflow_config_path" . }}
|
||||
subPath: airflow.cfg
|
||||
readOnly: true
|
||||
{{- if .Values.airflowLocalSettings }}
|
||||
- name: config
|
||||
mountPath: {{ template "airflow_local_setting_path" . }}
|
||||
subPath: airflow_local_settings.py
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/* Create the name of the webserver service account to use */}}
|
||||
{{- define "webserver.serviceAccountName" -}}
|
||||
{{- if .Values.webserver.serviceAccount.create }}
|
||||
{{- default (printf "%s-webserver" (include "airflow.serviceAccountName" .)) .Values.webserver.serviceAccount.name }}
|
||||
{{- else }}
|
||||
{{- default "default" .Values.webserver.serviceAccount.name }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/* Create the name of the redis service account to use */}}
|
||||
{{- define "redis.serviceAccountName" -}}
|
||||
{{- if .Values.redis.serviceAccount.create }}
|
||||
{{- default (printf "%s-redis" (include "airflow.serviceAccountName" .)) .Values.redis.serviceAccount.name }}
|
||||
{{- else }}
|
||||
{{- default "default" .Values.redis.serviceAccount.name }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/* Create the name of the flower service account to use */}}
|
||||
{{- define "flower.serviceAccountName" -}}
|
||||
{{- if .Values.flower.serviceAccount.create }}
|
||||
{{- default (printf "%s-flower" (include "airflow.serviceAccountName" .)) .Values.flower.serviceAccount.name }}
|
||||
{{- else }}
|
||||
{{- default "default" .Values.flower.serviceAccount.name }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/* Create the name of the scheduler service account to use */}}
|
||||
{{- define "scheduler.serviceAccountName" -}}
|
||||
{{- if .Values.scheduler.serviceAccount.create }}
|
||||
{{- default (printf "%s-scheduler" (include "airflow.serviceAccountName" .)) .Values.scheduler.serviceAccount.name }}
|
||||
{{- else }}
|
||||
{{- default "default" .Values.scheduler.serviceAccount.name }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/* Create the name of the StatsD service account to use */}}
|
||||
{{- define "statsd.serviceAccountName" -}}
|
||||
{{- if .Values.statsd.serviceAccount.create }}
|
||||
{{- default (printf "%s-statsd" (include "airflow.serviceAccountName" .)) .Values.statsd.serviceAccount.name }}
|
||||
{{- else }}
|
||||
{{- default "default" .Values.statsd.serviceAccount.name }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/* Create the name of the create user job service account to use */}}
|
||||
{{- define "createUserJob.serviceAccountName" -}}
|
||||
{{- if .Values.createUserJob.serviceAccount.create }}
|
||||
{{- default (printf "%s-create-user-job" (include "airflow.serviceAccountName" .)) .Values.createUserJob.serviceAccount.name }}
|
||||
{{- else }}
|
||||
{{- default "default" .Values.createUserJob.serviceAccount.name }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/* Create the name of the migrate database job service account to use */}}
|
||||
{{- define "migrateDatabaseJob.serviceAccountName" -}}
|
||||
{{- if .Values.migrateDatabaseJob.serviceAccount.create }}
|
||||
{{- default (printf "%s-migrate-database-job" (include "airflow.serviceAccountName" .)) .Values.migrateDatabaseJob.serviceAccount.name }}
|
||||
{{- else }}
|
||||
{{- default "default" .Values.migrateDatabaseJob.serviceAccount.name }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/* Create the name of the worker service account to use */}}
|
||||
{{- define "worker.serviceAccountName" -}}
|
||||
{{- if .Values.workers.serviceAccount.create }}
|
||||
{{- default (printf "%s-worker" (include "airflow.serviceAccountName" .)) .Values.workers.serviceAccount.name }}
|
||||
{{- else }}
|
||||
{{- default "default" .Values.workers.serviceAccount.name }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/* Create the name of the triggerer service account to use */}}
|
||||
{{- define "triggerer.serviceAccountName" -}}
|
||||
{{- if .Values.triggerer.serviceAccount.create }}
|
||||
{{- default (printf "%s-triggerer" (include "airflow.serviceAccountName" .)) .Values.triggerer.serviceAccount.name }}
|
||||
{{- else }}
|
||||
{{- default "default" .Values.triggerer.serviceAccount.name }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/* Create the name of the dag processor service account to use */}}
|
||||
{{- define "dagProcessor.serviceAccountName" -}}
|
||||
{{- if .Values.dagProcessor.serviceAccount.create }}
|
||||
{{- default (printf "%s-dag-processor" (include "airflow.serviceAccountName" .)) .Values.dagProcessor.serviceAccount.name }}
|
||||
{{- else }}
|
||||
{{- default "default" .Values.dagProcessor.serviceAccount.name }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/* Create the name of the pgbouncer service account to use */}}
|
||||
{{- define "pgbouncer.serviceAccountName" -}}
|
||||
{{- if .Values.pgbouncer.serviceAccount.create }}
|
||||
{{- default (printf "%s-pgbouncer" (include "airflow.serviceAccountName" .)) .Values.pgbouncer.serviceAccount.name }}
|
||||
{{- else }}
|
||||
{{- default "default" .Values.pgbouncer.serviceAccount.name }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/* Create the name of the cleanup service account to use */}}
|
||||
{{- define "cleanup.serviceAccountName" -}}
|
||||
{{- if .Values.cleanup.serviceAccount.create }}
|
||||
{{- default (printf "%s-cleanup" (include "airflow.serviceAccountName" .)) .Values.cleanup.serviceAccount.name }}
|
||||
{{- else }}
|
||||
{{- default "default" .Values.cleanup.serviceAccount.name }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "wait-for-migrations-command" -}}
|
||||
{{- if semverCompare ">=2.0.0" .Values.airflowVersion }}
|
||||
- airflow
|
||||
- db
|
||||
- check-migrations
|
||||
- --migration-wait-timeout={{ .Values.images.migrationsWaitTimeout }}
|
||||
{{- else }}
|
||||
- python
|
||||
- -c
|
||||
- |
|
||||
import airflow
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
|
||||
from alembic.config import Config
|
||||
from alembic.runtime.migration import MigrationContext
|
||||
from alembic.script import ScriptDirectory
|
||||
|
||||
from airflow import settings
|
||||
|
||||
package_dir = os.path.abspath(os.path.dirname(airflow.__file__))
|
||||
directory = os.path.join(package_dir, 'migrations')
|
||||
config = Config(os.path.join(package_dir, 'alembic.ini'))
|
||||
config.set_main_option('script_location', directory)
|
||||
config.set_main_option('sqlalchemy.url', settings.SQL_ALCHEMY_CONN.replace('%', '%%'))
|
||||
script_ = ScriptDirectory.from_config(config)
|
||||
|
||||
timeout=60
|
||||
|
||||
with settings.engine.connect() as connection:
|
||||
context = MigrationContext.configure(connection)
|
||||
ticker = 0
|
||||
while True:
|
||||
source_heads = set(script_.get_heads())
|
||||
|
||||
db_heads = set(context.get_current_heads())
|
||||
if source_heads == db_heads:
|
||||
break
|
||||
|
||||
if ticker >= timeout:
|
||||
raise TimeoutError("There are still unapplied migrations after {} seconds.".format(ticker))
|
||||
ticker += 1
|
||||
time.sleep(1)
|
||||
logging.info('Waiting for migrations... %s second(s)', ticker)
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "scheduler_liveness_check_command" }}
|
||||
{{- if semverCompare ">=2.5.0" .Values.airflowVersion }}
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
CONNECTION_CHECK_MAX_COUNT=0 AIRFLOW__LOGGING__LOGGING_LEVEL=ERROR exec /entrypoint \
|
||||
airflow jobs check --job-type SchedulerJob --local
|
||||
{{- else if semverCompare ">=2.1.0" .Values.airflowVersion }}
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
CONNECTION_CHECK_MAX_COUNT=0 AIRFLOW__LOGGING__LOGGING_LEVEL=ERROR exec /entrypoint \
|
||||
airflow jobs check --job-type SchedulerJob --hostname $(hostname)
|
||||
{{- else }}
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
CONNECTION_CHECK_MAX_COUNT=0 exec /entrypoint python -Wignore -c "
|
||||
import os
|
||||
os.environ['AIRFLOW__CORE__LOGGING_LEVEL'] = 'ERROR'
|
||||
os.environ['AIRFLOW__LOGGING__LOGGING_LEVEL'] = 'ERROR'
|
||||
from airflow.jobs.scheduler_job import SchedulerJob
|
||||
from airflow.utils.db import create_session
|
||||
from airflow.utils.net import get_hostname
|
||||
import sys
|
||||
with create_session() as session:
|
||||
job = session.query(SchedulerJob).filter_by(hostname=get_hostname()).order_by(
|
||||
SchedulerJob.latest_heartbeat.desc()).limit(1).first()
|
||||
sys.exit(0 if job.is_alive() else 1)"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
|
||||
{{- define "scheduler_startup_check_command" }}
|
||||
{{- if semverCompare ">=2.5.0" .Values.airflowVersion }}
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
CONNECTION_CHECK_MAX_COUNT=0 AIRFLOW__LOGGING__LOGGING_LEVEL=ERROR exec /entrypoint \
|
||||
airflow jobs check --job-type SchedulerJob --local
|
||||
{{- else if semverCompare ">=2.1.0" .Values.airflowVersion }}
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
CONNECTION_CHECK_MAX_COUNT=0 AIRFLOW__LOGGING__LOGGING_LEVEL=ERROR exec /entrypoint \
|
||||
airflow jobs check --job-type SchedulerJob --hostname $(hostname)
|
||||
{{- else }}
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
CONNECTION_CHECK_MAX_COUNT=0 exec /entrypoint python -Wignore -c "
|
||||
import os
|
||||
os.environ['AIRFLOW__CORE__LOGGING_LEVEL'] = 'ERROR'
|
||||
os.environ['AIRFLOW__LOGGING__LOGGING_LEVEL'] = 'ERROR'
|
||||
from airflow.jobs.scheduler_job import SchedulerJob
|
||||
from airflow.utils.db import create_session
|
||||
from airflow.utils.net import get_hostname
|
||||
import sys
|
||||
with create_session() as session:
|
||||
job = session.query(SchedulerJob).filter_by(hostname=get_hostname()).order_by(
|
||||
SchedulerJob.latest_heartbeat.desc()).limit(1).first()
|
||||
sys.exit(0 if job.is_alive() else 1)"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "triggerer_liveness_check_command" }}
|
||||
{{- if semverCompare ">=2.5.0" .Values.airflowVersion }}
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
CONNECTION_CHECK_MAX_COUNT=0 AIRFLOW__LOGGING__LOGGING_LEVEL=ERROR exec /entrypoint \
|
||||
airflow jobs check --job-type TriggererJob --local
|
||||
{{- else }}
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
CONNECTION_CHECK_MAX_COUNT=0 AIRFLOW__LOGGING__LOGGING_LEVEL=ERROR exec /entrypoint \
|
||||
airflow jobs check --job-type TriggererJob --hostname $(hostname)
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "dag_processor_liveness_check_command" }}
|
||||
{{- $commandArgs := (list) -}}
|
||||
{{- if semverCompare ">=2.5.0" .Values.airflowVersion }}
|
||||
{{- $commandArgs = append $commandArgs "--local" -}}
|
||||
{{- if semverCompare ">=2.5.2" .Values.airflowVersion }}
|
||||
{{- $commandArgs = concat $commandArgs (list "--job-type" "DagProcessorJob") -}}
|
||||
{{- end }}
|
||||
{{- else }}
|
||||
{{- $commandArgs = concat $commandArgs (list "--hostname" "$(hostname)") -}}
|
||||
{{- end }}
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
CONNECTION_CHECK_MAX_COUNT=0 AIRFLOW__LOGGING__LOGGING_LEVEL=ERROR exec /entrypoint \
|
||||
airflow jobs check {{ join " " $commandArgs }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "registry_docker_config" }}
|
||||
{{- $host := .Values.registry.connection.host }}
|
||||
{{- $email := .Values.registry.connection.email }}
|
||||
{{- $user := .Values.registry.connection.user }}
|
||||
{{- $pass := .Values.registry.connection.pass }}
|
||||
|
||||
{{- $config := dict "auths" }}
|
||||
{{- $auth := dict }}
|
||||
{{- $data := dict }}
|
||||
{{- $_ := set $data "username" $user }}
|
||||
{{- $_ := set $data "password" $pass }}
|
||||
{{- $_ := set $data "email" $email }}
|
||||
{{- $_ := set $data "auth" (printf "%v:%v" $user $pass | b64enc) }}
|
||||
{{- $_ := set $auth $host $data }}
|
||||
{{- $_ := set $config "auths" $auth }}
|
||||
{{ $config | toJson | print }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Set the default value for pod securityContext
|
||||
If no value is passed for securityContexts.pod or <node>.securityContexts.pod or legacy securityContext and <node>.securityContext, defaults to global uid and gid.
|
||||
|
||||
+-----------------------------+ +------------------------+ +----------------------+ +-----------------+ +-------------------------+
|
||||
| <node>.securityContexts.pod | -> | <node>.securityContext | -> | securityContexts.pod | -> | securityContext | -> | Values.uid + Values.gid |
|
||||
+-----------------------------+ +------------------------+ +----------------------+ +-----------------+ +-------------------------+
|
||||
|
||||
Values are not accumulated meaning that if runAsUser is set to 10 in <node>.securityContexts.pod,
|
||||
any extra values set to securityContext or uid+gid will be ignored.
|
||||
|
||||
The template can be called like so:
|
||||
include "airflowPodSecurityContext" (list . .Values.webserver)
|
||||
|
||||
Where `.` is the global variables scope and `.Values.webserver` the local variables scope for the webserver template.
|
||||
*/}}
|
||||
{{- define "airflowPodSecurityContext" -}}
|
||||
{{- $ := index . 0 -}}
|
||||
{{- with index . 1 }}
|
||||
{{- if .securityContexts.pod -}}
|
||||
{{ toYaml .securityContexts.pod | print }}
|
||||
{{- else if .securityContext -}}
|
||||
{{ toYaml .securityContext | print }}
|
||||
{{- else if $.Values.securityContexts.pod -}}
|
||||
{{ toYaml $.Values.securityContexts.pod | print }}
|
||||
{{- else if $.Values.securityContext -}}
|
||||
{{ toYaml $.Values.securityContext | print }}
|
||||
{{- else -}}
|
||||
runAsUser: {{ $.Values.uid }}
|
||||
fsGroup: {{ $.Values.gid }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Set the default value for pod securityContext
|
||||
If no value is passed for <node>.securityContexts.pod or <node>.securityContext, defaults to UID in the local node.
|
||||
|
||||
+-----------------------------+ +------------------------+ +-------------+
|
||||
| <node>.securityContexts.pod | -> | <node>.securityContext | -> | <node>.uid |
|
||||
+-----------------------------+ +------------------------+ +-------------+
|
||||
|
||||
The template can be called like so:
|
||||
include "localPodSecurityContext" (list . .Values.schedule)
|
||||
|
||||
It is important to pass the local variables scope to this template as it is used to determine the local node value for uid.
|
||||
*/}}
|
||||
{{- define "localPodSecurityContext" -}}
|
||||
{{- if .securityContexts.pod -}}
|
||||
{{ toYaml .securityContexts.pod | print }}
|
||||
{{- else if .securityContext -}}
|
||||
{{ toYaml .securityContext | print }}
|
||||
{{- else -}}
|
||||
runAsUser: {{ .uid }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Set the default value for container securityContext
|
||||
If no value is passed for <node>.securityContexts.container or <node>.securityContext, defaults to UID in the local node.
|
||||
|
||||
+-----------------------------------+ +------------------------+ +-------------+
|
||||
| <node>.securityContexts.container | -> | <node>.securityContext | -> | <node>.uid |
|
||||
+-----------------------------------+ +------------------------+ +-------------+
|
||||
|
||||
The template can be called like so:
|
||||
include "localContainerSecurityContext" .Values.statsd
|
||||
|
||||
It is important to pass the local variables scope to this template as it is used to determine the local node value for uid.
|
||||
*/}}
|
||||
{{- define "localContainerSecurityContext" -}}
|
||||
{{- if .securityContexts.container -}}
|
||||
{{ toYaml .securityContexts.container | print }}
|
||||
{{- else if .securityContext -}}
|
||||
{{ toYaml .securityContext | print }}
|
||||
{{- else -}}
|
||||
runAsUser: {{ .uid }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Set the default value for workers chown for persistent storage
|
||||
If no value is passed for securityContexts.pod or <node>.securityContexts.pod or legacy securityContext and <node>.securityContext, defaults to global uid and gid.
|
||||
The template looks for `runAsUser` and `fsGroup` specifically, any other parameter will be ignored.
|
||||
|
||||
+-----------------------------+ +----------------------------------------------------+ +------------------+ +-------------------------+
|
||||
| <node>.securityContexts.pod | -> | securityContexts.pod | <node>.securityContexts.pod | -> | securityContexts | -> | Values.uid + Values.gid |
|
||||
+-----------------------------+ +----------------------------------------------------+ +------------------+ +-------------------------+
|
||||
|
||||
Values are not accumulated meaning that if runAsUser is set to 10 in <node>.securityContexts.pod,
|
||||
any extra values set to securityContexts or uid+gid will be ignored.
|
||||
|
||||
The template can be called like so:
|
||||
include "airflowPodSecurityContextsIds" (list . .Values.webserver)
|
||||
|
||||
Where `.` is the global variables scope and `.Values.workers` the local variables scope for the workers template.
|
||||
*/}}
|
||||
{{- define "airflowPodSecurityContextsIds" -}}
|
||||
{{- $ := index . 0 -}}
|
||||
{{- with index . 1 }}
|
||||
{{- if .securityContexts.pod -}}
|
||||
{{ pluck "runAsUser" .securityContexts.pod | first | default $.Values.uid }}:{{ pluck "fsGroup" .securityContexts.pod | first | default $.Values.gid }}
|
||||
{{- else if $.Values.securityContext -}}
|
||||
{{ pluck "runAsUser" $.Values.securityContext | first | default $.Values.uid }}:{{ pluck "fsGroup" $.Values.securityContext | first | default $.Values.gid }}
|
||||
{{- else if $.Values.securityContexts.pod -}}
|
||||
{{ pluck "runAsUser" $.Values.securityContexts.pod | first | default $.Values.uid }}:{{ pluck "fsGroup" $.Values.securityContexts.pod | first | default $.Values.gid }}
|
||||
{{- else if $.Values.securityContext -}}
|
||||
{{ pluck "runAsUser" $.Values.securityContext | first | default $.Values.uid }}:{{ pluck "fsGroup" $.Values.securityContext | first | default $.Values.gid }}
|
||||
{{- else -}}
|
||||
{{ $.Values.uid }}:{{ $.Values.gid }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Set the default value for container securityContext
|
||||
If no value is passed for securityContexts.container or <node>.securityContexts.container, defaults to deny privileges escallation and dropping all POSIX capabilities.
|
||||
|
||||
+-----------------------------------+ +----------------------------+ +-----------------------------------------------------------+
|
||||
| <node>.securityContexts.container | -> | securityContexts.containers | -> | allowPrivilegesEscalation: false, capabilities.drop: [ALL]|
|
||||
+-----------------------------------+ +----------------------------+ +-----------------------------------------------------------+
|
||||
|
||||
The template can be called like so:
|
||||
include "containerSecurityContext" (list . .Values.webserver)
|
||||
|
||||
Where `.` is the global variables scope and `.Values.webserver` the local variables scope for the webserver template.
|
||||
*/}}
|
||||
{{- define "containerSecurityContext" -}}
|
||||
{{- $ := index . 0 -}}
|
||||
{{- with index . 1 }}
|
||||
{{- if .securityContexts.container -}}
|
||||
{{ toYaml .securityContexts.container | print }}
|
||||
{{- else if $.Values.securityContexts.containers -}}
|
||||
{{ toYaml $.Values.securityContexts.containers | print }}
|
||||
{{- else -}}
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Set the default value for external container securityContext(redis and statsd).
|
||||
If no value is passed for <node>.securityContexts.container, defaults to deny privileges escallation and dropping all POSIX capabilities.
|
||||
|
||||
+-----------------------------------+ +-----------------------------------------------------------+
|
||||
| <node>.securityContexts.container | -> | allowPrivilegesEscalation: false, capabilities.drop: [ALL]|
|
||||
+-----------------------------------+ +-----------------------------------------------------------+
|
||||
|
||||
The template can be called like so:
|
||||
include "externalContainerSecurityContext" .Values.statsd
|
||||
*/}}
|
||||
{{- define "externalContainerSecurityContext" -}}
|
||||
{{- if .securityContexts.container -}}
|
||||
{{ toYaml .securityContexts.container | print }}
|
||||
{{- else -}}
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "container_extra_envs" -}}
|
||||
{{- $ := index . 0 -}}
|
||||
{{- $env := index . 1 -}}
|
||||
{{- range $i, $config := $env }}
|
||||
- name: {{ $config.name }}
|
||||
value: {{ $config.value | quote }}
|
||||
{{- if or (eq $.Values.executor "KubernetesExecutor") (eq $.Values.executor "LocalKubernetesExecutor") (eq $.Values.executor "CeleryKubernetesExecutor") }}
|
||||
- name: AIRFLOW__KUBERNETES_ENVIRONMENT_VARIABLES__{{ $config.name }}
|
||||
value: {{ $config.value | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- define "kedaNetworkPolicySelector" }}
|
||||
{{- if .Values.workers.keda.enabled }}
|
||||
|
||||
{{- if .Values.workers.keda.namespaceLabels }}
|
||||
- namespaceSelector:
|
||||
matchLabels: {{- toYaml .Values.workers.keda.namespaceLabels | nindent 10 }}
|
||||
podSelector:
|
||||
{{- else }}
|
||||
- podSelector:
|
||||
{{- end }}
|
||||
matchLabels:
|
||||
app: keda-operator
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
64
helm/airflow/templates/check-values.yaml
Normal file
64
helm/airflow/templates/check-values.yaml
Normal file
@@ -0,0 +1,64 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
{{- /*
|
||||
The sole purpose of this yaml file is it to check the values file is consistent for some complexe combinations.
|
||||
*/ -}}
|
||||
|
||||
{{- /*
|
||||
##############################
|
||||
Redis related checks
|
||||
#############################
|
||||
*/ -}}
|
||||
|
||||
{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
|
||||
{{- if .Values.redis.enabled }}
|
||||
|
||||
{{- if and .Values.redis.passwordSecretName (not .Values.data.brokerUrlSecretName) }}
|
||||
{{ required "When using the internal redis of the chart and setting the value redis.passwordSecretName, you must also set the value data.brokerUrlSecretName." nil }}
|
||||
{{- end }}
|
||||
|
||||
{{- if and .Values.redis.passwordSecretName .Values.redis.password }}
|
||||
{{ required "You must not set both values redis.passwordSecretName and redis.password" nil }}
|
||||
{{- end }}
|
||||
|
||||
{{- else }}
|
||||
|
||||
{{- if not (or .Values.data.brokerUrlSecretName .Values.data.brokerUrl) }}
|
||||
{{ required "You must set one of the values data.brokerUrlSecretName or data.brokerUrl when using a Celery based executor with redis.enabled set to false (we need the url to the redis instance)." nil }}
|
||||
{{- end }}
|
||||
|
||||
{{- end }}
|
||||
|
||||
{{- if and .Values.data.brokerUrlSecretName .Values.data.brokerUrl }}
|
||||
{{ required "You must not set both values data.brokerUrlSecretName and data.brokerUrl" nil }}
|
||||
{{- end }}
|
||||
|
||||
{{- end }}
|
||||
|
||||
{{- if .Values.elasticsearch.enabled }}
|
||||
{{- if and .Values.elasticsearch.secretName .Values.elasticsearch.connection }}
|
||||
{{ required "You must not set both values elasticsearch.secretName and elasticsearch.connection" nil }}
|
||||
{{- end }}
|
||||
|
||||
{{- if not (or .Values.elasticsearch.secretName .Values.elasticsearch.connection) }}
|
||||
{{ required "You must set one of the values elasticsearch.secretName or elasticsearch.connection when using a Elasticsearch" nil }}
|
||||
{{- end }}
|
||||
|
||||
{{- end }}
|
||||
109
helm/airflow/templates/cleanup/cleanup-cronjob.yaml
Normal file
109
helm/airflow/templates/cleanup/cleanup-cronjob.yaml
Normal file
@@ -0,0 +1,109 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Cleanup Pods CronJob
|
||||
#################################
|
||||
{{- if .Values.cleanup.enabled }}
|
||||
{{- $nodeSelector := or .Values.cleanup.nodeSelector .Values.nodeSelector }}
|
||||
{{- $affinity := or .Values.cleanup.affinity .Values.affinity }}
|
||||
{{- $tolerations := or .Values.cleanup.tolerations .Values.tolerations }}
|
||||
{{- $topologySpreadConstraints := or .Values.cleanup.topologySpreadConstraints .Values.topologySpreadConstraints }}
|
||||
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.cleanup) }}
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-cleanup
|
||||
labels:
|
||||
tier: airflow
|
||||
component: airflow-cleanup-pods
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.cleanup.jobAnnotations }}
|
||||
annotations: {{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
schedule: "{{ tpl .Values.cleanup.schedule . }}"
|
||||
# The cron job does not allow concurrent runs; if it is time for a new job run and the previous job run hasn't finished yet, the cron job skips the new job run
|
||||
concurrencyPolicy: Forbid
|
||||
{{- if .Values.cleanup.failedJobsHistoryLimit }}
|
||||
failedJobsHistoryLimit: {{ .Values.cleanup.failedJobsHistoryLimit }}
|
||||
{{- end }}
|
||||
{{- if .Values.cleanup.successfulJobsHistoryLimit }}
|
||||
successfulJobsHistoryLimit: {{ .Values.cleanup.successfulJobsHistoryLimit }}
|
||||
{{- end }}
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 1
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
tier: airflow
|
||||
component: airflow-cleanup-pods
|
||||
release: {{ .Release.Name }}
|
||||
{{- if or (.Values.labels) (.Values.cleanup.labels) }}
|
||||
{{- mustMerge .Values.cleanup.labels .Values.labels | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
sidecar.istio.io/inject: "false"
|
||||
{{- if .Values.airflowPodAnnotations }}
|
||||
{{- toYaml .Values.airflowPodAnnotations | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.cleanup.podAnnotations }}
|
||||
{{- toYaml .Values.cleanup.podAnnotations | nindent 12 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
nodeSelector: {{- toYaml $nodeSelector | nindent 12 }}
|
||||
affinity: {{- toYaml $affinity | nindent 12 }}
|
||||
{{- if .Values.schedulerName }}
|
||||
schedulerName: {{ .Values.schedulerName }}
|
||||
{{- end }}
|
||||
tolerations: {{- toYaml $tolerations | nindent 12 }}
|
||||
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 12 }}
|
||||
serviceAccountName: {{ include "cleanup.serviceAccountName" . }}
|
||||
{{- if or .Values.registry.secretName .Values.registry.connection }}
|
||||
imagePullSecrets:
|
||||
- name: {{ template "registry_secret" . }}
|
||||
{{- end }}
|
||||
securityContext: {{ $securityContext | nindent 12 }}
|
||||
containers:
|
||||
- name: airflow-cleanup-pods
|
||||
image: {{ template "airflow_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
{{- if .Values.cleanup.command }}
|
||||
command: {{ tpl (toYaml .Values.cleanup.command) . | nindent 16 }}
|
||||
{{- end }}
|
||||
{{- if .Values.cleanup.args }}
|
||||
args: {{ tpl (toYaml .Values.cleanup.args) . | nindent 16 }}
|
||||
{{- end }}
|
||||
env:
|
||||
{{- include "standard_airflow_environment" . | indent 12 }}
|
||||
{{- include "container_extra_envs" (list . .Values.cleanup.env) | indent 12 }}
|
||||
volumeMounts: {{- include "airflow_config_mount" . | nindent 16 }}
|
||||
resources: {{- toYaml .Values.cleanup.resources | nindent 16 }}
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: {{ template "airflow_config" . }}
|
||||
{{- end }}
|
||||
40
helm/airflow/templates/cleanup/cleanup-serviceaccount.yaml
Normal file
40
helm/airflow/templates/cleanup/cleanup-serviceaccount.yaml
Normal file
@@ -0,0 +1,40 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Cleanup ServiceAccount
|
||||
#################################
|
||||
{{- if and .Values.cleanup.serviceAccount.create .Values.cleanup.enabled }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
automountServiceAccountToken: {{ .Values.cleanup.serviceAccount.automountServiceAccountToken }}
|
||||
metadata:
|
||||
name: {{ include "cleanup.serviceAccountName" . }}
|
||||
labels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- if or (.Values.labels) (.Values.cleanup.labels) }}
|
||||
{{- mustMerge .Values.cleanup.labels .Values.labels | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.cleanup.serviceAccount.annotations }}
|
||||
annotations: {{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
74
helm/airflow/templates/configmaps/configmap.yaml
Normal file
74
helm/airflow/templates/configmaps/configmap.yaml
Normal file
@@ -0,0 +1,74 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow ConfigMap
|
||||
#################################
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ template "airflow_config" . }}
|
||||
labels:
|
||||
tier: airflow
|
||||
component: config
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end -}}
|
||||
{{- if .Values.airflowConfigAnnotations }}
|
||||
annotations: {{- toYaml .Values.airflowConfigAnnotations | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- $Global := . }}
|
||||
data:
|
||||
# These are system-specified config overrides.
|
||||
airflow.cfg: |-
|
||||
{{- range $section, $settings := .Values.config }}
|
||||
[{{ $section }}]
|
||||
{{- range $key, $val := $settings }}
|
||||
{{ $key }} = {{ tpl ($val | toString) $Global }}
|
||||
{{- end }}
|
||||
{{ end }}
|
||||
|
||||
{{- if .Values.airflowLocalSettings }}
|
||||
airflow_local_settings.py: |-
|
||||
{{- tpl .Values.airflowLocalSettings . | nindent 4 }}
|
||||
{{- end }}
|
||||
|
||||
{{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.knownHosts }}
|
||||
known_hosts: |-
|
||||
{{- .Values.dags.gitSync.knownHosts | nindent 4 }}
|
||||
{{- end }}
|
||||
|
||||
{{/* {{- if or (eq $.Values.executor "LocalKubernetesExecutor") (eq $.Values.executor "KubernetesExecutor") (eq $.Values.executor "CeleryKubernetesExecutor") }}*/}}
|
||||
{{/* {{- if semverCompare ">=1.10.12" .Values.airflowVersion }}*/}}
|
||||
pod_template_file.yaml: |-
|
||||
{{- if .Values.podTemplate }}
|
||||
{{- tpl .Values.podTemplate . | nindent 4 }}
|
||||
{{- else }}
|
||||
{{- tpl (.Files.Get "files/pod-template-file.kubernetes-helm-yaml") . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{/* {{- end }}*/}}
|
||||
{{/* {{- end }}*/}}
|
||||
|
||||
{{- if .Values.kerberos.enabled }}
|
||||
krb5.conf: |-
|
||||
{{- tpl .Values.kerberos.config . | nindent 4 }}
|
||||
{{- end }}
|
||||
53
helm/airflow/templates/configmaps/extra-configmaps.yaml
Normal file
53
helm/airflow/templates/configmaps/extra-configmaps.yaml
Normal file
@@ -0,0 +1,53 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
####################################################
|
||||
## Extra ConfigMaps provisioned via the chart values
|
||||
####################################################
|
||||
{{- $Global := . }}
|
||||
{{- range $configMapName, $configMapContent := .Values.extraConfigMaps }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ tpl $configMapName $Global | quote }}
|
||||
labels:
|
||||
release: {{ $Global.Release.Name }}
|
||||
chart: "{{ $Global.Chart.Name }}-{{ $Global.Chart.Version }}"
|
||||
heritage: {{ $Global.Release.Service }}
|
||||
{{- with $Global.Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if $configMapContent.labels }}
|
||||
{{- toYaml $configMapContent.labels | nindent 4 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
"helm.sh/hook": "pre-install,pre-upgrade"
|
||||
"helm.sh/hook-delete-policy": "before-hook-creation"
|
||||
"helm.sh/hook-weight": "0"
|
||||
{{- if $configMapContent.annotations }}
|
||||
{{- toYaml $configMapContent.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if $configMapContent.data }}
|
||||
data:
|
||||
{{- with $configMapContent.data }}
|
||||
{{- tpl . $Global | nindent 2 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
52
helm/airflow/templates/configmaps/statsd-configmap.yaml
Normal file
52
helm/airflow/templates/configmaps/statsd-configmap.yaml
Normal file
@@ -0,0 +1,52 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow StatsD ConfigMap
|
||||
#################################
|
||||
{{- if and .Values.statsd.enabled }}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-statsd
|
||||
labels:
|
||||
tier: airflow
|
||||
component: config
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.statsd.configMapAnnotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
data:
|
||||
mappings.yml: |-
|
||||
{{- if .Values.statsd.overrideMappings }}
|
||||
mappings:
|
||||
{{- toYaml .Values.statsd.overrideMappings | nindent 6 }}
|
||||
{{- else }}
|
||||
{{- .Files.Get "files/statsd-mappings.yml" | nindent 4 }}
|
||||
{{- if .Values.statsd.extraMappings }}
|
||||
{{- toYaml .Values.statsd.extraMappings | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
44
helm/airflow/templates/configmaps/webserver-configmap.yaml
Normal file
44
helm/airflow/templates/configmaps/webserver-configmap.yaml
Normal file
@@ -0,0 +1,44 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow ConfigMap
|
||||
#################################
|
||||
{{- if and .Values.webserver.webserverConfig (not .Values.webserver.webserverConfigConfigMapName) }}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ template "airflow_webserver_config_configmap_name" . }}
|
||||
labels:
|
||||
tier: airflow
|
||||
component: config
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.webserver.configMapAnnotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
data:
|
||||
webserver_config.py: |-
|
||||
{{- tpl .Values.webserver.webserverConfig . | nindent 4 }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,261 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Dag Processor Deployment
|
||||
#################################
|
||||
{{- if semverCompare ">=2.3.0" .Values.airflowVersion }}
|
||||
{{- if .Values.dagProcessor.enabled }}
|
||||
{{- $nodeSelector := or .Values.dagProcessor.nodeSelector .Values.nodeSelector }}
|
||||
{{- $affinity := or .Values.dagProcessor.affinity .Values.affinity }}
|
||||
{{- $tolerations := or .Values.dagProcessor.tolerations .Values.tolerations }}
|
||||
{{- $topologySpreadConstraints := or .Values.dagProcessor.topologySpreadConstraints .Values.topologySpreadConstraints }}
|
||||
{{- $revisionHistoryLimit := or .Values.dagProcessor.revisionHistoryLimit .Values.revisionHistoryLimit }}
|
||||
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.dagProcessor) }}
|
||||
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.dagProcessor) }}
|
||||
{{- $containerLifecycleHooks := or .Values.dagProcessor.containerLifecycleHooks .Values.containerLifecycleHooks }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-dag-processor
|
||||
labels:
|
||||
tier: airflow
|
||||
component: dag-processor
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.dagProcessor.annotations }}
|
||||
annotations: {{- toYaml .Values.dagProcessor.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
replicas: {{ .Values.dagProcessor.replicas }}
|
||||
{{- if $revisionHistoryLimit }}
|
||||
revisionHistoryLimit: {{ $revisionHistoryLimit }}
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: dag-processor
|
||||
release: {{ .Release.Name }}
|
||||
{{- if .Values.dagProcessor.strategy }}
|
||||
strategy: {{- toYaml .Values.dagProcessor.strategy | nindent 4 }}
|
||||
{{- end }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
tier: airflow
|
||||
component: dag-processor
|
||||
release: {{ .Release.Name }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }}
|
||||
checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }}
|
||||
checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }}
|
||||
checksum/extra-configmaps: {{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }}
|
||||
checksum/extra-secrets: {{ include (print $.Template.BasePath "/secrets/extra-secrets.yaml") . | sha256sum }}
|
||||
{{- if .Values.dagProcessor.safeToEvict }}
|
||||
cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
|
||||
{{- end }}
|
||||
{{- if .Values.airflowPodAnnotations }}
|
||||
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.dagProcessor.podAnnotations }}
|
||||
{{- toYaml .Values.dagProcessor.podAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.dagProcessor.priorityClassName }}
|
||||
priorityClassName: {{ .Values.dagProcessor.priorityClassName }}
|
||||
{{- end }}
|
||||
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
|
||||
affinity:
|
||||
{{- if $affinity }}
|
||||
{{- toYaml $affinity | nindent 8 }}
|
||||
{{- else }}
|
||||
podAntiAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- podAffinityTerm:
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
component: dag-processor
|
||||
topologyKey: kubernetes.io/hostname
|
||||
weight: 100
|
||||
{{- end }}
|
||||
{{- if .Values.schedulerName }}
|
||||
schedulerName: {{ .Values.schedulerName }}
|
||||
{{- end }}
|
||||
tolerations: {{- toYaml $tolerations | nindent 8 }}
|
||||
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
|
||||
terminationGracePeriodSeconds: {{ .Values.dagProcessor.terminationGracePeriodSeconds }}
|
||||
restartPolicy: Always
|
||||
serviceAccountName: {{ include "dagProcessor.serviceAccountName" . }}
|
||||
securityContext: {{ $securityContext | nindent 8 }}
|
||||
{{- if or .Values.registry.secretName .Values.registry.connection }}
|
||||
imagePullSecrets:
|
||||
- name: {{ template "registry_secret" . }}
|
||||
{{- end }}
|
||||
initContainers:
|
||||
{{- if .Values.dagProcessor.waitForMigrations.enabled }}
|
||||
- name: wait-for-airflow-migrations
|
||||
resources: {{- toYaml .Values.dagProcessor.resources | nindent 12 }}
|
||||
image: {{ template "airflow_image_for_migrations" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
volumeMounts:
|
||||
{{- if .Values.volumeMounts }}
|
||||
{{- toYaml .Values.volumeMounts | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.dagProcessor.extraVolumeMounts }}
|
||||
{{- tpl (toYaml .Values.dagProcessor.extraVolumeMounts) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- include "airflow_config_mount" . | nindent 12 }}
|
||||
args: {{- include "wait-for-migrations-command" . | indent 10 }}
|
||||
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
|
||||
env:
|
||||
{{- include "custom_airflow_environment" . | indent 10 }}
|
||||
{{- include "standard_airflow_environment" . | indent 10 }}
|
||||
{{- if .Values.dagProcessor.waitForMigrations.env }}
|
||||
{{- tpl (toYaml .Values.dagProcessor.waitForMigrations.env) $ | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
|
||||
{{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.dagProcessor.extraInitContainers }}
|
||||
{{- toYaml .Values.dagProcessor.extraInitContainers | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: dag-processor
|
||||
image: {{ template "airflow_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
securityContext: {{ $containerSecurityContext | nindent 12 }}
|
||||
{{- if $containerLifecycleHooks }}
|
||||
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.dagProcessor.command }}
|
||||
command: {{ tpl (toYaml .Values.dagProcessor.command) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.dagProcessor.args }}
|
||||
args: {{ tpl (toYaml .Values.dagProcessor.args) . | nindent 12 }}
|
||||
{{- end }}
|
||||
resources: {{- toYaml .Values.dagProcessor.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
{{- if .Values.volumeMounts }}
|
||||
{{- toYaml .Values.volumeMounts | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.dagProcessor.extraVolumeMounts }}
|
||||
{{- tpl (toYaml .Values.dagProcessor.extraVolumeMounts) . | nindent 12 }}
|
||||
{{- end }}
|
||||
- name: logs
|
||||
mountPath: {{ template "airflow_logs" . }}
|
||||
{{- include "airflow_config_mount" . | nindent 12 }}
|
||||
{{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }}
|
||||
{{- include "airflow_dags_mount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
|
||||
env:
|
||||
{{- include "custom_airflow_environment" . | indent 10 }}
|
||||
{{- include "standard_airflow_environment" . | indent 10 }}
|
||||
{{- include "container_extra_envs" (list . .Values.dagProcessor.env) | indent 10 }}
|
||||
livenessProbe:
|
||||
initialDelaySeconds: {{ .Values.dagProcessor.livenessProbe.initialDelaySeconds }}
|
||||
timeoutSeconds: {{ .Values.dagProcessor.livenessProbe.timeoutSeconds }}
|
||||
failureThreshold: {{ .Values.dagProcessor.livenessProbe.failureThreshold }}
|
||||
periodSeconds: {{ .Values.dagProcessor.livenessProbe.periodSeconds }}
|
||||
exec:
|
||||
command:
|
||||
{{- if .Values.dagProcessor.livenessProbe.command }}
|
||||
{{- toYaml .Values.dagProcessor.livenessProbe.command | nindent 16 }}
|
||||
{{- else }}
|
||||
{{- include "dag_processor_liveness_check_command" . | indent 14 }}
|
||||
{{- end }}
|
||||
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
|
||||
{{- include "git_sync_container" . | indent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.dagProcessor.logGroomerSidecar.enabled }}
|
||||
- name: dag-processor-log-groomer
|
||||
resources: {{- toYaml .Values.dagProcessor.logGroomerSidecar.resources | nindent 12 }}
|
||||
image: {{ template "airflow_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
{{- if .Values.dagProcessor.logGroomerSidecar.command }}
|
||||
command: {{ tpl (toYaml .Values.dagProcessor.logGroomerSidecar.command) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.dagProcessor.logGroomerSidecar.args }}
|
||||
args: {{- tpl (toYaml .Values.dagProcessor.logGroomerSidecar.args) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.dagProcessor.logGroomerSidecar.retentionDays }}
|
||||
env:
|
||||
- name: AIRFLOW__LOG_RETENTION_DAYS
|
||||
value: "{{ .Values.dagProcessor.logGroomerSidecar.retentionDays }}"
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
- name: logs
|
||||
mountPath: {{ template "airflow_logs" . }}
|
||||
{{- if .Values.volumeMounts }}
|
||||
{{- toYaml .Values.volumeMounts | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.dagProcessor.extraVolumeMounts }}
|
||||
{{- tpl (toYaml .Values.dagProcessor.extraVolumeMounts) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
|
||||
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.dagProcessor.extraContainers }}
|
||||
{{- toYaml .Values.dagProcessor.extraContainers | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: {{ template "airflow_config" . }}
|
||||
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
|
||||
- name: webserver-config
|
||||
configMap:
|
||||
name: {{ template "airflow_webserver_config_configmap_name" . }}
|
||||
{{- end }}
|
||||
{{- if .Values.dags.persistence.enabled }}
|
||||
- name: dags
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ template "airflow_dags_volume_claim" . }}
|
||||
{{- else if .Values.dags.gitSync.enabled }}
|
||||
- name: dags
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.sshKeySecret }}
|
||||
{{- include "git_sync_ssh_key_volume" . | indent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.volumes }}
|
||||
{{- toYaml .Values.volumes | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.dagProcessor.extraVolumes }}
|
||||
{{- tpl (toYaml .Values.dagProcessor.extraVolumes) . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.logs.persistence.enabled }}
|
||||
- name: logs
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ template "airflow_logs_volume_claim" . }}
|
||||
{{- else }}
|
||||
- name: logs
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,43 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Dag Processor ServiceAccount
|
||||
#################################
|
||||
{{- if semverCompare ">=2.3.0" .Values.airflowVersion }}
|
||||
{{- if and .Values.dagProcessor.serviceAccount.create .Values.dagProcessor.enabled }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
automountServiceAccountToken: {{ .Values.dagProcessor.serviceAccount.automountServiceAccountToken }}
|
||||
metadata:
|
||||
name: {{ include "dagProcessor.serviceAccountName" . }}
|
||||
labels:
|
||||
tier: airflow
|
||||
component: dag-processor
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.dagProcessor.serviceAccount.annotations}}
|
||||
annotations: {{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
52
helm/airflow/templates/dags-persistent-volume-claim.yaml
Normal file
52
helm/airflow/templates/dags-persistent-volume-claim.yaml
Normal file
@@ -0,0 +1,52 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
######################################
|
||||
## Airflow DAGs PersistentVolumeClaim
|
||||
######################################
|
||||
{{- if and (not .Values.dags.persistence.existingClaim ) .Values.dags.persistence.enabled }}
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: {{ template "airflow_dags_volume_claim" . }}
|
||||
labels:
|
||||
tier: airflow
|
||||
component: dags-pvc
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.dags.persistence.annotations }}
|
||||
annotations: {{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
accessModes: [{{ .Values.dags.persistence.accessMode | quote }}]
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.dags.persistence.size | quote }}
|
||||
{{- if .Values.dags.persistence.storageClassName }}
|
||||
{{- if (eq "-" .Values.dags.persistence.storageClassName) }}
|
||||
storageClassName: ""
|
||||
{{- else }}
|
||||
storageClassName: "{{ .Values.dags.persistence.storageClassName }}"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
169
helm/airflow/templates/flower/flower-deployment.yaml
Normal file
169
helm/airflow/templates/flower/flower-deployment.yaml
Normal file
@@ -0,0 +1,169 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Flower Deployment
|
||||
#################################
|
||||
{{- if .Values.flower.enabled }}
|
||||
{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
|
||||
{{- $nodeSelector := or .Values.flower.nodeSelector .Values.nodeSelector }}
|
||||
{{- $affinity := or .Values.flower.affinity .Values.affinity }}
|
||||
{{- $tolerations := or .Values.flower.tolerations .Values.tolerations }}
|
||||
{{- $topologySpreadConstraints := or .Values.flower.topologySpreadConstraints .Values.topologySpreadConstraints }}
|
||||
{{- $revisionHistoryLimit := or .Values.flower.revisionHistoryLimit .Values.revisionHistoryLimit }}
|
||||
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.flower) }}
|
||||
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.flower) }}
|
||||
{{- $containerLifecycleHooks := or .Values.flower.containerLifecycleHooks .Values.containerLifecycleHooks }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-flower
|
||||
labels:
|
||||
tier: airflow
|
||||
component: flower
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.flower.annotations }}
|
||||
annotations: {{- toYaml .Values.flower.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
replicas: 1
|
||||
{{- if $revisionHistoryLimit }}
|
||||
revisionHistoryLimit: {{ $revisionHistoryLimit }}
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: flower
|
||||
release: {{ .Release.Name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
tier: airflow
|
||||
component: flower
|
||||
release: {{ .Release.Name }}
|
||||
{{- if or (.Values.labels) (.Values.flower.labels) }}
|
||||
{{- mustMerge .Values.flower.labels .Values.labels | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }}
|
||||
checksum/flower-secret: {{ include (print $.Template.BasePath "/secrets/flower-secret.yaml") . | sha256sum }}
|
||||
{{- if or (.Values.airflowPodAnnotations) (.Values.flower.podAnnotations) }}
|
||||
{{- mustMerge .Values.flower.podAnnotations .Values.airflowPodAnnotations | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
|
||||
affinity: {{- toYaml $affinity | nindent 8 }}
|
||||
{{- if .Values.schedulerName }}
|
||||
schedulerName: {{ .Values.schedulerName }}
|
||||
{{- end }}
|
||||
tolerations: {{- toYaml $tolerations | nindent 8 }}
|
||||
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
|
||||
serviceAccountName: {{ include "flower.serviceAccountName" . }}
|
||||
{{- if .Values.flower.priorityClassName }}
|
||||
priorityClassName: {{ .Values.flower.priorityClassName }}
|
||||
{{- end }}
|
||||
restartPolicy: Always
|
||||
securityContext: {{ $securityContext | nindent 8 }}
|
||||
{{- if or .Values.registry.secretName .Values.registry.connection }}
|
||||
imagePullSecrets:
|
||||
- name: {{ template "registry_secret" . }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: flower
|
||||
image: {{ template "flower_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.flower.pullPolicy }}
|
||||
securityContext: {{ $containerSecurityContext | nindent 12 }}
|
||||
{{- if $containerLifecycleHooks }}
|
||||
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.flower.command }}
|
||||
command: {{ tpl (toYaml .Values.flower.command) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.flower.args }}
|
||||
args: {{ tpl (toYaml .Values.flower.args) . | nindent 12 }}
|
||||
{{- end }}
|
||||
resources: {{- toYaml .Values.flower.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
{{- include "airflow_config_mount" . | nindent 12 }}
|
||||
{{- if .Values.volumeMounts }}
|
||||
{{- toYaml .Values.volumeMounts | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.flower.extraVolumeMounts }}
|
||||
{{- tpl (toYaml .Values.flower.extraVolumeMounts) . | nindent 12 }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- name: flower-ui
|
||||
containerPort: {{ .Values.ports.flowerUI }}
|
||||
livenessProbe:
|
||||
failureThreshold: 10
|
||||
exec:
|
||||
command:
|
||||
- curl
|
||||
{{- if (or .Values.flower.secretName (and .Values.flower.username .Values.flower.password))}}
|
||||
- "--user"
|
||||
- $AIRFLOW__CELERY__FLOWER_BASIC_AUTH
|
||||
{{- end }}
|
||||
- {{ printf "localhost:%s" (.Values.ports.flowerUI | toString) }}
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 5
|
||||
readinessProbe:
|
||||
failureThreshold: 10
|
||||
exec:
|
||||
command:
|
||||
- curl
|
||||
{{- if (or .Values.flower.secretName (and .Values.flower.username .Values.flower.password))}}
|
||||
- "--user"
|
||||
- $AIRFLOW__CELERY__FLOWER_BASIC_AUTH
|
||||
{{- end }}
|
||||
- {{ printf "localhost:%s" (.Values.ports.flowerUI | toString) }}
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 5
|
||||
envFrom:
|
||||
{{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
|
||||
env:
|
||||
{{- if (or .Values.flower.secretName (and .Values.flower.username .Values.flower.password))}}
|
||||
- name: AIRFLOW__CELERY__FLOWER_BASIC_AUTH
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ template "flower_secret" . }}
|
||||
key: basicAuth
|
||||
{{- end }}
|
||||
{{- include "standard_airflow_environment" . | indent 10 }}
|
||||
{{- include "custom_airflow_environment" . | indent 10 }}
|
||||
{{- include "container_extra_envs" (list . .Values.flower.env) | indent 10 }}
|
||||
{{- if .Values.flower.extraContainers }}
|
||||
{{- toYaml .Values.flower.extraContainers | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: {{ template "airflow_config" . }}
|
||||
{{- if .Values.volumes }}
|
||||
{{- toYaml .Values.volumes | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.flower.extraVolumes }}
|
||||
{{- tpl (toYaml .Values.flower.extraVolumes) . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
94
helm/airflow/templates/flower/flower-ingress.yaml
Normal file
94
helm/airflow/templates/flower/flower-ingress.yaml
Normal file
@@ -0,0 +1,94 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Flower Ingress
|
||||
#################################
|
||||
{{- if .Values.flower.enabled }}
|
||||
{{- if and (or .Values.ingress.flower.enabled .Values.ingress.enabled) (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")) }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-flower-ingress
|
||||
labels:
|
||||
tier: airflow
|
||||
component: flower-ingress
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- if or (.Values.labels) (.Values.flower.labels) }}
|
||||
{{- mustMerge .Values.flower.labels .Values.labels | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.ingress.flower.annotations }}
|
||||
annotations: {{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if and .Values.ingress.flower.hosts (.Values.ingress.flower.hosts | first | kindIs "string" | not) }}
|
||||
{{- $anyTlsHosts := false -}}
|
||||
{{- range .Values.ingress.flower.hosts }}
|
||||
{{- if .tls }}
|
||||
{{- if .tls.enabled }}
|
||||
{{- $anyTlsHosts = true -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if $anyTlsHosts }}
|
||||
tls:
|
||||
{{- range .Values.ingress.flower.hosts }}
|
||||
{{- if .tls }}
|
||||
{{- if .tls.enabled }}
|
||||
- hosts:
|
||||
- {{ .name | quote }}
|
||||
secretName: {{ .tls.secretName }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- else if .Values.ingress.flower.tls.enabled }}
|
||||
tls:
|
||||
- hosts:
|
||||
{{- .Values.ingress.flower.hosts | default (list .Values.ingress.flower.host) | toYaml | nindent 8 }}
|
||||
secretName: {{ .Values.ingress.flower.tls.secretName }}
|
||||
{{- end }}
|
||||
rules:
|
||||
{{- range .Values.ingress.flower.hosts | default (list .Values.ingress.flower.host) }}
|
||||
- http:
|
||||
paths:
|
||||
- backend:
|
||||
service:
|
||||
name: {{ $.Release.Name }}-flower
|
||||
port:
|
||||
name: flower-ui
|
||||
{{- if $.Values.ingress.flower.path }}
|
||||
path: {{ $.Values.ingress.flower.path }}
|
||||
pathType: {{ $.Values.ingress.flower.pathType }}
|
||||
{{- end }}
|
||||
{{- $hostname := . -}}
|
||||
{{- if . | kindIs "string" | not }}
|
||||
{{- $hostname = .name -}}
|
||||
{{- end }}
|
||||
{{- if $hostname }}
|
||||
host: {{ tpl $hostname $ | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.ingress.flower.ingressClassName }}
|
||||
ingressClassName: {{ .Values.ingress.flower.ingressClassName }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
60
helm/airflow/templates/flower/flower-networkpolicy.yaml
Normal file
60
helm/airflow/templates/flower/flower-networkpolicy.yaml
Normal file
@@ -0,0 +1,60 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Flower NetworkPolicy
|
||||
#################################
|
||||
{{- if .Values.flower.enabled }}
|
||||
{{- $celery_executors := list "CeleryExecutor" "CeleryKubernetesExecutor"}}
|
||||
{{- if and .Values.networkPolicies.enabled (has .Values.executor $celery_executors) }}
|
||||
{{- $from := or .Values.flower.networkPolicy.ingress.from .Values.flower.extraNetworkPolicies }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-flower-policy
|
||||
labels:
|
||||
tier: airflow
|
||||
component: airflow-flower-policy
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- if or (.Values.labels) (.Values.flower.labels) }}
|
||||
{{- mustMerge .Values.flower.labels .Values.labels | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: flower
|
||||
release: {{ .Release.Name }}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
{{- if $from }}
|
||||
ingress:
|
||||
- from: {{- toYaml $from | nindent 6 }}
|
||||
ports:
|
||||
{{ range .Values.flower.networkPolicy.ingress.ports }}
|
||||
-
|
||||
{{- range $key, $val := . }}
|
||||
{{ $key }}: {{ tpl (toString $val) $ }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
61
helm/airflow/templates/flower/flower-service.yaml
Normal file
61
helm/airflow/templates/flower/flower-service.yaml
Normal file
@@ -0,0 +1,61 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Flower Service Component
|
||||
#################################
|
||||
{{- if .Values.flower.enabled }}
|
||||
{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-flower
|
||||
labels:
|
||||
tier: airflow
|
||||
component: flower
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- if or (.Values.labels) (.Values.flower.labels) }}
|
||||
{{- mustMerge .Values.flower.labels .Values.labels | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.flower.service.annotations }}
|
||||
annotations: {{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
type: {{ .Values.flower.service.type }}
|
||||
selector:
|
||||
tier: airflow
|
||||
component: flower
|
||||
release: {{ .Release.Name }}
|
||||
ports:
|
||||
{{ range .Values.flower.service.ports }}
|
||||
-
|
||||
{{- range $key, $val := . }}
|
||||
{{ $key }}: {{ tpl (toString $val) $ }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.flower.service.loadBalancerIP }}
|
||||
loadBalancerIP: {{ .Values.flower.service.loadBalancerIP }}
|
||||
{{- end }}
|
||||
{{- if .Values.flower.service.loadBalancerSourceRanges }}
|
||||
loadBalancerSourceRanges: {{- toYaml .Values.flower.service.loadBalancerSourceRanges | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
41
helm/airflow/templates/flower/flower-serviceaccount.yaml
Normal file
41
helm/airflow/templates/flower/flower-serviceaccount.yaml
Normal file
@@ -0,0 +1,41 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
######################################
|
||||
## Airflow Flower ServiceAccount
|
||||
######################################
|
||||
{{- if and .Values.flower.enabled (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")) .Values.flower.serviceAccount.create }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
automountServiceAccountToken: {{ .Values.flower.serviceAccount.automountServiceAccountToken }}
|
||||
metadata:
|
||||
name: {{ include "flower.serviceAccountName" . }}
|
||||
labels:
|
||||
tier: airflow
|
||||
component: flower
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- if or (.Values.labels) (.Values.flower.labels) }}
|
||||
{{- mustMerge .Values.flower.labels .Values.labels | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.flower.serviceAccount.annotations }}
|
||||
annotations: {{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,41 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
###########################################
|
||||
## Airflow Create User Job ServiceAccount
|
||||
###########################################
|
||||
{{- if and .Values.createUserJob.serviceAccount.create .Values.webserver.defaultUser.enabled }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
automountServiceAccountToken: {{ .Values.createUserJob.serviceAccount.automountServiceAccountToken }}
|
||||
metadata:
|
||||
name: {{ include "createUserJob.serviceAccountName" . }}
|
||||
labels:
|
||||
tier: airflow
|
||||
component: create-user-job
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- if or (.Values.labels) (.Values.createUserJob.labels) }}
|
||||
{{- mustMerge .Values.createUserJob.labels .Values.labels | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.createUserJob.serviceAccount.annotations }}
|
||||
annotations: {{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
134
helm/airflow/templates/jobs/create-user-job.yaml
Normal file
134
helm/airflow/templates/jobs/create-user-job.yaml
Normal file
@@ -0,0 +1,134 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Create User Job
|
||||
#################################
|
||||
{{- if .Values.webserver.defaultUser.enabled }}
|
||||
{{- $nodeSelector := or .Values.createUserJob.nodeSelector .Values.nodeSelector }}
|
||||
{{- $affinity := or .Values.createUserJob.affinity .Values.affinity }}
|
||||
{{- $tolerations := or .Values.createUserJob.tolerations .Values.tolerations }}
|
||||
{{- $topologySpreadConstraints := or .Values.createUserJob.topologySpreadConstraints .Values.topologySpreadConstraints }}
|
||||
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.createUserJob) }}
|
||||
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.createUserJob) }}
|
||||
{{- $containerLifecycleHooks := or .Values.createUserJob.containerLifecycleHooks .Values.containerLifecycleHooks }}
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-create-user
|
||||
labels:
|
||||
tier: airflow
|
||||
component: create-user-job
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- $annotations := dict }}
|
||||
{{- if .Values.createUserJob.useHelmHooks }}
|
||||
{{- $_ := set $annotations "helm.sh/hook" "post-install,post-upgrade" }}
|
||||
{{- $_ := set $annotations "helm.sh/hook-weight" "2" }}
|
||||
{{- $_ := set $annotations "helm.sh/hook-delete-policy" "before-hook-creation,hook-succeeded" }}
|
||||
{{- end }}
|
||||
{{- with $annotations := merge $annotations .Values.createUserJob.jobAnnotations }}
|
||||
annotations: {{- $annotations | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if not (kindIs "invalid" .Values.createUserJob.ttlSecondsAfterFinished) }}
|
||||
ttlSecondsAfterFinished: {{ .Values.createUserJob.ttlSecondsAfterFinished }}
|
||||
{{- end }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
tier: airflow
|
||||
component: create-user-job
|
||||
release: {{ .Release.Name }}
|
||||
{{- if or (.Values.labels) (.Values.createUserJob.labels) }}
|
||||
{{- mustMerge .Values.createUserJob.labels .Values.labels | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if or .Values.airflowPodAnnotations .Values.createUserJob.annotations }}
|
||||
annotations:
|
||||
{{- if .Values.airflowPodAnnotations }}
|
||||
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.createUserJob.annotations }}
|
||||
{{- toYaml .Values.createUserJob.annotations | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
spec:
|
||||
securityContext: {{ $securityContext | nindent 8 }}
|
||||
restartPolicy: OnFailure
|
||||
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
|
||||
affinity: {{- toYaml $affinity | nindent 8 }}
|
||||
{{- if .Values.schedulerName }}
|
||||
schedulerName: {{ .Values.schedulerName }}
|
||||
{{- end }}
|
||||
tolerations: {{- toYaml $tolerations | nindent 8 }}
|
||||
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
|
||||
serviceAccountName: {{ include "createUserJob.serviceAccountName" . }}
|
||||
{{- if or .Values.registry.secretName .Values.registry.connection }}
|
||||
imagePullSecrets:
|
||||
- name: {{ template "registry_secret" . }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: create-user
|
||||
image: {{ template "airflow_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
securityContext: {{ $containerSecurityContext | nindent 12 }}
|
||||
{{- if $containerLifecycleHooks }}
|
||||
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.createUserJob.command }}
|
||||
command: {{ tpl (toYaml .Values.createUserJob.command) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.createUserJob.args }}
|
||||
args: {{ tpl (toYaml .Values.createUserJob.args) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.createUserJob.applyCustomEnv }}
|
||||
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
|
||||
env: {{- include "custom_airflow_environment" . | indent 10 }}
|
||||
{{- else }}
|
||||
env:
|
||||
{{- end }}
|
||||
{{- include "standard_airflow_environment" . | indent 10 }}
|
||||
{{- include "container_extra_envs" (list . .Values.createUserJob.env) | indent 10 }}
|
||||
resources: {{- toYaml .Values.createUserJob.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
{{- include "airflow_config_mount" . | nindent 12 }}
|
||||
{{- if .Values.volumeMounts }}
|
||||
{{- toYaml .Values.volumeMounts | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.createUserJob.extraVolumeMounts }}
|
||||
{{- tpl (toYaml .Values.createUserJob.extraVolumeMounts) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.createUserJob.extraContainers }}
|
||||
{{- toYaml .Values.createUserJob.extraContainers | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: {{ template "airflow_config" . }}
|
||||
{{- if .Values.volumes }}
|
||||
{{- toYaml .Values.volumes | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.createUserJob.extraVolumes }}
|
||||
{{- tpl (toYaml .Values.createUserJob.extraVolumes) . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,41 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
#############################################
|
||||
## Airflow Migrate Database Job ServiceAccount
|
||||
##############################################
|
||||
{{- if .Values.migrateDatabaseJob.serviceAccount.create }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
automountServiceAccountToken: {{ .Values.migrateDatabaseJob.serviceAccount.automountServiceAccountToken }}
|
||||
metadata:
|
||||
name: {{ include "migrateDatabaseJob.serviceAccountName" . }}
|
||||
labels:
|
||||
tier: airflow
|
||||
component: run-airflow-migrations
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.migrateDatabaseJob.serviceAccount.annotations }}
|
||||
annotations: {{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
135
helm/airflow/templates/jobs/migrate-database-job.yaml
Normal file
135
helm/airflow/templates/jobs/migrate-database-job.yaml
Normal file
@@ -0,0 +1,135 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Run Migrations
|
||||
#################################
|
||||
{{- if .Values.migrateDatabaseJob.enabled }}
|
||||
{{- $nodeSelector := or .Values.migrateDatabaseJob.nodeSelector .Values.nodeSelector }}
|
||||
{{- $affinity := or .Values.migrateDatabaseJob.affinity .Values.affinity }}
|
||||
{{- $tolerations := or .Values.migrateDatabaseJob.tolerations .Values.tolerations }}
|
||||
{{- $topologySpreadConstraints := or .Values.migrateDatabaseJob.topologySpreadConstraints .Values.topologySpreadConstraints }}
|
||||
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.migrateDatabaseJob) }}
|
||||
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.migrateDatabaseJob) }}
|
||||
{{- $containerLifecycleHooks := or .Values.migrateDatabaseJob.containerLifecycleHooks .Values.containerLifecycleHooks }}
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-run-airflow-migrations
|
||||
labels:
|
||||
tier: airflow
|
||||
component: run-airflow-migrations
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- $annotations := dict }}
|
||||
{{- if .Values.migrateDatabaseJob.useHelmHooks }}
|
||||
{{- $_ := set $annotations "helm.sh/hook" "post-install,post-upgrade" }}
|
||||
{{- $_ := set $annotations "helm.sh/hook-weight" "1" }}
|
||||
{{- $_ := set $annotations "helm.sh/hook-delete-policy" "before-hook-creation,hook-succeeded" }}
|
||||
{{- end }}
|
||||
{{- with $annotations := merge $annotations .Values.migrateDatabaseJob.jobAnnotations }}
|
||||
annotations: {{- $annotations | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if not (kindIs "invalid" .Values.migrateDatabaseJob.ttlSecondsAfterFinished) }}
|
||||
ttlSecondsAfterFinished: {{ .Values.migrateDatabaseJob.ttlSecondsAfterFinished }}
|
||||
{{- end }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
tier: airflow
|
||||
component: run-airflow-migrations
|
||||
release: {{ .Release.Name }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if or .Values.airflowPodAnnotations .Values.migrateDatabaseJob.annotations }}
|
||||
annotations:
|
||||
{{- if .Values.airflowPodAnnotations }}
|
||||
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.migrateDatabaseJob.annotations }}
|
||||
{{- toYaml .Values.migrateDatabaseJob.annotations | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
spec:
|
||||
securityContext: {{ $securityContext | nindent 8 }}
|
||||
restartPolicy: OnFailure
|
||||
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
|
||||
affinity: {{- toYaml $affinity | nindent 8 }}
|
||||
{{- if .Values.schedulerName }}
|
||||
schedulerName: {{ .Values.schedulerName }}
|
||||
{{- end }}
|
||||
tolerations: {{- toYaml $tolerations | nindent 8 }}
|
||||
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
|
||||
serviceAccountName: {{ include "migrateDatabaseJob.serviceAccountName" . }}
|
||||
{{- if or .Values.registry.secretName .Values.registry.connection }}
|
||||
imagePullSecrets:
|
||||
- name: {{ template "registry_secret" . }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: run-airflow-migrations
|
||||
image: {{ template "airflow_image_for_migrations" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
securityContext: {{ $containerSecurityContext | nindent 12 }}
|
||||
{{- if $containerLifecycleHooks }}
|
||||
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.migrateDatabaseJob.command }}
|
||||
command: {{- tpl (toYaml .Values.migrateDatabaseJob.command) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.migrateDatabaseJob.args }}
|
||||
args: {{- tpl (toYaml .Values.migrateDatabaseJob.args) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.migrateDatabaseJob.applyCustomEnv }}
|
||||
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
|
||||
env: {{- include "custom_airflow_environment" . | indent 10 }}
|
||||
{{- else }}
|
||||
env:
|
||||
{{- end }}
|
||||
- name: PYTHONUNBUFFERED
|
||||
value: "1"
|
||||
{{- include "standard_airflow_environment" . | indent 10 }}
|
||||
resources: {{- toYaml .Values.migrateDatabaseJob.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
{{- include "airflow_config_mount" . | nindent 12 }}
|
||||
{{- if .Values.volumeMounts }}
|
||||
{{- toYaml .Values.volumeMounts | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.migrateDatabaseJob.extraVolumeMounts }}
|
||||
{{- tpl (toYaml .Values.migrateDatabaseJob.extraVolumeMounts) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.migrateDatabaseJob.extraContainers }}
|
||||
{{- toYaml .Values.migrateDatabaseJob.extraContainers | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: {{ template "airflow_config" . }}
|
||||
{{- if .Values.volumes }}
|
||||
{{- toYaml .Values.volumes | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.migrateDatabaseJob.extraVolumes }}
|
||||
{{- tpl (toYaml .Values.migrateDatabaseJob.extraVolumes) . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
39
helm/airflow/templates/limitrange.yaml
Normal file
39
helm/airflow/templates/limitrange.yaml
Normal file
@@ -0,0 +1,39 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Namespace LimitRange
|
||||
#################################
|
||||
{{- if .Values.limits }}
|
||||
apiVersion: v1
|
||||
kind: LimitRange
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-limit-range
|
||||
labels:
|
||||
tier: resources
|
||||
component: limitrange
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
limits: {{- toYaml .Values.limits | nindent 4 }}
|
||||
{{- end }}
|
||||
52
helm/airflow/templates/logs-persistent-volume-claim.yaml
Normal file
52
helm/airflow/templates/logs-persistent-volume-claim.yaml
Normal file
@@ -0,0 +1,52 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
######################################
|
||||
## Airflow LOGs PersistentVolumeClaim
|
||||
######################################
|
||||
{{- if and (not .Values.logs.persistence.existingClaim ) .Values.logs.persistence.enabled }}
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: {{ template "airflow_logs_volume_claim" . }}
|
||||
labels:
|
||||
tier: airflow
|
||||
component: logs-pvc
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.logs.persistence.annotations }}
|
||||
annotations: {{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
accessModes: ["ReadWriteMany"]
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.logs.persistence.size | quote }}
|
||||
{{- if .Values.logs.persistence.storageClassName }}
|
||||
{{- if (eq "-" .Values.logs.persistence.storageClassName) }}
|
||||
storageClassName: ""
|
||||
{{- else }}
|
||||
storageClassName: "{{ .Values.logs.persistence.storageClassName }}"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
212
helm/airflow/templates/pgbouncer/pgbouncer-deployment.yaml
Normal file
212
helm/airflow/templates/pgbouncer/pgbouncer-deployment.yaml
Normal file
@@ -0,0 +1,212 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Pgbouncer Deployment
|
||||
#################################
|
||||
{{- if .Values.pgbouncer.enabled }}
|
||||
{{- $nodeSelector := or .Values.pgbouncer.nodeSelector .Values.nodeSelector }}
|
||||
{{- $affinity := or .Values.pgbouncer.affinity .Values.affinity }}
|
||||
{{- $tolerations := or .Values.pgbouncer.tolerations .Values.tolerations }}
|
||||
{{- $topologySpreadConstraints := or .Values.pgbouncer.topologySpreadConstraints .Values.topologySpreadConstraints }}
|
||||
{{- $revisionHistoryLimit := or .Values.pgbouncer.revisionHistoryLimit .Values.revisionHistoryLimit }}
|
||||
{{- $securityContext := include "localPodSecurityContext" .Values.pgbouncer }}
|
||||
{{- $containerSecurityContext := include "externalContainerSecurityContext" .Values.pgbouncer }}
|
||||
{{- $containerSecurityContextMetricsExporter := include "externalContainerSecurityContext" .Values.pgbouncer.metricsExporterSidecar }}
|
||||
{{- $containerLifecycleHooks := .Values.pgbouncer.containerLifecycleHooks }}
|
||||
{{- $containerLifecycleHooksMetricsExporter := .Values.pgbouncer.metricsExporterSidecar.containerLifecycleHooks }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-pgbouncer
|
||||
labels:
|
||||
tier: airflow
|
||||
component: pgbouncer
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.pgbouncer.annotations }}
|
||||
annotations: {{- toYaml .Values.pgbouncer.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
replicas: {{ .Values.pgbouncer.replicas | default "1" }}
|
||||
{{- if $revisionHistoryLimit }}
|
||||
revisionHistoryLimit: {{ $revisionHistoryLimit }}
|
||||
{{- end }}
|
||||
strategy:
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 0
|
||||
selector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: pgbouncer
|
||||
release: {{ .Release.Name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
tier: airflow
|
||||
component: pgbouncer
|
||||
release: {{ .Release.Name }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }}
|
||||
checksum/pgbouncer-certificates-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-certificates-secret.yaml") . | sha256sum }}
|
||||
{{- if .Values.pgbouncer.podAnnotations }}
|
||||
{{- toYaml .Values.pgbouncer.podAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.pgbouncer.priorityClassName }}
|
||||
priorityClassName: {{ .Values.pgbouncer.priorityClassName }}
|
||||
{{- end }}
|
||||
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
|
||||
affinity: {{- toYaml $affinity | nindent 8 }}
|
||||
{{- if .Values.schedulerName }}
|
||||
schedulerName: {{ .Values.schedulerName }}
|
||||
{{- end }}
|
||||
tolerations: {{- toYaml $tolerations | nindent 8 }}
|
||||
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
|
||||
serviceAccountName: {{ include "pgbouncer.serviceAccountName" . }}
|
||||
securityContext: {{ $securityContext | nindent 8 }}
|
||||
restartPolicy: Always
|
||||
{{- if or .Values.registry.secretName .Values.registry.connection }}
|
||||
imagePullSecrets:
|
||||
- name: {{ template "registry_secret" . }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: pgbouncer
|
||||
image: {{ template "pgbouncer_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.pgbouncer.pullPolicy }}
|
||||
securityContext: {{ $containerSecurityContext | nindent 12 }}
|
||||
{{- if .Values.pgbouncer.command }}
|
||||
command: {{ tpl (toYaml .Values.pgbouncer.command) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.pgbouncer.args }}
|
||||
args: {{ tpl (toYaml .Values.pgbouncer.args) . | nindent 12 }}
|
||||
{{- end }}
|
||||
resources: {{- toYaml .Values.pgbouncer.resources | nindent 12 }}
|
||||
{{- with .Values.pgbouncer.env }}
|
||||
env: {{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- name: pgbouncer
|
||||
containerPort: {{ .Values.ports.pgbouncer }}
|
||||
livenessProbe:
|
||||
tcpSocket:
|
||||
port: {{ .Values.ports.pgbouncer }}
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: {{ .Values.ports.pgbouncer }}
|
||||
volumeMounts:
|
||||
- name: pgbouncer-config
|
||||
subPath: pgbouncer.ini
|
||||
mountPath: /etc/pgbouncer/pgbouncer.ini
|
||||
readOnly: true
|
||||
- name: pgbouncer-config
|
||||
subPath: users.txt
|
||||
mountPath: /etc/pgbouncer/users.txt
|
||||
readOnly: true
|
||||
{{- if .Values.pgbouncer.ssl.ca }}
|
||||
- name: pgbouncer-certificates
|
||||
subPath: root.crt
|
||||
mountPath: /etc/pgbouncer/root.crt
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if .Values.pgbouncer.ssl.cert }}
|
||||
- name: pgbouncer-certificates
|
||||
subPath: server.crt
|
||||
mountPath: /etc/pgbouncer/server.crt
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if .Values.pgbouncer.ssl.key }}
|
||||
- name: pgbouncer-certificates
|
||||
subPath: server.key
|
||||
mountPath: /etc/pgbouncer/server.key
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if .Values.volumeMounts }}
|
||||
{{- toYaml .Values.volumeMounts | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.pgbouncer.extraVolumeMounts }}
|
||||
{{- tpl (toYaml .Values.pgbouncer.extraVolumeMounts) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if $containerLifecycleHooks }}
|
||||
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
|
||||
{{- end }}
|
||||
- name: metrics-exporter
|
||||
resources: {{- toYaml .Values.pgbouncer.metricsExporterSidecar.resources | nindent 12 }}
|
||||
image: {{ template "pgbouncer_exporter_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.pgbouncerExporter.pullPolicy }}
|
||||
securityContext: {{ $containerSecurityContextMetricsExporter | nindent 12 }}
|
||||
env:
|
||||
- name: DATABASE_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ template "pgbouncer_stats_secret" . }}
|
||||
{{- if (and .Values.pgbouncer.metricsExporterSidecar.statsSecretName .Values.pgbouncer.metricsExporterSidecar.statsSecretKey) }}
|
||||
key: {{ .Values.pgbouncer.metricsExporterSidecar.statsSecretKey }}
|
||||
{{- else }}
|
||||
key: "connection"
|
||||
{{- end }}
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: {{ .Values.ports.pgbouncerScrape }}
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- pgbouncer_exporter
|
||||
- health
|
||||
initialDelaySeconds: {{ .Values.pgbouncer.metricsExporterSidecar.livenessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.pgbouncer.metricsExporterSidecar.livenessProbe.periodSeconds }}
|
||||
timeoutSeconds: {{ .Values.pgbouncer.metricsExporterSidecar.livenessProbe.timeoutSeconds }}
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- pgbouncer_exporter
|
||||
- health
|
||||
initialDelaySeconds: {{ .Values.pgbouncer.metricsExporterSidecar.readinessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.pgbouncer.metricsExporterSidecar.readinessProbe.periodSeconds }}
|
||||
timeoutSeconds: {{ .Values.pgbouncer.metricsExporterSidecar.readinessProbe.timeoutSeconds }}
|
||||
{{- if $containerLifecycleHooksMetricsExporter }}
|
||||
lifecycle: {{- tpl (toYaml $containerLifecycleHooksMetricsExporter) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.pgbouncer.extraContainers }}
|
||||
{{- toYaml .Values.pgbouncer.extraContainers | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: pgbouncer-config
|
||||
secret:
|
||||
secretName: {{ template "pgbouncer_config_secret" . }}
|
||||
{{- if or .Values.pgbouncer.ssl.ca .Values.pgbouncer.ssl.cert .Values.pgbouncer.ssl.key }}
|
||||
- name: pgbouncer-certificates
|
||||
secret:
|
||||
secretName: {{ template "pgbouncer_certificates_secret" . }}
|
||||
{{- end }}
|
||||
{{- if .Values.volumes }}
|
||||
{{- toYaml .Values.volumes | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.pgbouncer.extraVolumes }}
|
||||
{{- tpl (toYaml .Values.pgbouncer.extraVolumes) . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,77 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Pgbouncer NetworkPolicy
|
||||
#################################
|
||||
{{- $workersKedaEnabled := and .Values.workers.keda.enabled (has .Values.executor (list "CeleryExecutor" "CeleryKubernetesExecutor")) }}
|
||||
{{- $triggererEnabled := and (semverCompare ">=2.2.0" .Values.airflowVersion) .Values.triggerer.enabled }}
|
||||
{{- $triggererKedaEnabled := and $triggererEnabled .Values.triggerer.keda.enabled }}
|
||||
{{- if and .Values.pgbouncer.enabled .Values.networkPolicies.enabled }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-pgbouncer-policy
|
||||
labels:
|
||||
tier: airflow
|
||||
component: airflow-pgbouncer-policy
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: pgbouncer
|
||||
release: {{ .Release.Name }}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
{{- if or $workersKedaEnabled $triggererKedaEnabled }}
|
||||
{{- if and $workersKedaEnabled .Values.workers.keda.namespaceLabels }}
|
||||
- namespaceSelector:
|
||||
matchLabels: {{- toYaml .Values.workers.keda.namespaceLabels | nindent 10 }}
|
||||
podSelector:
|
||||
{{- else if and $triggererEnabled .Values.triggerer.keda.namespaceLabels }}
|
||||
- namespaceSelector:
|
||||
matchLabels: {{- toYaml .Values.triggerer.keda.namespaceLabels | nindent 10 }}
|
||||
podSelector:
|
||||
{{- else }}
|
||||
- podSelector:
|
||||
{{- end }}
|
||||
matchLabels:
|
||||
app: keda-operator
|
||||
{{- end }}
|
||||
{{- if .Values.pgbouncer.extraNetworkPolicies}}
|
||||
{{- toYaml .Values.pgbouncer.extraNetworkPolicies | nindent 4 }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: {{ .Values.ports.pgbouncer }}
|
||||
- protocol: TCP
|
||||
port: {{ .Values.ports.pgbouncerScrape }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,44 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Pgbouncer PodDisruptionBudget
|
||||
#################################
|
||||
{{- if and .Values.pgbouncer.enabled .Values.pgbouncer.podDisruptionBudget.enabled }}
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-pgbouncer-pdb
|
||||
labels:
|
||||
tier: airflow
|
||||
component: pgbouncer
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: pgbouncer
|
||||
release: {{ .Release.Name }}
|
||||
{{- toYaml .Values.pgbouncer.podDisruptionBudget.config | nindent 2 }}
|
||||
{{- end }}
|
||||
56
helm/airflow/templates/pgbouncer/pgbouncer-service.yaml
Normal file
56
helm/airflow/templates/pgbouncer/pgbouncer-service.yaml
Normal file
@@ -0,0 +1,56 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Pgbouncer Service
|
||||
#################################
|
||||
{{- if .Values.pgbouncer.enabled }}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-pgbouncer
|
||||
labels:
|
||||
tier: airflow
|
||||
component: pgbouncer
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: {{ .Values.ports.pgbouncerScrape | quote }}
|
||||
{{- if .Values.pgbouncer.service.extraAnnotations }}
|
||||
{{- toYaml .Values.pgbouncer.service.extraAnnotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
tier: airflow
|
||||
component: pgbouncer
|
||||
release: {{ .Release.Name }}
|
||||
ports:
|
||||
- name: pgbouncer
|
||||
protocol: TCP
|
||||
port: {{ .Values.ports.pgbouncer }}
|
||||
- name: pgbouncer-metrics
|
||||
protocol: TCP
|
||||
port: {{ .Values.ports.pgbouncerScrape }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,41 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
######################################
|
||||
## Airflow Pgbouncer ServiceAccount
|
||||
######################################
|
||||
{{- if and .Values.pgbouncer.serviceAccount.create .Values.pgbouncer.enabled }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
automountServiceAccountToken: {{ .Values.pgbouncer.serviceAccount.automountServiceAccountToken }}
|
||||
metadata:
|
||||
name: {{ include "pgbouncer.serviceAccountName" . }}
|
||||
labels:
|
||||
tier: airflow
|
||||
component: pgbouncer
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.pgbouncer.serviceAccount.annotations }}
|
||||
annotations: {{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
34
helm/airflow/templates/priorityclasses/priority-classes.yaml
Normal file
34
helm/airflow/templates/priorityclasses/priority-classes.yaml
Normal file
@@ -0,0 +1,34 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
#################################################
|
||||
## Priority classes provisioned via the chart values
|
||||
#################################################
|
||||
{{- $Global := . }}
|
||||
{{- range $e := .Values.priorityClasses }}
|
||||
---
|
||||
apiVersion: scheduling.k8s.io/v1
|
||||
kind: PriorityClass
|
||||
metadata:
|
||||
name: {{ $Global.Release.Name }}-{{ $e.name }}
|
||||
labels:
|
||||
release: {{ $Global.Release.Name }}
|
||||
preemptionPolicy: {{ default "PreemptLowerPriority" $e.preemptionPolicy }}
|
||||
value: {{ $e.value | required "value is required" }}
|
||||
{{- end }}
|
||||
44
helm/airflow/templates/rbac/pod-cleanup-role.yaml
Normal file
44
helm/airflow/templates/rbac/pod-cleanup-role.yaml
Normal file
@@ -0,0 +1,44 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Cleanup Role
|
||||
#################################
|
||||
{{- if and .Values.rbac.create .Values.cleanup.enabled }}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-cleanup-role
|
||||
labels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- "pods"
|
||||
verbs:
|
||||
- "list"
|
||||
- "delete"
|
||||
{{- end }}
|
||||
44
helm/airflow/templates/rbac/pod-cleanup-rolebinding.yaml
Normal file
44
helm/airflow/templates/rbac/pod-cleanup-rolebinding.yaml
Normal file
@@ -0,0 +1,44 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Cleanup Role Binding
|
||||
#################################
|
||||
{{- if and .Values.rbac.create .Values.cleanup.enabled }}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-cleanup-rolebinding
|
||||
labels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: {{ include "airflow.fullname" . }}-cleanup-role
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "cleanup.serviceAccountName" . }}
|
||||
namespace: "{{ .Release.Namespace }}"
|
||||
{{- end }}
|
||||
74
helm/airflow/templates/rbac/pod-launcher-role.yaml
Normal file
74
helm/airflow/templates/rbac/pod-launcher-role.yaml
Normal file
@@ -0,0 +1,74 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Pod Launcher Role
|
||||
#################################
|
||||
{{- if and .Values.rbac.create .Values.allowPodLaunching }}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
{{- if .Values.multiNamespaceMode }}
|
||||
kind: ClusterRole
|
||||
{{- else }}
|
||||
kind: Role
|
||||
{{- end }}
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-pod-launcher-role
|
||||
{{- if not .Values.multiNamespaceMode }}
|
||||
namespace: "{{ .Release.Namespace }}"
|
||||
{{- end }}
|
||||
labels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- "pods"
|
||||
verbs:
|
||||
- "create"
|
||||
- "list"
|
||||
- "get"
|
||||
- "patch"
|
||||
- "watch"
|
||||
- "delete"
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- "pods/log"
|
||||
verbs:
|
||||
- "get"
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- "pods/exec"
|
||||
verbs:
|
||||
- "create"
|
||||
- "get"
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- "events"
|
||||
verbs:
|
||||
- "list"
|
||||
{{- end }}
|
||||
64
helm/airflow/templates/rbac/pod-launcher-rolebinding.yaml
Normal file
64
helm/airflow/templates/rbac/pod-launcher-rolebinding.yaml
Normal file
@@ -0,0 +1,64 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Pod Launcher Role Binding
|
||||
#################################
|
||||
{{- if and .Values.rbac.create .Values.allowPodLaunching }}
|
||||
{{- $schedulerLaunchExecutors := list "LocalExecutor" "LocalKubernetesExecutor" "KubernetesExecutor" "CeleryKubernetesExecutor" }}
|
||||
{{- $workerLaunchExecutors := list "CeleryExecutor" "LocalKubernetesExecutor" "KubernetesExecutor" "CeleryKubernetesExecutor" }}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
{{- if .Values.multiNamespaceMode }}
|
||||
kind: ClusterRoleBinding
|
||||
{{- else }}
|
||||
kind: RoleBinding
|
||||
{{- end }}
|
||||
metadata:
|
||||
{{- if not .Values.multiNamespaceMode }}
|
||||
namespace: "{{ .Release.Namespace }}"
|
||||
{{- end }}
|
||||
name: {{ include "airflow.fullname" . }}-pod-launcher-rolebinding
|
||||
labels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
{{- if .Values.multiNamespaceMode }}
|
||||
kind: ClusterRole
|
||||
{{- else }}
|
||||
kind: Role
|
||||
{{- end }}
|
||||
name: {{ include "airflow.fullname" . }}-pod-launcher-role
|
||||
subjects:
|
||||
{{- if has .Values.executor $schedulerLaunchExecutors }}
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "scheduler.serviceAccountName" . }}
|
||||
namespace: "{{ .Release.Namespace }}"
|
||||
{{- end }}
|
||||
{{- if has .Values.executor $workerLaunchExecutors }}
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "worker.serviceAccountName" . }}
|
||||
namespace: "{{ .Release.Namespace }}"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
59
helm/airflow/templates/rbac/pod-log-reader-role.yaml
Normal file
59
helm/airflow/templates/rbac/pod-log-reader-role.yaml
Normal file
@@ -0,0 +1,59 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Pod Reader Role
|
||||
#################################
|
||||
{{- if and .Values.rbac.create (or .Values.webserver.allowPodLogReading .Values.triggerer.enabled) }}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
{{- if .Values.multiNamespaceMode }}
|
||||
kind: ClusterRole
|
||||
{{- else }}
|
||||
kind: Role
|
||||
{{- end }}
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-pod-log-reader-role
|
||||
{{- if not .Values.multiNamespaceMode }}
|
||||
namespace: "{{ .Release.Namespace }}"
|
||||
{{- end }}
|
||||
labels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- "pods"
|
||||
verbs:
|
||||
- "list"
|
||||
- "get"
|
||||
- "watch"
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- "pods/log"
|
||||
verbs:
|
||||
- "get"
|
||||
- "list"
|
||||
{{- end }}
|
||||
62
helm/airflow/templates/rbac/pod-log-reader-rolebinding.yaml
Normal file
62
helm/airflow/templates/rbac/pod-log-reader-rolebinding.yaml
Normal file
@@ -0,0 +1,62 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Pod Reader Role Binding
|
||||
#################################
|
||||
{{- if and .Values.rbac.create (or .Values.webserver.allowPodLogReading .Values.triggerer.enabled) }}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
{{- if .Values.multiNamespaceMode }}
|
||||
kind: ClusterRoleBinding
|
||||
{{- else }}
|
||||
kind: RoleBinding
|
||||
{{- end }}
|
||||
metadata:
|
||||
{{- if not .Values.multiNamespaceMode }}
|
||||
namespace: "{{ .Release.Namespace }}"
|
||||
{{- end }}
|
||||
name: {{ include "airflow.fullname" . }}-pod-log-reader-rolebinding
|
||||
labels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
{{- if .Values.multiNamespaceMode }}
|
||||
kind: ClusterRole
|
||||
{{- else }}
|
||||
kind: Role
|
||||
{{- end }}
|
||||
name: {{ include "airflow.fullname" . }}-pod-log-reader-role
|
||||
subjects:
|
||||
{{- if .Values.webserver.allowPodLogReading }}
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "webserver.serviceAccountName" . }}
|
||||
namespace: "{{ .Release.Namespace }}"
|
||||
{{- end }}
|
||||
{{- if .Values.triggerer.enabled }}
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "triggerer.serviceAccountName" . }}
|
||||
namespace: "{{ .Release.Namespace }}"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,88 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow SCC Role Binding
|
||||
#################################
|
||||
{{- if and .Values.rbac.create .Values.rbac.createSCCRoleBinding }}
|
||||
{{- $hasWorkers := has .Values.executor (list "CeleryExecutor" "LocalKubernetesExecutor" "KubernetesExecutor" "CeleryKubernetesExecutor") }}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
{{- if .Values.multiNamespaceMode }}
|
||||
kind: ClusterRoleBinding
|
||||
{{- else }}
|
||||
kind: RoleBinding
|
||||
{{- end }}
|
||||
metadata:
|
||||
{{- if not .Values.multiNamespaceMode }}
|
||||
namespace: "{{ .Release.Namespace }}"
|
||||
{{- end }}
|
||||
name: {{ include "airflow.fullname" . }}-scc-rolebinding
|
||||
labels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: system:openshift:scc:anyuid
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "webserver.serviceAccountName" . }}
|
||||
namespace: "{{ .Release.Namespace }}"
|
||||
{{- if $hasWorkers }}
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "worker.serviceAccountName" . }}
|
||||
namespace: "{{ .Release.Namespace }}"
|
||||
{{- end }}
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "scheduler.serviceAccountName" . }}
|
||||
namespace: "{{ .Release.Namespace }}"
|
||||
{{- if and .Values.statsd.enabled }}
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "statsd.serviceAccountName" . }}
|
||||
namespace: "{{ .Release.Namespace }}"
|
||||
{{- end }}
|
||||
{{- if and .Values.flower.enabled (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")) }}
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "flower.serviceAccountName" . }}
|
||||
namespace: "{{ .Release.Namespace }}"
|
||||
{{- end }}
|
||||
{{- if and (semverCompare ">=2.2.0" .Values.airflowVersion) }}
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "triggerer.serviceAccountName" . }}
|
||||
namespace: "{{ .Release.Namespace }}"
|
||||
{{- end }}
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "migrateDatabaseJob.serviceAccountName" . }}
|
||||
namespace: "{{ .Release.Namespace }}"
|
||||
{{- if .Values.webserver.defaultUser.enabled }}
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "createUserJob.serviceAccountName" . }}
|
||||
namespace: "{{ .Release.Namespace }}"
|
||||
{{- end }}
|
||||
{{- if and .Values.cleanup.enabled }}
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "cleanup.serviceAccountName" . }}
|
||||
namespace: "{{ .Release.Namespace }}"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
65
helm/airflow/templates/redis/redis-networkpolicy.yaml
Normal file
65
helm/airflow/templates/redis/redis-networkpolicy.yaml
Normal file
@@ -0,0 +1,65 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Redis NetworkPolicy
|
||||
#################################
|
||||
{{- if and .Values.redis.enabled .Values.networkPolicies.enabled (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")) }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-redis-policy
|
||||
labels:
|
||||
tier: airflow
|
||||
component: redis-policy
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: redis
|
||||
release: {{ .Release.Name }}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: worker
|
||||
release: {{ .Release.Name }}
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: scheduler
|
||||
release: {{ .Release.Name }}
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: flower
|
||||
release: {{ .Release.Name }}
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: {{ .Values.ports.redisDB }}
|
||||
{{- end }}
|
||||
48
helm/airflow/templates/redis/redis-service.yaml
Normal file
48
helm/airflow/templates/redis/redis-service.yaml
Normal file
@@ -0,0 +1,48 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Redis Service
|
||||
#################################
|
||||
{{- if and .Values.redis.enabled (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")) }}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-redis
|
||||
labels:
|
||||
tier: airflow
|
||||
component: redis
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
tier: airflow
|
||||
component: redis
|
||||
release: {{ .Release.Name }}
|
||||
ports:
|
||||
- name: redis-db
|
||||
protocol: TCP
|
||||
port: {{ .Values.ports.redisDB }}
|
||||
targetPort: {{ .Values.ports.redisDB }}
|
||||
{{- end }}
|
||||
41
helm/airflow/templates/redis/redis-serviceaccount.yaml
Normal file
41
helm/airflow/templates/redis/redis-serviceaccount.yaml
Normal file
@@ -0,0 +1,41 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
######################################
|
||||
## Airflow Redis ServiceAccount
|
||||
######################################
|
||||
{{- if and .Values.redis.enabled .Values.redis.serviceAccount.create (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")) }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
automountServiceAccountToken: {{ .Values.redis.serviceAccount.automountServiceAccountToken }}
|
||||
metadata:
|
||||
name: {{ include "redis.serviceAccountName" . }}
|
||||
labels:
|
||||
tier: airflow
|
||||
component: redis
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.redis.serviceAccount.annotations }}
|
||||
annotations: {{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
126
helm/airflow/templates/redis/redis-statefulset.yaml
Normal file
126
helm/airflow/templates/redis/redis-statefulset.yaml
Normal file
@@ -0,0 +1,126 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Redis StatefulSet
|
||||
#################################
|
||||
{{- if and .Values.redis.enabled (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")) }}
|
||||
{{- $nodeSelector := or .Values.redis.nodeSelector .Values.nodeSelector }}
|
||||
{{- $affinity := or .Values.redis.affinity .Values.affinity }}
|
||||
{{- $tolerations := or .Values.redis.tolerations .Values.tolerations }}
|
||||
{{- $topologySpreadConstraints := or .Values.redis.topologySpreadConstraints .Values.topologySpreadConstraints }}
|
||||
{{- $securityContext := include "localPodSecurityContext" .Values.redis }}
|
||||
{{- $containerSecurityContext := include "externalContainerSecurityContext" .Values.redis }}
|
||||
{{- $containerLifecycleHooks := .Values.redis.containerLifecycleHooks }}
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-redis
|
||||
labels:
|
||||
tier: airflow
|
||||
component: redis
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
serviceName: {{ include "airflow.fullname" . }}-redis
|
||||
selector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: redis
|
||||
release: {{ .Release.Name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
tier: airflow
|
||||
component: redis
|
||||
release: {{ .Release.Name }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if or .Values.redis.safeToEvict .Values.redis.podAnnotations }}
|
||||
annotations:
|
||||
{{- if .Values.redis.podAnnotations }}
|
||||
{{- toYaml .Values.redis.podAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.redis.safeToEvict }}
|
||||
cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
spec:
|
||||
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
|
||||
affinity: {{- toYaml $affinity | nindent 8 }}
|
||||
tolerations: {{- toYaml $tolerations | nindent 8 }}
|
||||
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
|
||||
serviceAccountName: {{ include "redis.serviceAccountName" . }}
|
||||
{{- if .Values.schedulerName }}
|
||||
schedulerName: {{ .Values.schedulerName }}
|
||||
{{- end }}
|
||||
{{- if or .Values.registry.secretName .Values.registry.connection }}
|
||||
imagePullSecrets:
|
||||
- name: {{ template "registry_secret" . }}
|
||||
{{- end }}
|
||||
securityContext: {{ $securityContext | nindent 8 }}
|
||||
containers:
|
||||
- name: redis
|
||||
image: {{ template "redis_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.redis.pullPolicy }}
|
||||
securityContext: {{ $containerSecurityContext | nindent 12 }}
|
||||
{{- if $containerLifecycleHooks }}
|
||||
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
|
||||
{{- end }}
|
||||
command: ["/bin/sh"]
|
||||
resources: {{- toYaml .Values.redis.resources | nindent 12 }}
|
||||
args: ["-c", "redis-server --requirepass ${REDIS_PASSWORD}"]
|
||||
ports:
|
||||
- name: redis-db
|
||||
containerPort: {{ .Values.ports.redisDB }}
|
||||
volumeMounts:
|
||||
- name: redis-db
|
||||
mountPath: /data
|
||||
env:
|
||||
- name: REDIS_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ template "redis_password_secret" . }}
|
||||
key: password
|
||||
{{- if not .Values.redis.persistence.enabled }}
|
||||
volumes:
|
||||
- name: redis-db
|
||||
emptyDir: {}
|
||||
{{- else }}
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: redis-db
|
||||
{{- if .Values.redis.persistence.annotations }}
|
||||
annotations: {{- toYaml .Values.redis.persistence.annotations | nindent 10 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.redis.persistence.storageClassName }}
|
||||
storageClassName: {{ .Values.redis.persistence.storageClassName }}
|
||||
{{- end }}
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.redis.persistence.size }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
39
helm/airflow/templates/resourcequota.yaml
Normal file
39
helm/airflow/templates/resourcequota.yaml
Normal file
@@ -0,0 +1,39 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Namespace ResourceQuota
|
||||
#################################
|
||||
{{- if .Values.quotas }}
|
||||
apiVersion: v1
|
||||
kind: ResourceQuota
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-resource-quota
|
||||
labels:
|
||||
tier: resources
|
||||
component: resourcequota
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
hard: {{- toYaml .Values.quotas | nindent 4 }}
|
||||
{{- end }}
|
||||
339
helm/airflow/templates/scheduler/scheduler-deployment.yaml
Normal file
339
helm/airflow/templates/scheduler/scheduler-deployment.yaml
Normal file
@@ -0,0 +1,339 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Scheduler Deployment/StatefulSet
|
||||
#################################
|
||||
|
||||
# Are we using a local executor?
|
||||
{{- $local := contains "Local" .Values.executor }}
|
||||
# Is persistence enabled on the _workers_?
|
||||
# This is important because in $local mode, the scheduler assumes the role of the worker
|
||||
{{- $persistence := .Values.workers.persistence.enabled }}
|
||||
# If we're using a StatefulSet
|
||||
{{- $stateful := and $local $persistence }}
|
||||
# We can skip DAGs mounts on scheduler if dagProcessor is enabled, except with $local mode
|
||||
{{- $localOrDagProcessorDisabled := or (not .Values.dagProcessor.enabled) $local }}
|
||||
# If we're using elasticsearch logging
|
||||
{{- $elasticsearch := .Values.elasticsearch.enabled }}
|
||||
{{- $nodeSelector := or .Values.scheduler.nodeSelector .Values.nodeSelector }}
|
||||
{{- $affinity := or .Values.scheduler.affinity .Values.affinity }}
|
||||
{{- $tolerations := or .Values.scheduler.tolerations .Values.tolerations }}
|
||||
{{- $topologySpreadConstraints := or .Values.scheduler.topologySpreadConstraints .Values.topologySpreadConstraints }}
|
||||
{{- $revisionHistoryLimit := or .Values.scheduler.revisionHistoryLimit .Values.revisionHistoryLimit }}
|
||||
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.scheduler) }}
|
||||
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.scheduler) }}
|
||||
{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list . .Values.scheduler.waitForMigrations) }}
|
||||
{{- $containerSecurityContextLogGroomerSidecar := include "containerSecurityContext" (list . .Values.scheduler.logGroomerSidecar) }}
|
||||
{{- $containerLifecycleHooks := or .Values.scheduler.containerLifecycleHooks .Values.containerLifecycleHooks }}
|
||||
{{- $containerLifecycleHooksWaitForMigrations := or .Values.scheduler.waitForMigrations.containerLifecycleHooks .Values.containerLifecycleHooks }}
|
||||
{{- $containerLifecycleHooksLogGroomerSidecar := or .Values.scheduler.logGroomerSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }}
|
||||
apiVersion: apps/v1
|
||||
kind: {{ if $stateful }}StatefulSet{{ else }}Deployment{{ end }}
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-scheduler
|
||||
labels:
|
||||
tier: airflow
|
||||
component: scheduler
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
executor: {{ .Values.executor }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.annotations }}
|
||||
annotations: {{- toYaml .Values.scheduler.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if $stateful }}
|
||||
serviceName: {{ include "airflow.fullname" . }}-scheduler
|
||||
{{- end }}
|
||||
replicas: {{ .Values.scheduler.replicas }}
|
||||
{{- if $revisionHistoryLimit }}
|
||||
revisionHistoryLimit: {{ $revisionHistoryLimit }}
|
||||
{{- end }}
|
||||
{{- if and $stateful .Values.scheduler.updateStrategy }}
|
||||
updateStrategy: {{- toYaml .Values.scheduler.updateStrategy | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if and (not $stateful) .Values.scheduler.strategy }}
|
||||
strategy: {{- toYaml .Values.scheduler.strategy | nindent 4 }}
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: scheduler
|
||||
release: {{ .Release.Name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
tier: airflow
|
||||
component: scheduler
|
||||
release: {{ .Release.Name }}
|
||||
{{- if or (.Values.labels) (.Values.scheduler.labels) }}
|
||||
{{- mustMerge .Values.scheduler.labels .Values.labels | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }}
|
||||
checksum/result-backend-secret: {{ include (print $.Template.BasePath "/secrets/result-backend-connection-secret.yaml") . | sha256sum }}
|
||||
checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }}
|
||||
checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }}
|
||||
checksum/extra-configmaps: {{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }}
|
||||
checksum/extra-secrets: {{ include (print $.Template.BasePath "/secrets/extra-secrets.yaml") . | sha256sum }}
|
||||
{{- if .Values.scheduler.safeToEvict }}
|
||||
cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
|
||||
{{- end }}
|
||||
{{- if .Values.airflowPodAnnotations }}
|
||||
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.podAnnotations }}
|
||||
{{- toYaml .Values.scheduler.podAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.scheduler.priorityClassName }}
|
||||
priorityClassName: {{ .Values.scheduler.priorityClassName }}
|
||||
{{- end }}
|
||||
{{- if .Values.schedulerName }}
|
||||
schedulerName: {{ .Values.schedulerName }}
|
||||
{{- end }}
|
||||
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
|
||||
affinity:
|
||||
{{- if $affinity }}
|
||||
{{- toYaml $affinity | nindent 8 }}
|
||||
{{- else }}
|
||||
podAntiAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- podAffinityTerm:
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
component: scheduler
|
||||
topologyKey: kubernetes.io/hostname
|
||||
weight: 100
|
||||
{{- end }}
|
||||
tolerations: {{- toYaml $tolerations | nindent 8 }}
|
||||
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
|
||||
restartPolicy: Always
|
||||
terminationGracePeriodSeconds: 10
|
||||
serviceAccountName: {{ include "scheduler.serviceAccountName" . }}
|
||||
securityContext: {{ $securityContext | nindent 8 }}
|
||||
{{- if or .Values.registry.secretName .Values.registry.connection }}
|
||||
imagePullSecrets:
|
||||
- name: {{ template "registry_secret" . }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.hostAliases }}
|
||||
hostAliases: {{- toYaml .Values.scheduler.hostAliases | nindent 8 }}
|
||||
{{- end }}
|
||||
initContainers:
|
||||
{{- if .Values.scheduler.waitForMigrations.enabled }}
|
||||
- name: wait-for-airflow-migrations
|
||||
resources: {{- toYaml .Values.scheduler.resources | nindent 12 }}
|
||||
image: {{ template "airflow_image_for_migrations" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
securityContext: {{ $containerSecurityContextWaitForMigrations | nindent 12 }}
|
||||
{{- if $containerLifecycleHooksWaitForMigrations }}
|
||||
lifecycle: {{- tpl (toYaml $containerLifecycleHooksWaitForMigrations) . | nindent 12 }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
{{- include "airflow_config_mount" . | nindent 12 }}
|
||||
{{- if .Values.volumeMounts }}
|
||||
{{- toYaml .Values.volumeMounts | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.extraVolumeMounts }}
|
||||
{{- tpl (toYaml .Values.scheduler.extraVolumeMounts) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
|
||||
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
args: {{- include "wait-for-migrations-command" . | indent 10 }}
|
||||
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
|
||||
env:
|
||||
{{- include "custom_airflow_environment" . | indent 10 }}
|
||||
{{- include "standard_airflow_environment" . | indent 10 }}
|
||||
{{- if .Values.scheduler.waitForMigrations.env }}
|
||||
{{- tpl (toYaml .Values.scheduler.waitForMigrations.env) $ | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if and $localOrDagProcessorDisabled .Values.dags.gitSync.enabled }}
|
||||
{{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.extraInitContainers }}
|
||||
{{- toYaml .Values.scheduler.extraInitContainers | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
# Always run the main scheduler container.
|
||||
- name: scheduler
|
||||
image: {{ template "airflow_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
securityContext: {{ $containerSecurityContext | nindent 12 }}
|
||||
{{- if $containerLifecycleHooks }}
|
||||
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.command }}
|
||||
command: {{ tpl (toYaml .Values.scheduler.command) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.args }}
|
||||
args: {{ tpl (toYaml .Values.scheduler.args) . | nindent 12 }}
|
||||
{{- end }}
|
||||
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
|
||||
env:
|
||||
{{- include "custom_airflow_environment" . | indent 10 }}
|
||||
{{- include "standard_airflow_environment" . | indent 10 }}
|
||||
{{- include "container_extra_envs" (list . .Values.scheduler.env) | indent 10 }}
|
||||
livenessProbe:
|
||||
initialDelaySeconds: {{ .Values.scheduler.livenessProbe.initialDelaySeconds }}
|
||||
timeoutSeconds: {{ .Values.scheduler.livenessProbe.timeoutSeconds }}
|
||||
failureThreshold: {{ .Values.scheduler.livenessProbe.failureThreshold }}
|
||||
periodSeconds: {{ .Values.scheduler.livenessProbe.periodSeconds }}
|
||||
exec:
|
||||
command:
|
||||
{{- if .Values.scheduler.livenessProbe.command }}
|
||||
{{- toYaml .Values.scheduler.livenessProbe.command | nindent 16 }}
|
||||
{{- else }}
|
||||
{{- include "scheduler_liveness_check_command" . | indent 14 }}
|
||||
{{- end }}
|
||||
startupProbe:
|
||||
timeoutSeconds: {{ .Values.scheduler.startupProbe.timeoutSeconds }}
|
||||
failureThreshold: {{ .Values.scheduler.startupProbe.failureThreshold }}
|
||||
periodSeconds: {{ .Values.scheduler.startupProbe.periodSeconds }}
|
||||
exec:
|
||||
command:
|
||||
{{- if .Values.scheduler.startupProbe.command }}
|
||||
{{- toYaml .Values.scheduler.startupProbe.command | nindent 16 }}
|
||||
{{- else }}
|
||||
{{- include "scheduler_startup_check_command" . | indent 14 }}
|
||||
{{- end }}
|
||||
{{- if and $local (not $elasticsearch) }}
|
||||
# Serve logs if we're in local mode and we don't have elasticsearch enabled.
|
||||
ports:
|
||||
- name: worker-logs
|
||||
containerPort: {{ .Values.ports.workerLogs }}
|
||||
{{- end }}
|
||||
resources: {{- toYaml .Values.scheduler.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
{{- if semverCompare ">=1.10.12" .Values.airflowVersion }}
|
||||
- name: config
|
||||
mountPath: {{ include "airflow_pod_template_file" . }}/pod_template_file.yaml
|
||||
subPath: pod_template_file.yaml
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
- name: logs
|
||||
mountPath: {{ template "airflow_logs" . }}
|
||||
{{- include "airflow_config_mount" . | nindent 12 }}
|
||||
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
|
||||
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if and $localOrDagProcessorDisabled (or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled) }}
|
||||
{{- include "airflow_dags_mount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.volumeMounts }}
|
||||
{{- toYaml .Values.volumeMounts | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.extraVolumeMounts }}
|
||||
{{- tpl (toYaml .Values.scheduler.extraVolumeMounts) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if and $localOrDagProcessorDisabled .Values.dags.gitSync.enabled }}
|
||||
{{- include "git_sync_container" . | indent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.logGroomerSidecar.enabled }}
|
||||
- name: scheduler-log-groomer
|
||||
resources: {{- toYaml .Values.scheduler.logGroomerSidecar.resources | nindent 12 }}
|
||||
image: {{ template "airflow_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
securityContext: {{ $containerSecurityContextLogGroomerSidecar | nindent 12 }}
|
||||
{{- if $containerLifecycleHooksLogGroomerSidecar }}
|
||||
lifecycle: {{- tpl (toYaml $containerLifecycleHooksLogGroomerSidecar) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.logGroomerSidecar.command }}
|
||||
command: {{ tpl (toYaml .Values.scheduler.logGroomerSidecar.command) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.logGroomerSidecar.args }}
|
||||
args: {{- tpl (toYaml .Values.scheduler.logGroomerSidecar.args) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.logGroomerSidecar.retentionDays }}
|
||||
env:
|
||||
- name: AIRFLOW__LOG_RETENTION_DAYS
|
||||
value: "{{ .Values.scheduler.logGroomerSidecar.retentionDays }}"
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
- name: logs
|
||||
mountPath: {{ template "airflow_logs" . }}
|
||||
{{- if .Values.volumeMounts }}
|
||||
{{- toYaml .Values.volumeMounts | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.extraVolumeMounts }}
|
||||
{{- tpl (toYaml .Values.scheduler.extraVolumeMounts) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
|
||||
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.extraContainers }}
|
||||
{{- toYaml .Values.scheduler.extraContainers | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: {{ template "airflow_config" . }}
|
||||
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
|
||||
- name: webserver-config
|
||||
configMap:
|
||||
name: {{ template "airflow_webserver_config_configmap_name" . }}
|
||||
{{- end }}
|
||||
{{- if $localOrDagProcessorDisabled }}
|
||||
{{- if .Values.dags.persistence.enabled }}
|
||||
- name: dags
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ template "airflow_dags_volume_claim" . }}
|
||||
{{- else if .Values.dags.gitSync.enabled }}
|
||||
- name: dags
|
||||
emptyDir: {}
|
||||
{{- if .Values.dags.gitSync.sshKeySecret }}
|
||||
{{- include "git_sync_ssh_key_volume" . | indent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.volumes }}
|
||||
{{- toYaml .Values.volumes | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.extraVolumes }}
|
||||
{{- tpl (toYaml .Values.scheduler.extraVolumes) . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.logs.persistence.enabled }}
|
||||
- name: logs
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ template "airflow_logs_volume_claim" . }}
|
||||
{{- else if not $stateful }}
|
||||
- name: logs
|
||||
emptyDir: {}
|
||||
{{- else }}
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: logs
|
||||
{{- if .Values.workers.persistence.annotations }}
|
||||
annotations: {{- toYaml .Values.workers.persistence.annotations | nindent 10 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.workers.persistence.storageClassName }}
|
||||
storageClassName: {{ .Values.workers.persistence.storageClassName }}
|
||||
{{- end }}
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.workers.persistence.size }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,57 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Scheduler NetworkPolicy
|
||||
#################################
|
||||
{{- if .Values.networkPolicies.enabled }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-scheduler-policy
|
||||
labels:
|
||||
tier: airflow
|
||||
component: airflow-scheduler-policy
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- if or (.Values.labels) (.Values.scheduler.labels) }}
|
||||
{{- mustMerge .Values.scheduler.labels .Values.labels | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: scheduler
|
||||
release: {{ .Release.Name }}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
{{- if eq .Values.executor "LocalExecutor" }}
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: webserver
|
||||
release: {{ .Release.Name }}
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: {{ .Values.ports.workerLogs }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,44 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Scheduler PodDisruptionBudget
|
||||
#################################
|
||||
{{- if .Values.scheduler.podDisruptionBudget.enabled }}
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-scheduler-pdb
|
||||
labels:
|
||||
tier: airflow
|
||||
component: scheduler
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- if or (.Values.labels) (.Values.scheduler.labels) }}
|
||||
{{- mustMerge .Values.scheduler.labels .Values.labels | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: scheduler
|
||||
release: {{ .Release.Name }}
|
||||
{{- toYaml .Values.scheduler.podDisruptionBudget.config | nindent 2 }}
|
||||
{{- end }}
|
||||
48
helm/airflow/templates/scheduler/scheduler-service.yaml
Normal file
48
helm/airflow/templates/scheduler/scheduler-service.yaml
Normal file
@@ -0,0 +1,48 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Scheduler Service
|
||||
#################################
|
||||
{{- if or (eq .Values.executor "LocalExecutor") (eq .Values.executor "LocalKubernetesExecutor") }}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-scheduler
|
||||
labels:
|
||||
tier: airflow
|
||||
component: scheduler
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- if or (.Values.labels) (.Values.scheduler.labels) }}
|
||||
{{- mustMerge .Values.scheduler.labels .Values.labels | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
clusterIP: None
|
||||
selector:
|
||||
tier: airflow
|
||||
component: scheduler
|
||||
release: {{ .Release.Name }}
|
||||
ports:
|
||||
- name: task-logs
|
||||
protocol: TCP
|
||||
port: {{ .Values.ports.workerLogs }}
|
||||
targetPort: {{ .Values.ports.workerLogs }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,44 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Scheduler ServiceAccount
|
||||
#################################
|
||||
{{- if .Values.scheduler.serviceAccount.create }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
automountServiceAccountToken: {{ .Values.scheduler.serviceAccount.automountServiceAccountToken }}
|
||||
metadata:
|
||||
name: {{ include "scheduler.serviceAccountName" . }}
|
||||
labels:
|
||||
tier: airflow
|
||||
component: scheduler
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- if or (.Values.labels) (.Values.scheduler.labels) }}
|
||||
{{- mustMerge .Values.scheduler.labels .Values.labels | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.scheduler.serviceAccount.annotations }}
|
||||
annotations:
|
||||
{{- range $key, $value := . }}
|
||||
{{- printf "%s: %s" $key (tpl $value $ | quote) | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
44
helm/airflow/templates/secrets/elasticsearch-secret.yaml
Normal file
44
helm/airflow/templates/secrets/elasticsearch-secret.yaml
Normal file
@@ -0,0 +1,44 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Elasticsearch Secret
|
||||
#################################
|
||||
{{- if (and .Values.elasticsearch.enabled (not .Values.elasticsearch.secretName)) }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-elasticsearch
|
||||
labels:
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
type: Opaque
|
||||
data:
|
||||
{{- with .Values.elasticsearch.connection }}
|
||||
{{- if and .user .pass }}
|
||||
connection: {{ urlJoin (dict "scheme" (default "http" .scheme) "userinfo" (printf "%s:%s" (.user | urlquery) (.pass | urlquery)) "host" (printf "%s:%s" .host ((default 9200 .port) | toString) ) ) | b64enc | quote }}
|
||||
{{- else }}
|
||||
connection: {{ urlJoin (dict "scheme" (default "http" .scheme) "host" (printf "%s:%s" .host ((default 9200 .port) | toString))) | b64enc | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
62
helm/airflow/templates/secrets/extra-secrets.yaml
Normal file
62
helm/airflow/templates/secrets/extra-secrets.yaml
Normal file
@@ -0,0 +1,62 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
#################################################
|
||||
## Extra Secrets provisioned via the chart values
|
||||
#################################################
|
||||
{{- $Global := . }}
|
||||
{{- range $secretName, $secretContent := .Values.extraSecrets }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ tpl $secretName $Global | quote }}
|
||||
labels:
|
||||
release: {{ $Global.Release.Name }}
|
||||
chart: "{{ $Global.Chart.Name }}-{{ $Global.Chart.Version }}"
|
||||
heritage: {{ $Global.Release.Service }}
|
||||
{{- with $Global.Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if $secretContent.labels }}
|
||||
{{- toYaml $secretContent.labels | nindent 4 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
"helm.sh/hook": "pre-install,pre-upgrade"
|
||||
"helm.sh/hook-delete-policy": "before-hook-creation"
|
||||
"helm.sh/hook-weight": "0"
|
||||
{{- if $secretContent.annotations }}
|
||||
{{- toYaml $secretContent.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if $secretContent.type }}
|
||||
type: {{ $secretContent.type }}
|
||||
{{- end }}
|
||||
{{- if $secretContent.data }}
|
||||
data:
|
||||
{{- with $secretContent.data }}
|
||||
{{- tpl . $Global | nindent 2 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if $secretContent.stringData }}
|
||||
stringData:
|
||||
{{- with $secretContent.stringData }}
|
||||
{{- tpl . $Global | nindent 2 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
44
helm/airflow/templates/secrets/fernetkey-secret.yaml
Normal file
44
helm/airflow/templates/secrets/fernetkey-secret.yaml
Normal file
@@ -0,0 +1,44 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Fernet Key Secret
|
||||
#################################
|
||||
{{- if not .Values.fernetKeySecretName }}
|
||||
{{- $generated_fernet_key := (randAlphaNum 32 | b64enc) }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-fernet-key
|
||||
labels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
"helm.sh/hook": "pre-install"
|
||||
"helm.sh/hook-delete-policy": "before-hook-creation"
|
||||
"helm.sh/hook-weight": "0"
|
||||
type: Opaque
|
||||
data:
|
||||
fernet-key: {{ (default $generated_fernet_key .Values.fernetKey) | b64enc | quote }}
|
||||
{{- end }}
|
||||
38
helm/airflow/templates/secrets/flower-secret.yaml
Normal file
38
helm/airflow/templates/secrets/flower-secret.yaml
Normal file
@@ -0,0 +1,38 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Flower Secret
|
||||
#################################
|
||||
{{- if (and (not .Values.flower.secretName) .Values.flower.username .Values.flower.password) }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-flower
|
||||
labels:
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
type: Opaque
|
||||
data:
|
||||
basicAuth: {{ (printf "%s:%s" .Values.flower.username .Values.flower.password) | b64enc | quote }}
|
||||
{{- end }}
|
||||
40
helm/airflow/templates/secrets/kerberos-keytab-secret.yaml
Normal file
40
helm/airflow/templates/secrets/kerberos-keytab-secret.yaml
Normal file
@@ -0,0 +1,40 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Kerberos Secret
|
||||
#################################
|
||||
{{- if .Values.kerberos.keytabBase64Content }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ include "kerberos_keytab_secret" . | quote }}
|
||||
labels:
|
||||
tier: airflow
|
||||
component: webserver
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
type: Opaque
|
||||
data:
|
||||
kerberos.keytab: {{ .Values.kerberos.keytabBase64Content }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,53 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Metadata Secret
|
||||
#################################
|
||||
{{- if not .Values.data.metadataSecretName }}
|
||||
{{- $defaultMetadataHost := .Values.postgresql.nameOverride | default (printf "%s-%s.%s" .Release.Name "postgresql" .Release.Namespace) }}
|
||||
{{- $metadataHost := .Values.data.metadataConnection.host | default $defaultMetadataHost }}
|
||||
{{- $pgbouncerHost := (printf "%s-%s.%s" .Release.Name "pgbouncer" .Release.Namespace) }}
|
||||
{{- $host := ternary $pgbouncerHost $metadataHost .Values.pgbouncer.enabled }}
|
||||
{{- $port := ((ternary .Values.ports.pgbouncer .Values.data.metadataConnection.port .Values.pgbouncer.enabled) | toString) }}
|
||||
{{- $database := (ternary (printf "%s-%s" .Release.Name "metadata") .Values.data.metadataConnection.db .Values.pgbouncer.enabled) }}
|
||||
{{- $query := ternary (printf "sslmode=%s" .Values.data.metadataConnection.sslmode) "" (eq .Values.data.metadataConnection.protocol "postgresql") }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-metadata
|
||||
labels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
type: Opaque
|
||||
data:
|
||||
{{- with .Values.data.metadataConnection }}
|
||||
connection: {{ urlJoin (dict "scheme" .protocol "userinfo" (printf "%s:%s" (.user | urlquery) (.pass | urlquery) ) "host" (printf "%s:%s" $host $port) "path" (printf "/%s" $database) "query" $query) | b64enc | quote }}
|
||||
{{- end }}
|
||||
{{- if and .Values.workers.keda.enabled .Values.pgbouncer.enabled (not .Values.workers.keda.usePgbouncer) }}
|
||||
{{- with .Values.data.metadataConnection }}
|
||||
kedaConnection: {{ urlJoin (dict "scheme" .protocol "userinfo" (printf "%s:%s" (.user | urlquery) (.pass | urlquery) ) "host" (printf "%s:%s" $metadataHost $port) "path" (printf "/%s" $database) "query" $query) | b64enc | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,46 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Pgbouncer Certificate Secret
|
||||
#################################
|
||||
{{- if or .Values.pgbouncer.ssl.ca .Values.pgbouncer.ssl.cert .Values.pgbouncer.ssl.key }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ template "pgbouncer_certificates_secret" . }}
|
||||
labels:
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
type: Opaque
|
||||
data:
|
||||
{{- if .Values.pgbouncer.ssl.ca }}
|
||||
root.crt: {{ .Values.pgbouncer.ssl.ca | b64enc }}
|
||||
{{- end }}
|
||||
{{- if .Values.pgbouncer.ssl.cert }}
|
||||
server.crt: {{ .Values.pgbouncer.ssl.cert | b64enc }}
|
||||
{{- end }}
|
||||
{{- if .Values.pgbouncer.ssl.key }}
|
||||
server.key: {{ .Values.pgbouncer.ssl.key | b64enc }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
41
helm/airflow/templates/secrets/pgbouncer-config-secret.yaml
Normal file
41
helm/airflow/templates/secrets/pgbouncer-config-secret.yaml
Normal file
@@ -0,0 +1,41 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Pgbouncer Config Secret
|
||||
#################################
|
||||
{{- if (and .Values.pgbouncer.enabled (not .Values.pgbouncer.configSecretName)) }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ template "pgbouncer_config_secret" . }}
|
||||
labels:
|
||||
tier: airflow
|
||||
component: pgbouncer
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
type: Opaque
|
||||
data:
|
||||
pgbouncer.ini: {{ include "pgbouncer_config" . | b64enc }}
|
||||
users.txt: {{ include "pgbouncer_users" . | b64enc }}
|
||||
{{- end }}
|
||||
40
helm/airflow/templates/secrets/pgbouncer-stats-secret.yaml
Normal file
40
helm/airflow/templates/secrets/pgbouncer-stats-secret.yaml
Normal file
@@ -0,0 +1,40 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Pgbouncer Stats Secret
|
||||
#################################
|
||||
{{- if (and .Values.pgbouncer.enabled (not .Values.pgbouncer.metricsExporterSidecar.statsSecretName)) }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ template "pgbouncer_stats_secret" . }}
|
||||
labels:
|
||||
tier: airflow
|
||||
component: pgbouncer
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
type: Opaque
|
||||
data:
|
||||
connection: {{ urlJoin (dict "scheme" "postgresql" "userinfo" (printf "%s:%s" (.Values.data.metadataConnection.user | urlquery) (.Values.data.metadataConnection.pass | urlquery) ) "host" (printf "127.0.0.1:%s" (.Values.ports.pgbouncer | toString)) "path" "/pgbouncer" "query" (printf "sslmode=%s" (.Values.pgbouncer.metricsExporterSidecar.sslmode | toString ))) | b64enc | quote }}
|
||||
{{- end }}
|
||||
83
helm/airflow/templates/secrets/redis-secrets.yaml
Normal file
83
helm/airflow/templates/secrets/redis-secrets.yaml
Normal file
@@ -0,0 +1,83 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
# We will create these secrets (if necessary) _even if_ we aren't
|
||||
# currently using CeleryExecutor or CeleryKubernetesExecutor. As we are
|
||||
# relying on the "pre-install" hack to prevent changing randomly generated passwords,
|
||||
# updating the executor later doesn't give us the opportunity to deploy them
|
||||
# when we need them. We will always deploy them defensively to make the executor
|
||||
# update path actually work.
|
||||
|
||||
################################
|
||||
## Airflow Redis Password Secret
|
||||
#################################
|
||||
{{- $random_redis_password := randAlphaNum 10 }}
|
||||
{{- if and .Values.redis.enabled (not .Values.redis.passwordSecretName) }}
|
||||
# If passwordSecretName is not set, we will either use the set password, or use the generated one
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-redis-password
|
||||
labels:
|
||||
tier: airflow
|
||||
component: redis
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
"helm.sh/hook": "pre-install"
|
||||
"helm.sh/hook-delete-policy": "before-hook-creation"
|
||||
"helm.sh/hook-weight": "0"
|
||||
type: Opaque
|
||||
data:
|
||||
password: {{ (default $random_redis_password .Values.redis.password) | b64enc | quote }}
|
||||
---
|
||||
{{- end }}
|
||||
{{- if not .Values.data.brokerUrlSecretName }}
|
||||
##################################
|
||||
## Airflow Redis Connection Secret
|
||||
##################################
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-broker-url
|
||||
labels:
|
||||
tier: airflow
|
||||
component: redis
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
"helm.sh/hook": "pre-install"
|
||||
"helm.sh/hook-delete-policy": "before-hook-creation"
|
||||
"helm.sh/hook-weight": "0"
|
||||
type: Opaque
|
||||
data:
|
||||
{{- if .Values.redis.enabled }}
|
||||
connection: {{ urlJoin (dict "scheme" "redis" "userinfo" (printf ":%s" ((default $random_redis_password .Values.redis.password) | urlquery)) "host" (printf "%s-redis:6379" .Release.Name ) "path" "/0") | b64enc | quote }}
|
||||
{{- else }}
|
||||
connection: {{ (printf "%s" .Values.data.brokerUrl) | b64enc | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
38
helm/airflow/templates/secrets/registry-secret.yaml
Normal file
38
helm/airflow/templates/secrets/registry-secret.yaml
Normal file
@@ -0,0 +1,38 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Registry Secret
|
||||
#################################
|
||||
{{- if (and .Values.registry.connection (not .Values.registry.secretName)) }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-registry
|
||||
labels:
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
type: kubernetes.io/dockerconfigjson
|
||||
data:
|
||||
.dockerconfigjson: {{ include "registry_docker_config" . | b64enc }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,50 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Result Backend Secret
|
||||
#################################
|
||||
{{- if not .Values.data.resultBackendSecretName }}
|
||||
{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
|
||||
{{- if or (semverCompare "<2.4.0" .Values.airflowVersion) (and (semverCompare ">=2.4.0" .Values.airflowVersion) .Values.data.resultBackendConnection) }}
|
||||
{{- $connection := .Values.data.resultBackendConnection | default .Values.data.metadataConnection }}
|
||||
{{- $resultBackendHost := $connection.host | default (printf "%s-%s" .Release.Name "postgresql") }}
|
||||
{{- $pgbouncerHost := printf "%s-%s" .Release.Name "pgbouncer" }}
|
||||
{{- $host := ternary $pgbouncerHost $resultBackendHost .Values.pgbouncer.enabled }}
|
||||
{{- $port := (ternary .Values.ports.pgbouncer $connection.port .Values.pgbouncer.enabled) | toString }}
|
||||
{{- $database := ternary (printf "%s-%s" .Release.Name "result-backend") $connection.db .Values.pgbouncer.enabled }}
|
||||
{{- $query := ternary (printf "sslmode=%s" $connection.sslmode) "" (eq $connection.protocol "postgresql") }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-result-backend
|
||||
labels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
type: Opaque
|
||||
data:
|
||||
connection: {{ urlJoin (dict "scheme" (printf "db+%s" $connection.protocol) "userinfo" (printf "%s:%s" ($connection.user|urlquery) ($connection.pass | urlquery)) "host" (printf "%s:%s" $host $port) "path" (printf "/%s" $database) "query" $query) | b64enc | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,41 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
############################################
|
||||
## Airflow Webserver Flask Secret Key Secret
|
||||
############################################
|
||||
{{- if not .Values.webserverSecretKeySecretName }}
|
||||
{{ $generated_secret_key := (randAlphaNum 32 | b64enc) }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-webserver-secret-key
|
||||
labels:
|
||||
tier: airflow
|
||||
component: webserver
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
type: Opaque
|
||||
data:
|
||||
webserver-secret-key: {{ (default $generated_secret_key .Values.webserverSecretKey) | b64enc | quote }}
|
||||
{{- end }}
|
||||
138
helm/airflow/templates/statsd/statsd-deployment.yaml
Normal file
138
helm/airflow/templates/statsd/statsd-deployment.yaml
Normal file
@@ -0,0 +1,138 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow StatsD Deployment
|
||||
#################################
|
||||
{{- if .Values.statsd.enabled }}
|
||||
{{- $nodeSelector := or .Values.statsd.nodeSelector .Values.nodeSelector }}
|
||||
{{- $affinity := or .Values.statsd.affinity .Values.affinity }}
|
||||
{{- $tolerations := or .Values.statsd.tolerations .Values.tolerations }}
|
||||
{{- $topologySpreadConstraints := or .Values.statsd.topologySpreadConstraints .Values.topologySpreadConstraints }}
|
||||
{{- $revisionHistoryLimit := or .Values.statsd.revisionHistoryLimit .Values.revisionHistoryLimit }}
|
||||
{{- $securityContext := include "localPodSecurityContext" .Values.statsd }}
|
||||
{{- $containerSecurityContext := include "externalContainerSecurityContext" .Values.statsd }}
|
||||
{{- $containerLifecycleHooks := .Values.statsd.containerLifecycleHooks }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-statsd
|
||||
labels:
|
||||
tier: airflow
|
||||
component: statsd
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.statsd.annotations }}
|
||||
annotations: {{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
replicas: 1
|
||||
{{- if $revisionHistoryLimit }}
|
||||
revisionHistoryLimit: {{ $revisionHistoryLimit }}
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: statsd
|
||||
release: {{ .Release.Name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
tier: airflow
|
||||
component: statsd
|
||||
release: {{ .Release.Name }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if or .Values.statsd.extraMappings .Values.statsd.podAnnotations }}
|
||||
annotations:
|
||||
checksum/statsd-config: {{ include (print $.Template.BasePath "/configmaps/statsd-configmap.yaml") . | sha256sum }}
|
||||
{{- if .Values.statsd.podAnnotations }}
|
||||
{{- toYaml .Values.statsd.podAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.statsd.priorityClassName }}
|
||||
priorityClassName: {{ .Values.statsd.priorityClassName }}
|
||||
{{- end }}
|
||||
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
|
||||
affinity: {{- toYaml $affinity | nindent 8 }}
|
||||
tolerations: {{- toYaml $tolerations | nindent 8 }}
|
||||
{{- if .Values.schedulerName }}
|
||||
schedulerName: {{ .Values.schedulerName }}
|
||||
{{- end }}
|
||||
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
|
||||
serviceAccountName: {{ include "statsd.serviceAccountName" . }}
|
||||
securityContext: {{ $securityContext | nindent 8 }}
|
||||
restartPolicy: Always
|
||||
{{- if or .Values.registry.secretName .Values.registry.connection }}
|
||||
imagePullSecrets:
|
||||
- name: {{ template "registry_secret" . }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: statsd
|
||||
image: {{ template "statsd_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.statsd.pullPolicy }}
|
||||
securityContext: {{ $containerSecurityContext | nindent 12 }}
|
||||
{{- if $containerLifecycleHooks }}
|
||||
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.statsd.args }}
|
||||
args: {{ tpl (toYaml .Values.statsd.args) . | nindent 12 }}
|
||||
{{- else}}
|
||||
args:
|
||||
- "--statsd.mapping-config=/etc/statsd-exporter/mappings.yml"
|
||||
{{- end }}
|
||||
resources: {{- toYaml .Values.statsd.resources | nindent 12 }}
|
||||
{{- with .Values.statsd.env }}
|
||||
env: {{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- name: statsd-ingest
|
||||
protocol: UDP
|
||||
containerPort: {{ .Values.ports.statsdIngest }}
|
||||
- name: statsd-scrape
|
||||
containerPort: {{ .Values.ports.statsdScrape }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /metrics
|
||||
port: {{ .Values.ports.statsdScrape }}
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /metrics
|
||||
port: {{ .Values.ports.statsdScrape }}
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /etc/statsd-exporter/mappings.yml
|
||||
subPath: mappings.yml
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: {{ include "airflow.fullname" . }}-statsd
|
||||
{{- end }}
|
||||
59
helm/airflow/templates/statsd/statsd-networkpolicy.yaml
Normal file
59
helm/airflow/templates/statsd/statsd-networkpolicy.yaml
Normal file
@@ -0,0 +1,59 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow StatsD NetworkPolicy
|
||||
#################################
|
||||
{{- if and .Values.networkPolicies.enabled .Values.statsd.enabled }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-statsd-policy
|
||||
labels:
|
||||
tier: airflow
|
||||
component: statsd-policy
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: statsd
|
||||
release: {{ .Release.Name }}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
{{- if .Values.statsd.extraNetworkPolicies }}
|
||||
{{- toYaml .Values.statsd.extraNetworkPolicies | nindent 4 }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: {{ .Values.ports.statsdIngest }}
|
||||
- protocol: TCP
|
||||
port: {{ .Values.ports.statsdScrape }}
|
||||
{{- end }}
|
||||
58
helm/airflow/templates/statsd/statsd-service.yaml
Normal file
58
helm/airflow/templates/statsd/statsd-service.yaml
Normal file
@@ -0,0 +1,58 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow StatsD Service
|
||||
#################################
|
||||
{{- if .Values.statsd.enabled }}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-statsd
|
||||
labels:
|
||||
tier: airflow
|
||||
component: statsd
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: {{ .Values.ports.statsdScrape | quote }}
|
||||
{{- if .Values.statsd.service.extraAnnotations }}
|
||||
{{- toYaml .Values.statsd.service.extraAnnotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
tier: airflow
|
||||
component: statsd
|
||||
release: {{ .Release.Name }}
|
||||
ports:
|
||||
- name: statsd-ingest
|
||||
protocol: UDP
|
||||
port: {{ .Values.ports.statsdIngest }}
|
||||
targetPort: {{ .Values.ports.statsdIngest }}
|
||||
- name: statsd-scrape
|
||||
protocol: TCP
|
||||
port: {{ .Values.ports.statsdScrape }}
|
||||
targetPort: {{ .Values.ports.statsdScrape }}
|
||||
{{- end }}
|
||||
41
helm/airflow/templates/statsd/statsd-serviceaccount.yaml
Normal file
41
helm/airflow/templates/statsd/statsd-serviceaccount.yaml
Normal file
@@ -0,0 +1,41 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
######################################
|
||||
## Airflow StatsD ServiceAccount
|
||||
######################################
|
||||
{{- if and .Values.statsd.enabled .Values.statsd.serviceAccount.create }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
automountServiceAccountToken: {{ .Values.statsd.serviceAccount.automountServiceAccountToken }}
|
||||
metadata:
|
||||
name: {{ include "statsd.serviceAccountName" . }}
|
||||
labels:
|
||||
tier: airflow
|
||||
component: statsd
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.statsd.serviceAccount.annotations }}
|
||||
annotations: {{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
312
helm/airflow/templates/triggerer/triggerer-deployment.yaml
Normal file
312
helm/airflow/templates/triggerer/triggerer-deployment.yaml
Normal file
@@ -0,0 +1,312 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Triggerer Deployment
|
||||
#################################
|
||||
{{- if semverCompare ">=2.2.0" .Values.airflowVersion }}
|
||||
{{- if .Values.triggerer.enabled }}
|
||||
{{- /* Airflow version 2.6.0 is when triggerer logs serve introduced */ -}}
|
||||
{{- $persistence := and .Values.triggerer.persistence.enabled (semverCompare ">=2.6.0" .Values.airflowVersion) }}
|
||||
{{- $keda := .Values.triggerer.keda.enabled }}
|
||||
{{- $nodeSelector := or .Values.triggerer.nodeSelector .Values.nodeSelector }}
|
||||
{{- $affinity := or .Values.triggerer.affinity .Values.affinity }}
|
||||
{{- $tolerations := or .Values.triggerer.tolerations .Values.tolerations }}
|
||||
{{- $topologySpreadConstraints := or .Values.triggerer.topologySpreadConstraints .Values.topologySpreadConstraints }}
|
||||
{{- $revisionHistoryLimit := or .Values.triggerer.revisionHistoryLimit .Values.revisionHistoryLimit }}
|
||||
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.triggerer) }}
|
||||
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.triggerer) }}
|
||||
{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list . .Values.triggerer.waitForMigrations) }}
|
||||
{{- $containerSecurityContextLogGroomer := include "containerSecurityContext" (list . .Values.triggerer.logGroomerSidecar) }}
|
||||
{{- $containerLifecycleHooks := or .Values.triggerer.containerLifecycleHooks .Values.containerLifecycleHooks }}
|
||||
{{- $containerLifecycleHooksWaitForMigrations := or .Values.triggerer.waitForMigrations.containerLifecycleHooks .Values.containerLifecycleHooks }}
|
||||
{{- $containerLifecycleHooksLogGroomerSidecar := or .Values.triggerer.logGroomerSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }}
|
||||
apiVersion: apps/v1
|
||||
kind: {{ if $persistence }}StatefulSet{{ else }}Deployment{{ end }}
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-triggerer
|
||||
labels:
|
||||
tier: airflow
|
||||
component: triggerer
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.triggerer.annotations }}
|
||||
annotations: {{- toYaml .Values.triggerer.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if $persistence }}
|
||||
serviceName: {{ .Release.Name }}-triggerer
|
||||
{{- end }}
|
||||
{{- if not $keda }}
|
||||
replicas: {{ .Values.triggerer.replicas }}
|
||||
{{- end }}
|
||||
{{- if $revisionHistoryLimit }}
|
||||
revisionHistoryLimit: {{ $revisionHistoryLimit }}
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: triggerer
|
||||
release: {{ .Release.Name }}
|
||||
{{- if and $persistence .Values.triggerer.updateStrategy }}
|
||||
updateStrategy: {{- toYaml .Values.triggerer.updateStrategy | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if and (not $persistence) (.Values.triggerer.strategy) }}
|
||||
strategy: {{- toYaml .Values.triggerer.strategy | nindent 4 }}
|
||||
{{- end }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
tier: airflow
|
||||
component: triggerer
|
||||
release: {{ .Release.Name }}
|
||||
{{- if or (.Values.labels) (.Values.triggerer.labels) }}
|
||||
{{- mustMerge .Values.triggerer.labels .Values.labels | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }}
|
||||
checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }}
|
||||
checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }}
|
||||
checksum/extra-configmaps: {{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }}
|
||||
checksum/extra-secrets: {{ include (print $.Template.BasePath "/secrets/extra-secrets.yaml") . | sha256sum }}
|
||||
{{- if .Values.triggerer.safeToEvict }}
|
||||
cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
|
||||
{{- end }}
|
||||
{{- if .Values.airflowPodAnnotations }}
|
||||
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.triggerer.podAnnotations }}
|
||||
{{- toYaml .Values.triggerer.podAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.triggerer.priorityClassName }}
|
||||
priorityClassName: {{ .Values.triggerer.priorityClassName }}
|
||||
{{- end }}
|
||||
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
|
||||
{{- if .Values.schedulerName }}
|
||||
schedulerName: {{ .Values.schedulerName }}
|
||||
{{- end }}
|
||||
affinity:
|
||||
{{- if $affinity }}
|
||||
{{- toYaml $affinity | nindent 8 }}
|
||||
{{- else }}
|
||||
podAntiAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- podAffinityTerm:
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
component: triggerer
|
||||
topologyKey: kubernetes.io/hostname
|
||||
weight: 100
|
||||
{{- end }}
|
||||
tolerations: {{- toYaml $tolerations | nindent 8 }}
|
||||
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
|
||||
terminationGracePeriodSeconds: {{ .Values.triggerer.terminationGracePeriodSeconds }}
|
||||
restartPolicy: Always
|
||||
serviceAccountName: {{ include "triggerer.serviceAccountName" . }}
|
||||
securityContext: {{ $securityContext | nindent 8 }}
|
||||
{{- if or .Values.registry.secretName .Values.registry.connection }}
|
||||
imagePullSecrets:
|
||||
- name: {{ template "registry_secret" . }}
|
||||
{{- end }}
|
||||
initContainers:
|
||||
{{- if .Values.triggerer.waitForMigrations.enabled }}
|
||||
- name: wait-for-airflow-migrations
|
||||
resources:
|
||||
{{- toYaml .Values.triggerer.resources | nindent 12 }}
|
||||
image: {{ template "airflow_image_for_migrations" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
securityContext: {{ $containerSecurityContextWaitForMigrations | nindent 12 }}
|
||||
{{- if $containerLifecycleHooksWaitForMigrations }}
|
||||
lifecycle: {{- tpl (toYaml $containerLifecycleHooksWaitForMigrations) . | nindent 12 }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
{{- include "airflow_config_mount" . | nindent 12 }}
|
||||
{{- if .Values.volumeMounts }}
|
||||
{{- toYaml .Values.volumeMounts | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.triggerer.extraVolumeMounts }}
|
||||
{{- tpl (toYaml .Values.triggerer.extraVolumeMounts) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
|
||||
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
args: {{- include "wait-for-migrations-command" . | indent 10 }}
|
||||
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
|
||||
env:
|
||||
{{- include "custom_airflow_environment" . | indent 10 }}
|
||||
{{- include "standard_airflow_environment" . | indent 10 }}
|
||||
{{- if .Values.triggerer.waitForMigrations.env }}
|
||||
{{- tpl (toYaml .Values.triggerer.waitForMigrations.env) $ | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
|
||||
{{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.triggerer.extraInitContainers }}
|
||||
{{- toYaml .Values.triggerer.extraInitContainers | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: triggerer
|
||||
image: {{ template "airflow_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
securityContext: {{ $containerSecurityContext | nindent 12 }}
|
||||
{{- if $containerLifecycleHooks }}
|
||||
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.triggerer.command }}
|
||||
command: {{ tpl (toYaml .Values.triggerer.command) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.triggerer.args }}
|
||||
args: {{ tpl (toYaml .Values.triggerer.args) . | nindent 12 }}
|
||||
{{- end }}
|
||||
resources: {{- toYaml .Values.triggerer.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
{{- if .Values.volumeMounts }}
|
||||
{{- toYaml .Values.volumeMounts | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.triggerer.extraVolumeMounts }}
|
||||
{{- tpl (toYaml .Values.triggerer.extraVolumeMounts) . | nindent 12 }}
|
||||
{{- end }}
|
||||
- name: logs
|
||||
mountPath: {{ template "airflow_logs" . }}
|
||||
{{- include "airflow_config_mount" . | nindent 12 }}
|
||||
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
|
||||
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }}
|
||||
{{- include "airflow_dags_mount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
|
||||
env:
|
||||
{{- include "custom_airflow_environment" . | indent 10 }}
|
||||
{{- include "standard_airflow_environment" . | indent 10 }}
|
||||
{{- include "container_extra_envs" (list . .Values.triggerer.env) | nindent 10 }}
|
||||
livenessProbe:
|
||||
initialDelaySeconds: {{ .Values.triggerer.livenessProbe.initialDelaySeconds }}
|
||||
timeoutSeconds: {{ .Values.triggerer.livenessProbe.timeoutSeconds }}
|
||||
failureThreshold: {{ .Values.triggerer.livenessProbe.failureThreshold }}
|
||||
periodSeconds: {{ .Values.triggerer.livenessProbe.periodSeconds }}
|
||||
exec:
|
||||
command:
|
||||
{{- if .Values.triggerer.livenessProbe.command }}
|
||||
{{- toYaml .Values.triggerer.livenessProbe.command | nindent 16 }}
|
||||
{{- else }}
|
||||
{{- include "triggerer_liveness_check_command" . | indent 14 }}
|
||||
{{- end }}
|
||||
{{- /* Airflow version 2.6.0 is when triggerer logs serve introduced */ -}}
|
||||
{{- if semverCompare ">=2.6.0" .Values.airflowVersion }}
|
||||
ports:
|
||||
- name: triggerer-logs
|
||||
containerPort: {{ .Values.ports.triggererLogs }}
|
||||
{{- end }}
|
||||
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
|
||||
{{- include "git_sync_container" . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.triggerer.logGroomerSidecar.enabled }}
|
||||
- name: triggerer-log-groomer
|
||||
resources: {{- toYaml .Values.triggerer.logGroomerSidecar.resources | nindent 12 }}
|
||||
image: {{ template "airflow_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
securityContext: {{ $containerSecurityContextLogGroomer | nindent 12 }}
|
||||
{{- if $containerLifecycleHooksLogGroomerSidecar }}
|
||||
lifecycle: {{- tpl (toYaml $containerLifecycleHooksLogGroomerSidecar) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.triggerer.logGroomerSidecar.command }}
|
||||
command: {{ tpl (toYaml .Values.triggerer.logGroomerSidecar.command) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.triggerer.logGroomerSidecar.args }}
|
||||
args: {{- tpl (toYaml .Values.triggerer.logGroomerSidecar.args) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.triggerer.logGroomerSidecar.retentionDays }}
|
||||
env:
|
||||
- name: AIRFLOW__LOG_RETENTION_DAYS
|
||||
value: "{{ .Values.triggerer.logGroomerSidecar.retentionDays }}"
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
- name: logs
|
||||
mountPath: {{ template "airflow_logs" . }}
|
||||
{{- if .Values.volumeMounts }}
|
||||
{{- toYaml .Values.volumeMounts | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.triggerer.extraVolumeMounts }}
|
||||
{{- tpl (toYaml .Values.triggerer.extraVolumeMounts) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
|
||||
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.triggerer.extraContainers }}
|
||||
{{- toYaml .Values.triggerer.extraContainers | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: {{ template "airflow_config" . }}
|
||||
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
|
||||
- name: webserver-config
|
||||
configMap:
|
||||
name: {{ template "airflow_webserver_config_configmap_name" . }}
|
||||
{{- end }}
|
||||
{{- if .Values.dags.persistence.enabled }}
|
||||
- name: dags
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ template "airflow_dags_volume_claim" . }}
|
||||
{{- else if .Values.dags.gitSync.enabled }}
|
||||
- name: dags
|
||||
emptyDir: {}
|
||||
{{- if .Values.dags.gitSync.sshKeySecret }}
|
||||
{{- include "git_sync_ssh_key_volume" . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.volumes }}
|
||||
{{- toYaml .Values.volumes | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.triggerer.extraVolumes }}
|
||||
{{- tpl (toYaml .Values.triggerer.extraVolumes) . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.logs.persistence.enabled }}
|
||||
- name: logs
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ template "airflow_logs_volume_claim" . }}
|
||||
{{- else if not $persistence }}
|
||||
- name: logs
|
||||
emptyDir: {}
|
||||
{{- else }}
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: logs
|
||||
{{- if .Values.triggerer.persistence.annotations }}
|
||||
annotations: {{- toYaml .Values.triggerer.persistence.annotations | nindent 10 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.triggerer.persistence.storageClassName }}
|
||||
storageClassName: {{ .Values.triggerer.persistence.storageClassName }}
|
||||
{{- end }}
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.triggerer.persistence.size }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,57 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Triggerer KEDA Scaler
|
||||
#################################
|
||||
{{- if semverCompare ">=2.2.0" .Values.airflowVersion }}
|
||||
{{- if and .Values.triggerer.enabled .Values.triggerer.keda.enabled }}
|
||||
apiVersion: keda.sh/v1alpha1
|
||||
kind: ScaledObject
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-triggerer
|
||||
labels:
|
||||
tier: airflow
|
||||
component: triggerer-horizontalpodautoscaler
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
deploymentName: {{ .Release.Name }}-triggerer
|
||||
{{- if or (.Values.labels) (.Values.triggerer.labels) }}
|
||||
{{- mustMerge .Values.triggerer.labels .Values.labels | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
kind: {{ ternary "StatefulSet" "Deployment" .Values.triggerer.persistence.enabled }}
|
||||
name: {{ .Release.Name }}-triggerer
|
||||
pollingInterval: {{ .Values.triggerer.keda.pollingInterval }}
|
||||
cooldownPeriod: {{ .Values.triggerer.keda.cooldownPeriod }}
|
||||
minReplicaCount: {{ .Values.triggerer.keda.minReplicaCount }}
|
||||
maxReplicaCount: {{ .Values.triggerer.keda.maxReplicaCount }}
|
||||
{{- if .Values.triggerer.keda.advanced }}
|
||||
advanced: {{- toYaml .Values.triggerer.keda.advanced | nindent 4 }}
|
||||
{{- end }}
|
||||
triggers:
|
||||
- type: postgresql
|
||||
metadata:
|
||||
targetQueryValue: "1"
|
||||
connectionFromEnv: AIRFLOW_CONN_AIRFLOW_DB
|
||||
query: {{ tpl .Values.triggerer.keda.query . | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,60 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
##################################
|
||||
## Airflow triggerer NetworkPolicy
|
||||
##################################
|
||||
{{- /* Airflow version 2.6.0 is when triggerer logs serve introduced */ -}}
|
||||
{{- if semverCompare ">=2.6.0" .Values.airflowVersion }}
|
||||
{{- if .Values.networkPolicies.enabled }}
|
||||
{{- if .Values.triggerer.enabled }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-triggerer-policy
|
||||
labels:
|
||||
tier: airflow
|
||||
component: airflow-triggerer-policy
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- if or (.Values.labels) (.Values.triggerer.labels) }}
|
||||
{{- mustMerge .Values.triggerer.labels .Values.labels | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: triggerer
|
||||
release: {{ .Release.Name }}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
component: webserver
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: {{ .Values.ports.triggererLogs }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
51
helm/airflow/templates/triggerer/triggerer-service.yaml
Normal file
51
helm/airflow/templates/triggerer/triggerer-service.yaml
Normal file
@@ -0,0 +1,51 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow triggerer Service
|
||||
#################################
|
||||
{{- /* Airflow version 2.6.0 is when triggerer logs serve introduced */ -}}
|
||||
{{- if semverCompare ">=2.6.0" .Values.airflowVersion }}
|
||||
{{- if .Values.triggerer.enabled }}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-triggerer
|
||||
labels:
|
||||
tier: airflow
|
||||
component: triggerer
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- if or (.Values.labels) (.Values.triggerer.labels) }}
|
||||
{{- mustMerge .Values.triggerer.labels .Values.labels | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
clusterIP: None
|
||||
selector:
|
||||
tier: airflow
|
||||
component: triggerer
|
||||
release: {{ .Release.Name }}
|
||||
ports:
|
||||
- name: triggerer-logs
|
||||
protocol: TCP
|
||||
port: {{ .Values.ports.triggererLogs }}
|
||||
targetPort: {{ .Values.ports.triggererLogs }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,43 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Triggerer ServiceAccount
|
||||
#################################
|
||||
{{- if semverCompare ">=2.2.0" .Values.airflowVersion }}
|
||||
{{- if and .Values.triggerer.serviceAccount.create .Values.triggerer.enabled }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
automountServiceAccountToken: {{ .Values.triggerer.serviceAccount.automountServiceAccountToken }}
|
||||
metadata:
|
||||
name: {{ include "triggerer.serviceAccountName" . }}
|
||||
labels:
|
||||
tier: airflow
|
||||
component: triggerer
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- if or (.Values.labels) (.Values.triggerer.labels) }}
|
||||
{{- mustMerge .Values.triggerer.labels .Values.labels | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.triggerer.serviceAccount.annotations}}
|
||||
annotations: {{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
297
helm/airflow/templates/webserver/webserver-deployment.yaml
Normal file
297
helm/airflow/templates/webserver/webserver-deployment.yaml
Normal file
@@ -0,0 +1,297 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Webserver Deployment
|
||||
#################################
|
||||
{{- $nodeSelector := or .Values.webserver.nodeSelector .Values.nodeSelector }}
|
||||
{{- $affinity := or .Values.webserver.affinity .Values.affinity }}
|
||||
{{- $tolerations := or .Values.webserver.tolerations .Values.tolerations }}
|
||||
{{- $topologySpreadConstraints := or .Values.webserver.topologySpreadConstraints .Values.topologySpreadConstraints }}
|
||||
{{- $revisionHistoryLimit := or .Values.webserver.revisionHistoryLimit .Values.revisionHistoryLimit }}
|
||||
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.webserver) }}
|
||||
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.webserver) }}
|
||||
{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list . .Values.webserver.waitForMigrations) }}
|
||||
{{- $containerLifecycleHooks := or .Values.webserver.containerLifecycleHooks .Values.containerLifecycleHooks }}
|
||||
{{- $containerLifecycleHooksWaitForMigrations := or .Values.webserver.waitForMigrations.containerLifecycleHooks .Values.containerLifecycleHooks }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-webserver
|
||||
labels:
|
||||
tier: airflow
|
||||
component: webserver
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.webserver.annotations }}
|
||||
annotations: {{- toYaml .Values.webserver.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
replicas: {{ .Values.webserver.replicas }}
|
||||
{{- if $revisionHistoryLimit }}
|
||||
revisionHistoryLimit: {{ $revisionHistoryLimit }}
|
||||
{{- end }}
|
||||
strategy:
|
||||
{{- if .Values.webserver.strategy }}
|
||||
{{- toYaml .Values.webserver.strategy | nindent 4 }}
|
||||
{{- else }}
|
||||
{{- if semverCompare ">=2.0.0" .Values.airflowVersion }}
|
||||
# Here we define the rolling update strategy
|
||||
# - maxSurge define how many pod we can add at a time
|
||||
# - maxUnavailable define how many pod can be unavailable
|
||||
# during the rolling update
|
||||
# Setting maxUnavailable to 0 would make sure we have the appropriate
|
||||
# capacity during the rolling update.
|
||||
# You can also use percentage based value instead of integer.
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 0
|
||||
{{- else }}
|
||||
type: Recreate
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: webserver
|
||||
release: {{ .Release.Name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
tier: airflow
|
||||
component: webserver
|
||||
release: {{ .Release.Name }}
|
||||
{{- if or (.Values.labels) (.Values.webserver.labels) }}
|
||||
{{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }}
|
||||
checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }}
|
||||
checksum/webserver-secret-key: {{ include (print $.Template.BasePath "/secrets/webserver-secret-key-secret.yaml") . | sha256sum }}
|
||||
checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }}
|
||||
checksum/webserver-config: {{ include (print $.Template.BasePath "/configmaps/webserver-configmap.yaml") . | sha256sum }}
|
||||
checksum/extra-configmaps: {{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }}
|
||||
checksum/extra-secrets: {{ include (print $.Template.BasePath "/secrets/extra-secrets.yaml") . | sha256sum }}
|
||||
{{- if .Values.airflowPodAnnotations }}
|
||||
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.webserver.podAnnotations }}
|
||||
{{- toYaml .Values.webserver.podAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.webserver.hostAliases }}
|
||||
hostAliases: {{- toYaml .Values.webserver.hostAliases | nindent 8 }}
|
||||
{{- end }}
|
||||
serviceAccountName: {{ include "webserver.serviceAccountName" . }}
|
||||
{{- if .Values.webserver.priorityClassName }}
|
||||
priorityClassName: {{ .Values.webserver.priorityClassName }}
|
||||
{{- end }}
|
||||
{{- if .Values.schedulerName }}
|
||||
schedulerName: {{ .Values.schedulerName }}
|
||||
{{- end }}
|
||||
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
|
||||
affinity:
|
||||
{{- if $affinity }}
|
||||
{{- toYaml $affinity | nindent 8 }}
|
||||
{{- else }}
|
||||
podAntiAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- podAffinityTerm:
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
component: webserver
|
||||
topologyKey: kubernetes.io/hostname
|
||||
weight: 100
|
||||
{{- end }}
|
||||
tolerations: {{- toYaml $tolerations | nindent 8 }}
|
||||
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
|
||||
restartPolicy: Always
|
||||
securityContext: {{ $securityContext | nindent 8 }}
|
||||
{{- if or .Values.registry.secretName .Values.registry.connection }}
|
||||
imagePullSecrets:
|
||||
- name: {{ template "registry_secret" . }}
|
||||
{{- end }}
|
||||
initContainers:
|
||||
{{- if .Values.webserver.waitForMigrations.enabled }}
|
||||
- name: wait-for-airflow-migrations
|
||||
resources: {{- toYaml .Values.webserver.resources | nindent 12 }}
|
||||
image: {{ template "airflow_image_for_migrations" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
securityContext: {{ $containerSecurityContextWaitForMigrations | nindent 12 }}
|
||||
{{- if $containerLifecycleHooksWaitForMigrations }}
|
||||
lifecycle: {{- tpl (toYaml $containerLifecycleHooksWaitForMigrations) . | nindent 12 }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
{{- include "airflow_config_mount" . | nindent 12 }}
|
||||
{{- if .Values.volumeMounts }}
|
||||
{{- toYaml .Values.volumeMounts | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.webserver.extraVolumeMounts }}
|
||||
{{- tpl (toYaml .Values.webserver.extraVolumeMounts) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
|
||||
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
args: {{- include "wait-for-migrations-command" . | indent 10 }}
|
||||
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
|
||||
env:
|
||||
{{- include "custom_airflow_environment" . | indent 10 }}
|
||||
{{- include "standard_airflow_environment" . | indent 10 }}
|
||||
{{- if .Values.webserver.waitForMigrations.env }}
|
||||
{{- tpl (toYaml .Values.webserver.waitForMigrations.env) $ | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) (semverCompare "<2.0.0" .Values.airflowVersion) }}
|
||||
{{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.webserver.extraInitContainers }}
|
||||
{{- toYaml .Values.webserver.extraInitContainers | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: webserver
|
||||
image: {{ template "airflow_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
securityContext: {{ or $containerSecurityContext .Values.webserver.securityContexts.container .Values.securityContexts.container | nindent 12 }}
|
||||
{{- if $containerLifecycleHooks }}
|
||||
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.webserver.command }}
|
||||
command: {{ tpl (toYaml .Values.webserver.command) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.webserver.args }}
|
||||
args: {{- tpl (toYaml .Values.webserver.args) . | nindent 12 }}
|
||||
{{- end }}
|
||||
resources: {{- toYaml .Values.webserver.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
{{- if semverCompare ">=1.10.12" .Values.airflowVersion }}
|
||||
- name: config
|
||||
mountPath: {{ include "airflow_pod_template_file" . }}/pod_template_file.yaml
|
||||
subPath: pod_template_file.yaml
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- include "airflow_config_mount" . | nindent 12 }}
|
||||
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
|
||||
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if and (semverCompare "<2.0.0" .Values.airflowVersion) (or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled) }}
|
||||
{{- include "airflow_dags_mount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.logs.persistence.enabled }}
|
||||
- name: logs
|
||||
mountPath: {{ template "airflow_logs" . }}
|
||||
{{- end }}
|
||||
{{- if .Values.volumeMounts }}
|
||||
{{- toYaml .Values.volumeMounts | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.webserver.extraVolumeMounts }}
|
||||
{{- tpl (toYaml .Values.webserver.extraVolumeMounts) . | nindent 12 }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- name: airflow-ui
|
||||
containerPort: {{ .Values.ports.airflowUI }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: {{ if .Values.config.webserver.base_url }}{{- with urlParse (tpl .Values.config.webserver.base_url .) }}{{ .path }}{{ end }}{{ end }}/health
|
||||
port: {{ .Values.ports.airflowUI }}
|
||||
{{- if .Values.config.webserver.base_url}}
|
||||
httpHeaders:
|
||||
- name: Host
|
||||
value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values.config.webserver.base_url .)).host "" }}
|
||||
{{- end }}
|
||||
scheme: {{ .Values.webserver.livenessProbe.scheme | default "http" }}
|
||||
initialDelaySeconds: {{ .Values.webserver.livenessProbe.initialDelaySeconds }}
|
||||
timeoutSeconds: {{ .Values.webserver.livenessProbe.timeoutSeconds }}
|
||||
failureThreshold: {{ .Values.webserver.livenessProbe.failureThreshold }}
|
||||
periodSeconds: {{ .Values.webserver.livenessProbe.periodSeconds }}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: {{ if .Values.config.webserver.base_url }}{{- with urlParse (tpl .Values.config.webserver.base_url .) }}{{ .path }}{{ end }}{{ end }}/health
|
||||
port: {{ .Values.ports.airflowUI }}
|
||||
{{- if .Values.config.webserver.base_url }}
|
||||
httpHeaders:
|
||||
- name: Host
|
||||
value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values.config.webserver.base_url .)).host "" }}
|
||||
{{- end }}
|
||||
scheme: {{ .Values.webserver.readinessProbe.scheme | default "http" }}
|
||||
initialDelaySeconds: {{ .Values.webserver.readinessProbe.initialDelaySeconds }}
|
||||
timeoutSeconds: {{ .Values.webserver.readinessProbe.timeoutSeconds }}
|
||||
failureThreshold: {{ .Values.webserver.readinessProbe.failureThreshold }}
|
||||
periodSeconds: {{ .Values.webserver.readinessProbe.periodSeconds }}
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: {{ if .Values.config.webserver.base_url }}{{- with urlParse (tpl .Values.config.webserver.base_url .) }}{{ .path }}{{ end }}{{ end }}/health
|
||||
port: {{ .Values.ports.airflowUI }}
|
||||
{{- if .Values.config.webserver.base_url}}
|
||||
httpHeaders:
|
||||
- name: Host
|
||||
value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values.config.webserver.base_url .)).host "" }}
|
||||
{{- end }}
|
||||
scheme: {{ .Values.webserver.startupProbe.scheme | default "http" }}
|
||||
timeoutSeconds: {{ .Values.webserver.startupProbe.timeoutSeconds }}
|
||||
failureThreshold: {{ .Values.webserver.startupProbe.failureThreshold }}
|
||||
periodSeconds: {{ .Values.webserver.startupProbe.periodSeconds }}
|
||||
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
|
||||
env:
|
||||
{{- include "custom_airflow_environment" . | indent 10 }}
|
||||
{{- include "standard_airflow_environment" . | indent 10 }}
|
||||
{{- include "container_extra_envs" (list . .Values.webserver.env) | indent 10 }}
|
||||
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) (semverCompare "<2.0.0" .Values.airflowVersion) }}
|
||||
{{- include "git_sync_container" . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.webserver.extraContainers }}
|
||||
{{- toYaml .Values.webserver.extraContainers | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: {{ template "airflow_config" . }}
|
||||
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
|
||||
- name: webserver-config
|
||||
configMap:
|
||||
name: {{ template "airflow_webserver_config_configmap_name" . }}
|
||||
{{- end }}
|
||||
{{- if (semverCompare "<2.0.0" .Values.airflowVersion) }}
|
||||
{{- if .Values.dags.persistence.enabled }}
|
||||
- name: dags
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ template "airflow_dags_volume_claim" . }}
|
||||
{{- else if .Values.dags.gitSync.enabled }}
|
||||
- name: dags
|
||||
emptyDir: {}
|
||||
{{- if .Values.dags.gitSync.sshKeySecret }}
|
||||
{{- include "git_sync_ssh_key_volume" . | indent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.logs.persistence.enabled }}
|
||||
- name: logs
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ template "airflow_logs_volume_claim" . }}
|
||||
{{- end }}
|
||||
{{- if .Values.volumes }}
|
||||
{{- toYaml .Values.volumes | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.webserver.extraVolumes }}
|
||||
{{- tpl (toYaml .Values.webserver.extraVolumes) . | nindent 8 }}
|
||||
{{- end }}
|
||||
111
helm/airflow/templates/webserver/webserver-ingress.yaml
Normal file
111
helm/airflow/templates/webserver/webserver-ingress.yaml
Normal file
@@ -0,0 +1,111 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Webserver Ingress
|
||||
#################################
|
||||
{{- if or .Values.ingress.web.enabled .Values.ingress.enabled }}
|
||||
{{- $fullname := (include "airflow.fullname" .) }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: {{ $fullname }}-ingress
|
||||
labels:
|
||||
tier: airflow
|
||||
component: airflow-ingress
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- if or (.Values.labels) (.Values.webserver.labels) }}
|
||||
{{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.ingress.web.annotations }}
|
||||
annotations: {{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if and .Values.ingress.web.hosts (.Values.ingress.web.hosts | first | kindIs "string" | not) }}
|
||||
{{- $anyTlsHosts := false -}}
|
||||
{{- range .Values.ingress.web.hosts }}
|
||||
{{- if .tls }}
|
||||
{{- if .tls.enabled }}
|
||||
{{- $anyTlsHosts = true -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if $anyTlsHosts }}
|
||||
tls:
|
||||
{{- range .Values.ingress.web.hosts }}
|
||||
{{- if .tls }}
|
||||
{{- if .tls.enabled }}
|
||||
- hosts:
|
||||
- {{ .name | quote }}
|
||||
secretName: {{ .tls.secretName }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- else if .Values.ingress.web.tls.enabled }}
|
||||
tls:
|
||||
- hosts:
|
||||
{{- .Values.ingress.web.hosts | default (list .Values.ingress.web.host) | toYaml | nindent 8 }}
|
||||
secretName: {{ .Values.ingress.web.tls.secretName }}
|
||||
{{- end }}
|
||||
rules:
|
||||
{{- range .Values.ingress.web.hosts | default (list .Values.ingress.web.host) }}
|
||||
- http:
|
||||
paths:
|
||||
{{- range $.Values.ingress.web.precedingPaths }}
|
||||
- path: {{ .path }}
|
||||
pathType: {{ .pathType }}
|
||||
backend:
|
||||
service:
|
||||
name: {{ .serviceName }}
|
||||
port:
|
||||
name: {{ .servicePort }}
|
||||
{{- end }}
|
||||
- backend:
|
||||
service:
|
||||
name: {{ $fullname }}-webserver
|
||||
port:
|
||||
name: airflow-ui
|
||||
{{- if $.Values.ingress.web.path }}
|
||||
path: {{ $.Values.ingress.web.path }}
|
||||
pathType: {{ $.Values.ingress.web.pathType }}
|
||||
{{- end }}
|
||||
{{- range $.Values.ingress.web.succeedingPaths }}
|
||||
- path: {{ .path }}
|
||||
pathType: {{ .pathType }}
|
||||
backend:
|
||||
service:
|
||||
name: {{ .serviceName }}
|
||||
port:
|
||||
name: {{ .servicePort }}
|
||||
{{- end }}
|
||||
{{- $hostname := . -}}
|
||||
{{- if . | kindIs "string" | not }}
|
||||
{{- $hostname = .name -}}
|
||||
{{- end }}
|
||||
{{- if $hostname }}
|
||||
host: {{ tpl $hostname $ | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.ingress.web.ingressClassName }}
|
||||
ingressClassName: {{ .Values.ingress.web.ingressClassName }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,57 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Webserver NetworkPolicy
|
||||
#################################
|
||||
{{- if .Values.networkPolicies.enabled }}
|
||||
{{- $from := or .Values.webserver.networkPolicy.ingress.from .Values.webserver.extraNetworkPolicies }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-webserver-policy
|
||||
labels:
|
||||
tier: airflow
|
||||
component: airflow-webserver-policy
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- if or (.Values.labels) (.Values.webserver.labels) }}
|
||||
{{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: webserver
|
||||
release: {{ .Release.Name }}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
{{- if $from }}
|
||||
ingress:
|
||||
- from: {{- toYaml $from | nindent 6 }}
|
||||
ports:
|
||||
{{ range .Values.webserver.networkPolicy.ingress.ports }}
|
||||
-
|
||||
{{- range $key, $val := . }}
|
||||
{{ $key }}: {{ tpl (toString $val) $ }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,44 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Webserver PodDisruptionBudget
|
||||
#################################
|
||||
{{- if .Values.webserver.podDisruptionBudget.enabled }}
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-webserver-pdb
|
||||
labels:
|
||||
tier: airflow
|
||||
component: webserver
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- if or (.Values.labels) (.Values.webserver.labels) }}
|
||||
{{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: webserver
|
||||
release: {{ .Release.Name }}
|
||||
{{- toYaml .Values.webserver.podDisruptionBudget.config | nindent 2 }}
|
||||
{{- end }}
|
||||
56
helm/airflow/templates/webserver/webserver-service.yaml
Normal file
56
helm/airflow/templates/webserver/webserver-service.yaml
Normal file
@@ -0,0 +1,56 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Webserver Service
|
||||
#################################
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-webserver
|
||||
labels:
|
||||
tier: airflow
|
||||
component: webserver
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- if or (.Values.labels) (.Values.webserver.labels) }}
|
||||
{{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.webserver.service.annotations }}
|
||||
annotations: {{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
type: {{ .Values.webserver.service.type }}
|
||||
selector:
|
||||
tier: airflow
|
||||
component: webserver
|
||||
release: {{ .Release.Name }}
|
||||
ports:
|
||||
{{- range .Values.webserver.service.ports }}
|
||||
- {{ range $key, $val := . }}
|
||||
{{- $key }}: {{ tpl (toString $val) $ }}
|
||||
{{ end }}
|
||||
{{- end }}
|
||||
{{- if .Values.webserver.service.loadBalancerIP }}
|
||||
loadBalancerIP: {{ .Values.webserver.service.loadBalancerIP }}
|
||||
{{- end }}
|
||||
{{- if .Values.webserver.service.loadBalancerSourceRanges }}
|
||||
loadBalancerSourceRanges: {{- toYaml .Values.webserver.service.loadBalancerSourceRanges | nindent 4 }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,41 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
######################################
|
||||
## Airflow Webserver ServiceAccount
|
||||
######################################
|
||||
{{- if .Values.webserver.serviceAccount.create }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
automountServiceAccountToken: {{ .Values.webserver.serviceAccount.automountServiceAccountToken }}
|
||||
metadata:
|
||||
name: {{ include "webserver.serviceAccountName" . }}
|
||||
labels:
|
||||
tier: airflow
|
||||
component: webserver
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- if or (.Values.labels) (.Values.webserver.labels) }}
|
||||
{{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.webserver.serviceAccount.annotations }}
|
||||
annotations: {{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
412
helm/airflow/templates/workers/worker-deployment.yaml
Normal file
412
helm/airflow/templates/workers/worker-deployment.yaml
Normal file
@@ -0,0 +1,412 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Worker Deployment
|
||||
#################################
|
||||
{{- $persistence := .Values.workers.persistence.enabled }}
|
||||
{{- $keda := .Values.workers.keda.enabled }}
|
||||
{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
|
||||
{{- $nodeSelector := or .Values.workers.nodeSelector .Values.nodeSelector }}
|
||||
{{- $affinity := or .Values.workers.affinity .Values.affinity }}
|
||||
{{- $tolerations := or .Values.workers.tolerations .Values.tolerations }}
|
||||
{{- $topologySpreadConstraints := or .Values.workers.topologySpreadConstraints .Values.topologySpreadConstraints }}
|
||||
{{- $revisionHistoryLimit := or .Values.workers.revisionHistoryLimit .Values.revisionHistoryLimit }}
|
||||
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.workers) }}
|
||||
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.workers) }}
|
||||
{{- $containerSecurityContextPersistence := include "containerSecurityContext" (list . .Values.workers.persistence) }}
|
||||
{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list . .Values.workers.waitForMigrations) }}
|
||||
{{- $containerSecurityContextLogGroomerSidecar := include "containerSecurityContext" (list . .Values.workers.logGroomerSidecar) }}
|
||||
{{- $containerSecurityContextKerberosSidecar := include "containerSecurityContext" (list . .Values.workers.kerberosSidecar) }}
|
||||
{{- $containerLifecycleHooks := or .Values.workers.containerLifecycleHooks .Values.containerLifecycleHooks }}
|
||||
{{- $containerLifecycleHooksPersistence := or .Values.workers.persistence.containerLifecycleHooks .Values.containerLifecycleHooks }}
|
||||
{{- $containerLifecycleHooksWaitForMigrations := or .Values.workers.waitForMigrations.containerLifecycleHooks .Values.containerLifecycleHooks }}
|
||||
{{- $containerLifecycleHooksLogGroomerSidecar := or .Values.workers.logGroomerSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }}
|
||||
{{- $containerLifecycleHooksKerberosSidecar := or .Values.workers.kerberosSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }}
|
||||
apiVersion: apps/v1
|
||||
kind: {{ if $persistence }}StatefulSet{{ else }}Deployment{{ end }}
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-worker
|
||||
labels:
|
||||
tier: airflow
|
||||
component: worker
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.workers.annotations }}
|
||||
annotations: {{- toYaml .Values.workers.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if $persistence }}
|
||||
serviceName: {{ include "airflow.fullname" . }}-worker
|
||||
{{- end }}
|
||||
{{- if not $keda }}
|
||||
replicas: {{ .Values.workers.replicas }}
|
||||
{{- end }}
|
||||
{{- if $revisionHistoryLimit }}
|
||||
revisionHistoryLimit: {{ $revisionHistoryLimit }}
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: worker
|
||||
release: {{ .Release.Name }}
|
||||
{{- if and $persistence .Values.workers.updateStrategy }}
|
||||
updateStrategy: {{- toYaml .Values.workers.updateStrategy | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if and (not $persistence) (.Values.workers.strategy) }}
|
||||
strategy: {{- toYaml .Values.workers.strategy | nindent 4 }}
|
||||
{{- end }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
tier: airflow
|
||||
component: worker
|
||||
release: {{ .Release.Name }}
|
||||
{{- if or (.Values.labels) (.Values.workers.labels) }}
|
||||
{{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }}
|
||||
checksum/result-backend-secret: {{ include (print $.Template.BasePath "/secrets/result-backend-connection-secret.yaml") . | sha256sum }}
|
||||
checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }}
|
||||
checksum/webserver-secret-key: {{ include (print $.Template.BasePath "/secrets/webserver-secret-key-secret.yaml") . | sha256sum }}
|
||||
checksum/kerberos-keytab: {{ include (print $.Template.BasePath "/secrets/kerberos-keytab-secret.yaml") . | sha256sum }}
|
||||
checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }}
|
||||
checksum/extra-configmaps: {{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }}
|
||||
checksum/extra-secrets: {{ include (print $.Template.BasePath "/secrets/extra-secrets.yaml") . | sha256sum }}
|
||||
{{- if .Values.workers.safeToEvict }}
|
||||
cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
|
||||
{{- end }}
|
||||
{{- if .Values.airflowPodAnnotations }}
|
||||
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.workers.podAnnotations }}
|
||||
{{- toYaml .Values.workers.podAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.workers.runtimeClassName }}
|
||||
runtimeClassName: {{ .Values.workers.runtimeClassName }}
|
||||
{{- end }}
|
||||
{{- if .Values.workers.priorityClassName }}
|
||||
priorityClassName: {{ .Values.workers.priorityClassName }}
|
||||
{{- end }}
|
||||
{{- if .Values.schedulerName }}
|
||||
schedulerName: {{ .Values.schedulerName }}
|
||||
{{- end }}
|
||||
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
|
||||
affinity:
|
||||
{{- if $affinity }}
|
||||
{{- toYaml $affinity | nindent 8 }}
|
||||
{{- else }}
|
||||
podAntiAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- podAffinityTerm:
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
component: worker
|
||||
topologyKey: kubernetes.io/hostname
|
||||
weight: 100
|
||||
{{- end }}
|
||||
tolerations: {{- toYaml $tolerations | nindent 8 }}
|
||||
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
|
||||
{{- if .Values.workers.hostAliases }}
|
||||
hostAliases: {{- toYaml .Values.workers.hostAliases | nindent 8 }}
|
||||
{{- end }}
|
||||
terminationGracePeriodSeconds: {{ .Values.workers.terminationGracePeriodSeconds }}
|
||||
restartPolicy: Always
|
||||
serviceAccountName: {{ include "worker.serviceAccountName" . }}
|
||||
securityContext: {{ $securityContext | nindent 8 }}
|
||||
{{- if or .Values.registry.secretName .Values.registry.connection }}
|
||||
imagePullSecrets:
|
||||
- name: {{ template "registry_secret" . }}
|
||||
{{- end }}
|
||||
initContainers:
|
||||
{{- if and $persistence .Values.workers.persistence.fixPermissions }}
|
||||
- name: volume-permissions
|
||||
resources: {{- toYaml .Values.workers.resources | nindent 12 }}
|
||||
image: {{ template "airflow_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
command:
|
||||
- chown
|
||||
- -R
|
||||
- "{{ include "airflowPodSecurityContextsIds" (list . .Values.workers) }}"
|
||||
- {{ template "airflow_logs" . }}
|
||||
securityContext: {{ $containerSecurityContextPersistence | nindent 12 }}
|
||||
{{- if $containerLifecycleHooksPersistence }}
|
||||
lifecycle: {{- tpl (toYaml $containerLifecycleHooksPersistence) . | nindent 12 }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
- name: logs
|
||||
mountPath: {{ template "airflow_logs" . }}
|
||||
{{- end }}
|
||||
{{- if .Values.workers.waitForMigrations.enabled }}
|
||||
- name: wait-for-airflow-migrations
|
||||
resources: {{- toYaml .Values.workers.resources | nindent 12 }}
|
||||
image: {{ template "airflow_image_for_migrations" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
securityContext: {{ $containerSecurityContextWaitForMigrations | nindent 12 }}
|
||||
{{- if $containerLifecycleHooksWaitForMigrations }}
|
||||
lifecycle: {{- tpl (toYaml $containerLifecycleHooksWaitForMigrations) . | nindent 12 }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
{{- include "airflow_config_mount" . | nindent 12 }}
|
||||
{{- if .Values.volumeMounts }}
|
||||
{{- toYaml .Values.volumeMounts | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.workers.extraVolumeMounts }}
|
||||
{{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
|
||||
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
args: {{- include "wait-for-migrations-command" . | indent 10 }}
|
||||
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
|
||||
env:
|
||||
{{- include "custom_airflow_environment" . | indent 10 }}
|
||||
{{- include "standard_airflow_environment" . | indent 10 }}
|
||||
{{- if .Values.workers.waitForMigrations.env }}
|
||||
{{- tpl (toYaml .Values.workers.waitForMigrations.env) $ | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
|
||||
{{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.workers.extraInitContainers }}
|
||||
{{- toYaml .Values.workers.extraInitContainers | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: worker
|
||||
image: {{ template "airflow_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
securityContext: {{ $containerSecurityContext | nindent 12 }}
|
||||
{{- if $containerLifecycleHooks }}
|
||||
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.workers.command }}
|
||||
command: {{ tpl (toYaml .Values.workers.command) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.workers.args }}
|
||||
args: {{ tpl (toYaml .Values.workers.args) . | nindent 12 }}
|
||||
{{- end }}
|
||||
resources: {{- toYaml .Values.workers.resources | nindent 12 }}
|
||||
{{- if .Values.workers.livenessProbe.enabled }}
|
||||
livenessProbe:
|
||||
initialDelaySeconds: {{ .Values.workers.livenessProbe.initialDelaySeconds }}
|
||||
timeoutSeconds: {{ .Values.workers.livenessProbe.timeoutSeconds }}
|
||||
failureThreshold: {{ .Values.workers.livenessProbe.failureThreshold }}
|
||||
periodSeconds: {{ .Values.workers.livenessProbe.periodSeconds }}
|
||||
exec:
|
||||
command:
|
||||
{{- if .Values.workers.livenessProbe.command }}
|
||||
{{- toYaml .Values.workers.livenessProbe.command | nindent 16 }}
|
||||
{{- else }}
|
||||
- sh
|
||||
- -c
|
||||
- CONNECTION_CHECK_MAX_COUNT=0 exec /entrypoint python -m celery --app {{ include "celery_executor_namespace" . }} inspect ping -d celery@$(hostname)
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- name: worker-logs
|
||||
containerPort: {{ .Values.ports.workerLogs }}
|
||||
volumeMounts:
|
||||
{{- if .Values.volumeMounts }}
|
||||
{{- toYaml .Values.volumeMounts | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.workers.extraVolumeMounts }}
|
||||
{{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 12 }}
|
||||
{{- end }}
|
||||
- name: logs
|
||||
mountPath: {{ template "airflow_logs" . }}
|
||||
{{- include "airflow_config_mount" . | nindent 12 }}
|
||||
{{- if .Values.kerberos.enabled }}
|
||||
- name: kerberos-keytab
|
||||
subPath: "kerberos.keytab"
|
||||
mountPath: {{ .Values.kerberos.keytabPath | quote }}
|
||||
readOnly: true
|
||||
- name: config
|
||||
mountPath: {{ .Values.kerberos.configPath | quote }}
|
||||
subPath: krb5.conf
|
||||
readOnly: true
|
||||
- name: kerberos-ccache
|
||||
mountPath: {{ .Values.kerberos.ccacheMountPath | quote }}
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }}
|
||||
{{- include "airflow_dags_mount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
|
||||
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
|
||||
env:
|
||||
# Only signal the main process, not the process group, to make Warm Shutdown work properly
|
||||
- name: DUMB_INIT_SETSID
|
||||
value: "0"
|
||||
{{- include "custom_airflow_environment" . | indent 10 }}
|
||||
{{- include "standard_airflow_environment" . | indent 10 }}
|
||||
{{- include "container_extra_envs" (list . .Values.workers.env) | indent 10 }}
|
||||
{{- if .Values.workers.kerberosSidecar.enabled }}
|
||||
- name: KRB5_CONFIG
|
||||
value: {{ .Values.kerberos.configPath | quote }}
|
||||
- name: KRB5CCNAME
|
||||
value: {{ include "kerberos_ccache_path" . | quote }}
|
||||
{{- end }}
|
||||
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
|
||||
{{- include "git_sync_container" . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if and $persistence .Values.workers.logGroomerSidecar.enabled }}
|
||||
- name: worker-log-groomer
|
||||
image: {{ template "airflow_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
securityContext: {{ $containerSecurityContextLogGroomerSidecar | nindent 12 }}
|
||||
{{- if $containerLifecycleHooksLogGroomerSidecar }}
|
||||
lifecycle: {{- tpl (toYaml $containerLifecycleHooksLogGroomerSidecar) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.workers.logGroomerSidecar.command }}
|
||||
command: {{ tpl (toYaml .Values.workers.logGroomerSidecar.command) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.workers.logGroomerSidecar.args }}
|
||||
args: {{ tpl (toYaml .Values.workers.logGroomerSidecar.args) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.workers.logGroomerSidecar.retentionDays }}
|
||||
env:
|
||||
- name: AIRFLOW__LOG_RETENTION_DAYS
|
||||
value: "{{ .Values.workers.logGroomerSidecar.retentionDays }}"
|
||||
{{- end }}
|
||||
resources: {{- toYaml .Values.workers.logGroomerSidecar.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: logs
|
||||
mountPath: {{ template "airflow_logs" . }}
|
||||
{{- if .Values.volumeMounts }}
|
||||
{{- toYaml .Values.volumeMounts | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.workers.extraVolumeMounts }}
|
||||
{{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
|
||||
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.workers.kerberosSidecar.enabled }}
|
||||
- name: worker-kerberos
|
||||
image: {{ template "airflow_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
securityContext: {{ $containerSecurityContextKerberosSidecar | nindent 12 }}
|
||||
{{- if $containerLifecycleHooksKerberosSidecar }}
|
||||
lifecycle: {{- tpl (toYaml $containerLifecycleHooksKerberosSidecar) . | nindent 12 }}
|
||||
{{- end }}
|
||||
args: ["kerberos"]
|
||||
resources: {{- toYaml .Values.workers.kerberosSidecar.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: logs
|
||||
mountPath: {{ template "airflow_logs" . }}
|
||||
{{- include "airflow_config_mount" . | nindent 12 }}
|
||||
- name: config
|
||||
mountPath: {{ .Values.kerberos.configPath | quote }}
|
||||
subPath: krb5.conf
|
||||
readOnly: true
|
||||
- name: kerberos-keytab
|
||||
subPath: "kerberos.keytab"
|
||||
mountPath: {{ .Values.kerberos.keytabPath | quote }}
|
||||
readOnly: true
|
||||
- name: kerberos-ccache
|
||||
mountPath: {{ .Values.kerberos.ccacheMountPath | quote }}
|
||||
readOnly: false
|
||||
{{- if .Values.volumeMounts }}
|
||||
{{- toYaml .Values.volumeMounts | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.workers.extraVolumeMounts }}
|
||||
{{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
|
||||
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
|
||||
env:
|
||||
- name: KRB5_CONFIG
|
||||
value: {{ .Values.kerberos.configPath | quote }}
|
||||
- name: KRB5CCNAME
|
||||
value: {{ include "kerberos_ccache_path" . | quote }}
|
||||
{{- include "custom_airflow_environment" . | indent 10 }}
|
||||
{{- include "standard_airflow_environment" . | indent 10 }}
|
||||
{{- end }}
|
||||
{{- if .Values.workers.extraContainers }}
|
||||
{{- toYaml .Values.workers.extraContainers | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if .Values.volumes }}
|
||||
{{- toYaml .Values.volumes | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if .Values.workers.extraVolumes }}
|
||||
{{- tpl (toYaml .Values.workers.extraVolumes) . | nindent 8 }}
|
||||
{{- end }}
|
||||
- name: config
|
||||
configMap:
|
||||
name: {{ template "airflow_config" . }}
|
||||
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
|
||||
- name: webserver-config
|
||||
configMap:
|
||||
name: {{ template "airflow_webserver_config_configmap_name" . }}
|
||||
{{- end }}
|
||||
{{- if .Values.kerberos.enabled }}
|
||||
- name: kerberos-keytab
|
||||
secret:
|
||||
secretName: {{ include "kerberos_keytab_secret" . | quote }}
|
||||
- name: kerberos-ccache
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- if .Values.dags.persistence.enabled }}
|
||||
- name: dags
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ template "airflow_dags_volume_claim" . }}
|
||||
{{- else if .Values.dags.gitSync.enabled }}
|
||||
- name: dags
|
||||
emptyDir: {}
|
||||
{{- if .Values.dags.gitSync.sshKeySecret }}
|
||||
{{- include "git_sync_ssh_key_volume" . | indent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.logs.persistence.enabled }}
|
||||
- name: logs
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ template "airflow_logs_volume_claim" . }}
|
||||
{{- else if not $persistence }}
|
||||
- name: logs
|
||||
emptyDir: {}
|
||||
{{- else }}
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: logs
|
||||
{{- if .Values.workers.persistence.annotations }}
|
||||
annotations: {{- toYaml .Values.workers.persistence.annotations | nindent 10 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.workers.persistence.storageClassName }}
|
||||
storageClassName: {{ .Values.workers.persistence.storageClassName }}
|
||||
{{- end }}
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.workers.persistence.size }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
59
helm/airflow/templates/workers/worker-kedaautoscaler.yaml
Normal file
59
helm/airflow/templates/workers/worker-kedaautoscaler.yaml
Normal file
@@ -0,0 +1,59 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Worker KEDA Scaler
|
||||
#################################
|
||||
{{- if and .Values.workers.keda.enabled (has .Values.executor (list "CeleryExecutor" "CeleryKubernetesExecutor")) }}
|
||||
apiVersion: keda.sh/v1alpha1
|
||||
kind: ScaledObject
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-worker
|
||||
labels:
|
||||
tier: airflow
|
||||
component: worker-horizontalpodautoscaler
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
deploymentName: {{ .Release.Name }}-worker
|
||||
{{- if or (.Values.labels) (.Values.workers.labels) }}
|
||||
{{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
kind: {{ ternary "StatefulSet" "Deployment" .Values.workers.persistence.enabled }}
|
||||
name: {{ include "airflow.fullname" . }}-worker
|
||||
pollingInterval: {{ .Values.workers.keda.pollingInterval }}
|
||||
cooldownPeriod: {{ .Values.workers.keda.cooldownPeriod }}
|
||||
minReplicaCount: {{ .Values.workers.keda.minReplicaCount }}
|
||||
maxReplicaCount: {{ .Values.workers.keda.maxReplicaCount }}
|
||||
{{- if .Values.workers.keda.advanced }}
|
||||
advanced: {{- toYaml .Values.workers.keda.advanced | nindent 4 }}
|
||||
{{- end }}
|
||||
triggers:
|
||||
- type: postgresql
|
||||
metadata:
|
||||
targetQueryValue: "1"
|
||||
{{- if and .Values.pgbouncer.enabled (not .Values.workers.keda.usePgbouncer) }}
|
||||
connectionFromEnv: KEDA_DB_CONN
|
||||
{{- else }}
|
||||
connectionFromEnv: AIRFLOW_CONN_AIRFLOW_DB
|
||||
{{- end }}
|
||||
query: {{ tpl .Values.workers.keda.query . | quote }}
|
||||
{{- end }}
|
||||
55
helm/airflow/templates/workers/worker-networkpolicy.yaml
Normal file
55
helm/airflow/templates/workers/worker-networkpolicy.yaml
Normal file
@@ -0,0 +1,55 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Worker NetworkPolicy
|
||||
#################################
|
||||
{{- if and .Values.networkPolicies.enabled (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")) }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-worker-policy
|
||||
labels:
|
||||
tier: airflow
|
||||
component: airflow-worker-policy
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- if or (.Values.labels) (.Values.workers.labels) }}
|
||||
{{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: worker
|
||||
release: {{ .Release.Name }}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
component: webserver
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: {{ .Values.ports.workerLogs }}
|
||||
{{- end }}
|
||||
48
helm/airflow/templates/workers/worker-service.yaml
Normal file
48
helm/airflow/templates/workers/worker-service.yaml
Normal file
@@ -0,0 +1,48 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Worker Service
|
||||
#################################
|
||||
{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "airflow.fullname" . }}-worker
|
||||
labels:
|
||||
tier: airflow
|
||||
component: worker
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- if or (.Values.labels) (.Values.workers.labels) }}
|
||||
{{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
clusterIP: None
|
||||
selector:
|
||||
tier: airflow
|
||||
component: worker
|
||||
release: {{ .Release.Name }}
|
||||
ports:
|
||||
- name: worker-logs
|
||||
protocol: TCP
|
||||
port: {{ .Values.ports.workerLogs }}
|
||||
targetPort: {{ .Values.ports.workerLogs }}
|
||||
{{- end }}
|
||||
41
helm/airflow/templates/workers/worker-serviceaccount.yaml
Normal file
41
helm/airflow/templates/workers/worker-serviceaccount.yaml
Normal file
@@ -0,0 +1,41 @@
|
||||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
*/}}
|
||||
|
||||
################################
|
||||
## Airflow Worker ServiceAccount
|
||||
#################################
|
||||
{{- if and .Values.workers.serviceAccount.create (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") (eq .Values.executor "KubernetesExecutor") (eq .Values.executor "LocalKubernetesExecutor")) }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
automountServiceAccountToken: {{ .Values.workers.serviceAccount.automountServiceAccountToken }}
|
||||
metadata:
|
||||
name: {{ include "worker.serviceAccountName" . }}
|
||||
labels:
|
||||
tier: airflow
|
||||
component: worker
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- if or (.Values.labels) (.Values.workers.labels) }}
|
||||
{{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.workers.serviceAccount.annotations}}
|
||||
annotations: {{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
Reference in New Issue
Block a user