sa_8001/dsk-devops-toolchains
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

디렉토리 구조 및 각 서비스 추가

Browse Source
This commit is contained in:
dsk-minchulahn
2024-01-03 17:29:11 +09:00
parent 98de2a7627
commit d601d0f259
1632 changed files with 207616 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
helm/airflow/.gitignore vendored Normal file
View File

@@ -0,0 +1,9 @@
# User overrides
config.yaml
config.yml
# Build dir
repository
# Never check in tmpcharts
tmpcharts

42
helm/airflow/.helmignore Normal file
View File

@@ -0,0 +1,42 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*~
# Various IDEs
.project
.idea/
*.tmproj
bin
# We do not want to include our Python Helm Chart Unit test files
tests

6
helm/airflow/Chart.lock Normal file
View File

@@ -0,0 +1,6 @@
dependencies:
- name: postgresql
repository: https://charts.bitnami.com/bitnami
version: 12.10.0
digest: sha256:731562ef1f62ee687121df2d44ff8131a73aa63841f6cac858c30748ad349d55
generated: "2023-08-25T13:23:48.02337-06:00"

137
helm/airflow/Chart.yaml Normal file
View File

@@ -0,0 +1,137 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# apiVersion v2 is Helm 3
---
apiVersion: v2
name: airflow
version: 1.11.0-dev
appVersion: 2.7.1
description: The official Helm chart to deploy Apache Airflow, a platform to
programmatically author, schedule, and monitor workflows
home: https://airflow.apache.org/
sources:
- https://github.com/apache/airflow
icon: https://airflow.apache.org/images/airflow_dark_bg.png
keywords:
- apache
- airflow
- workflow
- scheduler
dependencies:
- name: postgresql
version: 12.10.0
repository: https://charts.bitnami.com/bitnami
condition: postgresql.enabled
maintainers:
- email: dev@airflow.apache.org
name: Apache Airflow PMC
type: application
annotations:
artifacthub.io/links: |
- name: Documentation
url: https://airflow.apache.org/docs/helm-chart/1.8.0/
artifacthub.io/screenshots: |
- title: DAGs View
url: https://airflow.apache.org/docs/apache-airflow/2.7.1/_images/dags.png
- title: Datasets View
url: https://airflow.apache.org/docs/apache-airflow/2.7.1/_images/datasets.png
- title: Grid View
url: https://airflow.apache.org/docs/apache-airflow/2.7.1/_images/grid.png
- title: Graph View
url: https://airflow.apache.org/docs/apache-airflow/2.7.1/_images/graph.png
- title: Calendar View
url: https://airflow.apache.org/docs/apache-airflow/2.7.1/_images/calendar.png
- title: Variable View
url: https://airflow.apache.org/docs/apache-airflow/2.7.1/_images/variable_hidden.png
- title: Gantt Chart
url: https://airflow.apache.org/docs/apache-airflow/2.7.1/_images/gantt.png
- title: Task Duration
url: https://airflow.apache.org/docs/apache-airflow/2.7.1/_images/duration.png
- title: Code View
url: https://airflow.apache.org/docs/apache-airflow/2.7.1/_images/code.png
artifacthub.io/changes: |
- description: Add support for container security context
kind: added
links:
- name: '#31043'
url: https://github.com/apache/airflow/pull/31043
- description: Validate ``executor`` and ``config.core.executor`` match
kind: changed
links:
- name: '#30693'
url: https://github.com/apache/airflow/pull/30693
- description: Support ``minAvailable`` property for PodDisruptionBudget
kind: changed
links:
- name: '#30603'
url: https://github.com/apache/airflow/pull/30603
- description: Add ``volumeMounts`` to dag processor ``waitForMigrations``
kind: changed
links:
- name: '#30990'
url: https://github.com/apache/airflow/pull/30990
- description: Template extra volumes
kind: changed
links:
- name: '#30773'
url: https://github.com/apache/airflow/pull/30773
- description: Fix webserver probes timeout and period
kind: fixed
links:
- name: '#30609'
url: https://github.com/apache/airflow/pull/30609
- description: Add missing ``waitForMigrations`` for workers
kind: fixed
links:
- name: '#31625'
url: https://github.com/apache/airflow/pull/31625
- description: Add missing ``priorityClassName`` to K8S worker pod template
kind: fixed
links:
- name: '#31328'
url: https://github.com/apache/airflow/pull/31328
- description: Adding log groomer sidecar to dag processor
kind: fixed
links:
- name: '#30726'
url: https://github.com/apache/airflow/pull/30726
- description: Do not propagate global security context to statsd and redis
kind: fixed
links:
- name: '#31865'
url: https://github.com/apache/airflow/pull/31865
- description: 'Misc: Default Airflow version to 2.6.3'
kind: changed
links:
- name: '#31979'
url: https://github.com/apache/airflow/pull/31979
- description: 'Misc: Use template comments for the chart license header'
kind: changed
links:
- name: '#30569'
url: https://github.com/apache/airflow/pull/30569
- description: 'Misc: Align ``apiVersion`` and ``kind`` order in chart templates'
kind: changed
links:
- name: '#31850'
url: https://github.com/apache/airflow/pull/31850
- description: 'Misc: Cleanup Kubernetes < 1.23 support'
kind: changed
links:
- name: '#31847'
url: https://github.com/apache/airflow/pull/31847

14
helm/airflow/INSTALL Normal file
View File

@@ -0,0 +1,14 @@
## INSTALL / BUILD instructions for Apache Airflow Chart
# The Assumption here is that you have a running Kubernetes cluster
# and helm installed & configured to talk with the cluster
# Run `helm install` Command
helm install airflow .
# If you want to install in a particular namespace
## Create that namespace (example 'airflow' here, change it as needed)
kubectl create namespace airflow
## Install the chart in that namespace
helm install airflow -n airflow .

201
helm/airflow/LICENSE Normal file
View File

@@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

17
helm/airflow/NOTICE Normal file
View File

@@ -0,0 +1,17 @@
Apache Airflow
Copyright 2016-2021 The Apache Software Foundation
This product includes software developed at The Apache Software
Foundation (http://www.apache.org/).
=======================================================================
postgresql:
-----
This product contains vendored-in postgresql Helm chart.
Copyright © 2022 Bitnami
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

63
helm/airflow/README.md Normal file
View File

@@ -0,0 +1,63 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
# Helm Chart for Apache Airflow
[![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/apache-airflow)](https://artifacthub.io/packages/search?repo=apache-airflow)
[Apache Airflow](https://airflow.apache.org/) is a platform to programmatically author, schedule and monitor workflows.
## Introduction
This chart will bootstrap an [Airflow](https://airflow.apache.org) deployment on a [Kubernetes](http://kubernetes.io)
cluster using the [Helm](https://helm.sh) package manager.
## Requirements
- Kubernetes 1.24+ cluster
- Helm 3.0+
- PV provisioner support in the underlying infrastructure (optionally)
## Features
* Supported executors: ``LocalExecutor``, ``CeleryExecutor``, ``KubernetesExecutor``, ``LocalKubernetesExecutor``, ``CeleryKubernetesExecutor``
* Supported Airflow version: ``1.10+``, ``2.0+``
* Supported database backend: ``PostgresSQL``, ``MySQL``
* Autoscaling for ``CeleryExecutor`` provided by KEDA
* PostgreSQL and PgBouncer with a battle-tested configuration
* Monitoring:
* StatsD/Prometheus metrics for Airflow
* Prometheus metrics for PgBouncer
* Flower
* Automatic database migration after a new deployment
* Administrator account creation during deployment
* Kerberos secure configuration
* One-command deployment for any type of executor. You don't need to provide other services e.g. Redis/Database to test the Airflow.
## Documentation
Full documentation for Helm Chart (latest **stable** release) lives [on the website](https://airflow.apache.org/docs/helm-chart/).
> Note: If you're looking for documentation for main branch (latest development branch): you can find it on [s.apache.org/airflow-docs/](http://apache-airflow-docs.s3-website.eu-central-1.amazonaws.com/docs/helm-chart/latest/index.html).
> Source code for documentation is in [../docs/helm-chart](https://github.com/apache/airflow/tree/main/docs/helm-chart)
>
## Contributing
Want to help build Apache Airflow? Check out our [contributing documentation](https://github.com/apache/airflow/blob/main/CONTRIBUTING.rst).

749
helm/airflow/RELEASE_NOTES.rst Normal file
View File

@@ -0,0 +1,749 @@
.. Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
.. http://www.apache.org/licenses/LICENSE-2.0
.. Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
.. contents:: Apache Airflow Helm Chart Releases
:local:
:depth: 1
Run ``helm repo update`` before upgrading the chart to the latest version.
.. towncrier release notes start
Airflow Helm Chart 1.10.0 (2023-06-26)
--------------------------------------
Significant Changes
^^^^^^^^^^^^^^^^^^^
Default Airflow image is updated to ``2.6.2`` (#31979)
""""""""""""""""""""""""""""""""""""""""""""""""""""""
The default Airflow image that is used with the Chart is now ``2.6.2``, previously it was ``2.5.3``.
New Features
^^^^^^^^^^^^
- Add support for container security context (#31043)
Improvements
^^^^^^^^^^^^
- Validate ``executor`` and ``config.core.executor`` match (#30693)
- Support ``minAvailable`` property for PodDisruptionBudget (#30603)
- Add ``volumeMounts`` to dag processor ``waitForMigrations`` (#30990)
- Template extra volumes (#30773)
Bug Fixes
^^^^^^^^^
- Fix webserver probes timeout and period (#30609)
- Add missing ``waitForMigrations`` for workers (#31625)
- Add missing ``priorityClassName`` to K8S worker pod template (#31328)
- Adding log groomer sidecar to dag processor (#30726)
- Do not propagate global security context to statsd and redis (#31865)
Misc
^^^^
- Default Airflow version to 2.6.2 (#31979)
- Use template comments for the chart license header (#30569)
- Align ``apiVersion`` and ``kind`` order in chart templates (#31850)
- Cleanup Kubernetes < 1.23 support (#31847)
Airflow Helm Chart 1.9.0 (2023-04-14)
-------------------------------------
Significant Changes
^^^^^^^^^^^^^^^^^^^
Default PgBouncer and PgBouncer Exporter images have been updated (#29919)
""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
The PgBouncer and PgBouncer Exporter images are based on newer software/os. They are also multi-platform AMD/ARM images:
* ``pgbouncer``: 1.16.1 based on alpine 3.14 (``airflow-pgbouncer-2023.02.24-1.16.1``)
* ``pgbouncer-exporter``: 0.14.0 based on alpine 3.17 (``apache/airflow:airflow-pgbouncer-exporter-2023.02.21-0.14.0``)
Default Airflow image is updated to ``2.5.3`` (#30411)
""""""""""""""""""""""""""""""""""""""""""""""""""""""
The default Airflow image that is used with the Chart is now ``2.5.3``, previously it was ``2.5.1``.
New Features
^^^^^^^^^^^^
- Add support for ``hostAliases`` for Airflow webserver and scheduler (#30051)
- Add support for annotations on StatsD Deployment and cleanup CronJob (#30126)
- Add support for annotations in logs PVC (#29270)
- Add support for annotations in extra ConfigMap and Secrets (#30303)
- Add support for pod annotations to PgBouncer (#30168)
- Add support for ``ttlSecondsAfterFinished`` on ``migrateDatabaseJob`` and ``createUserJob`` (#29314)
- Add support for using SHA digest of Docker images (#30214)
Improvements
^^^^^^^^^^^^
- Template extra volumes in Helm Chart (#29357)
- Make Liveness/Readiness Probe timeouts configurable for PgBouncer Exporter (#29752)
- Enable individual trigger logging (#29482)
Bug Fixes
^^^^^^^^^
- Add ``config.kubernetes_executor`` to values (#29818)
- Block extra properties in image config (#30217)
- Remove replicas if KEDA is enabled (#29838)
- Mount ``kerberos.keytab`` to worker when enabled (#29526)
- Fix adding annotations for dag persistence PVC (#29622)
- Fix ``bitnami/postgresql`` default username and password (#29478)
- Add global volumes in pod template file (#29295)
- Add log groomer sidecar to triggerer service (#29392)
- Helm deployment fails when ``postgresql.nameOverride`` is used (#29214)
Doc only changes
^^^^^^^^^^^^^^^^
- Add gitSync optional env description (#29378)
- Add webserver NodePort example (#29460)
- Include Rancher in Helm chart install instructions (#28416)
- Change RSA SSH host key to reflect update from Github (#30286)
Misc
^^^^
- Update Airflow version to 2.5.3 (#30411)
- Switch to newer versions of PgBouncer and PgBouncer Exporter in chart (#29919)
- Reformat chart templates (#29917)
- Reformat chart templates part 2 (#29941)
- Reformat chart templates part 3 (#30312)
- Replace deprecated k8s registry references (#29938)
- Fix ``airflow_dags_mount`` formatting (#29296)
- Fix ``webserver.service.ports`` formatting (#29297)
Airflow Helm Chart 1.8.0 (2023-02-06)
-------------------------------------
Significant Changes
^^^^^^^^^^^^^^^^^^^
``bitnami/postgresql`` subchart updated to ``12.1.9`` (#29071)
""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
The version of postgresql installed is still version 11.
If you are upgrading an existing helm release with the built-in postgres database, you will either need to delete your release and reinstall fresh, or manually delete these 2 objects:
.. code-block::
kubectl delete secret {RELEASE_NAME}-postgresql
kubectl delete statefulset {RELEASE_NAME}-postgresql
As a reminder, it is recommended to `set up an external database <https://airflow.apache.org/docs/helm-chart/stable/production-guide.html#database>`_ in production.
This version of the chart uses different variable names for setting usernames and passwords in the postgres database.
- ``postgresql.auth.enablePostgresUser`` is used to determine if the "postgres" admin account will be created.
- ``postgresql.auth.postgresPassword`` sets the password for the "postgres" user.
- ``postgresql.auth.username`` and ``postrgesql.auth.password`` are used to set credentials for a non-admin account if desired.
- ``postgresql.postgresqlUsername`` and ``postgresql.postresqlPassword``, which were used in the previous version of the chart, are no longer used.
Users will need to make those changes in their values files if they are changing the Postgres configuration.
Previously the subchart version was ``10.5.3``.
Default ``dags.gitSync.wait`` reduced to ``5`` seconds (#27625)
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
The default for ``dags.gitSync.wait`` has been reduced from ``60`` seconds to ``5`` seconds to reduce the likelihood of DAGs
becoming inconsistent between Airflow components. This will, however, increase traffic to the remote git repository.
Default Airflow image is updated to ``2.5.1`` (#29074)
""""""""""""""""""""""""""""""""""""""""""""""""""""""
The default Airflow image that is used with the Chart is now ``2.5.1``, previously it was ``2.4.1``.
Default git-sync image is updated to ``3.6.3`` (#27848)
"""""""""""""""""""""""""""""""""""""""""""""""""""""""
The default git-sync image that is used with the Chart is now ``3.6.3``, previously it was ``3.4.0``.
Default redis image is updated to ``7-bullseye`` (#27443)
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""
The default redis image that is used with the Chart is now ``7-bullseye``, previously it was ``6-bullseye``.
New Features
^^^^^^^^^^^^
- Add annotations on deployments (#28688)
- Add global volume & volumeMounts to the chart (#27781)
Improvements
^^^^^^^^^^^^
- Add support for ``webserverConfigConfigMapName`` (#27419)
- Enhance chart to allow overriding command-line args to statsd exporter (#28041)
- Add support for NodePort in Services (#26945)
- Add worker log-groomer-sidecar enable option (#27178)
- Add HostAliases to Pod template file (#27544)
- Allow PgBouncer replicas to be configurable (#27439)
Bug Fixes
^^^^^^^^^
- Create scheduler service to serve task logs for LocalKubernetesExecutor (#28828)
- Fix NOTES.txt to show correct URL (#28264)
- Add worker service account for LocalKubernetesExecutor (#28813)
- Remove checks for 1.19 api checks (#28461)
- Add airflow_local_settings to all airflow containers (#27779)
- Make custom env vars optional for job templates (#27148)
- Decrease default gitSync wait (#27625)
- Add ``extraVolumeMounts`` to sidecars too (#27420)
- Fix PgBouncer after PostgreSQL subchart upgrade (#29207)
Doc only changes
^^^^^^^^^^^^^^^^
- Enhance production guide with a few Argo specific guidelines (#29078)
- Add doc note about Pod template images (#29032)
- Update production guide db section (#28610)
- Fix to LoadBalancer snippet (#28014)
- Fix gitSync example code (#28083)
- Correct repo example for cloning via ssh (#27671)
Misc
^^^^
- Update Airflow version to 2.5.1 (#29074)
- Update git-sync to 3.6.3 (#27848)
- Upgrade ``bitnami/postgresql`` subchart to 12.1.9 (#29071)
- Update redis to 7 (#27443)
- Replace helm chart icon (#27704)
Airflow Helm Chart 1.7.0 (2022-10-14)
-------------------------------------
Significant Changes
^^^^^^^^^^^^^^^^^^^
Default Airflow image is updated to ``2.4.1`` (#26485)
""""""""""""""""""""""""""""""""""""""""""""""""""""""
The default Airflow image that is used with the Chart is now ``2.4.1``, previously it was ``2.3.2``.
New Features
^^^^^^^^^^^^
- Make cleanup job history configurable (#26838)
- Added labels to specific Airflow components (#25031)
- Add StatsD ``overrideMappings`` in Helm chart values (#26598)
- Adding ``podAnnotations`` to StatsD deployment template (#25732)
- Container specific extra environment variables (#24784)
- Custom labels for extra Secrets and ConfigMaps (#25283)
- Add ``revisionHistoryLimit`` to all deployments (#25059)
- Adding ``podAnnotations`` to Redis StatefulSet (#23708)
- Provision Standalone Dag Processor (#23711)
- Add configurable scheme for webserver probes (#22815)
- Add support for KEDA HPA config to Helm chart (#24220)
Improvements
^^^^^^^^^^^^
- Add 'executor' label to Airflow scheduler deployment (#25684)
- Add default ``flower_url_prefix`` in Helm chart values (#26415)
- Add liveness probe to Celery workers (#25561)
- Use ``sql_alchemy_conn`` for celery result backend when ``result_backend`` is not set (#24496)
Bug Fixes
^^^^^^^^^
- Fix pod template ``imagePullPolicy`` (#26423)
- Do not declare a volume for ``sshKeySecret`` if dag persistence is enabled (#22913)
- Pass worker annotations to generated pod template (#24647)
- Fix semver compare number for ``jobs check`` command (#24480)
- Use ``--local`` flag for liveness probes in Airflow 2.5+ (#24999)
Doc only changes
^^^^^^^^^^^^^^^^
- Improve documentation on helm hooks disabling (#26747)
- Remove ``ssh://`` prefix from git repo value (#26632)
- Fix ``defaultAirflowRepository`` comment (#26428)
- Baking DAGs into Docker image (#26401)
- Reload pods when using the same DAG tag (#24576)
- Minor clarifications about ``result_backend``, dag processor, and ``helm uninstall`` (#24929)
- Add hyperlinks to GitHub PRs for Release Notes (#24532)
- Terraform should not use Helm hooks for starting jobs (#26604)
- Flux should not use Helm hooks for starting jobs (#24288)
- Provide details on how to pull Airflow image from a private repository (#24394)
- Helm logo no longer a link (#23977)
- Document LocalKubernetesExecutor support in chart (#23876)
- Update Production Guide (#23836)
Misc
^^^^
- Default Airflow version to 2.4.1 (#26485)
- Vendor in the Bitnami chart (#24395)
- Remove kubernetes 1.20 support (#25871)
Airflow Helm Chart 1.6.0 (2022-05-20)
-------------------------------------
Significant Changes
^^^^^^^^^^^^^^^^^^^
Default Airflow image is updated to ``2.3.0`` (#23386)
""""""""""""""""""""""""""""""""""""""""""""""""""""""
The default Airflow image that is used with the Chart is now ``2.3.0``, previously it was ``2.2.4``.
``ingress.enabled`` is deprecated
"""""""""""""""""""""""""""""""""
Instead of having a single flag to control ingress resources for both the webserver and flower, there
are now separate flags to control them individually, ``ingress.web.enabled`` and ``ingress.flower.enabled``.
``ingress.enabled`` is now deprecated, but will still continue to control them both.
Flower disabled by default
""""""""""""""""""""""""""
Flower is no longer enabled by default when using CeleryExecutor. If you'd like to deploy it, set
``flower.enabed`` to true in your values file.
New Features
^^^^^^^^^^^^
- Support ``annotations`` on ``volumeClaimTemplates`` (#23433)
- Add support for ``topologySpreadConstraints`` to Helm Chart (#22712)
- Helm support for LocalKubernetesExecutor (#22388)
- Add ``securityContext`` config for Redis to Helm chart (#22182)
- Allow ``annotations`` on Helm DAG PVC (#22261)
- enable optional ``subPath`` for DAGs volume mount (#22323)
- Added support to override ``auth_type`` in ``auth_file`` in PgBouncer Helm configuration (#21999)
- Add ``extraVolumeMounts`` to Flower (#22414)
- Add webserver ``PodDisruptionBudget`` (#21735)
Improvements
^^^^^^^^^^^^
- Ensure the messages from migration job show up early (#23479)
- Allow migration jobs and init containers to be optional (#22195)
- Use jobs check command for liveness probe check in Airflow 2 (#22143)
Doc only changes
^^^^^^^^^^^^^^^^
- Adds ``resultBackendSecretName`` warning in Helm production docs (#23307)
Misc
^^^^
- Update default Airflow version to ``2.3.0`` (#23386)
- Move the database configuration to a new section (#22284)
- Disable flower in chart by default (#23737)
Airflow Helm Chart 1.5.0, (2022-03-07)
--------------------------------------
Significant changes
^^^^^^^^^^^^^^^^^^^
Default Airflow image is updated to ``2.2.4``
"""""""""""""""""""""""""""""""""""""""""""""
The default Airflow image that is used with the Chart is now ``2.2.4``, previously it was ``2.2.3``.
Removed ``config.api``
""""""""""""""""""""""
This section configured the authentication backend for the Airflow API but used the same values as the Airflow default setting, which made it unnecessary to
declare the same again.
New Features
^^^^^^^^^^^^
- Add support for custom command and args in jobs (#20864)
- Support for ``priorityClassName`` (#20794)
- Add ``envFrom`` to the Flower deployment (#21401)
- Add annotations to cleanup pods (#21484)
Improvements
^^^^^^^^^^^^
- Speedup liveness probe for scheduler and triggerer (#20833, #21108)
- Update git-sync to v3.4.0 (#21309)
- Remove default auth backend setting (#21640)
Bug Fixes
^^^^^^^^^
- Fix elasticsearch URL when username/password are empty (#21222)
- Mount ``airflow.cfg`` in wait-for-airflow-migrations containers (#20609)
- Grant pod log reader to triggerer ServiceAccount (#21111)
Doc only changes
^^^^^^^^^^^^^^^^
- Simplify chart docs for configuring Airflow (#21747)
- Add extra information about time synchronization needed (#21685)
- Fix extra containers docs (#20787)
Misc
^^^^
- Use ``2.2.4`` as default Airflow version (#21745)
- Change Redis image to bullseye (#21875)
Airflow Helm Chart 1.4.0, (2022-01-10)
--------------------------------------
Significant changes
^^^^^^^^^^^^^^^^^^^
Default Airflow image is updated to ``2.2.3``
"""""""""""""""""""""""""""""""""""""""""""""
The default Airflow image that is used with the Chart is now ``2.2.3``, previously it was ``2.2.1``.
``ingress.web.hosts`` and ``ingress.flower.hosts`` parameters data type has changed and ``ingress.web.tls`` and ``ingress.flower.tls`` have moved
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
``ingress.web.hosts`` and ``ingress.flower.hosts`` have had their types have been changed from an array of strings to an array of objects. ``ingress.web.tls`` and ``ingress.flower.tls`` can now be specified per host in ``ingress.web.hosts`` and ``ingress.flower.hosts`` respectively.
The old parameter names will continue to work, however support for them will be removed in a future release so please update your values file.
Fixed precedence of ``nodeSelector``, ``affinity`` and ``tolerations`` params
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
``nodeSelector``, ``affinity`` and ``tolerations`` params precedence has been fixed on all components. Now component-specific params
(e.g. ``webserver.affinity``) takes precedence over the global param (e.g. ``affinity``).
Default ``KubernetesExecutor`` worker affinity removed
""""""""""""""""""""""""""""""""""""""""""""""""""""""
Previously a default affinity was added to ``KubernetesExecutor`` workers to spread the workers out across nodes. This default affinity is no
longer set because, in general, there is no reason to spread task-specific workers across nodes.
Changes in webserver and flower ``NetworkPolicy`` default ports
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
The defaults for ``webserver.networkPolicy.ingress.ports`` and ``flower.networkPolicy.ingress.ports`` moved away from using named ports to numerical ports to avoid issues with OpenShift.
Increase default ``livenessProbe`` ``timeoutSeconds`` for scheduler and triggerer
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
The default timeout for the scheduler and triggerer ``livenessProbe`` has been increased from 10 seconds to 20 seconds.
New Features
^^^^^^^^^^^^
- Add ``type`` to extra secrets param (#20599)
- Support elasticsearch connection ``scheme`` (#20564)
- Allows to disable built-in secret variables individually (#18974)
- Add support for ``securityContext`` (#18249)
- Add extra containers, volumes and volume mounts for jobs (#18808)
- Allow ingress multiple hostnames w/diff secrets (#18542)
- PgBouncer extra volumes, volume mounts, and ``sslmode`` (#19749)
- Allow specifying kerberos keytab (#19054)
- Allow disabling the Helm hooks (#18776, #20018)
- Add ``migration-wait-timeout`` (#20069)
Improvements
^^^^^^^^^^^^
- Increase default ``livenessProbe`` timeout (#20698)
- Strict schema for k8s objects for values.yaml (#19181)
- Remove unnecessary ``pod_template_file`` defaults (#19690)
- Use built-in ``check-migrations`` command for Airflow>=2 (#19676)
Bug Fixes
^^^^^^^^^
- Fix precedence of ``affinity``, ``nodeSelector``, and ``tolerations`` (#20641)
- Fix chart elasticsearch default port 80 to 9200. (#20616)
- Fix network policy issue for webserver and flower ui (#20199)
- Use local definitions for k8s schema validation (#20544)
- Add custom labels for ingresses/PVCs (#20535)
- Fix extra secrets/configmaps labels (#20464)
- Fix flower restarts on update (#20316)
- Properly quote namespace names (#20266)
Doc only changes
^^^^^^^^^^^^^^^^
- Add ``helm dependency update`` step to chart INSTALL (#20702)
- Reword section covering the envvar secrets (#20566)
- Add "Customizing Workers" page (#20331)
- Include Datadog example in production guide (#17996)
- Update production Helm guide database section to use k8s secret (#19892)
- Fix ``multiNamespaceMode`` docs to also cover KPO (#19879)
- Clarify Helm behaviour when it comes to loading default connections (#19708)
Misc
^^^^
- Use ``2.2.3`` as default Airflow version (#20450)
- Add ArtifactHUB annotations for docs and screenshots (#20558)
- Add kubernetes 1.21 support (#19557)
Airflow Helm Chart 1.3.0 (2021-11-08)
-------------------------------------
Significant changes
^^^^^^^^^^^^^^^^^^^
Default Airflow image is updated to ``2.2.1``
"""""""""""""""""""""""""""""""""""""""""""""
The default Airflow image that is used with the Chart is now ``2.2.1`` (which is Python ``3.7``), previously it was ``2.1.4`` (which is Python ``3.6``).
The triggerer component requires Python ``3.7``. If you require Python ``3.6`` and Airflow ``2.2.0`` or later, use a ``3.6`` based image and set ``triggerer.enabled=False`` in your values.
Resources made configurable for ``airflow-run-airflow-migrations`` job
""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
Now it's possible to set resources requests and limits for migration job through ``migrateDatabaseJob.resources`` value.
New Features
^^^^^^^^^^^^
- Chart: Add resources for ``cleanup`` and ``createuser`` jobs (#19263)
- Chart: Add labels to jobs created by cleanup pods (#19225)
- Add migration job resources (#19175)
- Allow custom pod annotations to all components (#18481)
- Chart: Make PgBouncer cmd/args configurable (#18910)
- Chart: Use python 3.7 by default; support disabling triggerer (#18920)
Improvements
^^^^^^^^^^^^
- Chart: Increase default liveness probe timeout (#19003)
- Chart: Mount DAGs in triggerer (#18753)
Bug Fixes
^^^^^^^^^
- Allow Airflow UI to create worker pod via Clear > Run (#18272)
- Allow Airflow standard images to run in OpenShift utilizing the official Helm chart #18136 (#18147)
Doc only changes
^^^^^^^^^^^^^^^^
- Chart: Fix ``extraEnvFrom`` examples (#19144)
- Chart docs: Update webserver secret key reference configuration (#18595)
- Fix helm chart links in source install guide (#18588)
Misc
^^^^
- Chart: Update default Airflow version to ``2.2.1`` (#19326)
- Modernize dockerfiles builds (#19327)
- Chart: Use strict k8s schemas for template validation (#19379)
Airflow Helm Chart 1.2.0 (2021-09-28)
-------------------------------------
Significant Changes
^^^^^^^^^^^^^^^^^^^
``ingress.web.host`` and ``ingress.flower.host`` parameters have been renamed and data type changed
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
``ingress.web.host`` and ``ingress.flower.host`` parameters have been renamed to ``ingress.web.hosts`` and ``ingress.flower.hosts``, respectively. Their types have been changed from a string to an array of strings.
The old parameter names will continue to work, however support for them will be removed in a future release so please update your values file.
Default Airflow version is updated to ``2.1.4``
"""""""""""""""""""""""""""""""""""""""""""""""
The default Airflow version that is installed with the Chart is now ``2.1.4``, previously it was ``2.1.2``.
Removed ``ingress.flower.precedingPaths`` and ``ingress.flower.succeedingPaths`` parameters
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
``ingress.flower.precedingPaths`` and ``ingress.flower.succeedingPaths`` parameters have been removed as they had previously had no effect on rendered YAML output.
Change of default ``path`` on Ingress
"""""""""""""""""""""""""""""""""""""
With the move to support the stable Kubernetes Ingress API the default path has been changed from being unset to ``/``. For most Ingress controllers this should not change the behavior of the resulting Ingress resource.
New Features
^^^^^^^^^^^^
- Add Triggerer to Helm Chart (#17743)
- Chart: warn when webserver secret key isn't set (#18306)
- add ``extraContainers`` for ``migrateDatabaseJob`` (#18379)
- Labels on job templates (#18403)
- Chart: Allow running and waiting for DB Migrations using default image (#18218)
- Chart: Make cleanup cronjob cmd/args configurable (#17970)
- Chart: configurable number of retention days for log groomers (#17764)
- Chart: Add ``loadBalancerSourceRanges`` in webserver and flower services (#17666)
- Chart: Support ``extraContainers`` in k8s workers (#17562)
Improvements
^^^^^^^^^^^^
- Switch to latest version of PGBouncer-Exporter (#18429)
- Chart: Ability to access http k8s via multiple hostnames (#18257)
- Chart: Use stable API versions where available (#17211)
- Chart: Allow ``podTemplate`` to be templated (#17560)
Bug Fixes
^^^^^^^^^
- Chart: Fix applying ``labels`` on Triggerer (#18299)
- Fixes warm shutdown for celery worker. (#18068)
- Chart: Fix minor Triggerer issues (#18105)
- Chart: fix webserver secret key update (#18079)
- Chart: fix running with ``uid`` ``0`` (#17688)
- Chart: use ServiceAccount template for log reader RoleBinding (#17645)
- Chart: Fix elasticsearch-secret template port default function (#17428)
- KEDA task count query should ignore k8s queue (#17433)
Doc only changes
^^^^^^^^^^^^^^^^
- Chart Doc: Delete extra space in adding connections doc (#18424)
- Improves installing from sources pages for all components (#18251)
- Chart docs: Format ``loadBalancerSourceRanges`` using code-block (#17763)
- Doc: Fix a broken link in an ssh-related warning message (#17294)
- Chart: Add instructions to Update Helm Repo before upgrade (#17282)
- Chart docs: better note for logs existing PVC permissions (#17177)
Misc
^^^^
- Chart: Update the default Airflow version to ``2.1.4`` (#18354)
Airflow Helm Chart 1.1.0 (2021-07-26)
-------------------------------------
Significant Changes
^^^^^^^^^^^^^^^^^^^
Run ``helm repo update`` before upgrading the chart to the latest version.
Default Airflow version is updated to ``2.1.2``
"""""""""""""""""""""""""""""""""""""""""""""""
The default Airflow version that is installed with the Chart is now ``2.1.2``, previously it was ``2.0.2``.
Helm 2 no longer supported
""""""""""""""""""""""""""
This chart has dropped support for `Helm 2 as it has been deprecated <https://helm.sh/blog/helm-v2-deprecation-timeline/>`__ and no longer receiving security updates since November 2020.
``webserver.extraNetworkPolicies`` and ``flower.extraNetworkPolicies`` parameters have been renamed
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
``webserver.extraNetworkPolicies`` and ``flower.extraNetworkPolicies`` have been renamed to ``webserver.networkPolicy.ingress.from`` and ``flower.networkPolicy.ingress.from``, respectively. Their values and behavior are the same.
The old parameter names will continue to work, however support for them will be removed in a future release so please update your values file.
Removed ``dags.gitSync.root``, ``dags.gitSync.dest``, and ``dags.gitSync.excludeWebserver`` parameters
""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
The ``dags.gitSync.root`` and ``dags.gitSync.dest`` parameters did not provide any useful behaviors to chart users so they have been removed.
If you have them set in your values file you can safely remove them.
The ``dags.gitSync.excludeWebserver`` parameter was mistakenly included in the charts ``values.schema.json``. If you have it set in your values file,
you can safely remove it.
``nodeSelector``, ``affinity`` and ``tolerations`` on ``migrateDatabaseJob`` and ``createUserJob`` jobs
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
The ``migrateDatabaseJob`` and ``createUserJob`` jobs were incorrectly using the ``webserver``'s ``nodeSelector``, ``affinity``
and ``tolerations`` (if set). Each job is now configured separately.
New Features
^^^^^^^^^^^^
- Chart: Allow using ``krb5.conf`` with ``CeleryExecutor`` (#16822)
- Chart: Refactor webserver and flower NetworkPolicy (#16619)
- Chart: Apply worker's node assigning settings to Pod Template File (#16663)
- Chart: Support for overriding webserver and flower service ports (#16572)
- Chart: Support ``extraContainers`` and ``extraVolumes`` in flower (#16515)
- Chart: Allow configuration of pod resources in helm chart (#16425)
- Chart: Support job level annotations; fix jobs scheduling config (#16331)
- feat: Helm chart adding ``minReplicaCount`` to the KEDA ``worker-kedaautoscaler.yaml`` (#16262)
- Chart: Adds support for custom command and args (#16153)
- Chart: Add extra ini config to ``pgbouncer`` (#16120)
- Chart: Add ``extraInitContainers`` to scheduler/webserver/workers (#16098)
- Configurable resources for git-sync sidecar (#16080)
- Chart: Template ``airflowLocalSettings`` and ``webserver.webserverConfig`` (#16074)
- Support ``strategy``/``updateStrategy`` on scheduler (#16069)
- Chart: Add both airflow and extra annotations to jobs (#16058)
- ``loadBalancerIP`` and ``annotations`` for both Flower and Webserver (#15972)
Improvements
^^^^^^^^^^^^
- Chart: Update Postgres subchart to 10.5.3 (#17041)
- Chart: Update the default Airflow version to ``2.1.2`` (#17013)
- Update default image as ``2.1.1`` for Helm Chart (#16785)
- Chart: warn when using default logging with ``KubernetesExecutor`` (#16784)
- Drop support for Helm 2 (#16575)
- Chart: ``podAntiAffinity`` for scheduler, webserver, and workers (#16315)
- Chart: Update the default Airflow Version to ``2.1.0`` (#16273)
- Chart: Only mount DAGs in webserver when required (#16229)
- Chart: Remove ``git-sync``: ``root`` and ``dest`` params (#15955)
- Chart: Add warning about missing ``knownHosts`` (#15950)
Bug Fixes
^^^^^^^^^
- Chart: Create a random secret for Webserver's flask secret key (#17142)
- Chart: fix labels on cleanup ServiceAccount (#16722)
- Chart: Fix overriding node assigning settings on Worker Deployment (#16670)
- Chart: Always deploy a ``gitsync`` init container (#16339)
- Chart: Fix updating from ``KubernetesExecutor`` to ``CeleryExecutor`` (#16242)
- Chart: Adds labels to Kubernetes worker pods (#16203)
- Chart: Allow ``webserver.base_url`` to be templated (#16126)
- Chart: Fix ``PgBouncer`` exporter sidecar (#16099)
- Remove ``dags.gitSync.excludeWebserver`` from chart ``values.schema.json`` (#16070)
- Chart: Fix Elasticsearch secret created without Elasticsearch enabled (#16015)
- Handle special characters in passwords for Helm Chart (#16004)
- Fix flower ServiceAccount created without flower enable (#16011)
- Chart: ``gitsync`` Clean Up for ``KubernetesExecutor`` (#15925)
- Mount DAGs read only when using ``gitsync`` (#15953)
Doc only changes
^^^^^^^^^^^^^^^^
- Chart docs: note uid write permissions for existing PVC (#17170)
- Chart Docs: Add single-line description for ``multiNamespaceMode`` (#17147)
- Chart: Update description for Helm chart to include 'official' (#17040)
- Chart: Better comment and example for ``podTemplate`` (#16859)
- Chart: Add more clear docs for setting ``pod_template_file.yaml`` (#16632)
- Fix description on ``scheduler.livenessprobe.periodSeconds`` (#16486)
- Chart docs: Fix ``extrasecrets`` example (#16305)
- Small improvements for ``README.md`` files (#16244)
Misc
^^^^
- Removes pylint from our toolchain (#16682)
- Update link to match what is in pre-commit (#16408)
- Chart: Update the ``appVersion`` to 2.1.0 in ``Chart.yaml`` (#16337)
- Rename the main branch of the Airflow repo to be ``main`` (#16149)
- Update Chart version to ``1.1.0-rc1`` (#16124)

BIN
helm/airflow/charts/postgresql-12.10.0.tgz Normal file
View File

Binary file not shown.

28
helm/airflow/dockerfiles/README.md Normal file
View File

@@ -0,0 +1,28 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
Those are images that are needed for the Helm Chart.
In each of the images you can find "build_and_push.sh" script that builds and pushes the image.
You need to be a PMC with direct push access to "apache/airflow" DockerHub registry
to be able to push to the Airflow DockerHub registry.
You can set the DOCKERHUB_USER variable to push to your own DockerHub user if you want
to test the image or build your own image.

57
helm/airflow/dockerfiles/pgbouncer-exporter/Dockerfile Normal file
View File

@@ -0,0 +1,57 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
ARG ALPINE_VERSION="3.17"
ARG GO_VERSION
FROM golang:${GO_VERSION} AS builder
ARG PGBOUNCER_EXPORTER_VERSION
WORKDIR /usr/src/myapp
SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
RUN URL="https://github.com/jbub/pgbouncer_exporter/archive/v${PGBOUNCER_EXPORTER_VERSION}.tar.gz" \
&& curl -L "${URL}" | tar -zx --strip-components 1 \
&& PLATFORM=$([ "$(uname -m)" = "aarch64" ] && echo "arm64" || echo "amd64" )\
&& GOOS=linux GOARCH="${PLATFORM}" CGO_ENABLED=0 go build -v
FROM alpine:${ALPINE_VERSION} AS final
# We want to make sure this one includes latest security fixes.
# "Pin versions in apk add" https://github.com/hadolint/hadolint/wiki/DL3018
# hadolint ignore=DL3018
RUN apk --no-cache add libressl libressl-dev openssl
COPY --from=builder /usr/src/myapp/pgbouncer_exporter /bin
ARG PGBOUNCER_EXPORTER_VERSION
ARG AIRFLOW_PGBOUNCER_EXPORTER_VERSION
ARG GO_VERSION
ARG COMMIT_SHA
LABEL org.apache.airflow.component="pgbouncer-exporter" \
org.apache.airflow.pgbouncer-exporter.version="${PGBOUNCER_EXPORTER_VERSION}" \
org.apache.airflow.go.version="${GO_VERSION}" \
org.apache.airflow.airflow-pgbouncer-exporter.version="${AIRFLOW_PGBOUNCER_EXPORTER_VERSION}" \
org.apache.airflow.commit-sha="${COMMIT_SHA}" \
maintainer="Apache Airflow Community <dev@airflow.apache.org>"
HEALTHCHECK CMD ["/bin/pgbouncer_exporter", "health"]
USER nobody
ENTRYPOINT ["/bin/pgbouncer_exporter"]
CMD ["server"]

68
helm/airflow/dockerfiles/pgbouncer-exporter/build_and_push.sh Executable file
View File

@@ -0,0 +1,68 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
DOCKERHUB_USER=${DOCKERHUB_USER:="apache"}
readonly DOCKERHUB_USER
DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow"}
readonly DOCKERHUB_REPO
PGBOUNCER_EXPORTER_VERSION="0.14.0"
readonly PGBOUNCER_EXPORTER_VERSION
AIRFLOW_PGBOUNCER_EXPORTER_VERSION="2023.02.21"
readonly AIRFLOW_PGBOUNCER_EXPORTER_VERSION
EXPECTED_GO_VERSION="1.19.6"
readonly EXPECTED_GO_VERSION
COMMIT_SHA=$(git rev-parse HEAD)
readonly COMMIT_SHA
TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:airflow-pgbouncer-exporter-${AIRFLOW_PGBOUNCER_EXPORTER_VERSION}-${PGBOUNCER_EXPORTER_VERSION}"
readonly TAG
function center_text() {
columns=$(tput cols || echo 80)
printf "%*s\n" $(( (${#1} + columns) / 2)) "$1"
}
cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1
center_text "Building image"
# Note, you need buildx and qemu installed for your docker. They come pre-installed with docker-desktop, but
# as described in:
# * https://docs.docker.com/build/install-buildx/
# * https://docs.docker.com/build/building/multi-platform/
# You can also install them easily on all docker-based systems
# You might also need to create a different builder to build multi-platform images
# For example by running `docker buildx create --use`
docker buildx build . \
--platform linux/amd64,linux/arm64 \
--pull \
--push \
--build-arg "PGBOUNCER_EXPORTER_VERSION=${PGBOUNCER_EXPORTER_VERSION}" \
--build-arg "AIRFLOW_PGBOUNCER_EXPORTER_VERSION=${AIRFLOW_PGBOUNCER_EXPORTER_VERSION}"\
--build-arg "COMMIT_SHA=${COMMIT_SHA}" \
--build-arg "GO_VERSION=${EXPECTED_GO_VERSION}" \
--tag "${TAG}"
center_text "Checking image"
docker run --rm "${TAG}" --version

77
helm/airflow/dockerfiles/pgbouncer/Dockerfile Normal file
View File

@@ -0,0 +1,77 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
ARG ALPINE_VERSION="3.14"
FROM alpine:${ALPINE_VERSION} AS builder
SHELL ["/bin/ash", "-e", "-x", "-c", "-o", "pipefail"]
ARG PGBOUNCER_VERSION
ARG AIRFLOW_PGBOUNCER_VERSION
ARG PGBOUNCER_SHA256
# Those are build deps only but still we want the latest versions of those
# "Pin versions in apk add" https://github.com/hadolint/hadolint/wiki/DL3018
# hadolint ignore=DL3018
RUN apk --no-cache add make pkgconfig build-base libtool wget gcc g++ libevent-dev libressl-dev c-ares-dev ca-certificates
# We are not using Dash so we can safely ignore the "Dash warning"
# "In dash, something is not supported." https://github.com/koalaman/shellcheck/wiki/SC2169
# hadolint ignore=SC2169,SC3060
RUN wget --progress=dot:giga "https://github.com/pgbouncer/pgbouncer/releases/download/pgbouncer_${PGBOUNCER_VERSION//\./_}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" \
&& echo "${PGBOUNCER_SHA256} pgbouncer-${PGBOUNCER_VERSION}.tar.gz" | sha256sum -c - \
&& tar -xzvf pgbouncer-$PGBOUNCER_VERSION.tar.gz
WORKDIR /pgbouncer-$PGBOUNCER_VERSION
RUN ./configure --prefix=/usr --disable-debug && make && make install \
&& mkdir /etc/pgbouncer \
&& cp ./etc/pgbouncer.ini /etc/pgbouncer/ \
&& touch /etc/pgbouncer/userlist.txt \
&& sed -i -e "s|logfile = |#logfile = |" \
-e "s|pidfile = |#pidfile = |" \
-e "s|listen_addr = .*|listen_addr = 0.0.0.0|" \
-e "s|auth_type = .*|auth_type = md5|" \
/etc/pgbouncer/pgbouncer.ini
FROM alpine:${ALPINE_VERSION}
ARG PGBOUNCER_VERSION
ARG AIRFLOW_PGBOUNCER_VERSION
ARG COMMIT_SHA
# We want to make sure this one includes latest security fixes.
# "Pin versions in apk add" https://github.com/hadolint/hadolint/wiki/DL3018
# hadolint ignore=DL3018
RUN apk --no-cache add libevent libressl c-ares
COPY --from=builder /etc/pgbouncer /etc/pgbouncer
COPY --from=builder /usr/bin/pgbouncer /usr/bin/pgbouncer
LABEL org.apache.airflow.component="pgbouncer" \
org.apache.airflow.pgbouncer.version="${PGBOUNCER_VERSION}" \
org.apache.airflow.airflow-pgbouncer.version="${AIRFLOW_PGBOUNCER_VERSION}" \
org.apache.airflow.commit-sha="${COMMIT_SHA}" \
maintainer="Apache Airflow Community <dev@airflow.apache.org>"
# Healthcheck
HEALTHCHECK --interval=10s --timeout=3s CMD stat /tmp/.s.PGSQL.*
EXPOSE 6432
USER nobody
# pgbouncer can't run as root, so let's drop to 'nobody'
ENTRYPOINT ["/usr/bin/pgbouncer", "-u", "nobody", "/etc/pgbouncer/pgbouncer.ini" ]

69
helm/airflow/dockerfiles/pgbouncer/build_and_push.sh Executable file
View File

@@ -0,0 +1,69 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
DOCKERHUB_USER=${DOCKERHUB_USER:="apache"}
readonly DOCKERHUB_USER
DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow"}
readonly DOCKERHUB_REPO
PGBOUNCER_VERSION="1.16.1"
readonly PGBOUNCER_VERSION
PGBOUNCER_SHA256="087477e9e4766d032b04b7b006c0c8d64160a54141a7bfc2c6e5ae7ae11bf7fc"
readonly PGBOUNCER_SHA256
AIRFLOW_PGBOUNCER_VERSION="2023.02.24"
readonly AIRFLOW_PGBOUNCER_VERSION
COMMIT_SHA=$(git rev-parse HEAD)
readonly COMMIT_SHA
TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:airflow-pgbouncer-${AIRFLOW_PGBOUNCER_VERSION}-${PGBOUNCER_VERSION}"
readonly TAG
function center_text() {
columns=$(tput cols || echo 80)
printf "%*s\n" $(( (${#1} + columns) / 2)) "$1"
}
cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1
center_text "Building image"
# Note, you need buildx and qemu installed for your docker. They come pre-installed with docker-desktop, but
# as described in:
# * https://docs.docker.com/build/install-buildx/
# * https://docs.docker.com/build/building/multi-platform/
# You can also install them easily on all docker-based systems
# You might also need to create a different builder to build multi-platform images
# For example by running `docker buildx create --use`
docker buildx build . \
--platform linux/amd64,linux/arm64 \
--pull \
--push \
--build-arg "PGBOUNCER_VERSION=${PGBOUNCER_VERSION}" \
--build-arg "AIRFLOW_PGBOUNCER_VERSION=${AIRFLOW_PGBOUNCER_VERSION}"\
--build-arg "PGBOUNCER_SHA256=${PGBOUNCER_SHA256}"\
--build-arg "COMMIT_SHA=${COMMIT_SHA}" \
--tag "${TAG}"
center_text "Checking image"
docker run --rm "${TAG}" pgbouncer --version

141
helm/airflow/files/pod-template-file.kubernetes-helm-yaml Normal file
View File

@@ -0,0 +1,141 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
---
{{- $nodeSelector := or .Values.workers.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.workers.affinity .Values.affinity }}
{{- $tolerations := or .Values.workers.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.workers.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.workers) }}
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.workers) }}
{{- $containerLifecycleHooks := or .Values.workers.containerLifecycleHooks .Values.containerLifecycleHooks }}
apiVersion: v1
kind: Pod
metadata:
name: placeholder-name
labels:
tier: airflow
component: worker
release: {{ .Release.Name }}
{{- if or (.Values.labels) (.Values.workers.labels) }}
{{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- if or .Values.airflowPodAnnotations .Values.workers.podAnnotations }}
annotations:
{{- if .Values.airflowPodAnnotations }}
{{- toYaml .Values.airflowPodAnnotations | nindent 4 }}
{{- end }}
{{- if .Values.workers.podAnnotations }}
{{- toYaml .Values.workers.podAnnotations | nindent 4 }}
{{- end }}
{{- end }}
spec:
{{- if or (and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled)) .Values.workers.extraInitContainers }}
initContainers:
{{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }}
{{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 4 }}
{{- end }}
{{- if .Values.workers.extraInitContainers }}
{{- toYaml .Values.workers.extraInitContainers | nindent 4 }}
{{- end }}
{{- end }}
containers:
- envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 6 }}
env:
- name: AIRFLOW__CORE__EXECUTOR
value: LocalExecutor
{{- include "standard_airflow_environment" . | indent 6}}
{{- include "custom_airflow_environment" . | indent 6 }}
{{- include "container_extra_envs" (list . .Values.workers.env) | indent 6 }}
image: {{ template "pod_template_image" . }}
imagePullPolicy: {{ .Values.images.pod_template.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 8 }}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 8 }}
{{- end }}
name: base
resources: {{- toYaml .Values.workers.resources | nindent 8 }}
volumeMounts:
- mountPath: {{ template "airflow_logs" . }}
name: logs
{{- include "airflow_config_mount" . | nindent 8 }}
{{- if or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled }}
{{- include "airflow_dags_mount" . | nindent 8 }}
{{- end }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 8 }}
{{- end }}
{{- if .Values.workers.extraVolumeMounts }}
{{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 8 }}
{{- end }}
{{- if .Values.workers.extraContainers }}
{{- toYaml .Values.workers.extraContainers | nindent 4 }}
{{- end }}
{{- if .Values.workers.priorityClassName }}
priorityClassName: {{ .Values.workers.priorityClassName }}
{{- end }}
{{- if .Values.workers.runtimeClassName }}
priorityClassName: {{ .Values.workers.runtimeClassName }}
{{- end }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
{{- if .Values.workers.hostAliases }}
hostAliases: {{- toYaml .Values.workers.hostAliases | nindent 4 }}
{{- end }}
restartPolicy: Never
securityContext: {{ $securityContext | nindent 4 }}
nodeSelector: {{- toYaml $nodeSelector | nindent 4 }}
affinity: {{- toYaml $affinity | nindent 4 }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
terminationGracePeriodSeconds: {{ .Values.workers.terminationGracePeriodSeconds }}
tolerations: {{- toYaml $tolerations | nindent 4 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 4 }}
serviceAccountName: {{ include "worker.serviceAccountName" . }}
volumes:
{{- if .Values.dags.persistence.enabled }}
- name: dags
persistentVolumeClaim:
claimName: {{ template "airflow_dags_volume_claim" . }}
{{- else if .Values.dags.gitSync.enabled }}
- name: dags
emptyDir: {}
{{- end }}
{{- if .Values.logs.persistence.enabled }}
- name: logs
persistentVolumeClaim:
claimName: {{ template "airflow_logs_volume_claim" . }}
{{- else }}
- emptyDir: {}
name: logs
{{- end }}
{{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.sshKeySecret }}
{{- include "git_sync_ssh_key_volume" . | nindent 2 }}
{{- end }}
- configMap:
name: {{ include "airflow_config" . }}
name: config
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 2 }}
{{- end }}
{{- if .Values.workers.extraVolumes }}
{{- tpl (toYaml .Values.workers.extraVolumes) . | nindent 2 }}
{{- end }}

87
helm/airflow/files/statsd-mappings.yml Normal file
View File

@@ -0,0 +1,87 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
---
mappings:
# Map dot separated stats to labels
- match: airflow.dagrun.dependency-check.*.*
name: "airflow_dagrun_dependency_check"
labels:
dag_id: "$1"
- match: airflow.operator_successes_(.*)
match_type: regex
name: "airflow_operator_successes"
labels:
operator: "$1"
- match: airflow.operator_failures_(.*)
match_type: regex
name: "airflow_operator_failures"
labels:
operator: "$1"
- match: airflow.scheduler_heartbeat
match_type: regex
name: "airflow_scheduler_heartbeat"
labels:
type: counter
- match: airflow.dag.*.*.duration
name: "airflow_task_duration"
labels:
dag_id: "$1"
task_id: "$2"
- match: airflow.dagrun.duration.success.*
name: "airflow_dagrun_duration"
labels:
dag_id: "$1"
- match: airflow.dagrun.duration.failed.*
name: "airflow_dagrun_failed"
labels:
dag_id: "$1"
- match: airflow.dagrun.schedule_delay.*
name: "airflow_dagrun_schedule_delay"
labels:
dag_id: "$1"
- match: airflow.dag_processing.last_runtime.*
name: "airflow_dag_processing_last_runtime"
labels:
dag_file: "$1"
- match: airflow.dag_processing.last_run.seconds_ago.*
name: "airflow_dag_processing_last_run_seconds_ago"
labels:
dag_file: "$1"
- match: airflow.pool.open_slots.*
name: "airflow_pool_open_slots"
labels:
pool: "$1"
- match: airflow.pool.used_slots.*
name: "airflow_pool_used_slots"
labels:
pool: "$1"
- match: airflow.pool.starving_tasks.*
name: "airflow_pool_starving_tasks"
labels:
pool: "$1"

23
helm/airflow/newsfragments/31066.significant.rst Normal file
View File

@@ -0,0 +1,23 @@
Support naming customization on helm chart resources, some resources may be renamed during upgrade
This is a new opt-in switch ``useStandardNaming``, for backwards compatibility, to leverage the standard naming convention, which allows full use of fullnameOverride and nameOverride in all resources.
Only the following resources will be renamed using default of ``useStandardNaming=false``:
- ConfigMap {release}-airflow-config to {release}-config
- Secret {release}-airflow-metadata to {release}-metadata
- Secret {release}-airflow-result-backend to {release}-result-backend
For existing installations, all your resources will be recreated with a new name and helm will delete previous resources.
This won't delete existing PVCs for logs used by statefulset/deployments, but it will recreate them with brand new PVCs.
If you do want to preserve logs history you'll need to manually copy the data of these volumes into the new volumes after
deployment. Depending on what storage backend/class you're using this procedure may vary. If you don't mind starting
with fresh logs/redis volumes, you can just delete the old pvcs that will be names, for example:
.. code-block:: bash
kubectl delete pvc -n airflow logs-gta-triggerer-0
kubectl delete pvc -n airflow logs-gta-worker-0
kubectl delete pvc -n airflow redis-db-gta-redis-0
If you do not change ``useStandardNaming`` or ``fullnameOverride`` after upgrade, you can proceed as usual and no unexpected behaviours will be presented.

3
helm/airflow/newsfragments/33747.significant.rst Normal file
View File

@@ -0,0 +1,3 @@
``bitnami/postgresql`` subchart updated to ``12.10.0``
The PostgreSQL subchart that is used with the Chart is now ``12.10.0``, previously it was ``12.1.9``.

3
helm/airflow/newsfragments/33748.significant.rst Normal file
View File

@@ -0,0 +1,3 @@
Default git-sync image is updated to ``3.6.9``
The default git-sync image that is used with the Chart is now ``3.6.9``, previously it was ``3.6.3``.

3
helm/airflow/newsfragments/34186.significant.rst Normal file
View File

@@ -0,0 +1,3 @@
Default Airflow image is updated to ``2.7.1``
The default Airflow image that is used with the Chart is now ``2.7.1``, previously it was ``2.6.2``.

50
helm/airflow/newsfragments/config.toml Normal file
View File

@@ -0,0 +1,50 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
[tool.towncrier]
name = "Airflow Helm Chart"
filename = "RELEASE_NOTES.rst"
underlines = ["-", '^']
[[tool.towncrier.type]]
directory = "significant"
name = "Significant Changes"
showcontent = true
[[tool.towncrier.type]]
directory = "feature"
name = "Features"
showcontent = true
[[tool.towncrier.type]]
directory = "improvement"
name = "Improvements"
showcontent = true
[[tool.towncrier.type]]
directory = "bugfix"
name = "Bug Fixes"
showcontent = true
[[tool.towncrier.type]]
directory = "doc"
name = "Doc only Changes"
showcontent = true
[[tool.towncrier.type]]
directory = "misc"
name = "Misc"
showcontent = true

197
helm/airflow/override_values.yaml Normal file
View File

@@ -0,0 +1,197 @@
## Prerequirements
## Airflow Namespace 생성
# kubectl create namespace airflow
## Web Server Secret Key 생성
# kubectl create secret -n airflow generic webserver-secret --from-literal="webserver-secret-key=$(python3 -c 'import secrets; print(secrets.token_hex(16))')"
## GitHub 연동을 위한 Secret Key 생성
# kubectl create secret generic airflow-git-ssh-secret \
# --from-file=gitSshKey=[사용할 SSH Private key] \
# -n airflow
## keda 활용 시 설치
# helm repo add kedacore https://kedacore.github.io/charts
# helm repo update
# kubectl create namespace keda
# helm install keda kedacore/keda --namespace keda --version "v2.0.0"
---
webserverSecretKeySecretName: webserver-secret
executor: "CeleryKubernetesExecutor"
workers:
replicas: 1
resources:
limits:
cpu: 1
memory: 3Gi
requests:
cpu: 1
memory: 2Gi
keda:
enabled: true
minReplicaCount: 1
maxReplicaCount: 3
# advanced:
# horizontalPodAutoscalerConfig:
# behavior:
# scaleDown:
# stabilizationWindowSeconds: 600
# policies:
# - type: Percent
# value: 100
# periodSeconds: 15
scheduler:
replicas: 1
resources:
limits:
cpu: 1
memory: 1Gi
requests:
cpu: 500m
memory: 1Gi
webserver:
replicas: 1
resources:
limits:
cpu: 1
memory: 2Gi
requests:
cpu: 500m
memory: 1Gi
service:
type: NodePort
ports:
- name: airflow-ui
port: 8080
targetPort: 8080
nodePort: 30180
defaultUser:
email: minchulahn@ex-em.com
triggerer:
replicas: 1
resources:
limits:
cpu: 1
memory: 1Gi
requests:
cpu: 500m
memory: 1Gi
statsd:
resources:
limits:
cpu: 100m
memory: 128Mi
requests:
cpu: 100m
memory: 128Mi
redis:
resources:
limits:
cpu: 100m
memory: 128Mi
requests:
cpu: 100m
memory: 128Mi
dags:
gitSync:
enabled: true
repo: git@github.com:cloudmoa/dsk-airflow-pipeline.git
branch: main
subPath: dags
sshKeySecret: airflow-git-ssh-secret
postgresql:
primary:
service:
type: NodePort
nodePorts:
postgresql: 30185
logs:
persistence:
enabled: true
config:
core:
dags_folder: '{{ include "airflow_dags" . }}'
# This is ignored when used with the official Docker image
load_examples: 'False'
executor: '{{ .Values.executor }}'
# For Airflow 1.10, backward compatibility; moved to [logging] in 2.0
colored_console_log: 'False'
remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}'
logging:
remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}'
colored_console_log: 'False'
metrics:
statsd_on: '{{ ternary "True" "False" .Values.statsd.enabled }}'
statsd_port: 9125
statsd_prefix: airflow
statsd_host: '{{ printf "%s-statsd" .Release.Name }}'
webserver:
enable_proxy_fix: 'True'
# For Airflow 1.10
rbac: 'True'
celery:
flower_url_prefix: '{{ .Values.ingress.flower.path }}'
worker_concurrency: 16
scheduler:
standalone_dag_processor: '{{ ternary "True" "False" .Values.dagProcessor.enabled }}'
# statsd params included for Airflow 1.10 backward compatibility; moved to [metrics] in 2.0
statsd_on: '{{ ternary "True" "False" .Values.statsd.enabled }}'
statsd_port: 9125
statsd_prefix: airflow
statsd_host: '{{ printf "%s-statsd" .Release.Name }}'
# `run_duration` included for Airflow 1.10 backward compatibility; removed in 2.0.
run_duration: 41460
elasticsearch:
json_format: 'True'
log_id_template: "{dag_id}_{task_id}_{execution_date}_{try_number}"
elasticsearch_configs:
max_retries: 3
timeout: 30
retry_timeout: 'True'
kerberos:
keytab: '{{ .Values.kerberos.keytabPath }}'
reinit_frequency: '{{ .Values.kerberos.reinitFrequency }}'
principal: '{{ .Values.kerberos.principal }}'
ccache: '{{ .Values.kerberos.ccacheMountPath }}/{{ .Values.kerberos.ccacheFileName }}'
celery_kubernetes_executor:
kubernetes_queue: 'kubernetes'
# The `kubernetes` section is deprecated in Airflow >= 2.5.0 due to an airflow.cfg schema change.
# The `kubernetes` section can be removed once the helm chart no longer supports Airflow < 2.5.0.
kubernetes:
namespace: '{{ .Release.Namespace }}'
# The following `airflow_` entries are for Airflow 1, and can be removed when it is no longer supported.
airflow_configmap: '{{ include "airflow_config" . }}'
airflow_local_settings_configmap: '{{ include "airflow_config" . }}'
pod_template_file: '{{ include "airflow_pod_template_file" . }}/pod_template_file.yaml'
worker_container_repository: '{{ .Values.images.airflow.repository | default .Values.defaultAirflowRepository }}'
worker_container_tag: '{{ .Values.images.airflow.tag | default .Values.defaultAirflowTag }}'
multi_namespace_mode: '{{ ternary "True" "False" .Values.multiNamespaceMode }}'
# The `kubernetes_executor` section duplicates the `kubernetes` section in Airflow >= 2.5.0 due to an airflow.cfg schema change.
kubernetes_executor:
namespace: '{{ .Release.Namespace }}'
pod_template_file: '{{ include "airflow_pod_template_file" . }}/pod_template_file.yaml'
worker_container_repository: '{{ .Values.images.airflow.repository | default .Values.defaultAirflowRepository }}'
worker_container_tag: '{{ .Values.images.airflow.tag | default .Values.defaultAirflowTag }}'
multi_namespace_mode: '{{ ternary "True" "False" .Values.multiNamespaceMode }}'
email:
email_backend: airflow.providers.amazon.aws.utils.emailer.send_email
email_conn_id: aws_ses
default_email_on_retry: True
default_email_on_failure: True
from_email: DataSaker <notice@datasaker.io>
api:
auth_backends: airflow.api.auth.backend.basic_auth

206
helm/airflow/templates/NOTES.txt Normal file
View File

@@ -0,0 +1,206 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
Thank you for installing Apache {{ title .Chart.Name }} {{ .Values.airflowVersion }}!
Your release is named {{ .Release.Name }}.
{{- if or .Values.ingress.web.enabled .Values.ingress.flower.enabled .Values.ingress.enabled }}
You can now access your service(s) by following defined Ingress urls:
{{- if .Values.ingress.web.host }}
DEPRECATION WARNING:
`ingress.web.host` has been renamed to `ingress.web.hosts` and is now an array.
Please change your values as support for the old name will be dropped in a future release.
{{- end }}
{{- if .Values.ingress.web.tls }}
DEPRECATION WARNING:
`ingress.web.tls` has been renamed to `ingress.web.hosts[*].tls` and can be set per host.
Please change your values as support for the old name will be dropped in a future release.
{{- end }}
{{- if .Values.ingress.flower.host }}
DEPRECATION WARNING:
`ingress.flower.host` has been renamed to `ingress.flower.hosts` and is now an array.
Please change your values as support for the old name will be dropped in a future release.
{{- end }}
{{- if .Values.ingress.flower.tls }}
DEPRECATION WARNING:
`ingress.flower.tls` has been renamed to `ingress.flower.hosts[*].tls` and can be set per host.
Please change your values as support for the old name will be dropped in a future release.
{{- end }}
{{- if .Values.ingress.enabled }}
DEPRECATION WARNING:
`ingress.enabled` has been deprecated. There are now separate flags to control the webserver and
flower individually, ``ingress.web.enabled`` and ``ingress.flower.enabled``.
Please change your values as support for the old name will be dropped in a future release.
{{- end }}
{{- if or .Values.ingress.web.enabled .Values.ingress.enabled }}
Airflow Webserver:
{{- range .Values.ingress.web.hosts | default (list .Values.ingress.web.host) }}
{{- $tlsEnabled := $.Values.ingress.web.tls.enabled -}}
{{- $hostname := $.Values.ingress.web.host -}}
{{- if . | kindIs "string" | not }}
{{- if .tls }}
{{- $tlsEnabled = .tls.enabled -}}
{{- $hostname = .name -}}
{{- end }}
{{- end }}
http{{ if $tlsEnabled }}s{{ end }}://{{ $hostname }}{{ $.Values.ingress.web.path }}/
{{- end }}
{{- end }}
{{- if and (or .Values.ingress.flower.enabled .Values.ingress.enabled) (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")) }}
Flower dashboard:
{{- range .Values.ingress.flower.hosts | default (list .Values.ingress.flower.host) }}
{{- $tlsEnabled := $.Values.ingress.flower.tls.enabled -}}
{{- $hostname := $.Values.ingress.flower.host -}}
{{- if . | kindIs "string" | not }}
{{- if .tls }}
{{- $tlsEnabled = .tls.enabled -}}
{{- $hostname = .name -}}
{{- end }}
{{- end }}
http{{ if $tlsEnabled }}s{{ end }}://{{ $hostname }}{{ $.Values.ingress.flower.path }}/
{{- end }}
{{- end }}
{{- else }}
You can now access your dashboard(s) by executing the following command(s) and visiting the corresponding port at localhost in your browser:
Airflow Webserver: kubectl port-forward svc/{{ include "airflow.fullname" . }}-webserver {{ .Values.ports.airflowUI }}:{{ .Values.ports.airflowUI }} --namespace {{ .Release.Namespace }}
{{- if .Values.flower.enabled }}
{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")}}
Flower dashboard: kubectl port-forward svc/{{ include "airflow.fullname" . }}-flower {{ .Values.ports.flowerUI }}:{{ .Values.ports.flowerUI }} --namespace {{ .Release.Namespace }}
{{- end }}
{{- end }}
{{- end }}
{{- if .Values.webserver.defaultUser.enabled}}
Default Webserver (Airflow UI) Login credentials:
username: {{ .Values.webserver.defaultUser.username }}
password: {{ .Values.webserver.defaultUser.password }}
{{- end }}
{{- if .Values.postgresql.enabled }}
Default Postgres connection credentials:
username: {{ .Values.data.metadataConnection.user }}
password: {{ .Values.data.metadataConnection.pass }}
port: {{ .Values.data.metadataConnection.port }}
{{- end }}
{{- if not .Values.fernetKeySecretName }}
You can get Fernet Key value by running the following:
echo Fernet Key: $(kubectl get secret --namespace {{ .Release.Namespace }} {{ .Release.Name }}-fernet-key -o jsonpath="{.data.fernet-key}" | base64 --decode)
{{- end }}
{{- if or (eq .Values.executor "KubernetesExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
{{- if and (not .Values.logs.persistence.enabled) (eq (lower (tpl .Values.config.logging.remote_logging .)) "false") }}
WARNING:
Kubernetes workers task logs may not persist unless you configure log persistence or remote logging!
Logging options can be found at: https://airflow.apache.org/docs/helm-chart/stable/manage-logs.html
(This warning can be ignored if logging is configured with environment variables or secrets backend)
{{- end }}
{{- end }}
{{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.sshKeySecret (not .Values.dags.gitSync.knownHosts)}}
#####################################################
# WARNING: You should set dags.gitSync.knownHosts #
#####################################################
You are using ssh authentication for your gitsync repo, however you currently have SSH known_hosts verification disabled,
making you susceptible to man-in-the-middle attacks!
Information on how to set knownHosts can be found here:
https://airflow.apache.org/docs/helm-chart/stable/production-guide.html#knownhosts
{{- end }}
{{- if .Values.flower.extraNetworkPolicies }}
DEPRECATION WARNING:
`flower.extraNetworkPolicies` has been renamed to `flower.networkPolicy.peers`.
Please change your values as support for the old name will be dropped in a future release.
{{- end }}
{{- if .Values.webserver.extraNetworkPolicies }}
DEPRECATION WARNING:
`webserver.extraNetworkPolicies` has been renamed to `webserver.networkPolicy.peers`.
Please change your values as support for the old name will be dropped in a future release.
{{- end }}
{{- if not (or .Values.webserverSecretKey .Values.webserverSecretKeySecretName) }}
{{- if .Values.securityContext }}
DEPRECATION WARNING:
`securityContext` has been renamed to `securityContexts`, to be enabled on container and pod level.
Please change your values as support for the old name will be dropped in a future release.
{{- end }}
###########################################################
# WARNING: You should set a static webserver secret key #
###########################################################
You are using a dynamically generated webserver secret key, which can lead to
unnecessary restarts of your Airflow components.
Information on how to set a static webserver secret key can be found here:
https://airflow.apache.org/docs/helm-chart/stable/production-guide.html#webserver-secret-key
{{- end }}
{{- if or .Values.postgresql.postgresqlUsername .Values.postgresql.postgresqlPassword }}
{{ fail "postgresql.postgresqlUsername and postgresql.postgresqlPassword are no longer supported. If you wish to use the 'postgres' user, set its password with postgresql.auth.postgresPassword. If you wish to create a different user, do so with postgresql.auth.username and postgresql.auth.password." }}
{{- end }}
{{- if ne .Values.executor (tpl .Values.config.core.executor $) }}
{{ fail "Please configure the executor with `executor`, not `config.core.executor`." }}
{{- end }}

985
helm/airflow/templates/_helpers.yaml Normal file
View File

@@ -0,0 +1,985 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "airflow.fullname" -}}
{{- if not .Values.useStandardNaming }}
{{- .Release.Name }}
{{- else if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{- define "airflow.serviceAccountName" -}}
{{ if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/* Standard Airflow environment variables */}}
{{- define "standard_airflow_environment" }}
# Hard Coded Airflow Envs
{{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__CORE__FERNET_KEY }}
- name: AIRFLOW__CORE__FERNET_KEY
valueFrom:
secretKeyRef:
name: {{ template "fernet_key_secret" . }}
key: fernet-key
{{- end }}
# For Airflow <2.3, backward compatibility; moved to [database] in 2.3
{{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__CORE__SQL_ALCHEMY_CONN }}
- name: AIRFLOW__CORE__SQL_ALCHEMY_CONN
valueFrom:
secretKeyRef:
name: {{ template "airflow_metadata_secret" . }}
key: connection
{{- end }}
{{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__DATABASE__SQL_ALCHEMY_CONN }}
- name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN
valueFrom:
secretKeyRef:
name: {{ template "airflow_metadata_secret" . }}
key: connection
{{- end }}
{{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW_CONN_AIRFLOW_DB }}
- name: AIRFLOW_CONN_AIRFLOW_DB
valueFrom:
secretKeyRef:
name: {{ template "airflow_metadata_secret" . }}
key: connection
{{- end }}
{{- if and .Values.workers.keda.enabled .Values.pgbouncer.enabled (not .Values.workers.keda.usePgbouncer) }}
- name: KEDA_DB_CONN
valueFrom:
secretKeyRef:
name: {{ template "airflow_metadata_secret" . }}
key: kedaConnection
{{- end }}
{{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__WEBSERVER__SECRET_KEY }}
- name: AIRFLOW__WEBSERVER__SECRET_KEY
valueFrom:
secretKeyRef:
name: {{ template "webserver_secret_key_secret" . }}
key: webserver-secret-key
{{- end }}
{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
{{- if or (semverCompare "<2.4.0" .Values.airflowVersion) (.Values.data.resultBackendSecretName) (.Values.data.resultBackendConnection) }}
{{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__CELERY__CELERY_RESULT_BACKEND }}
# (Airflow 1.10.* variant)
- name: AIRFLOW__CELERY__CELERY_RESULT_BACKEND
valueFrom:
secretKeyRef:
name: {{ template "airflow_result_backend_secret" . }}
key: connection
{{- end }}
{{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__CELERY__RESULT_BACKEND }}
- name: AIRFLOW__CELERY__RESULT_BACKEND
valueFrom:
secretKeyRef:
name: {{ template "airflow_result_backend_secret" . }}
key: connection
{{- end }}
{{- end }}
{{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__CELERY__BROKER_URL }}
- name: AIRFLOW__CELERY__BROKER_URL
valueFrom:
secretKeyRef:
name: {{ default (printf "%s-broker-url" .Release.Name) .Values.data.brokerUrlSecretName }}
key: connection
{{- end }}
{{- end }}
{{- if .Values.elasticsearch.enabled }}
# The elasticsearch variables were updated to the shorter names in v1.10.4
{{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__ELASTICSEARCH__HOST }}
- name: AIRFLOW__ELASTICSEARCH__HOST
valueFrom:
secretKeyRef:
name: {{ template "elasticsearch_secret" . }}
key: connection
{{- end }}
{{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__ELASTICSEARCH__ELASTICSEARCH_HOST }}
# This is the older format for these variable names, kept here for backward compatibility
- name: AIRFLOW__ELASTICSEARCH__ELASTICSEARCH_HOST
valueFrom:
secretKeyRef:
name: {{ template "elasticsearch_secret" . }}
key: connection
{{- end }}
{{- end }}
{{- end }}
{{/* User defined Airflow environment variables */}}
{{- define "custom_airflow_environment" }}
# Dynamically created environment variables
{{- range $i, $config := .Values.env }}
- name: {{ $config.name }}
value: {{ $config.value | quote }}
{{- if or (eq $.Values.executor "KubernetesExecutor") (eq $.Values.executor "LocalKubernetesExecutor") (eq $.Values.executor "CeleryKubernetesExecutor") }}
- name: AIRFLOW__KUBERNETES_ENVIRONMENT_VARIABLES__{{ $config.name }}
value: {{ $config.value | quote }}
{{- end }}
{{- end }}
# Dynamically created secret envs
{{- range $i, $config := .Values.secret }}
- name: {{ $config.envName }}
valueFrom:
secretKeyRef:
name: {{ $config.secretName }}
key: {{ default "value" $config.secretKey }}
{{- end }}
{{- if or (eq $.Values.executor "LocalKubernetesExecutor") (eq $.Values.executor "KubernetesExecutor") (eq $.Values.executor "CeleryKubernetesExecutor") }}
{{- range $i, $config := .Values.secret }}
- name: AIRFLOW__KUBERNETES_SECRETS__{{ $config.envName }}
value: {{ printf "%s=%s" $config.secretName $config.secretKey }}
{{- end }}
{{ end }}
# Extra env
{{- $Global := . }}
{{- with .Values.extraEnv }}
{{- tpl . $Global | nindent 2 }}
{{- end }}
{{- end }}
{{/* User defined Airflow environment from */}}
{{- define "custom_airflow_environment_from" }}
{{- $Global := . }}
{{- with .Values.extraEnvFrom }}
{{- tpl . $Global | nindent 2 }}
{{- end }}
{{- end }}
{{/* Git ssh key volume */}}
{{- define "git_sync_ssh_key_volume" }}
- name: git-sync-ssh-key
secret:
secretName: {{ .Values.dags.gitSync.sshKeySecret }}
defaultMode: 288
{{- end }}
{{/* Git sync container */}}
{{- define "git_sync_container" }}
- name: {{ .Values.dags.gitSync.containerName }}{{ if .is_init }}-init{{ end }}
image: {{ template "git_sync_image" . }}
imagePullPolicy: {{ .Values.images.gitSync.pullPolicy }}
securityContext: {{- include "localContainerSecurityContext" .Values.dags.gitSync | nindent 4 }}
env:
{{- if .Values.dags.gitSync.sshKeySecret }}
- name: GIT_SSH_KEY_FILE
value: "/etc/git-secret/ssh"
- name: GIT_SYNC_SSH
value: "true"
{{- if .Values.dags.gitSync.knownHosts }}
- name: GIT_KNOWN_HOSTS
value: "true"
- name: GIT_SSH_KNOWN_HOSTS_FILE
value: "/etc/git-secret/known_hosts"
{{- else }}
- name: GIT_KNOWN_HOSTS
value: "false"
{{- end }}
{{ else if .Values.dags.gitSync.credentialsSecret }}
- name: GIT_SYNC_USERNAME
valueFrom:
secretKeyRef:
name: {{ .Values.dags.gitSync.credentialsSecret | quote }}
key: GIT_SYNC_USERNAME
- name: GIT_SYNC_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.dags.gitSync.credentialsSecret | quote }}
key: GIT_SYNC_PASSWORD
{{- end }}
- name: GIT_SYNC_REV
value: {{ .Values.dags.gitSync.rev | quote }}
- name: GIT_SYNC_BRANCH
value: {{ .Values.dags.gitSync.branch | quote }}
- name: GIT_SYNC_REPO
value: {{ .Values.dags.gitSync.repo | quote }}
- name: GIT_SYNC_DEPTH
value: {{ .Values.dags.gitSync.depth | quote }}
- name: GIT_SYNC_ROOT
value: "/git"
- name: GIT_SYNC_DEST
value: "repo"
- name: GIT_SYNC_ADD_USER
value: "true"
- name: GIT_SYNC_WAIT
value: {{ .Values.dags.gitSync.wait | quote }}
- name: GIT_SYNC_MAX_SYNC_FAILURES
value: {{ .Values.dags.gitSync.maxFailures | quote }}
{{- if .is_init }}
- name: GIT_SYNC_ONE_TIME
value: "true"
{{- end }}
{{- with .Values.dags.gitSync.env }}
{{- toYaml . | nindent 4 }}
{{- end }}
resources: {{ toYaml .Values.dags.gitSync.resources | nindent 6 }}
volumeMounts:
- name: dags
mountPath: /git
{{- if .Values.dags.gitSync.sshKeySecret }}
- name: git-sync-ssh-key
mountPath: /etc/git-secret/ssh
readOnly: true
subPath: gitSshKey
{{- if .Values.dags.gitSync.knownHosts }}
- name: config
mountPath: /etc/git-secret/known_hosts
readOnly: true
subPath: known_hosts
{{- end }}
{{- end }}
{{- if .Values.dags.gitSync.extraVolumeMounts }}
{{- tpl (toYaml .Values.dags.gitSync.extraVolumeMounts) . | nindent 2 }}
{{- end }}
{{- end }}
{{/* This helper will change when customers deploy a new image */}}
{{- define "airflow_image" -}}
{{- $repository := .Values.images.airflow.repository | default .Values.defaultAirflowRepository -}}
{{- $tag := .Values.images.airflow.tag | default .Values.defaultAirflowTag -}}
{{- $digest := .Values.images.airflow.digest | default .Values.defaultAirflowDigest -}}
{{- if $digest }}
{{- printf "%s@%s" $repository $digest -}}
{{- else }}
{{- printf "%s:%s" $repository $tag -}}
{{- end }}
{{- end }}
{{- define "pod_template_image" -}}
{{- printf "%s:%s" (.Values.images.pod_template.repository | default .Values.defaultAirflowRepository) (.Values.images.pod_template.tag | default .Values.defaultAirflowTag) }}
{{- end }}
{{/* This helper is used for airflow containers that do not need the users code */}}
{{ define "default_airflow_image" -}}
{{- $repository := .Values.defaultAirflowRepository -}}
{{- $tag := .Values.defaultAirflowTag -}}
{{- $digest := .Values.defaultAirflowDigest -}}
{{- if $digest }}
{{- printf "%s@%s" $repository $digest -}}
{{- else }}
{{- printf "%s:%s" $repository $tag -}}
{{- end }}
{{- end }}
{{ define "airflow_image_for_migrations" -}}
{{- if .Values.images.useDefaultImageForMigration }}
{{- template "default_airflow_image" . }}
{{- else }}
{{- template "airflow_image" . }}
{{- end }}
{{- end }}
{{- define "flower_image" -}}
{{- printf "%s:%s" (.Values.images.flower.repository | default .Values.defaultAirflowRepository) (.Values.images.flower.tag | default .Values.defaultAirflowTag) }}
{{- end }}
{{- define "statsd_image" -}}
{{- printf "%s:%s" .Values.images.statsd.repository .Values.images.statsd.tag }}
{{- end }}
{{- define "redis_image" -}}
{{- printf "%s:%s" .Values.images.redis.repository .Values.images.redis.tag }}
{{- end }}
{{- define "pgbouncer_image" -}}
{{- printf "%s:%s" .Values.images.pgbouncer.repository .Values.images.pgbouncer.tag }}
{{- end }}
{{- define "pgbouncer_exporter_image" -}}
{{- printf "%s:%s" .Values.images.pgbouncerExporter.repository .Values.images.pgbouncerExporter.tag }}
{{- end }}
{{- define "git_sync_image" -}}
{{- printf "%s:%s" .Values.images.gitSync.repository .Values.images.gitSync.tag }}
{{- end }}
{{- define "fernet_key_secret" -}}
{{- default (printf "%s-fernet-key" .Release.Name) .Values.fernetKeySecretName }}
{{- end }}
{{- define "webserver_secret_key_secret" -}}
{{- default (printf "%s-webserver-secret-key" (include "airflow.fullname" .)) .Values.webserverSecretKeySecretName }}
{{- end }}
{{- define "redis_password_secret" -}}
{{- default (printf "%s-redis-password" .Release.Name) .Values.redis.passwordSecretName }}
{{- end }}
{{- define "airflow_metadata_secret" -}}
{{- default (printf "%s-metadata" (include "airflow.fullname" .)) .Values.data.metadataSecretName }}
{{- end }}
{{- define "airflow_result_backend_secret" -}}
{{- default (printf "%s-result-backend" (include "airflow.fullname" .)) .Values.data.resultBackendSecretName }}
{{- end }}
{{- define "airflow_pod_template_file" -}}
{{- printf "%s/pod_templates" .Values.airflowHome }}
{{- end }}
{{- define "pgbouncer_config_secret" -}}
{{- default (printf "%s-pgbouncer-config" (include "airflow.fullname" .)) .Values.pgbouncer.configSecretName }}
{{- end }}
{{- define "pgbouncer_certificates_secret" -}}
{{- printf "%s-pgbouncer-certificates" (include "airflow.fullname" .) }}
{{- end }}
{{- define "pgbouncer_stats_secret" -}}
{{- default (printf "%s-pgbouncer-stats" (include "airflow.fullname" .)) .Values.pgbouncer.metricsExporterSidecar.statsSecretName }}
{{- end }}
{{- define "registry_secret" -}}
{{- default (printf "%s-registry" (include "airflow.fullname" .)) .Values.registry.secretName }}
{{- end }}
{{- define "elasticsearch_secret" -}}
{{- default (printf "%s-elasticsearch" (include "airflow.fullname" .)) .Values.elasticsearch.secretName }}
{{- end }}
{{- define "flower_secret" -}}
{{- default (printf "%s-flower" (include "airflow.fullname" .)) .Values.flower.secretName }}
{{- end }}
{{- define "kerberos_keytab_secret" -}}
{{- printf "%s-kerberos-keytab" (include "airflow.fullname" .) }}
{{- end }}
{{- define "kerberos_ccache_path" -}}
{{- printf "%s/%s" .Values.kerberos.ccacheMountPath .Values.kerberos.ccacheFileName }}
{{- end }}
{{- define "celery_executor_namespace" -}}
{{- if semverCompare ">=2.7.0" .Values.airflowVersion }}
{{- print "airflow.providers.celery.executors.celery_executor.app" -}}
{{- else }}
{{- print "airflow.executors.celery_executor.app" -}}
{{- end }}
{{- end }}
{{- define "pgbouncer_config" -}}
{{ $resultBackendConnection := .Values.data.resultBackendConnection | default .Values.data.metadataConnection }}
{{ $pgMetadataHost := .Values.data.metadataConnection.host | default (printf "%s-%s.%s" .Release.Name "postgresql" .Release.Namespace) }}
{{ $pgResultBackendHost := $resultBackendConnection.host | default (printf "%s-%s.%s" .Release.Name "postgresql" .Release.Namespace) }}
[databases]
{{ .Release.Name }}-metadata = host={{ $pgMetadataHost }} dbname={{ .Values.data.metadataConnection.db }} port={{ .Values.data.metadataConnection.port }} pool_size={{ .Values.pgbouncer.metadataPoolSize }} {{ .Values.pgbouncer.extraIniMetadata | default "" }}
{{ .Release.Name }}-result-backend = host={{ $pgResultBackendHost }} dbname={{ $resultBackendConnection.db }} port={{ $resultBackendConnection.port }} pool_size={{ .Values.pgbouncer.resultBackendPoolSize }} {{ .Values.pgbouncer.extraIniResultBackend | default "" }}
[pgbouncer]
pool_mode = transaction
listen_port = {{ .Values.ports.pgbouncer }}
listen_addr = *
auth_type = {{ .Values.pgbouncer.auth_type }}
auth_file = {{ .Values.pgbouncer.auth_file }}
stats_users = {{ .Values.data.metadataConnection.user }}
ignore_startup_parameters = extra_float_digits
max_client_conn = {{ .Values.pgbouncer.maxClientConn }}
verbose = {{ .Values.pgbouncer.verbose }}
log_disconnections = {{ .Values.pgbouncer.logDisconnections }}
log_connections = {{ .Values.pgbouncer.logConnections }}
server_tls_sslmode = {{ .Values.pgbouncer.sslmode }}
server_tls_ciphers = {{ .Values.pgbouncer.ciphers }}
{{- if .Values.pgbouncer.ssl.ca }}
server_tls_ca_file = /etc/pgbouncer/root.crt
{{- end }}
{{- if .Values.pgbouncer.ssl.cert }}
server_tls_cert_file = /etc/pgbouncer/server.crt
{{- end }}
{{- if .Values.pgbouncer.ssl.key }}
server_tls_key_file = /etc/pgbouncer/server.key
{{- end }}
{{- if .Values.pgbouncer.extraIni }}
{{ .Values.pgbouncer.extraIni }}
{{- end }}
{{- end }}
{{ define "pgbouncer_users" }}
{{- $resultBackendConnection := .Values.data.resultBackendConnection | default .Values.data.metadataConnection }}
{{ .Values.data.metadataConnection.user | quote }} {{ .Values.data.metadataConnection.pass | quote }}
{{ $resultBackendConnection.user | quote }} {{ $resultBackendConnection.pass | quote }}
{{- end }}
{{- define "airflow_logs" -}}
{{- printf "%s/logs" .Values.airflowHome | quote }}
{{- end }}
{{- define "airflow_logs_no_quote" -}}
{{- printf "%s/logs" .Values.airflowHome }}
{{- end }}
{{- define "airflow_logs_volume_claim" -}}
{{- if .Values.logs.persistence.existingClaim }}
{{- .Values.logs.persistence.existingClaim }}
{{- else }}
{{- printf "%s-logs" .Release.Name }}
{{- end }}
{{- end }}
{{- define "airflow_dags" -}}
{{- if .Values.dags.gitSync.enabled }}
{{- printf "%s/dags/repo/%s" .Values.airflowHome .Values.dags.gitSync.subPath }}
{{- else }}
{{- printf "%s/dags" .Values.airflowHome }}
{{- end }}
{{- end }}
{{- define "airflow_dags_volume_claim" -}}
{{- if .Values.dags.persistence.existingClaim }}
{{- .Values.dags.persistence.existingClaim }}
{{- else }}
{{- printf "%s-dags" .Release.Name }}
{{- end }}
{{- end }}
{{- define "airflow_dags_mount" -}}
- name: dags
mountPath: {{ printf "%s/dags" .Values.airflowHome }}
{{- if .Values.dags.persistence.subPath }}
subPath: {{ .Values.dags.persistence.subPath }}
{{- end }}
readOnly: {{ .Values.dags.gitSync.enabled | ternary "True" "False" }}
{{- end }}
{{- define "airflow_config_path" -}}
{{- printf "%s/airflow.cfg" .Values.airflowHome | quote }}
{{- end }}
{{- define "airflow_webserver_config_path" -}}
{{- printf "%s/webserver_config.py" .Values.airflowHome | quote }}
{{- end }}
{{- define "airflow_webserver_config_configmap_name" -}}
{{- default (printf "%s-webserver-config" .Release.Name) .Values.webserver.webserverConfigConfigMapName }}
{{- end }}
{{- define "airflow_webserver_config_mount" -}}
- name: webserver-config
mountPath: {{ template "airflow_webserver_config_path" . }}
subPath: webserver_config.py
readOnly: True
{{- end }}
{{- define "airflow_local_setting_path" -}}
{{- printf "%s/config/airflow_local_settings.py" .Values.airflowHome | quote }}
{{- end }}
{{- define "airflow_config" -}}
{{- printf "%s-config" (include "airflow.fullname" .) }}
{{- end }}
{{- define "airflow_config_mount" -}}
- name: config
mountPath: {{ template "airflow_config_path" . }}
subPath: airflow.cfg
readOnly: true
{{- if .Values.airflowLocalSettings }}
- name: config
mountPath: {{ template "airflow_local_setting_path" . }}
subPath: airflow_local_settings.py
readOnly: true
{{- end }}
{{- end }}
{{/* Create the name of the webserver service account to use */}}
{{- define "webserver.serviceAccountName" -}}
{{- if .Values.webserver.serviceAccount.create }}
{{- default (printf "%s-webserver" (include "airflow.serviceAccountName" .)) .Values.webserver.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.webserver.serviceAccount.name }}
{{- end }}
{{- end }}
{{/* Create the name of the redis service account to use */}}
{{- define "redis.serviceAccountName" -}}
{{- if .Values.redis.serviceAccount.create }}
{{- default (printf "%s-redis" (include "airflow.serviceAccountName" .)) .Values.redis.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.redis.serviceAccount.name }}
{{- end }}
{{- end }}
{{/* Create the name of the flower service account to use */}}
{{- define "flower.serviceAccountName" -}}
{{- if .Values.flower.serviceAccount.create }}
{{- default (printf "%s-flower" (include "airflow.serviceAccountName" .)) .Values.flower.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.flower.serviceAccount.name }}
{{- end }}
{{- end }}
{{/* Create the name of the scheduler service account to use */}}
{{- define "scheduler.serviceAccountName" -}}
{{- if .Values.scheduler.serviceAccount.create }}
{{- default (printf "%s-scheduler" (include "airflow.serviceAccountName" .)) .Values.scheduler.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.scheduler.serviceAccount.name }}
{{- end }}
{{- end }}
{{/* Create the name of the StatsD service account to use */}}
{{- define "statsd.serviceAccountName" -}}
{{- if .Values.statsd.serviceAccount.create }}
{{- default (printf "%s-statsd" (include "airflow.serviceAccountName" .)) .Values.statsd.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.statsd.serviceAccount.name }}
{{- end }}
{{- end }}
{{/* Create the name of the create user job service account to use */}}
{{- define "createUserJob.serviceAccountName" -}}
{{- if .Values.createUserJob.serviceAccount.create }}
{{- default (printf "%s-create-user-job" (include "airflow.serviceAccountName" .)) .Values.createUserJob.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.createUserJob.serviceAccount.name }}
{{- end }}
{{- end }}
{{/* Create the name of the migrate database job service account to use */}}
{{- define "migrateDatabaseJob.serviceAccountName" -}}
{{- if .Values.migrateDatabaseJob.serviceAccount.create }}
{{- default (printf "%s-migrate-database-job" (include "airflow.serviceAccountName" .)) .Values.migrateDatabaseJob.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.migrateDatabaseJob.serviceAccount.name }}
{{- end }}
{{- end }}
{{/* Create the name of the worker service account to use */}}
{{- define "worker.serviceAccountName" -}}
{{- if .Values.workers.serviceAccount.create }}
{{- default (printf "%s-worker" (include "airflow.serviceAccountName" .)) .Values.workers.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.workers.serviceAccount.name }}
{{- end }}
{{- end }}
{{/* Create the name of the triggerer service account to use */}}
{{- define "triggerer.serviceAccountName" -}}
{{- if .Values.triggerer.serviceAccount.create }}
{{- default (printf "%s-triggerer" (include "airflow.serviceAccountName" .)) .Values.triggerer.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.triggerer.serviceAccount.name }}
{{- end }}
{{- end }}
{{/* Create the name of the dag processor service account to use */}}
{{- define "dagProcessor.serviceAccountName" -}}
{{- if .Values.dagProcessor.serviceAccount.create }}
{{- default (printf "%s-dag-processor" (include "airflow.serviceAccountName" .)) .Values.dagProcessor.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.dagProcessor.serviceAccount.name }}
{{- end }}
{{- end }}
{{/* Create the name of the pgbouncer service account to use */}}
{{- define "pgbouncer.serviceAccountName" -}}
{{- if .Values.pgbouncer.serviceAccount.create }}
{{- default (printf "%s-pgbouncer" (include "airflow.serviceAccountName" .)) .Values.pgbouncer.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.pgbouncer.serviceAccount.name }}
{{- end }}
{{- end }}
{{/* Create the name of the cleanup service account to use */}}
{{- define "cleanup.serviceAccountName" -}}
{{- if .Values.cleanup.serviceAccount.create }}
{{- default (printf "%s-cleanup" (include "airflow.serviceAccountName" .)) .Values.cleanup.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.cleanup.serviceAccount.name }}
{{- end }}
{{- end }}
{{- define "wait-for-migrations-command" -}}
{{- if semverCompare ">=2.0.0" .Values.airflowVersion }}
- airflow
- db
- check-migrations
- --migration-wait-timeout={{ .Values.images.migrationsWaitTimeout }}
{{- else }}
- python
- -c
- |
import airflow
import logging
import os
import time
from alembic.config import Config
from alembic.runtime.migration import MigrationContext
from alembic.script import ScriptDirectory
from airflow import settings
package_dir = os.path.abspath(os.path.dirname(airflow.__file__))
directory = os.path.join(package_dir, 'migrations')
config = Config(os.path.join(package_dir, 'alembic.ini'))
config.set_main_option('script_location', directory)
config.set_main_option('sqlalchemy.url', settings.SQL_ALCHEMY_CONN.replace('%', '%%'))
script_ = ScriptDirectory.from_config(config)
timeout=60
with settings.engine.connect() as connection:
context = MigrationContext.configure(connection)
ticker = 0
while True:
source_heads = set(script_.get_heads())
db_heads = set(context.get_current_heads())
if source_heads == db_heads:
break
if ticker >= timeout:
raise TimeoutError("There are still unapplied migrations after {} seconds.".format(ticker))
ticker += 1
time.sleep(1)
logging.info('Waiting for migrations... %s second(s)', ticker)
{{- end }}
{{- end }}
{{- define "scheduler_liveness_check_command" }}
{{- if semverCompare ">=2.5.0" .Values.airflowVersion }}
- sh
- -c
- |
CONNECTION_CHECK_MAX_COUNT=0 AIRFLOW__LOGGING__LOGGING_LEVEL=ERROR exec /entrypoint \
airflow jobs check --job-type SchedulerJob --local
{{- else if semverCompare ">=2.1.0" .Values.airflowVersion }}
- sh
- -c
- |
CONNECTION_CHECK_MAX_COUNT=0 AIRFLOW__LOGGING__LOGGING_LEVEL=ERROR exec /entrypoint \
airflow jobs check --job-type SchedulerJob --hostname $(hostname)
{{- else }}
- sh
- -c
- |
CONNECTION_CHECK_MAX_COUNT=0 exec /entrypoint python -Wignore -c "
import os
os.environ['AIRFLOW__CORE__LOGGING_LEVEL'] = 'ERROR'
os.environ['AIRFLOW__LOGGING__LOGGING_LEVEL'] = 'ERROR'
from airflow.jobs.scheduler_job import SchedulerJob
from airflow.utils.db import create_session
from airflow.utils.net import get_hostname
import sys
with create_session() as session:
job = session.query(SchedulerJob).filter_by(hostname=get_hostname()).order_by(
SchedulerJob.latest_heartbeat.desc()).limit(1).first()
sys.exit(0 if job.is_alive() else 1)"
{{- end }}
{{- end }}
{{- define "scheduler_startup_check_command" }}
{{- if semverCompare ">=2.5.0" .Values.airflowVersion }}
- sh
- -c
- |
CONNECTION_CHECK_MAX_COUNT=0 AIRFLOW__LOGGING__LOGGING_LEVEL=ERROR exec /entrypoint \
airflow jobs check --job-type SchedulerJob --local
{{- else if semverCompare ">=2.1.0" .Values.airflowVersion }}
- sh
- -c
- |
CONNECTION_CHECK_MAX_COUNT=0 AIRFLOW__LOGGING__LOGGING_LEVEL=ERROR exec /entrypoint \
airflow jobs check --job-type SchedulerJob --hostname $(hostname)
{{- else }}
- sh
- -c
- |
CONNECTION_CHECK_MAX_COUNT=0 exec /entrypoint python -Wignore -c "
import os
os.environ['AIRFLOW__CORE__LOGGING_LEVEL'] = 'ERROR'
os.environ['AIRFLOW__LOGGING__LOGGING_LEVEL'] = 'ERROR'
from airflow.jobs.scheduler_job import SchedulerJob
from airflow.utils.db import create_session
from airflow.utils.net import get_hostname
import sys
with create_session() as session:
job = session.query(SchedulerJob).filter_by(hostname=get_hostname()).order_by(
SchedulerJob.latest_heartbeat.desc()).limit(1).first()
sys.exit(0 if job.is_alive() else 1)"
{{- end }}
{{- end }}
{{- define "triggerer_liveness_check_command" }}
{{- if semverCompare ">=2.5.0" .Values.airflowVersion }}
- sh
- -c
- |
CONNECTION_CHECK_MAX_COUNT=0 AIRFLOW__LOGGING__LOGGING_LEVEL=ERROR exec /entrypoint \
airflow jobs check --job-type TriggererJob --local
{{- else }}
- sh
- -c
- |
CONNECTION_CHECK_MAX_COUNT=0 AIRFLOW__LOGGING__LOGGING_LEVEL=ERROR exec /entrypoint \
airflow jobs check --job-type TriggererJob --hostname $(hostname)
{{- end }}
{{- end }}
{{- define "dag_processor_liveness_check_command" }}
{{- $commandArgs := (list) -}}
{{- if semverCompare ">=2.5.0" .Values.airflowVersion }}
{{- $commandArgs = append $commandArgs "--local" -}}
{{- if semverCompare ">=2.5.2" .Values.airflowVersion }}
{{- $commandArgs = concat $commandArgs (list "--job-type" "DagProcessorJob") -}}
{{- end }}
{{- else }}
{{- $commandArgs = concat $commandArgs (list "--hostname" "$(hostname)") -}}
{{- end }}
- sh
- -c
- |
CONNECTION_CHECK_MAX_COUNT=0 AIRFLOW__LOGGING__LOGGING_LEVEL=ERROR exec /entrypoint \
airflow jobs check {{ join " " $commandArgs }}
{{- end }}
{{- define "registry_docker_config" }}
{{- $host := .Values.registry.connection.host }}
{{- $email := .Values.registry.connection.email }}
{{- $user := .Values.registry.connection.user }}
{{- $pass := .Values.registry.connection.pass }}
{{- $config := dict "auths" }}
{{- $auth := dict }}
{{- $data := dict }}
{{- $_ := set $data "username" $user }}
{{- $_ := set $data "password" $pass }}
{{- $_ := set $data "email" $email }}
{{- $_ := set $data "auth" (printf "%v:%v" $user $pass | b64enc) }}
{{- $_ := set $auth $host $data }}
{{- $_ := set $config "auths" $auth }}
{{ $config | toJson | print }}
{{- end }}
{{/*
Set the default value for pod securityContext
If no value is passed for securityContexts.pod or <node>.securityContexts.pod or legacy securityContext and <node>.securityContext, defaults to global uid and gid.
+-----------------------------+ +------------------------+ +----------------------+ +-----------------+ +-------------------------+
| <node>.securityContexts.pod | -> | <node>.securityContext | -> | securityContexts.pod | -> | securityContext | -> | Values.uid + Values.gid |
+-----------------------------+ +------------------------+ +----------------------+ +-----------------+ +-------------------------+
Values are not accumulated meaning that if runAsUser is set to 10 in <node>.securityContexts.pod,
any extra values set to securityContext or uid+gid will be ignored.
The template can be called like so:
include "airflowPodSecurityContext" (list . .Values.webserver)
Where `.` is the global variables scope and `.Values.webserver` the local variables scope for the webserver template.
*/}}
{{- define "airflowPodSecurityContext" -}}
{{- $ := index . 0 -}}
{{- with index . 1 }}
{{- if .securityContexts.pod -}}
{{ toYaml .securityContexts.pod | print }}
{{- else if .securityContext -}}
{{ toYaml .securityContext | print }}
{{- else if $.Values.securityContexts.pod -}}
{{ toYaml $.Values.securityContexts.pod | print }}
{{- else if $.Values.securityContext -}}
{{ toYaml $.Values.securityContext | print }}
{{- else -}}
runAsUser: {{ $.Values.uid }}
fsGroup: {{ $.Values.gid }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Set the default value for pod securityContext
If no value is passed for <node>.securityContexts.pod or <node>.securityContext, defaults to UID in the local node.
+-----------------------------+ +------------------------+ +-------------+
| <node>.securityContexts.pod | -> | <node>.securityContext | -> | <node>.uid |
+-----------------------------+ +------------------------+ +-------------+
The template can be called like so:
include "localPodSecurityContext" (list . .Values.schedule)
It is important to pass the local variables scope to this template as it is used to determine the local node value for uid.
*/}}
{{- define "localPodSecurityContext" -}}
{{- if .securityContexts.pod -}}
{{ toYaml .securityContexts.pod | print }}
{{- else if .securityContext -}}
{{ toYaml .securityContext | print }}
{{- else -}}
runAsUser: {{ .uid }}
{{- end -}}
{{- end -}}
{{/*
Set the default value for container securityContext
If no value is passed for <node>.securityContexts.container or <node>.securityContext, defaults to UID in the local node.
+-----------------------------------+ +------------------------+ +-------------+
| <node>.securityContexts.container | -> | <node>.securityContext | -> | <node>.uid |
+-----------------------------------+ +------------------------+ +-------------+
The template can be called like so:
include "localContainerSecurityContext" .Values.statsd
It is important to pass the local variables scope to this template as it is used to determine the local node value for uid.
*/}}
{{- define "localContainerSecurityContext" -}}
{{- if .securityContexts.container -}}
{{ toYaml .securityContexts.container | print }}
{{- else if .securityContext -}}
{{ toYaml .securityContext | print }}
{{- else -}}
runAsUser: {{ .uid }}
{{- end -}}
{{- end -}}
{{/*
Set the default value for workers chown for persistent storage
If no value is passed for securityContexts.pod or <node>.securityContexts.pod or legacy securityContext and <node>.securityContext, defaults to global uid and gid.
The template looks for `runAsUser` and `fsGroup` specifically, any other parameter will be ignored.
+-----------------------------+ +----------------------------------------------------+ +------------------+ +-------------------------+
| <node>.securityContexts.pod | -> | securityContexts.pod | <node>.securityContexts.pod | -> | securityContexts | -> | Values.uid + Values.gid |
+-----------------------------+ +----------------------------------------------------+ +------------------+ +-------------------------+
Values are not accumulated meaning that if runAsUser is set to 10 in <node>.securityContexts.pod,
any extra values set to securityContexts or uid+gid will be ignored.
The template can be called like so:
include "airflowPodSecurityContextsIds" (list . .Values.webserver)
Where `.` is the global variables scope and `.Values.workers` the local variables scope for the workers template.
*/}}
{{- define "airflowPodSecurityContextsIds" -}}
{{- $ := index . 0 -}}
{{- with index . 1 }}
{{- if .securityContexts.pod -}}
{{ pluck "runAsUser" .securityContexts.pod | first | default $.Values.uid }}:{{ pluck "fsGroup" .securityContexts.pod | first | default $.Values.gid }}
{{- else if $.Values.securityContext -}}
{{ pluck "runAsUser" $.Values.securityContext | first | default $.Values.uid }}:{{ pluck "fsGroup" $.Values.securityContext | first | default $.Values.gid }}
{{- else if $.Values.securityContexts.pod -}}
{{ pluck "runAsUser" $.Values.securityContexts.pod | first | default $.Values.uid }}:{{ pluck "fsGroup" $.Values.securityContexts.pod | first | default $.Values.gid }}
{{- else if $.Values.securityContext -}}
{{ pluck "runAsUser" $.Values.securityContext | first | default $.Values.uid }}:{{ pluck "fsGroup" $.Values.securityContext | first | default $.Values.gid }}
{{- else -}}
{{ $.Values.uid }}:{{ $.Values.gid }}
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Set the default value for container securityContext
If no value is passed for securityContexts.container or <node>.securityContexts.container, defaults to deny privileges escallation and dropping all POSIX capabilities.
+-----------------------------------+ +----------------------------+ +-----------------------------------------------------------+
| <node>.securityContexts.container | -> | securityContexts.containers | -> | allowPrivilegesEscalation: false, capabilities.drop: [ALL]|
+-----------------------------------+ +----------------------------+ +-----------------------------------------------------------+
The template can be called like so:
include "containerSecurityContext" (list . .Values.webserver)
Where `.` is the global variables scope and `.Values.webserver` the local variables scope for the webserver template.
*/}}
{{- define "containerSecurityContext" -}}
{{- $ := index . 0 -}}
{{- with index . 1 }}
{{- if .securityContexts.container -}}
{{ toYaml .securityContexts.container | print }}
{{- else if $.Values.securityContexts.containers -}}
{{ toYaml $.Values.securityContexts.containers | print }}
{{- else -}}
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Set the default value for external container securityContext(redis and statsd).
If no value is passed for <node>.securityContexts.container, defaults to deny privileges escallation and dropping all POSIX capabilities.
+-----------------------------------+ +-----------------------------------------------------------+
| <node>.securityContexts.container | -> | allowPrivilegesEscalation: false, capabilities.drop: [ALL]|
+-----------------------------------+ +-----------------------------------------------------------+
The template can be called like so:
include "externalContainerSecurityContext" .Values.statsd
*/}}
{{- define "externalContainerSecurityContext" -}}
{{- if .securityContexts.container -}}
{{ toYaml .securityContexts.container | print }}
{{- else -}}
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
{{- end -}}
{{- end -}}
{{- define "container_extra_envs" -}}
{{- $ := index . 0 -}}
{{- $env := index . 1 -}}
{{- range $i, $config := $env }}
- name: {{ $config.name }}
value: {{ $config.value | quote }}
{{- if or (eq $.Values.executor "KubernetesExecutor") (eq $.Values.executor "LocalKubernetesExecutor") (eq $.Values.executor "CeleryKubernetesExecutor") }}
- name: AIRFLOW__KUBERNETES_ENVIRONMENT_VARIABLES__{{ $config.name }}
value: {{ $config.value | quote }}
{{- end }}
{{- end }}
{{- end }}
{{- define "kedaNetworkPolicySelector" }}
{{- if .Values.workers.keda.enabled }}
{{- if .Values.workers.keda.namespaceLabels }}
- namespaceSelector:
matchLabels: {{- toYaml .Values.workers.keda.namespaceLabels | nindent 10 }}
podSelector:
{{- else }}
- podSelector:
{{- end }}
matchLabels:
app: keda-operator
{{- end }}
{{- end }}

64
helm/airflow/templates/check-values.yaml Normal file
View File

@@ -0,0 +1,64 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
{{- /*
The sole purpose of this yaml file is it to check the values file is consistent for some complexe combinations.
*/ -}}
{{- /*
##############################
Redis related checks
#############################
*/ -}}
{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
{{- if .Values.redis.enabled }}
{{- if and .Values.redis.passwordSecretName (not .Values.data.brokerUrlSecretName) }}
{{ required "When using the internal redis of the chart and setting the value redis.passwordSecretName, you must also set the value data.brokerUrlSecretName." nil }}
{{- end }}
{{- if and .Values.redis.passwordSecretName .Values.redis.password }}
{{ required "You must not set both values redis.passwordSecretName and redis.password" nil }}
{{- end }}
{{- else }}
{{- if not (or .Values.data.brokerUrlSecretName .Values.data.brokerUrl) }}
{{ required "You must set one of the values data.brokerUrlSecretName or data.brokerUrl when using a Celery based executor with redis.enabled set to false (we need the url to the redis instance)." nil }}
{{- end }}
{{- end }}
{{- if and .Values.data.brokerUrlSecretName .Values.data.brokerUrl }}
{{ required "You must not set both values data.brokerUrlSecretName and data.brokerUrl" nil }}
{{- end }}
{{- end }}
{{- if .Values.elasticsearch.enabled }}
{{- if and .Values.elasticsearch.secretName .Values.elasticsearch.connection }}
{{ required "You must not set both values elasticsearch.secretName and elasticsearch.connection" nil }}
{{- end }}
{{- if not (or .Values.elasticsearch.secretName .Values.elasticsearch.connection) }}
{{ required "You must set one of the values elasticsearch.secretName or elasticsearch.connection when using a Elasticsearch" nil }}
{{- end }}
{{- end }}

109
helm/airflow/templates/cleanup/cleanup-cronjob.yaml Normal file
View File

@@ -0,0 +1,109 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Cleanup Pods CronJob
#################################
{{- if .Values.cleanup.enabled }}
{{- $nodeSelector := or .Values.cleanup.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.cleanup.affinity .Values.affinity }}
{{- $tolerations := or .Values.cleanup.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.cleanup.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.cleanup) }}
apiVersion: batch/v1
kind: CronJob
metadata:
name: {{ include "airflow.fullname" . }}-cleanup
labels:
tier: airflow
component: airflow-cleanup-pods
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.cleanup.jobAnnotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
spec:
schedule: "{{ tpl .Values.cleanup.schedule . }}"
# The cron job does not allow concurrent runs; if it is time for a new job run and the previous job run hasn't finished yet, the cron job skips the new job run
concurrencyPolicy: Forbid
{{- if .Values.cleanup.failedJobsHistoryLimit }}
failedJobsHistoryLimit: {{ .Values.cleanup.failedJobsHistoryLimit }}
{{- end }}
{{- if .Values.cleanup.successfulJobsHistoryLimit }}
successfulJobsHistoryLimit: {{ .Values.cleanup.successfulJobsHistoryLimit }}
{{- end }}
jobTemplate:
spec:
backoffLimit: 1
template:
metadata:
labels:
tier: airflow
component: airflow-cleanup-pods
release: {{ .Release.Name }}
{{- if or (.Values.labels) (.Values.cleanup.labels) }}
{{- mustMerge .Values.cleanup.labels .Values.labels | toYaml | nindent 12 }}
{{- end }}
annotations:
sidecar.istio.io/inject: "false"
{{- if .Values.airflowPodAnnotations }}
{{- toYaml .Values.airflowPodAnnotations | nindent 12 }}
{{- end }}
{{- if .Values.cleanup.podAnnotations }}
{{- toYaml .Values.cleanup.podAnnotations | nindent 12 }}
{{- end }}
spec:
restartPolicy: Never
nodeSelector: {{- toYaml $nodeSelector | nindent 12 }}
affinity: {{- toYaml $affinity | nindent 12 }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
tolerations: {{- toYaml $tolerations | nindent 12 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 12 }}
serviceAccountName: {{ include "cleanup.serviceAccountName" . }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
securityContext: {{ $securityContext | nindent 12 }}
containers:
- name: airflow-cleanup-pods
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
{{- if .Values.cleanup.command }}
command: {{ tpl (toYaml .Values.cleanup.command) . | nindent 16 }}
{{- end }}
{{- if .Values.cleanup.args }}
args: {{ tpl (toYaml .Values.cleanup.args) . | nindent 16 }}
{{- end }}
env:
{{- include "standard_airflow_environment" . | indent 12 }}
{{- include "container_extra_envs" (list . .Values.cleanup.env) | indent 12 }}
volumeMounts: {{- include "airflow_config_mount" . | nindent 16 }}
resources: {{- toYaml .Values.cleanup.resources | nindent 16 }}
volumes:
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- end }}

40
helm/airflow/templates/cleanup/cleanup-serviceaccount.yaml Normal file
View File

@@ -0,0 +1,40 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Cleanup ServiceAccount
#################################
{{- if and .Values.cleanup.serviceAccount.create .Values.cleanup.enabled }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{ .Values.cleanup.serviceAccount.automountServiceAccountToken }}
metadata:
name: {{ include "cleanup.serviceAccountName" . }}
labels:
tier: airflow
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.cleanup.labels) }}
{{- mustMerge .Values.cleanup.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.cleanup.serviceAccount.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}

74
helm/airflow/templates/configmaps/configmap.yaml Normal file
View File

@@ -0,0 +1,74 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow ConfigMap
#################################
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ template "airflow_config" . }}
labels:
tier: airflow
component: config
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end -}}
{{- if .Values.airflowConfigAnnotations }}
annotations: {{- toYaml .Values.airflowConfigAnnotations | nindent 4 }}
{{- end }}
{{- $Global := . }}
data:
# These are system-specified config overrides.
airflow.cfg: |-
{{- range $section, $settings := .Values.config }}
[{{ $section }}]
{{- range $key, $val := $settings }}
{{ $key }} = {{ tpl ($val | toString) $Global }}
{{- end }}
{{ end }}
{{- if .Values.airflowLocalSettings }}
airflow_local_settings.py: |-
{{- tpl .Values.airflowLocalSettings . | nindent 4 }}
{{- end }}
{{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.knownHosts }}
known_hosts: |-
{{- .Values.dags.gitSync.knownHosts | nindent 4 }}
{{- end }}
{{/* {{- if or (eq $.Values.executor "LocalKubernetesExecutor") (eq $.Values.executor "KubernetesExecutor") (eq $.Values.executor "CeleryKubernetesExecutor") }}*/}}
{{/* {{- if semverCompare ">=1.10.12" .Values.airflowVersion }}*/}}
pod_template_file.yaml: |-
{{- if .Values.podTemplate }}
{{- tpl .Values.podTemplate . | nindent 4 }}
{{- else }}
{{- tpl (.Files.Get "files/pod-template-file.kubernetes-helm-yaml") . | nindent 4 }}
{{- end }}
{{/* {{- end }}*/}}
{{/* {{- end }}*/}}
{{- if .Values.kerberos.enabled }}
krb5.conf: |-
{{- tpl .Values.kerberos.config . | nindent 4 }}
{{- end }}

53
helm/airflow/templates/configmaps/extra-configmaps.yaml Normal file
View File

@@ -0,0 +1,53 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
####################################################
## Extra ConfigMaps provisioned via the chart values
####################################################
{{- $Global := . }}
{{- range $configMapName, $configMapContent := .Values.extraConfigMaps }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ tpl $configMapName $Global | quote }}
labels:
release: {{ $Global.Release.Name }}
chart: "{{ $Global.Chart.Name }}-{{ $Global.Chart.Version }}"
heritage: {{ $Global.Release.Service }}
{{- with $Global.Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if $configMapContent.labels }}
{{- toYaml $configMapContent.labels | nindent 4 }}
{{- end }}
annotations:
"helm.sh/hook": "pre-install,pre-upgrade"
"helm.sh/hook-delete-policy": "before-hook-creation"
"helm.sh/hook-weight": "0"
{{- if $configMapContent.annotations }}
{{- toYaml $configMapContent.annotations | nindent 4 }}
{{- end }}
{{- if $configMapContent.data }}
data:
{{- with $configMapContent.data }}
{{- tpl . $Global | nindent 2 }}
{{- end }}
{{- end }}
{{- end }}

52
helm/airflow/templates/configmaps/statsd-configmap.yaml Normal file
View File

@@ -0,0 +1,52 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow StatsD ConfigMap
#################################
{{- if and .Values.statsd.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "airflow.fullname" . }}-statsd
labels:
tier: airflow
component: config
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.statsd.configMapAnnotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
data:
mappings.yml: |-
{{- if .Values.statsd.overrideMappings }}
mappings:
{{- toYaml .Values.statsd.overrideMappings | nindent 6 }}
{{- else }}
{{- .Files.Get "files/statsd-mappings.yml" | nindent 4 }}
{{- if .Values.statsd.extraMappings }}
{{- toYaml .Values.statsd.extraMappings | nindent 6 }}
{{- end }}
{{- end }}
{{- end }}

44
helm/airflow/templates/configmaps/webserver-configmap.yaml Normal file
View File

@@ -0,0 +1,44 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow ConfigMap
#################################
{{- if and .Values.webserver.webserverConfig (not .Values.webserver.webserverConfigConfigMapName) }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ template "airflow_webserver_config_configmap_name" . }}
labels:
tier: airflow
component: config
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.webserver.configMapAnnotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
data:
webserver_config.py: |-
{{- tpl .Values.webserver.webserverConfig . | nindent 4 }}
{{- end }}

261
helm/airflow/templates/dag-processor/dag-processor-deployment.yaml Normal file
View File

@@ -0,0 +1,261 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Dag Processor Deployment
#################################
{{- if semverCompare ">=2.3.0" .Values.airflowVersion }}
{{- if .Values.dagProcessor.enabled }}
{{- $nodeSelector := or .Values.dagProcessor.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.dagProcessor.affinity .Values.affinity }}
{{- $tolerations := or .Values.dagProcessor.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.dagProcessor.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $revisionHistoryLimit := or .Values.dagProcessor.revisionHistoryLimit .Values.revisionHistoryLimit }}
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.dagProcessor) }}
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.dagProcessor) }}
{{- $containerLifecycleHooks := or .Values.dagProcessor.containerLifecycleHooks .Values.containerLifecycleHooks }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "airflow.fullname" . }}-dag-processor
labels:
tier: airflow
component: dag-processor
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if .Values.dagProcessor.annotations }}
annotations: {{- toYaml .Values.dagProcessor.annotations | nindent 4 }}
{{- end }}
spec:
replicas: {{ .Values.dagProcessor.replicas }}
{{- if $revisionHistoryLimit }}
revisionHistoryLimit: {{ $revisionHistoryLimit }}
{{- end }}
selector:
matchLabels:
tier: airflow
component: dag-processor
release: {{ .Release.Name }}
{{- if .Values.dagProcessor.strategy }}
strategy: {{- toYaml .Values.dagProcessor.strategy | nindent 4 }}
{{- end }}
template:
metadata:
labels:
tier: airflow
component: dag-processor
release: {{ .Release.Name }}
{{- with .Values.labels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }}
checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }}
checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }}
checksum/extra-configmaps: {{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }}
checksum/extra-secrets: {{ include (print $.Template.BasePath "/secrets/extra-secrets.yaml") . | sha256sum }}
{{- if .Values.dagProcessor.safeToEvict }}
cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
{{- end }}
{{- if .Values.airflowPodAnnotations }}
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
{{- end }}
{{- if .Values.dagProcessor.podAnnotations }}
{{- toYaml .Values.dagProcessor.podAnnotations | nindent 8 }}
{{- end }}
spec:
{{- if .Values.dagProcessor.priorityClassName }}
priorityClassName: {{ .Values.dagProcessor.priorityClassName }}
{{- end }}
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
affinity:
{{- if $affinity }}
{{- toYaml $affinity | nindent 8 }}
{{- else }}
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchLabels:
component: dag-processor
topologyKey: kubernetes.io/hostname
weight: 100
{{- end }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
tolerations: {{- toYaml $tolerations | nindent 8 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
terminationGracePeriodSeconds: {{ .Values.dagProcessor.terminationGracePeriodSeconds }}
restartPolicy: Always
serviceAccountName: {{ include "dagProcessor.serviceAccountName" . }}
securityContext: {{ $securityContext | nindent 8 }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
initContainers:
{{- if .Values.dagProcessor.waitForMigrations.enabled }}
- name: wait-for-airflow-migrations
resources: {{- toYaml .Values.dagProcessor.resources | nindent 12 }}
image: {{ template "airflow_image_for_migrations" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
volumeMounts:
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.dagProcessor.extraVolumeMounts }}
{{- tpl (toYaml .Values.dagProcessor.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- include "airflow_config_mount" . | nindent 12 }}
args: {{- include "wait-for-migrations-command" . | indent 10 }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- if .Values.dagProcessor.waitForMigrations.env }}
{{- tpl (toYaml .Values.dagProcessor.waitForMigrations.env) $ | nindent 12 }}
{{- end }}
{{- end }}
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
{{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }}
{{- end }}
{{- if .Values.dagProcessor.extraInitContainers }}
{{- toYaml .Values.dagProcessor.extraInitContainers | nindent 8 }}
{{- end }}
containers:
- name: dag-processor
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 12 }}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
{{- end }}
{{- if .Values.dagProcessor.command }}
command: {{ tpl (toYaml .Values.dagProcessor.command) . | nindent 12 }}
{{- end }}
{{- if .Values.dagProcessor.args }}
args: {{ tpl (toYaml .Values.dagProcessor.args) . | nindent 12 }}
{{- end }}
resources: {{- toYaml .Values.dagProcessor.resources | nindent 12 }}
volumeMounts:
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.dagProcessor.extraVolumeMounts }}
{{- tpl (toYaml .Values.dagProcessor.extraVolumeMounts) . | nindent 12 }}
{{- end }}
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }}
{{- include "airflow_dags_mount" . | nindent 12 }}
{{- end }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- include "container_extra_envs" (list . .Values.dagProcessor.env) | indent 10 }}
livenessProbe:
initialDelaySeconds: {{ .Values.dagProcessor.livenessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.dagProcessor.livenessProbe.timeoutSeconds }}
failureThreshold: {{ .Values.dagProcessor.livenessProbe.failureThreshold }}
periodSeconds: {{ .Values.dagProcessor.livenessProbe.periodSeconds }}
exec:
command:
{{- if .Values.dagProcessor.livenessProbe.command }}
{{- toYaml .Values.dagProcessor.livenessProbe.command | nindent 16 }}
{{- else }}
{{- include "dag_processor_liveness_check_command" . | indent 14 }}
{{- end }}
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
{{- include "git_sync_container" . | indent 8 }}
{{- end }}
{{- if .Values.dagProcessor.logGroomerSidecar.enabled }}
- name: dag-processor-log-groomer
resources: {{- toYaml .Values.dagProcessor.logGroomerSidecar.resources | nindent 12 }}
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
{{- if .Values.dagProcessor.logGroomerSidecar.command }}
command: {{ tpl (toYaml .Values.dagProcessor.logGroomerSidecar.command) . | nindent 12 }}
{{- end }}
{{- if .Values.dagProcessor.logGroomerSidecar.args }}
args: {{- tpl (toYaml .Values.dagProcessor.logGroomerSidecar.args) . | nindent 12 }}
{{- end }}
{{- if .Values.dagProcessor.logGroomerSidecar.retentionDays }}
env:
- name: AIRFLOW__LOG_RETENTION_DAYS
value: "{{ .Values.dagProcessor.logGroomerSidecar.retentionDays }}"
{{- end }}
volumeMounts:
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.dagProcessor.extraVolumeMounts }}
{{- tpl (toYaml .Values.dagProcessor.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
{{- end }}
{{- if .Values.dagProcessor.extraContainers }}
{{- toYaml .Values.dagProcessor.extraContainers | nindent 8 }}
{{- end }}
volumes:
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
- name: webserver-config
configMap:
name: {{ template "airflow_webserver_config_configmap_name" . }}
{{- end }}
{{- if .Values.dags.persistence.enabled }}
- name: dags
persistentVolumeClaim:
claimName: {{ template "airflow_dags_volume_claim" . }}
{{- else if .Values.dags.gitSync.enabled }}
- name: dags
emptyDir: {}
{{- end }}
{{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.sshKeySecret }}
{{- include "git_sync_ssh_key_volume" . | indent 8 }}
{{- end }}
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 8 }}
{{- end }}
{{- if .Values.dagProcessor.extraVolumes }}
{{- tpl (toYaml .Values.dagProcessor.extraVolumes) . | nindent 8 }}
{{- end }}
{{- if .Values.logs.persistence.enabled }}
- name: logs
persistentVolumeClaim:
claimName: {{ template "airflow_logs_volume_claim" . }}
{{- else }}
- name: logs
emptyDir: {}
{{- end }}
{{- end }}
{{- end }}

43
helm/airflow/templates/dag-processor/dag-processor-serviceaccount.yaml Normal file
View File

@@ -0,0 +1,43 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Dag Processor ServiceAccount
#################################
{{- if semverCompare ">=2.3.0" .Values.airflowVersion }}
{{- if and .Values.dagProcessor.serviceAccount.create .Values.dagProcessor.enabled }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{ .Values.dagProcessor.serviceAccount.automountServiceAccountToken }}
metadata:
name: {{ include "dagProcessor.serviceAccountName" . }}
labels:
tier: airflow
component: dag-processor
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.dagProcessor.serviceAccount.annotations}}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
{{- end }}

52
helm/airflow/templates/dags-persistent-volume-claim.yaml Normal file
View File

@@ -0,0 +1,52 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
######################################
## Airflow DAGs PersistentVolumeClaim
######################################
{{- if and (not .Values.dags.persistence.existingClaim ) .Values.dags.persistence.enabled }}
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: {{ template "airflow_dags_volume_claim" . }}
labels:
tier: airflow
component: dags-pvc
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.dags.persistence.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
spec:
accessModes: [{{ .Values.dags.persistence.accessMode | quote }}]
resources:
requests:
storage: {{ .Values.dags.persistence.size | quote }}
{{- if .Values.dags.persistence.storageClassName }}
{{- if (eq "-" .Values.dags.persistence.storageClassName) }}
storageClassName: ""
{{- else }}
storageClassName: "{{ .Values.dags.persistence.storageClassName }}"
{{- end }}
{{- end }}
{{- end }}

169
helm/airflow/templates/flower/flower-deployment.yaml Normal file
View File

@@ -0,0 +1,169 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Flower Deployment
#################################
{{- if .Values.flower.enabled }}
{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
{{- $nodeSelector := or .Values.flower.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.flower.affinity .Values.affinity }}
{{- $tolerations := or .Values.flower.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.flower.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $revisionHistoryLimit := or .Values.flower.revisionHistoryLimit .Values.revisionHistoryLimit }}
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.flower) }}
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.flower) }}
{{- $containerLifecycleHooks := or .Values.flower.containerLifecycleHooks .Values.containerLifecycleHooks }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "airflow.fullname" . }}-flower
labels:
tier: airflow
component: flower
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if .Values.flower.annotations }}
annotations: {{- toYaml .Values.flower.annotations | nindent 4 }}
{{- end }}
spec:
replicas: 1
{{- if $revisionHistoryLimit }}
revisionHistoryLimit: {{ $revisionHistoryLimit }}
{{- end }}
selector:
matchLabels:
tier: airflow
component: flower
release: {{ .Release.Name }}
template:
metadata:
labels:
tier: airflow
component: flower
release: {{ .Release.Name }}
{{- if or (.Values.labels) (.Values.flower.labels) }}
{{- mustMerge .Values.flower.labels .Values.labels | toYaml | nindent 8 }}
{{- end }}
annotations:
checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }}
checksum/flower-secret: {{ include (print $.Template.BasePath "/secrets/flower-secret.yaml") . | sha256sum }}
{{- if or (.Values.airflowPodAnnotations) (.Values.flower.podAnnotations) }}
{{- mustMerge .Values.flower.podAnnotations .Values.airflowPodAnnotations | toYaml | nindent 8 }}
{{- end }}
spec:
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
affinity: {{- toYaml $affinity | nindent 8 }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
tolerations: {{- toYaml $tolerations | nindent 8 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
serviceAccountName: {{ include "flower.serviceAccountName" . }}
{{- if .Values.flower.priorityClassName }}
priorityClassName: {{ .Values.flower.priorityClassName }}
{{- end }}
restartPolicy: Always
securityContext: {{ $securityContext | nindent 8 }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
containers:
- name: flower
image: {{ template "flower_image" . }}
imagePullPolicy: {{ .Values.images.flower.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 12 }}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
{{- end }}
{{- if .Values.flower.command }}
command: {{ tpl (toYaml .Values.flower.command) . | nindent 12 }}
{{- end }}
{{- if .Values.flower.args }}
args: {{ tpl (toYaml .Values.flower.args) . | nindent 12 }}
{{- end }}
resources: {{- toYaml .Values.flower.resources | nindent 12 }}
volumeMounts:
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.flower.extraVolumeMounts }}
{{- tpl (toYaml .Values.flower.extraVolumeMounts) . | nindent 12 }}
{{- end }}
ports:
- name: flower-ui
containerPort: {{ .Values.ports.flowerUI }}
livenessProbe:
failureThreshold: 10
exec:
command:
- curl
{{- if (or .Values.flower.secretName (and .Values.flower.username .Values.flower.password))}}
- "--user"
- $AIRFLOW__CELERY__FLOWER_BASIC_AUTH
{{- end }}
- {{ printf "localhost:%s" (.Values.ports.flowerUI | toString) }}
initialDelaySeconds: 10
periodSeconds: 5
readinessProbe:
failureThreshold: 10
exec:
command:
- curl
{{- if (or .Values.flower.secretName (and .Values.flower.username .Values.flower.password))}}
- "--user"
- $AIRFLOW__CELERY__FLOWER_BASIC_AUTH
{{- end }}
- {{ printf "localhost:%s" (.Values.ports.flowerUI | toString) }}
initialDelaySeconds: 10
periodSeconds: 5
envFrom:
{{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
{{- if (or .Values.flower.secretName (and .Values.flower.username .Values.flower.password))}}
- name: AIRFLOW__CELERY__FLOWER_BASIC_AUTH
valueFrom:
secretKeyRef:
name: {{ template "flower_secret" . }}
key: basicAuth
{{- end }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "container_extra_envs" (list . .Values.flower.env) | indent 10 }}
{{- if .Values.flower.extraContainers }}
{{- toYaml .Values.flower.extraContainers | nindent 8 }}
{{- end }}
volumes:
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 8 }}
{{- end }}
{{- if .Values.flower.extraVolumes }}
{{- tpl (toYaml .Values.flower.extraVolumes) . | nindent 8 }}
{{- end }}
{{- end }}
{{- end }}

94
helm/airflow/templates/flower/flower-ingress.yaml Normal file
View File

@@ -0,0 +1,94 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Flower Ingress
#################################
{{- if .Values.flower.enabled }}
{{- if and (or .Values.ingress.flower.enabled .Values.ingress.enabled) (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")) }}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ include "airflow.fullname" . }}-flower-ingress
labels:
tier: airflow
component: flower-ingress
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.flower.labels) }}
{{- mustMerge .Values.flower.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.ingress.flower.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if and .Values.ingress.flower.hosts (.Values.ingress.flower.hosts | first | kindIs "string" | not) }}
{{- $anyTlsHosts := false -}}
{{- range .Values.ingress.flower.hosts }}
{{- if .tls }}
{{- if .tls.enabled }}
{{- $anyTlsHosts = true -}}
{{- end }}
{{- end }}
{{- end }}
{{- if $anyTlsHosts }}
tls:
{{- range .Values.ingress.flower.hosts }}
{{- if .tls }}
{{- if .tls.enabled }}
- hosts:
- {{ .name | quote }}
secretName: {{ .tls.secretName }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- else if .Values.ingress.flower.tls.enabled }}
tls:
- hosts:
{{- .Values.ingress.flower.hosts | default (list .Values.ingress.flower.host) | toYaml | nindent 8 }}
secretName: {{ .Values.ingress.flower.tls.secretName }}
{{- end }}
rules:
{{- range .Values.ingress.flower.hosts | default (list .Values.ingress.flower.host) }}
- http:
paths:
- backend:
service:
name: {{ $.Release.Name }}-flower
port:
name: flower-ui
{{- if $.Values.ingress.flower.path }}
path: {{ $.Values.ingress.flower.path }}
pathType: {{ $.Values.ingress.flower.pathType }}
{{- end }}
{{- $hostname := . -}}
{{- if . | kindIs "string" | not }}
{{- $hostname = .name -}}
{{- end }}
{{- if $hostname }}
host: {{ tpl $hostname $ | quote }}
{{- end }}
{{- end }}
{{- if .Values.ingress.flower.ingressClassName }}
ingressClassName: {{ .Values.ingress.flower.ingressClassName }}
{{- end }}
{{- end }}
{{- end }}

60
helm/airflow/templates/flower/flower-networkpolicy.yaml Normal file
View File

@@ -0,0 +1,60 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Flower NetworkPolicy
#################################
{{- if .Values.flower.enabled }}
{{- $celery_executors := list "CeleryExecutor" "CeleryKubernetesExecutor"}}
{{- if and .Values.networkPolicies.enabled (has .Values.executor $celery_executors) }}
{{- $from := or .Values.flower.networkPolicy.ingress.from .Values.flower.extraNetworkPolicies }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: {{ include "airflow.fullname" . }}-flower-policy
labels:
tier: airflow
component: airflow-flower-policy
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.flower.labels) }}
{{- mustMerge .Values.flower.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
podSelector:
matchLabels:
tier: airflow
component: flower
release: {{ .Release.Name }}
policyTypes:
- Ingress
{{- if $from }}
ingress:
- from: {{- toYaml $from | nindent 6 }}
ports:
{{ range .Values.flower.networkPolicy.ingress.ports }}
-
{{- range $key, $val := . }}
{{ $key }}: {{ tpl (toString $val) $ }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}

61
helm/airflow/templates/flower/flower-service.yaml Normal file
View File

@@ -0,0 +1,61 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Flower Service Component
#################################
{{- if .Values.flower.enabled }}
{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "airflow.fullname" . }}-flower
labels:
tier: airflow
component: flower
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.flower.labels) }}
{{- mustMerge .Values.flower.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.flower.service.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
spec:
type: {{ .Values.flower.service.type }}
selector:
tier: airflow
component: flower
release: {{ .Release.Name }}
ports:
{{ range .Values.flower.service.ports }}
-
{{- range $key, $val := . }}
{{ $key }}: {{ tpl (toString $val) $ }}
{{- end }}
{{- end }}
{{- if .Values.flower.service.loadBalancerIP }}
loadBalancerIP: {{ .Values.flower.service.loadBalancerIP }}
{{- end }}
{{- if .Values.flower.service.loadBalancerSourceRanges }}
loadBalancerSourceRanges: {{- toYaml .Values.flower.service.loadBalancerSourceRanges | nindent 4 }}
{{- end }}
{{- end }}
{{- end }}

41
helm/airflow/templates/flower/flower-serviceaccount.yaml Normal file
View File

@@ -0,0 +1,41 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
######################################
## Airflow Flower ServiceAccount
######################################
{{- if and .Values.flower.enabled (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")) .Values.flower.serviceAccount.create }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{ .Values.flower.serviceAccount.automountServiceAccountToken }}
metadata:
name: {{ include "flower.serviceAccountName" . }}
labels:
tier: airflow
component: flower
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.flower.labels) }}
{{- mustMerge .Values.flower.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.flower.serviceAccount.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}

41
helm/airflow/templates/jobs/create-user-job-serviceaccount.yaml Normal file
View File

@@ -0,0 +1,41 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
###########################################
## Airflow Create User Job ServiceAccount
###########################################
{{- if and .Values.createUserJob.serviceAccount.create .Values.webserver.defaultUser.enabled }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{ .Values.createUserJob.serviceAccount.automountServiceAccountToken }}
metadata:
name: {{ include "createUserJob.serviceAccountName" . }}
labels:
tier: airflow
component: create-user-job
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.createUserJob.labels) }}
{{- mustMerge .Values.createUserJob.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.createUserJob.serviceAccount.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}

134
helm/airflow/templates/jobs/create-user-job.yaml Normal file
View File

@@ -0,0 +1,134 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Create User Job
#################################
{{- if .Values.webserver.defaultUser.enabled }}
{{- $nodeSelector := or .Values.createUserJob.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.createUserJob.affinity .Values.affinity }}
{{- $tolerations := or .Values.createUserJob.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.createUserJob.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.createUserJob) }}
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.createUserJob) }}
{{- $containerLifecycleHooks := or .Values.createUserJob.containerLifecycleHooks .Values.containerLifecycleHooks }}
apiVersion: batch/v1
kind: Job
metadata:
name: {{ include "airflow.fullname" . }}-create-user
labels:
tier: airflow
component: create-user-job
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- $annotations := dict }}
{{- if .Values.createUserJob.useHelmHooks }}
{{- $_ := set $annotations "helm.sh/hook" "post-install,post-upgrade" }}
{{- $_ := set $annotations "helm.sh/hook-weight" "2" }}
{{- $_ := set $annotations "helm.sh/hook-delete-policy" "before-hook-creation,hook-succeeded" }}
{{- end }}
{{- with $annotations := merge $annotations .Values.createUserJob.jobAnnotations }}
annotations: {{- $annotations | toYaml | nindent 4 }}
{{- end }}
spec:
{{- if not (kindIs "invalid" .Values.createUserJob.ttlSecondsAfterFinished) }}
ttlSecondsAfterFinished: {{ .Values.createUserJob.ttlSecondsAfterFinished }}
{{- end }}
template:
metadata:
labels:
tier: airflow
component: create-user-job
release: {{ .Release.Name }}
{{- if or (.Values.labels) (.Values.createUserJob.labels) }}
{{- mustMerge .Values.createUserJob.labels .Values.labels | toYaml | nindent 8 }}
{{- end }}
{{- if or .Values.airflowPodAnnotations .Values.createUserJob.annotations }}
annotations:
{{- if .Values.airflowPodAnnotations }}
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
{{- end }}
{{- if .Values.createUserJob.annotations }}
{{- toYaml .Values.createUserJob.annotations | nindent 8 }}
{{- end }}
{{- end }}
spec:
securityContext: {{ $securityContext | nindent 8 }}
restartPolicy: OnFailure
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
affinity: {{- toYaml $affinity | nindent 8 }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
tolerations: {{- toYaml $tolerations | nindent 8 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
serviceAccountName: {{ include "createUserJob.serviceAccountName" . }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
containers:
- name: create-user
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 12 }}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
{{- end }}
{{- if .Values.createUserJob.command }}
command: {{ tpl (toYaml .Values.createUserJob.command) . | nindent 12 }}
{{- end }}
{{- if .Values.createUserJob.args }}
args: {{ tpl (toYaml .Values.createUserJob.args) . | nindent 12 }}
{{- end }}
{{- if .Values.createUserJob.applyCustomEnv }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env: {{- include "custom_airflow_environment" . | indent 10 }}
{{- else }}
env:
{{- end }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- include "container_extra_envs" (list . .Values.createUserJob.env) | indent 10 }}
resources: {{- toYaml .Values.createUserJob.resources | nindent 12 }}
volumeMounts:
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.createUserJob.extraVolumeMounts }}
{{- tpl (toYaml .Values.createUserJob.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if .Values.createUserJob.extraContainers }}
{{- toYaml .Values.createUserJob.extraContainers | nindent 8 }}
{{- end }}
volumes:
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 8 }}
{{- end }}
{{- if .Values.createUserJob.extraVolumes }}
{{- tpl (toYaml .Values.createUserJob.extraVolumes) . | nindent 8 }}
{{- end }}
{{- end }}

41
helm/airflow/templates/jobs/migrate-database-job-serviceaccount.yaml Normal file
View File

@@ -0,0 +1,41 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
#############################################
## Airflow Migrate Database Job ServiceAccount
##############################################
{{- if .Values.migrateDatabaseJob.serviceAccount.create }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{ .Values.migrateDatabaseJob.serviceAccount.automountServiceAccountToken }}
metadata:
name: {{ include "migrateDatabaseJob.serviceAccountName" . }}
labels:
tier: airflow
component: run-airflow-migrations
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.migrateDatabaseJob.serviceAccount.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}

135
helm/airflow/templates/jobs/migrate-database-job.yaml Normal file
View File

@@ -0,0 +1,135 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Run Migrations
#################################
{{- if .Values.migrateDatabaseJob.enabled }}
{{- $nodeSelector := or .Values.migrateDatabaseJob.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.migrateDatabaseJob.affinity .Values.affinity }}
{{- $tolerations := or .Values.migrateDatabaseJob.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.migrateDatabaseJob.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.migrateDatabaseJob) }}
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.migrateDatabaseJob) }}
{{- $containerLifecycleHooks := or .Values.migrateDatabaseJob.containerLifecycleHooks .Values.containerLifecycleHooks }}
apiVersion: batch/v1
kind: Job
metadata:
name: {{ include "airflow.fullname" . }}-run-airflow-migrations
labels:
tier: airflow
component: run-airflow-migrations
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- $annotations := dict }}
{{- if .Values.migrateDatabaseJob.useHelmHooks }}
{{- $_ := set $annotations "helm.sh/hook" "post-install,post-upgrade" }}
{{- $_ := set $annotations "helm.sh/hook-weight" "1" }}
{{- $_ := set $annotations "helm.sh/hook-delete-policy" "before-hook-creation,hook-succeeded" }}
{{- end }}
{{- with $annotations := merge $annotations .Values.migrateDatabaseJob.jobAnnotations }}
annotations: {{- $annotations | toYaml | nindent 4 }}
{{- end }}
spec:
{{- if not (kindIs "invalid" .Values.migrateDatabaseJob.ttlSecondsAfterFinished) }}
ttlSecondsAfterFinished: {{ .Values.migrateDatabaseJob.ttlSecondsAfterFinished }}
{{- end }}
template:
metadata:
labels:
tier: airflow
component: run-airflow-migrations
release: {{ .Release.Name }}
{{- with .Values.labels }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- if or .Values.airflowPodAnnotations .Values.migrateDatabaseJob.annotations }}
annotations:
{{- if .Values.airflowPodAnnotations }}
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
{{- end }}
{{- if .Values.migrateDatabaseJob.annotations }}
{{- toYaml .Values.migrateDatabaseJob.annotations | nindent 8 }}
{{- end }}
{{- end }}
spec:
securityContext: {{ $securityContext | nindent 8 }}
restartPolicy: OnFailure
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
affinity: {{- toYaml $affinity | nindent 8 }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
tolerations: {{- toYaml $tolerations | nindent 8 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
serviceAccountName: {{ include "migrateDatabaseJob.serviceAccountName" . }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
containers:
- name: run-airflow-migrations
image: {{ template "airflow_image_for_migrations" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 12 }}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
{{- end }}
{{- if .Values.migrateDatabaseJob.command }}
command: {{- tpl (toYaml .Values.migrateDatabaseJob.command) . | nindent 12 }}
{{- end }}
{{- if .Values.migrateDatabaseJob.args }}
args: {{- tpl (toYaml .Values.migrateDatabaseJob.args) . | nindent 12 }}
{{- end }}
{{- if .Values.migrateDatabaseJob.applyCustomEnv }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env: {{- include "custom_airflow_environment" . | indent 10 }}
{{- else }}
env:
{{- end }}
- name: PYTHONUNBUFFERED
value: "1"
{{- include "standard_airflow_environment" . | indent 10 }}
resources: {{- toYaml .Values.migrateDatabaseJob.resources | nindent 12 }}
volumeMounts:
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.migrateDatabaseJob.extraVolumeMounts }}
{{- tpl (toYaml .Values.migrateDatabaseJob.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if .Values.migrateDatabaseJob.extraContainers }}
{{- toYaml .Values.migrateDatabaseJob.extraContainers | nindent 8 }}
{{- end }}
volumes:
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 8 }}
{{- end }}
{{- if .Values.migrateDatabaseJob.extraVolumes }}
{{- tpl (toYaml .Values.migrateDatabaseJob.extraVolumes) . | nindent 8 }}
{{- end }}
{{- end }}

39
helm/airflow/templates/limitrange.yaml Normal file
View File

@@ -0,0 +1,39 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Namespace LimitRange
#################################
{{- if .Values.limits }}
apiVersion: v1
kind: LimitRange
metadata:
name: {{ .Release.Name }}-limit-range
labels:
tier: resources
component: limitrange
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
limits: {{- toYaml .Values.limits | nindent 4 }}
{{- end }}

52
helm/airflow/templates/logs-persistent-volume-claim.yaml Normal file
View File

@@ -0,0 +1,52 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
######################################
## Airflow LOGs PersistentVolumeClaim
######################################
{{- if and (not .Values.logs.persistence.existingClaim ) .Values.logs.persistence.enabled }}
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: {{ template "airflow_logs_volume_claim" . }}
labels:
tier: airflow
component: logs-pvc
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.logs.persistence.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
spec:
accessModes: ["ReadWriteMany"]
resources:
requests:
storage: {{ .Values.logs.persistence.size | quote }}
{{- if .Values.logs.persistence.storageClassName }}
{{- if (eq "-" .Values.logs.persistence.storageClassName) }}
storageClassName: ""
{{- else }}
storageClassName: "{{ .Values.logs.persistence.storageClassName }}"
{{- end }}
{{- end }}
{{- end }}

212
helm/airflow/templates/pgbouncer/pgbouncer-deployment.yaml Normal file
View File

@@ -0,0 +1,212 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Pgbouncer Deployment
#################################
{{- if .Values.pgbouncer.enabled }}
{{- $nodeSelector := or .Values.pgbouncer.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.pgbouncer.affinity .Values.affinity }}
{{- $tolerations := or .Values.pgbouncer.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.pgbouncer.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $revisionHistoryLimit := or .Values.pgbouncer.revisionHistoryLimit .Values.revisionHistoryLimit }}
{{- $securityContext := include "localPodSecurityContext" .Values.pgbouncer }}
{{- $containerSecurityContext := include "externalContainerSecurityContext" .Values.pgbouncer }}
{{- $containerSecurityContextMetricsExporter := include "externalContainerSecurityContext" .Values.pgbouncer.metricsExporterSidecar }}
{{- $containerLifecycleHooks := .Values.pgbouncer.containerLifecycleHooks }}
{{- $containerLifecycleHooksMetricsExporter := .Values.pgbouncer.metricsExporterSidecar.containerLifecycleHooks }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "airflow.fullname" . }}-pgbouncer
labels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if .Values.pgbouncer.annotations }}
annotations: {{- toYaml .Values.pgbouncer.annotations | nindent 4 }}
{{- end }}
spec:
replicas: {{ .Values.pgbouncer.replicas | default "1" }}
{{- if $revisionHistoryLimit }}
revisionHistoryLimit: {{ $revisionHistoryLimit }}
{{- end }}
strategy:
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
selector:
matchLabels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
template:
metadata:
labels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
{{- with .Values.labels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }}
checksum/pgbouncer-certificates-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-certificates-secret.yaml") . | sha256sum }}
{{- if .Values.pgbouncer.podAnnotations }}
{{- toYaml .Values.pgbouncer.podAnnotations | nindent 8 }}
{{- end }}
spec:
{{- if .Values.pgbouncer.priorityClassName }}
priorityClassName: {{ .Values.pgbouncer.priorityClassName }}
{{- end }}
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
affinity: {{- toYaml $affinity | nindent 8 }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
tolerations: {{- toYaml $tolerations | nindent 8 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
serviceAccountName: {{ include "pgbouncer.serviceAccountName" . }}
securityContext: {{ $securityContext | nindent 8 }}
restartPolicy: Always
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
containers:
- name: pgbouncer
image: {{ template "pgbouncer_image" . }}
imagePullPolicy: {{ .Values.images.pgbouncer.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 12 }}
{{- if .Values.pgbouncer.command }}
command: {{ tpl (toYaml .Values.pgbouncer.command) . | nindent 12 }}
{{- end }}
{{- if .Values.pgbouncer.args }}
args: {{ tpl (toYaml .Values.pgbouncer.args) . | nindent 12 }}
{{- end }}
resources: {{- toYaml .Values.pgbouncer.resources | nindent 12 }}
{{- with .Values.pgbouncer.env }}
env: {{- toYaml . | nindent 12 }}
{{- end }}
ports:
- name: pgbouncer
containerPort: {{ .Values.ports.pgbouncer }}
livenessProbe:
tcpSocket:
port: {{ .Values.ports.pgbouncer }}
readinessProbe:
tcpSocket:
port: {{ .Values.ports.pgbouncer }}
volumeMounts:
- name: pgbouncer-config
subPath: pgbouncer.ini
mountPath: /etc/pgbouncer/pgbouncer.ini
readOnly: true
- name: pgbouncer-config
subPath: users.txt
mountPath: /etc/pgbouncer/users.txt
readOnly: true
{{- if .Values.pgbouncer.ssl.ca }}
- name: pgbouncer-certificates
subPath: root.crt
mountPath: /etc/pgbouncer/root.crt
readOnly: true
{{- end }}
{{- if .Values.pgbouncer.ssl.cert }}
- name: pgbouncer-certificates
subPath: server.crt
mountPath: /etc/pgbouncer/server.crt
readOnly: true
{{- end }}
{{- if .Values.pgbouncer.ssl.key }}
- name: pgbouncer-certificates
subPath: server.key
mountPath: /etc/pgbouncer/server.key
readOnly: true
{{- end }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.pgbouncer.extraVolumeMounts }}
{{- tpl (toYaml .Values.pgbouncer.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
{{- end }}
- name: metrics-exporter
resources: {{- toYaml .Values.pgbouncer.metricsExporterSidecar.resources | nindent 12 }}
image: {{ template "pgbouncer_exporter_image" . }}
imagePullPolicy: {{ .Values.images.pgbouncerExporter.pullPolicy }}
securityContext: {{ $containerSecurityContextMetricsExporter | nindent 12 }}
env:
- name: DATABASE_URL
valueFrom:
secretKeyRef:
name: {{ template "pgbouncer_stats_secret" . }}
{{- if (and .Values.pgbouncer.metricsExporterSidecar.statsSecretName .Values.pgbouncer.metricsExporterSidecar.statsSecretKey) }}
key: {{ .Values.pgbouncer.metricsExporterSidecar.statsSecretKey }}
{{- else }}
key: "connection"
{{- end }}
ports:
- name: metrics
containerPort: {{ .Values.ports.pgbouncerScrape }}
livenessProbe:
exec:
command:
- pgbouncer_exporter
- health
initialDelaySeconds: {{ .Values.pgbouncer.metricsExporterSidecar.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.pgbouncer.metricsExporterSidecar.livenessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.pgbouncer.metricsExporterSidecar.livenessProbe.timeoutSeconds }}
readinessProbe:
exec:
command:
- pgbouncer_exporter
- health
initialDelaySeconds: {{ .Values.pgbouncer.metricsExporterSidecar.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.pgbouncer.metricsExporterSidecar.readinessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.pgbouncer.metricsExporterSidecar.readinessProbe.timeoutSeconds }}
{{- if $containerLifecycleHooksMetricsExporter }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooksMetricsExporter) . | nindent 12 }}
{{- end }}
{{- if .Values.pgbouncer.extraContainers }}
{{- toYaml .Values.pgbouncer.extraContainers | nindent 8 }}
{{- end }}
volumes:
- name: pgbouncer-config
secret:
secretName: {{ template "pgbouncer_config_secret" . }}
{{- if or .Values.pgbouncer.ssl.ca .Values.pgbouncer.ssl.cert .Values.pgbouncer.ssl.key }}
- name: pgbouncer-certificates
secret:
secretName: {{ template "pgbouncer_certificates_secret" . }}
{{- end }}
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 8 }}
{{- end }}
{{- if .Values.pgbouncer.extraVolumes }}
{{- tpl (toYaml .Values.pgbouncer.extraVolumes) . | nindent 8 }}
{{- end }}
{{- end }}

77
helm/airflow/templates/pgbouncer/pgbouncer-networkpolicy.yaml Normal file
View File

@@ -0,0 +1,77 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Pgbouncer NetworkPolicy
#################################
{{- $workersKedaEnabled := and .Values.workers.keda.enabled (has .Values.executor (list "CeleryExecutor" "CeleryKubernetesExecutor")) }}
{{- $triggererEnabled := and (semverCompare ">=2.2.0" .Values.airflowVersion) .Values.triggerer.enabled }}
{{- $triggererKedaEnabled := and $triggererEnabled .Values.triggerer.keda.enabled }}
{{- if and .Values.pgbouncer.enabled .Values.networkPolicies.enabled }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: {{ include "airflow.fullname" . }}-pgbouncer-policy
labels:
tier: airflow
component: airflow-pgbouncer-policy
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
podSelector:
matchLabels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
policyTypes:
- Ingress
ingress:
- from:
- podSelector:
matchLabels:
tier: airflow
release: {{ .Release.Name }}
{{- if or $workersKedaEnabled $triggererKedaEnabled }}
{{- if and $workersKedaEnabled .Values.workers.keda.namespaceLabels }}
- namespaceSelector:
matchLabels: {{- toYaml .Values.workers.keda.namespaceLabels | nindent 10 }}
podSelector:
{{- else if and $triggererEnabled .Values.triggerer.keda.namespaceLabels }}
- namespaceSelector:
matchLabels: {{- toYaml .Values.triggerer.keda.namespaceLabels | nindent 10 }}
podSelector:
{{- else }}
- podSelector:
{{- end }}
matchLabels:
app: keda-operator
{{- end }}
{{- if .Values.pgbouncer.extraNetworkPolicies}}
{{- toYaml .Values.pgbouncer.extraNetworkPolicies | nindent 4 }}
{{- end }}
ports:
- protocol: TCP
port: {{ .Values.ports.pgbouncer }}
- protocol: TCP
port: {{ .Values.ports.pgbouncerScrape }}
{{- end }}

44
helm/airflow/templates/pgbouncer/pgbouncer-poddisruptionbudget.yaml Normal file
View File

@@ -0,0 +1,44 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Pgbouncer PodDisruptionBudget
#################################
{{- if and .Values.pgbouncer.enabled .Values.pgbouncer.podDisruptionBudget.enabled }}
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: {{ include "airflow.fullname" . }}-pgbouncer-pdb
labels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
selector:
matchLabels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
{{- toYaml .Values.pgbouncer.podDisruptionBudget.config | nindent 2 }}
{{- end }}

56
helm/airflow/templates/pgbouncer/pgbouncer-service.yaml Normal file
View File

@@ -0,0 +1,56 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Pgbouncer Service
#################################
{{- if .Values.pgbouncer.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "airflow.fullname" . }}-pgbouncer
labels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: {{ .Values.ports.pgbouncerScrape | quote }}
{{- if .Values.pgbouncer.service.extraAnnotations }}
{{- toYaml .Values.pgbouncer.service.extraAnnotations | nindent 4 }}
{{- end }}
spec:
type: ClusterIP
selector:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
ports:
- name: pgbouncer
protocol: TCP
port: {{ .Values.ports.pgbouncer }}
- name: pgbouncer-metrics
protocol: TCP
port: {{ .Values.ports.pgbouncerScrape }}
{{- end }}

41
helm/airflow/templates/pgbouncer/pgbouncer-serviceaccount.yaml Normal file
View File

@@ -0,0 +1,41 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
######################################
## Airflow Pgbouncer ServiceAccount
######################################
{{- if and .Values.pgbouncer.serviceAccount.create .Values.pgbouncer.enabled }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{ .Values.pgbouncer.serviceAccount.automountServiceAccountToken }}
metadata:
name: {{ include "pgbouncer.serviceAccountName" . }}
labels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.pgbouncer.serviceAccount.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}

34
helm/airflow/templates/priorityclasses/priority-classes.yaml Normal file
View File

@@ -0,0 +1,34 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
#################################################
## Priority classes provisioned via the chart values
#################################################
{{- $Global := . }}
{{- range $e := .Values.priorityClasses }}
---
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
name: {{ $Global.Release.Name }}-{{ $e.name }}
labels:
release: {{ $Global.Release.Name }}
preemptionPolicy: {{ default "PreemptLowerPriority" $e.preemptionPolicy }}
value: {{ $e.value | required "value is required" }}
{{- end }}

44
helm/airflow/templates/rbac/pod-cleanup-role.yaml Normal file
View File

@@ -0,0 +1,44 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Cleanup Role
#################################
{{- if and .Values.rbac.create .Values.cleanup.enabled }}
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ include "airflow.fullname" . }}-cleanup-role
labels:
tier: airflow
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
rules:
- apiGroups:
- ""
resources:
- "pods"
verbs:
- "list"
- "delete"
{{- end }}

44
helm/airflow/templates/rbac/pod-cleanup-rolebinding.yaml Normal file
View File

@@ -0,0 +1,44 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Cleanup Role Binding
#################################
{{- if and .Values.rbac.create .Values.cleanup.enabled }}
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ include "airflow.fullname" . }}-cleanup-rolebinding
labels:
tier: airflow
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: {{ include "airflow.fullname" . }}-cleanup-role
subjects:
- kind: ServiceAccount
name: {{ include "cleanup.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- end }}

74
helm/airflow/templates/rbac/pod-launcher-role.yaml Normal file
View File

@@ -0,0 +1,74 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Pod Launcher Role
#################################
{{- if and .Values.rbac.create .Values.allowPodLaunching }}
apiVersion: rbac.authorization.k8s.io/v1
{{- if .Values.multiNamespaceMode }}
kind: ClusterRole
{{- else }}
kind: Role
{{- end }}
metadata:
name: {{ include "airflow.fullname" . }}-pod-launcher-role
{{- if not .Values.multiNamespaceMode }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
labels:
tier: airflow
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
rules:
- apiGroups:
- ""
resources:
- "pods"
verbs:
- "create"
- "list"
- "get"
- "patch"
- "watch"
- "delete"
- apiGroups:
- ""
resources:
- "pods/log"
verbs:
- "get"
- apiGroups:
- ""
resources:
- "pods/exec"
verbs:
- "create"
- "get"
- apiGroups:
- ""
resources:
- "events"
verbs:
- "list"
{{- end }}

64
helm/airflow/templates/rbac/pod-launcher-rolebinding.yaml Normal file
View File

@@ -0,0 +1,64 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Pod Launcher Role Binding
#################################
{{- if and .Values.rbac.create .Values.allowPodLaunching }}
{{- $schedulerLaunchExecutors := list "LocalExecutor" "LocalKubernetesExecutor" "KubernetesExecutor" "CeleryKubernetesExecutor" }}
{{- $workerLaunchExecutors := list "CeleryExecutor" "LocalKubernetesExecutor" "KubernetesExecutor" "CeleryKubernetesExecutor" }}
apiVersion: rbac.authorization.k8s.io/v1
{{- if .Values.multiNamespaceMode }}
kind: ClusterRoleBinding
{{- else }}
kind: RoleBinding
{{- end }}
metadata:
{{- if not .Values.multiNamespaceMode }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
name: {{ include "airflow.fullname" . }}-pod-launcher-rolebinding
labels:
tier: airflow
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
roleRef:
apiGroup: rbac.authorization.k8s.io
{{- if .Values.multiNamespaceMode }}
kind: ClusterRole
{{- else }}
kind: Role
{{- end }}
name: {{ include "airflow.fullname" . }}-pod-launcher-role
subjects:
{{- if has .Values.executor $schedulerLaunchExecutors }}
- kind: ServiceAccount
name: {{ include "scheduler.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
{{- if has .Values.executor $workerLaunchExecutors }}
- kind: ServiceAccount
name: {{ include "worker.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
{{- end }}

59
helm/airflow/templates/rbac/pod-log-reader-role.yaml Normal file
View File

@@ -0,0 +1,59 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Pod Reader Role
#################################
{{- if and .Values.rbac.create (or .Values.webserver.allowPodLogReading .Values.triggerer.enabled) }}
apiVersion: rbac.authorization.k8s.io/v1
{{- if .Values.multiNamespaceMode }}
kind: ClusterRole
{{- else }}
kind: Role
{{- end }}
metadata:
name: {{ include "airflow.fullname" . }}-pod-log-reader-role
{{- if not .Values.multiNamespaceMode }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
labels:
tier: airflow
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
rules:
- apiGroups:
- ""
resources:
- "pods"
verbs:
- "list"
- "get"
- "watch"
- apiGroups:
- ""
resources:
- "pods/log"
verbs:
- "get"
- "list"
{{- end }}

62
helm/airflow/templates/rbac/pod-log-reader-rolebinding.yaml Normal file
View File

@@ -0,0 +1,62 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Pod Reader Role Binding
#################################
{{- if and .Values.rbac.create (or .Values.webserver.allowPodLogReading .Values.triggerer.enabled) }}
apiVersion: rbac.authorization.k8s.io/v1
{{- if .Values.multiNamespaceMode }}
kind: ClusterRoleBinding
{{- else }}
kind: RoleBinding
{{- end }}
metadata:
{{- if not .Values.multiNamespaceMode }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
name: {{ include "airflow.fullname" . }}-pod-log-reader-rolebinding
labels:
tier: airflow
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
roleRef:
apiGroup: rbac.authorization.k8s.io
{{- if .Values.multiNamespaceMode }}
kind: ClusterRole
{{- else }}
kind: Role
{{- end }}
name: {{ include "airflow.fullname" . }}-pod-log-reader-role
subjects:
{{- if .Values.webserver.allowPodLogReading }}
- kind: ServiceAccount
name: {{ include "webserver.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
{{- if .Values.triggerer.enabled }}
- kind: ServiceAccount
name: {{ include "triggerer.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
{{- end }}

88
helm/airflow/templates/rbac/security-context-constraint-rolebinding.yaml Normal file
View File

@@ -0,0 +1,88 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow SCC Role Binding
#################################
{{- if and .Values.rbac.create .Values.rbac.createSCCRoleBinding }}
{{- $hasWorkers := has .Values.executor (list "CeleryExecutor" "LocalKubernetesExecutor" "KubernetesExecutor" "CeleryKubernetesExecutor") }}
apiVersion: rbac.authorization.k8s.io/v1
{{- if .Values.multiNamespaceMode }}
kind: ClusterRoleBinding
{{- else }}
kind: RoleBinding
{{- end }}
metadata:
{{- if not .Values.multiNamespaceMode }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
name: {{ include "airflow.fullname" . }}-scc-rolebinding
labels:
tier: airflow
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:openshift:scc:anyuid
subjects:
- kind: ServiceAccount
name: {{ include "webserver.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- if $hasWorkers }}
- kind: ServiceAccount
name: {{ include "worker.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
- kind: ServiceAccount
name: {{ include "scheduler.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- if and .Values.statsd.enabled }}
- kind: ServiceAccount
name: {{ include "statsd.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
{{- if and .Values.flower.enabled (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")) }}
- kind: ServiceAccount
name: {{ include "flower.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
{{- if and (semverCompare ">=2.2.0" .Values.airflowVersion) }}
- kind: ServiceAccount
name: {{ include "triggerer.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
- kind: ServiceAccount
name: {{ include "migrateDatabaseJob.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- if .Values.webserver.defaultUser.enabled }}
- kind: ServiceAccount
name: {{ include "createUserJob.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
{{- if and .Values.cleanup.enabled }}
- kind: ServiceAccount
name: {{ include "cleanup.serviceAccountName" . }}
namespace: "{{ .Release.Namespace }}"
{{- end }}
{{- end }}

65
helm/airflow/templates/redis/redis-networkpolicy.yaml Normal file
View File

@@ -0,0 +1,65 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Redis NetworkPolicy
#################################
{{- if and .Values.redis.enabled .Values.networkPolicies.enabled (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")) }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: {{ include "airflow.fullname" . }}-redis-policy
labels:
tier: airflow
component: redis-policy
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
podSelector:
matchLabels:
tier: airflow
component: redis
release: {{ .Release.Name }}
policyTypes:
- Ingress
ingress:
- from:
- podSelector:
matchLabels:
tier: airflow
component: worker
release: {{ .Release.Name }}
- podSelector:
matchLabels:
tier: airflow
component: scheduler
release: {{ .Release.Name }}
- podSelector:
matchLabels:
tier: airflow
component: flower
release: {{ .Release.Name }}
ports:
- protocol: TCP
port: {{ .Values.ports.redisDB }}
{{- end }}

48
helm/airflow/templates/redis/redis-service.yaml Normal file
View File

@@ -0,0 +1,48 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Redis Service
#################################
{{- if and .Values.redis.enabled (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")) }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "airflow.fullname" . }}-redis
labels:
tier: airflow
component: redis
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
type: ClusterIP
selector:
tier: airflow
component: redis
release: {{ .Release.Name }}
ports:
- name: redis-db
protocol: TCP
port: {{ .Values.ports.redisDB }}
targetPort: {{ .Values.ports.redisDB }}
{{- end }}

41
helm/airflow/templates/redis/redis-serviceaccount.yaml Normal file
View File

@@ -0,0 +1,41 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
######################################
## Airflow Redis ServiceAccount
######################################
{{- if and .Values.redis.enabled .Values.redis.serviceAccount.create (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")) }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{ .Values.redis.serviceAccount.automountServiceAccountToken }}
metadata:
name: {{ include "redis.serviceAccountName" . }}
labels:
tier: airflow
component: redis
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.redis.serviceAccount.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}

126
helm/airflow/templates/redis/redis-statefulset.yaml Normal file
View File

@@ -0,0 +1,126 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Redis StatefulSet
#################################
{{- if and .Values.redis.enabled (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")) }}
{{- $nodeSelector := or .Values.redis.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.redis.affinity .Values.affinity }}
{{- $tolerations := or .Values.redis.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.redis.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $securityContext := include "localPodSecurityContext" .Values.redis }}
{{- $containerSecurityContext := include "externalContainerSecurityContext" .Values.redis }}
{{- $containerLifecycleHooks := .Values.redis.containerLifecycleHooks }}
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: {{ include "airflow.fullname" . }}-redis
labels:
tier: airflow
component: redis
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
serviceName: {{ include "airflow.fullname" . }}-redis
selector:
matchLabels:
tier: airflow
component: redis
release: {{ .Release.Name }}
template:
metadata:
labels:
tier: airflow
component: redis
release: {{ .Release.Name }}
{{- with .Values.labels }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- if or .Values.redis.safeToEvict .Values.redis.podAnnotations }}
annotations:
{{- if .Values.redis.podAnnotations }}
{{- toYaml .Values.redis.podAnnotations | nindent 8 }}
{{- end }}
{{- if .Values.redis.safeToEvict }}
cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
{{- end }}
{{- end }}
spec:
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
affinity: {{- toYaml $affinity | nindent 8 }}
tolerations: {{- toYaml $tolerations | nindent 8 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
serviceAccountName: {{ include "redis.serviceAccountName" . }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
securityContext: {{ $securityContext | nindent 8 }}
containers:
- name: redis
image: {{ template "redis_image" . }}
imagePullPolicy: {{ .Values.images.redis.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 12 }}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
{{- end }}
command: ["/bin/sh"]
resources: {{- toYaml .Values.redis.resources | nindent 12 }}
args: ["-c", "redis-server --requirepass ${REDIS_PASSWORD}"]
ports:
- name: redis-db
containerPort: {{ .Values.ports.redisDB }}
volumeMounts:
- name: redis-db
mountPath: /data
env:
- name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: {{ template "redis_password_secret" . }}
key: password
{{- if not .Values.redis.persistence.enabled }}
volumes:
- name: redis-db
emptyDir: {}
{{- else }}
volumeClaimTemplates:
- metadata:
name: redis-db
{{- if .Values.redis.persistence.annotations }}
annotations: {{- toYaml .Values.redis.persistence.annotations | nindent 10 }}
{{- end }}
spec:
{{- if .Values.redis.persistence.storageClassName }}
storageClassName: {{ .Values.redis.persistence.storageClassName }}
{{- end }}
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: {{ .Values.redis.persistence.size }}
{{- end }}
{{- end }}

39
helm/airflow/templates/resourcequota.yaml Normal file
View File

@@ -0,0 +1,39 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Namespace ResourceQuota
#################################
{{- if .Values.quotas }}
apiVersion: v1
kind: ResourceQuota
metadata:
name: {{ .Release.Name }}-resource-quota
labels:
tier: resources
component: resourcequota
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
hard: {{- toYaml .Values.quotas | nindent 4 }}
{{- end }}

339
helm/airflow/templates/scheduler/scheduler-deployment.yaml Normal file
View File

@@ -0,0 +1,339 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Scheduler Deployment/StatefulSet
#################################
# Are we using a local executor?
{{- $local := contains "Local" .Values.executor }}
# Is persistence enabled on the _workers_?
# This is important because in $local mode, the scheduler assumes the role of the worker
{{- $persistence := .Values.workers.persistence.enabled }}
# If we're using a StatefulSet
{{- $stateful := and $local $persistence }}
# We can skip DAGs mounts on scheduler if dagProcessor is enabled, except with $local mode
{{- $localOrDagProcessorDisabled := or (not .Values.dagProcessor.enabled) $local }}
# If we're using elasticsearch logging
{{- $elasticsearch := .Values.elasticsearch.enabled }}
{{- $nodeSelector := or .Values.scheduler.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.scheduler.affinity .Values.affinity }}
{{- $tolerations := or .Values.scheduler.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.scheduler.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $revisionHistoryLimit := or .Values.scheduler.revisionHistoryLimit .Values.revisionHistoryLimit }}
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.scheduler) }}
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.scheduler) }}
{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list . .Values.scheduler.waitForMigrations) }}
{{- $containerSecurityContextLogGroomerSidecar := include "containerSecurityContext" (list . .Values.scheduler.logGroomerSidecar) }}
{{- $containerLifecycleHooks := or .Values.scheduler.containerLifecycleHooks .Values.containerLifecycleHooks }}
{{- $containerLifecycleHooksWaitForMigrations := or .Values.scheduler.waitForMigrations.containerLifecycleHooks .Values.containerLifecycleHooks }}
{{- $containerLifecycleHooksLogGroomerSidecar := or .Values.scheduler.logGroomerSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }}
apiVersion: apps/v1
kind: {{ if $stateful }}StatefulSet{{ else }}Deployment{{ end }}
metadata:
name: {{ include "airflow.fullname" . }}-scheduler
labels:
tier: airflow
component: scheduler
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
executor: {{ .Values.executor }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if .Values.scheduler.annotations }}
annotations: {{- toYaml .Values.scheduler.annotations | nindent 4 }}
{{- end }}
spec:
{{- if $stateful }}
serviceName: {{ include "airflow.fullname" . }}-scheduler
{{- end }}
replicas: {{ .Values.scheduler.replicas }}
{{- if $revisionHistoryLimit }}
revisionHistoryLimit: {{ $revisionHistoryLimit }}
{{- end }}
{{- if and $stateful .Values.scheduler.updateStrategy }}
updateStrategy: {{- toYaml .Values.scheduler.updateStrategy | nindent 4 }}
{{- end }}
{{- if and (not $stateful) .Values.scheduler.strategy }}
strategy: {{- toYaml .Values.scheduler.strategy | nindent 4 }}
{{- end }}
selector:
matchLabels:
tier: airflow
component: scheduler
release: {{ .Release.Name }}
template:
metadata:
labels:
tier: airflow
component: scheduler
release: {{ .Release.Name }}
{{- if or (.Values.labels) (.Values.scheduler.labels) }}
{{- mustMerge .Values.scheduler.labels .Values.labels | toYaml | nindent 8 }}
{{- end }}
annotations:
checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }}
checksum/result-backend-secret: {{ include (print $.Template.BasePath "/secrets/result-backend-connection-secret.yaml") . | sha256sum }}
checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }}
checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }}
checksum/extra-configmaps: {{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }}
checksum/extra-secrets: {{ include (print $.Template.BasePath "/secrets/extra-secrets.yaml") . | sha256sum }}
{{- if .Values.scheduler.safeToEvict }}
cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
{{- end }}
{{- if .Values.airflowPodAnnotations }}
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
{{- end }}
{{- if .Values.scheduler.podAnnotations }}
{{- toYaml .Values.scheduler.podAnnotations | nindent 8 }}
{{- end }}
spec:
{{- if .Values.scheduler.priorityClassName }}
priorityClassName: {{ .Values.scheduler.priorityClassName }}
{{- end }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
affinity:
{{- if $affinity }}
{{- toYaml $affinity | nindent 8 }}
{{- else }}
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchLabels:
component: scheduler
topologyKey: kubernetes.io/hostname
weight: 100
{{- end }}
tolerations: {{- toYaml $tolerations | nindent 8 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
restartPolicy: Always
terminationGracePeriodSeconds: 10
serviceAccountName: {{ include "scheduler.serviceAccountName" . }}
securityContext: {{ $securityContext | nindent 8 }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
{{- if .Values.scheduler.hostAliases }}
hostAliases: {{- toYaml .Values.scheduler.hostAliases | nindent 8 }}
{{- end }}
initContainers:
{{- if .Values.scheduler.waitForMigrations.enabled }}
- name: wait-for-airflow-migrations
resources: {{- toYaml .Values.scheduler.resources | nindent 12 }}
image: {{ template "airflow_image_for_migrations" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContextWaitForMigrations | nindent 12 }}
{{- if $containerLifecycleHooksWaitForMigrations }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooksWaitForMigrations) . | nindent 12 }}
{{- end }}
volumeMounts:
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.scheduler.extraVolumeMounts }}
{{- tpl (toYaml .Values.scheduler.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
args: {{- include "wait-for-migrations-command" . | indent 10 }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- if .Values.scheduler.waitForMigrations.env }}
{{- tpl (toYaml .Values.scheduler.waitForMigrations.env) $ | nindent 12 }}
{{- end }}
{{- end }}
{{- if and $localOrDagProcessorDisabled .Values.dags.gitSync.enabled }}
{{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }}
{{- end }}
{{- if .Values.scheduler.extraInitContainers }}
{{- toYaml .Values.scheduler.extraInitContainers | nindent 8 }}
{{- end }}
containers:
# Always run the main scheduler container.
- name: scheduler
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 12 }}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
{{- end }}
{{- if .Values.scheduler.command }}
command: {{ tpl (toYaml .Values.scheduler.command) . | nindent 12 }}
{{- end }}
{{- if .Values.scheduler.args }}
args: {{ tpl (toYaml .Values.scheduler.args) . | nindent 12 }}
{{- end }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- include "container_extra_envs" (list . .Values.scheduler.env) | indent 10 }}
livenessProbe:
initialDelaySeconds: {{ .Values.scheduler.livenessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.scheduler.livenessProbe.timeoutSeconds }}
failureThreshold: {{ .Values.scheduler.livenessProbe.failureThreshold }}
periodSeconds: {{ .Values.scheduler.livenessProbe.periodSeconds }}
exec:
command:
{{- if .Values.scheduler.livenessProbe.command }}
{{- toYaml .Values.scheduler.livenessProbe.command | nindent 16 }}
{{- else }}
{{- include "scheduler_liveness_check_command" . | indent 14 }}
{{- end }}
startupProbe:
timeoutSeconds: {{ .Values.scheduler.startupProbe.timeoutSeconds }}
failureThreshold: {{ .Values.scheduler.startupProbe.failureThreshold }}
periodSeconds: {{ .Values.scheduler.startupProbe.periodSeconds }}
exec:
command:
{{- if .Values.scheduler.startupProbe.command }}
{{- toYaml .Values.scheduler.startupProbe.command | nindent 16 }}
{{- else }}
{{- include "scheduler_startup_check_command" . | indent 14 }}
{{- end }}
{{- if and $local (not $elasticsearch) }}
# Serve logs if we're in local mode and we don't have elasticsearch enabled.
ports:
- name: worker-logs
containerPort: {{ .Values.ports.workerLogs }}
{{- end }}
resources: {{- toYaml .Values.scheduler.resources | nindent 12 }}
volumeMounts:
{{- if semverCompare ">=1.10.12" .Values.airflowVersion }}
- name: config
mountPath: {{ include "airflow_pod_template_file" . }}/pod_template_file.yaml
subPath: pod_template_file.yaml
readOnly: true
{{- end }}
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
{{- if and $localOrDagProcessorDisabled (or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled) }}
{{- include "airflow_dags_mount" . | nindent 12 }}
{{- end }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.scheduler.extraVolumeMounts }}
{{- tpl (toYaml .Values.scheduler.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if and $localOrDagProcessorDisabled .Values.dags.gitSync.enabled }}
{{- include "git_sync_container" . | indent 8 }}
{{- end }}
{{- if .Values.scheduler.logGroomerSidecar.enabled }}
- name: scheduler-log-groomer
resources: {{- toYaml .Values.scheduler.logGroomerSidecar.resources | nindent 12 }}
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContextLogGroomerSidecar | nindent 12 }}
{{- if $containerLifecycleHooksLogGroomerSidecar }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooksLogGroomerSidecar) . | nindent 12 }}
{{- end }}
{{- if .Values.scheduler.logGroomerSidecar.command }}
command: {{ tpl (toYaml .Values.scheduler.logGroomerSidecar.command) . | nindent 12 }}
{{- end }}
{{- if .Values.scheduler.logGroomerSidecar.args }}
args: {{- tpl (toYaml .Values.scheduler.logGroomerSidecar.args) . | nindent 12 }}
{{- end }}
{{- if .Values.scheduler.logGroomerSidecar.retentionDays }}
env:
- name: AIRFLOW__LOG_RETENTION_DAYS
value: "{{ .Values.scheduler.logGroomerSidecar.retentionDays }}"
{{- end }}
volumeMounts:
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.scheduler.extraVolumeMounts }}
{{- tpl (toYaml .Values.scheduler.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
{{- end }}
{{- if .Values.scheduler.extraContainers }}
{{- toYaml .Values.scheduler.extraContainers | nindent 8 }}
{{- end }}
volumes:
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
- name: webserver-config
configMap:
name: {{ template "airflow_webserver_config_configmap_name" . }}
{{- end }}
{{- if $localOrDagProcessorDisabled }}
{{- if .Values.dags.persistence.enabled }}
- name: dags
persistentVolumeClaim:
claimName: {{ template "airflow_dags_volume_claim" . }}
{{- else if .Values.dags.gitSync.enabled }}
- name: dags
emptyDir: {}
{{- if .Values.dags.gitSync.sshKeySecret }}
{{- include "git_sync_ssh_key_volume" . | indent 8 }}
{{- end }}
{{- end }}
{{- end }}
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 8 }}
{{- end }}
{{- if .Values.scheduler.extraVolumes }}
{{- tpl (toYaml .Values.scheduler.extraVolumes) . | nindent 8 }}
{{- end }}
{{- if .Values.logs.persistence.enabled }}
- name: logs
persistentVolumeClaim:
claimName: {{ template "airflow_logs_volume_claim" . }}
{{- else if not $stateful }}
- name: logs
emptyDir: {}
{{- else }}
volumeClaimTemplates:
- metadata:
name: logs
{{- if .Values.workers.persistence.annotations }}
annotations: {{- toYaml .Values.workers.persistence.annotations | nindent 10 }}
{{- end }}
spec:
{{- if .Values.workers.persistence.storageClassName }}
storageClassName: {{ .Values.workers.persistence.storageClassName }}
{{- end }}
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: {{ .Values.workers.persistence.size }}
{{- end }}

57
helm/airflow/templates/scheduler/scheduler-networkpolicy.yaml Normal file
View File

@@ -0,0 +1,57 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Scheduler NetworkPolicy
#################################
{{- if .Values.networkPolicies.enabled }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: {{ include "airflow.fullname" . }}-scheduler-policy
labels:
tier: airflow
component: airflow-scheduler-policy
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.scheduler.labels) }}
{{- mustMerge .Values.scheduler.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
podSelector:
matchLabels:
tier: airflow
component: scheduler
release: {{ .Release.Name }}
policyTypes:
- Ingress
{{- if eq .Values.executor "LocalExecutor" }}
ingress:
- from:
- podSelector:
matchLabels:
tier: airflow
component: webserver
release: {{ .Release.Name }}
ports:
- protocol: TCP
port: {{ .Values.ports.workerLogs }}
{{- end }}
{{- end }}

44
helm/airflow/templates/scheduler/scheduler-poddisruptionbudget.yaml Normal file
View File

@@ -0,0 +1,44 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Scheduler PodDisruptionBudget
#################################
{{- if .Values.scheduler.podDisruptionBudget.enabled }}
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: {{ include "airflow.fullname" . }}-scheduler-pdb
labels:
tier: airflow
component: scheduler
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.scheduler.labels) }}
{{- mustMerge .Values.scheduler.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
selector:
matchLabels:
tier: airflow
component: scheduler
release: {{ .Release.Name }}
{{- toYaml .Values.scheduler.podDisruptionBudget.config | nindent 2 }}
{{- end }}

48
helm/airflow/templates/scheduler/scheduler-service.yaml Normal file
View File

@@ -0,0 +1,48 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Scheduler Service
#################################
{{- if or (eq .Values.executor "LocalExecutor") (eq .Values.executor "LocalKubernetesExecutor") }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "airflow.fullname" . }}-scheduler
labels:
tier: airflow
component: scheduler
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.scheduler.labels) }}
{{- mustMerge .Values.scheduler.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
clusterIP: None
selector:
tier: airflow
component: scheduler
release: {{ .Release.Name }}
ports:
- name: task-logs
protocol: TCP
port: {{ .Values.ports.workerLogs }}
targetPort: {{ .Values.ports.workerLogs }}
{{- end }}

44
helm/airflow/templates/scheduler/scheduler-serviceaccount.yaml Normal file
View File

@@ -0,0 +1,44 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Scheduler ServiceAccount
#################################
{{- if .Values.scheduler.serviceAccount.create }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{ .Values.scheduler.serviceAccount.automountServiceAccountToken }}
metadata:
name: {{ include "scheduler.serviceAccountName" . }}
labels:
tier: airflow
component: scheduler
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.scheduler.labels) }}
{{- mustMerge .Values.scheduler.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.scheduler.serviceAccount.annotations }}
annotations:
{{- range $key, $value := . }}
{{- printf "%s: %s" $key (tpl $value $ | quote) | nindent 4 }}
{{- end }}
{{- end }}
{{- end }}

44
helm/airflow/templates/secrets/elasticsearch-secret.yaml Normal file
View File

@@ -0,0 +1,44 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Elasticsearch Secret
#################################
{{- if (and .Values.elasticsearch.enabled (not .Values.elasticsearch.secretName)) }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "airflow.fullname" . }}-elasticsearch
labels:
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
type: Opaque
data:
{{- with .Values.elasticsearch.connection }}
{{- if and .user .pass }}
connection: {{ urlJoin (dict "scheme" (default "http" .scheme) "userinfo" (printf "%s:%s" (.user | urlquery) (.pass | urlquery)) "host" (printf "%s:%s" .host ((default 9200 .port) | toString) ) ) | b64enc | quote }}
{{- else }}
connection: {{ urlJoin (dict "scheme" (default "http" .scheme) "host" (printf "%s:%s" .host ((default 9200 .port) | toString))) | b64enc | quote }}
{{- end }}
{{- end }}
{{- end }}

62
helm/airflow/templates/secrets/extra-secrets.yaml Normal file
View File

@@ -0,0 +1,62 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
#################################################
## Extra Secrets provisioned via the chart values
#################################################
{{- $Global := . }}
{{- range $secretName, $secretContent := .Values.extraSecrets }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ tpl $secretName $Global | quote }}
labels:
release: {{ $Global.Release.Name }}
chart: "{{ $Global.Chart.Name }}-{{ $Global.Chart.Version }}"
heritage: {{ $Global.Release.Service }}
{{- with $Global.Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if $secretContent.labels }}
{{- toYaml $secretContent.labels | nindent 4 }}
{{- end }}
annotations:
"helm.sh/hook": "pre-install,pre-upgrade"
"helm.sh/hook-delete-policy": "before-hook-creation"
"helm.sh/hook-weight": "0"
{{- if $secretContent.annotations }}
{{- toYaml $secretContent.annotations | nindent 4 }}
{{- end }}
{{- if $secretContent.type }}
type: {{ $secretContent.type }}
{{- end }}
{{- if $secretContent.data }}
data:
{{- with $secretContent.data }}
{{- tpl . $Global | nindent 2 }}
{{- end }}
{{- end }}
{{- if $secretContent.stringData }}
stringData:
{{- with $secretContent.stringData }}
{{- tpl . $Global | nindent 2 }}
{{- end }}
{{- end }}
{{- end }}

44
helm/airflow/templates/secrets/fernetkey-secret.yaml Normal file
View File

@@ -0,0 +1,44 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Fernet Key Secret
#################################
{{- if not .Values.fernetKeySecretName }}
{{- $generated_fernet_key := (randAlphaNum 32 | b64enc) }}
apiVersion: v1
kind: Secret
metadata:
name: {{ .Release.Name }}-fernet-key
labels:
tier: airflow
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
annotations:
"helm.sh/hook": "pre-install"
"helm.sh/hook-delete-policy": "before-hook-creation"
"helm.sh/hook-weight": "0"
type: Opaque
data:
fernet-key: {{ (default $generated_fernet_key .Values.fernetKey) | b64enc | quote }}
{{- end }}

38
helm/airflow/templates/secrets/flower-secret.yaml Normal file
View File

@@ -0,0 +1,38 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Flower Secret
#################################
{{- if (and (not .Values.flower.secretName) .Values.flower.username .Values.flower.password) }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "airflow.fullname" . }}-flower
labels:
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
type: Opaque
data:
basicAuth: {{ (printf "%s:%s" .Values.flower.username .Values.flower.password) | b64enc | quote }}
{{- end }}

40
helm/airflow/templates/secrets/kerberos-keytab-secret.yaml Normal file
View File

@@ -0,0 +1,40 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Kerberos Secret
#################################
{{- if .Values.kerberos.keytabBase64Content }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "kerberos_keytab_secret" . | quote }}
labels:
tier: airflow
component: webserver
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
type: Opaque
data:
kerberos.keytab: {{ .Values.kerberos.keytabBase64Content }}
{{- end }}

53
helm/airflow/templates/secrets/metadata-connection-secret.yaml Normal file
View File

@@ -0,0 +1,53 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Metadata Secret
#################################
{{- if not .Values.data.metadataSecretName }}
{{- $defaultMetadataHost := .Values.postgresql.nameOverride | default (printf "%s-%s.%s" .Release.Name "postgresql" .Release.Namespace) }}
{{- $metadataHost := .Values.data.metadataConnection.host | default $defaultMetadataHost }}
{{- $pgbouncerHost := (printf "%s-%s.%s" .Release.Name "pgbouncer" .Release.Namespace) }}
{{- $host := ternary $pgbouncerHost $metadataHost .Values.pgbouncer.enabled }}
{{- $port := ((ternary .Values.ports.pgbouncer .Values.data.metadataConnection.port .Values.pgbouncer.enabled) | toString) }}
{{- $database := (ternary (printf "%s-%s" .Release.Name "metadata") .Values.data.metadataConnection.db .Values.pgbouncer.enabled) }}
{{- $query := ternary (printf "sslmode=%s" .Values.data.metadataConnection.sslmode) "" (eq .Values.data.metadataConnection.protocol "postgresql") }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "airflow.fullname" . }}-metadata
labels:
tier: airflow
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
type: Opaque
data:
{{- with .Values.data.metadataConnection }}
connection: {{ urlJoin (dict "scheme" .protocol "userinfo" (printf "%s:%s" (.user | urlquery) (.pass | urlquery) ) "host" (printf "%s:%s" $host $port) "path" (printf "/%s" $database) "query" $query) | b64enc | quote }}
{{- end }}
{{- if and .Values.workers.keda.enabled .Values.pgbouncer.enabled (not .Values.workers.keda.usePgbouncer) }}
{{- with .Values.data.metadataConnection }}
kedaConnection: {{ urlJoin (dict "scheme" .protocol "userinfo" (printf "%s:%s" (.user | urlquery) (.pass | urlquery) ) "host" (printf "%s:%s" $metadataHost $port) "path" (printf "/%s" $database) "query" $query) | b64enc | quote }}
{{- end }}
{{- end }}
{{- end }}

46
helm/airflow/templates/secrets/pgbouncer-certificates-secret.yaml Normal file
View File

@@ -0,0 +1,46 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Pgbouncer Certificate Secret
#################################
{{- if or .Values.pgbouncer.ssl.ca .Values.pgbouncer.ssl.cert .Values.pgbouncer.ssl.key }}
apiVersion: v1
kind: Secret
metadata:
name: {{ template "pgbouncer_certificates_secret" . }}
labels:
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
type: Opaque
data:
{{- if .Values.pgbouncer.ssl.ca }}
root.crt: {{ .Values.pgbouncer.ssl.ca | b64enc }}
{{- end }}
{{- if .Values.pgbouncer.ssl.cert }}
server.crt: {{ .Values.pgbouncer.ssl.cert | b64enc }}
{{- end }}
{{- if .Values.pgbouncer.ssl.key }}
server.key: {{ .Values.pgbouncer.ssl.key | b64enc }}
{{- end }}
{{- end }}

41
helm/airflow/templates/secrets/pgbouncer-config-secret.yaml Normal file
View File

@@ -0,0 +1,41 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Pgbouncer Config Secret
#################################
{{- if (and .Values.pgbouncer.enabled (not .Values.pgbouncer.configSecretName)) }}
apiVersion: v1
kind: Secret
metadata:
name: {{ template "pgbouncer_config_secret" . }}
labels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
type: Opaque
data:
pgbouncer.ini: {{ include "pgbouncer_config" . | b64enc }}
users.txt: {{ include "pgbouncer_users" . | b64enc }}
{{- end }}

40
helm/airflow/templates/secrets/pgbouncer-stats-secret.yaml Normal file
View File

@@ -0,0 +1,40 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Pgbouncer Stats Secret
#################################
{{- if (and .Values.pgbouncer.enabled (not .Values.pgbouncer.metricsExporterSidecar.statsSecretName)) }}
apiVersion: v1
kind: Secret
metadata:
name: {{ template "pgbouncer_stats_secret" . }}
labels:
tier: airflow
component: pgbouncer
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
type: Opaque
data:
connection: {{ urlJoin (dict "scheme" "postgresql" "userinfo" (printf "%s:%s" (.Values.data.metadataConnection.user | urlquery) (.Values.data.metadataConnection.pass | urlquery) ) "host" (printf "127.0.0.1:%s" (.Values.ports.pgbouncer | toString)) "path" "/pgbouncer" "query" (printf "sslmode=%s" (.Values.pgbouncer.metricsExporterSidecar.sslmode | toString ))) | b64enc | quote }}
{{- end }}

83
helm/airflow/templates/secrets/redis-secrets.yaml Normal file
View File

@@ -0,0 +1,83 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
# We will create these secrets (if necessary) _even if_ we aren't
# currently using CeleryExecutor or CeleryKubernetesExecutor. As we are
# relying on the "pre-install" hack to prevent changing randomly generated passwords,
# updating the executor later doesn't give us the opportunity to deploy them
# when we need them. We will always deploy them defensively to make the executor
# update path actually work.
################################
## Airflow Redis Password Secret
#################################
{{- $random_redis_password := randAlphaNum 10 }}
{{- if and .Values.redis.enabled (not .Values.redis.passwordSecretName) }}
# If passwordSecretName is not set, we will either use the set password, or use the generated one
apiVersion: v1
kind: Secret
metadata:
name: {{ .Release.Name }}-redis-password
labels:
tier: airflow
component: redis
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
annotations:
"helm.sh/hook": "pre-install"
"helm.sh/hook-delete-policy": "before-hook-creation"
"helm.sh/hook-weight": "0"
type: Opaque
data:
password: {{ (default $random_redis_password .Values.redis.password) | b64enc | quote }}
---
{{- end }}
{{- if not .Values.data.brokerUrlSecretName }}
##################################
## Airflow Redis Connection Secret
##################################
apiVersion: v1
kind: Secret
metadata:
name: {{ .Release.Name }}-broker-url
labels:
tier: airflow
component: redis
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
annotations:
"helm.sh/hook": "pre-install"
"helm.sh/hook-delete-policy": "before-hook-creation"
"helm.sh/hook-weight": "0"
type: Opaque
data:
{{- if .Values.redis.enabled }}
connection: {{ urlJoin (dict "scheme" "redis" "userinfo" (printf ":%s" ((default $random_redis_password .Values.redis.password) | urlquery)) "host" (printf "%s-redis:6379" .Release.Name ) "path" "/0") | b64enc | quote }}
{{- else }}
connection: {{ (printf "%s" .Values.data.brokerUrl) | b64enc | quote }}
{{- end }}
{{- end }}

38
helm/airflow/templates/secrets/registry-secret.yaml Normal file
View File

@@ -0,0 +1,38 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Registry Secret
#################################
{{- if (and .Values.registry.connection (not .Values.registry.secretName)) }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "airflow.fullname" . }}-registry
labels:
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
type: kubernetes.io/dockerconfigjson
data:
.dockerconfigjson: {{ include "registry_docker_config" . | b64enc }}
{{- end }}

50
helm/airflow/templates/secrets/result-backend-connection-secret.yaml Normal file
View File

@@ -0,0 +1,50 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Result Backend Secret
#################################
{{- if not .Values.data.resultBackendSecretName }}
{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
{{- if or (semverCompare "<2.4.0" .Values.airflowVersion) (and (semverCompare ">=2.4.0" .Values.airflowVersion) .Values.data.resultBackendConnection) }}
{{- $connection := .Values.data.resultBackendConnection | default .Values.data.metadataConnection }}
{{- $resultBackendHost := $connection.host | default (printf "%s-%s" .Release.Name "postgresql") }}
{{- $pgbouncerHost := printf "%s-%s" .Release.Name "pgbouncer" }}
{{- $host := ternary $pgbouncerHost $resultBackendHost .Values.pgbouncer.enabled }}
{{- $port := (ternary .Values.ports.pgbouncer $connection.port .Values.pgbouncer.enabled) | toString }}
{{- $database := ternary (printf "%s-%s" .Release.Name "result-backend") $connection.db .Values.pgbouncer.enabled }}
{{- $query := ternary (printf "sslmode=%s" $connection.sslmode) "" (eq $connection.protocol "postgresql") }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "airflow.fullname" . }}-result-backend
labels:
tier: airflow
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
type: Opaque
data:
connection: {{ urlJoin (dict "scheme" (printf "db+%s" $connection.protocol) "userinfo" (printf "%s:%s" ($connection.user|urlquery) ($connection.pass | urlquery)) "host" (printf "%s:%s" $host $port) "path" (printf "/%s" $database) "query" $query) | b64enc | quote }}
{{- end }}
{{- end }}
{{- end }}

41
helm/airflow/templates/secrets/webserver-secret-key-secret.yaml Normal file
View File

@@ -0,0 +1,41 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
############################################
## Airflow Webserver Flask Secret Key Secret
############################################
{{- if not .Values.webserverSecretKeySecretName }}
{{ $generated_secret_key := (randAlphaNum 32 | b64enc) }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "airflow.fullname" . }}-webserver-secret-key
labels:
tier: airflow
component: webserver
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
type: Opaque
data:
webserver-secret-key: {{ (default $generated_secret_key .Values.webserverSecretKey) | b64enc | quote }}
{{- end }}

138
helm/airflow/templates/statsd/statsd-deployment.yaml Normal file
View File

@@ -0,0 +1,138 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow StatsD Deployment
#################################
{{- if .Values.statsd.enabled }}
{{- $nodeSelector := or .Values.statsd.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.statsd.affinity .Values.affinity }}
{{- $tolerations := or .Values.statsd.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.statsd.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $revisionHistoryLimit := or .Values.statsd.revisionHistoryLimit .Values.revisionHistoryLimit }}
{{- $securityContext := include "localPodSecurityContext" .Values.statsd }}
{{- $containerSecurityContext := include "externalContainerSecurityContext" .Values.statsd }}
{{- $containerLifecycleHooks := .Values.statsd.containerLifecycleHooks }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "airflow.fullname" . }}-statsd
labels:
tier: airflow
component: statsd
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.statsd.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
spec:
replicas: 1
{{- if $revisionHistoryLimit }}
revisionHistoryLimit: {{ $revisionHistoryLimit }}
{{- end }}
selector:
matchLabels:
tier: airflow
component: statsd
release: {{ .Release.Name }}
template:
metadata:
labels:
tier: airflow
component: statsd
release: {{ .Release.Name }}
{{- with .Values.labels }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- if or .Values.statsd.extraMappings .Values.statsd.podAnnotations }}
annotations:
checksum/statsd-config: {{ include (print $.Template.BasePath "/configmaps/statsd-configmap.yaml") . | sha256sum }}
{{- if .Values.statsd.podAnnotations }}
{{- toYaml .Values.statsd.podAnnotations | nindent 8 }}
{{- end }}
{{- end }}
spec:
{{- if .Values.statsd.priorityClassName }}
priorityClassName: {{ .Values.statsd.priorityClassName }}
{{- end }}
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
affinity: {{- toYaml $affinity | nindent 8 }}
tolerations: {{- toYaml $tolerations | nindent 8 }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
serviceAccountName: {{ include "statsd.serviceAccountName" . }}
securityContext: {{ $securityContext | nindent 8 }}
restartPolicy: Always
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
containers:
- name: statsd
image: {{ template "statsd_image" . }}
imagePullPolicy: {{ .Values.images.statsd.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 12 }}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
{{- end }}
{{- if .Values.statsd.args }}
args: {{ tpl (toYaml .Values.statsd.args) . | nindent 12 }}
{{- else}}
args:
- "--statsd.mapping-config=/etc/statsd-exporter/mappings.yml"
{{- end }}
resources: {{- toYaml .Values.statsd.resources | nindent 12 }}
{{- with .Values.statsd.env }}
env: {{- toYaml . | nindent 12 }}
{{- end }}
ports:
- name: statsd-ingest
protocol: UDP
containerPort: {{ .Values.ports.statsdIngest }}
- name: statsd-scrape
containerPort: {{ .Values.ports.statsdScrape }}
livenessProbe:
httpGet:
path: /metrics
port: {{ .Values.ports.statsdScrape }}
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
readinessProbe:
httpGet:
path: /metrics
port: {{ .Values.ports.statsdScrape }}
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
volumeMounts:
- name: config
mountPath: /etc/statsd-exporter/mappings.yml
subPath: mappings.yml
volumes:
- name: config
configMap:
name: {{ include "airflow.fullname" . }}-statsd
{{- end }}

59
helm/airflow/templates/statsd/statsd-networkpolicy.yaml Normal file
View File

@@ -0,0 +1,59 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow StatsD NetworkPolicy
#################################
{{- if and .Values.networkPolicies.enabled .Values.statsd.enabled }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: {{ include "airflow.fullname" . }}-statsd-policy
labels:
tier: airflow
component: statsd-policy
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
podSelector:
matchLabels:
tier: airflow
component: statsd
release: {{ .Release.Name }}
policyTypes:
- Ingress
ingress:
- from:
- podSelector:
matchLabels:
tier: airflow
release: {{ .Release.Name }}
{{- if .Values.statsd.extraNetworkPolicies }}
{{- toYaml .Values.statsd.extraNetworkPolicies | nindent 4 }}
{{- end }}
ports:
- protocol: UDP
port: {{ .Values.ports.statsdIngest }}
- protocol: TCP
port: {{ .Values.ports.statsdScrape }}
{{- end }}

58
helm/airflow/templates/statsd/statsd-service.yaml Normal file
View File

@@ -0,0 +1,58 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow StatsD Service
#################################
{{- if .Values.statsd.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "airflow.fullname" . }}-statsd
labels:
tier: airflow
component: statsd
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: {{ .Values.ports.statsdScrape | quote }}
{{- if .Values.statsd.service.extraAnnotations }}
{{- toYaml .Values.statsd.service.extraAnnotations | nindent 4 }}
{{- end }}
spec:
type: ClusterIP
selector:
tier: airflow
component: statsd
release: {{ .Release.Name }}
ports:
- name: statsd-ingest
protocol: UDP
port: {{ .Values.ports.statsdIngest }}
targetPort: {{ .Values.ports.statsdIngest }}
- name: statsd-scrape
protocol: TCP
port: {{ .Values.ports.statsdScrape }}
targetPort: {{ .Values.ports.statsdScrape }}
{{- end }}

41
helm/airflow/templates/statsd/statsd-serviceaccount.yaml Normal file
View File

@@ -0,0 +1,41 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
######################################
## Airflow StatsD ServiceAccount
######################################
{{- if and .Values.statsd.enabled .Values.statsd.serviceAccount.create }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{ .Values.statsd.serviceAccount.automountServiceAccountToken }}
metadata:
name: {{ include "statsd.serviceAccountName" . }}
labels:
tier: airflow
component: statsd
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.statsd.serviceAccount.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}

312
helm/airflow/templates/triggerer/triggerer-deployment.yaml Normal file
View File

@@ -0,0 +1,312 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Triggerer Deployment
#################################
{{- if semverCompare ">=2.2.0" .Values.airflowVersion }}
{{- if .Values.triggerer.enabled }}
{{- /* Airflow version 2.6.0 is when triggerer logs serve introduced */ -}}
{{- $persistence := and .Values.triggerer.persistence.enabled (semverCompare ">=2.6.0" .Values.airflowVersion) }}
{{- $keda := .Values.triggerer.keda.enabled }}
{{- $nodeSelector := or .Values.triggerer.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.triggerer.affinity .Values.affinity }}
{{- $tolerations := or .Values.triggerer.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.triggerer.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $revisionHistoryLimit := or .Values.triggerer.revisionHistoryLimit .Values.revisionHistoryLimit }}
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.triggerer) }}
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.triggerer) }}
{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list . .Values.triggerer.waitForMigrations) }}
{{- $containerSecurityContextLogGroomer := include "containerSecurityContext" (list . .Values.triggerer.logGroomerSidecar) }}
{{- $containerLifecycleHooks := or .Values.triggerer.containerLifecycleHooks .Values.containerLifecycleHooks }}
{{- $containerLifecycleHooksWaitForMigrations := or .Values.triggerer.waitForMigrations.containerLifecycleHooks .Values.containerLifecycleHooks }}
{{- $containerLifecycleHooksLogGroomerSidecar := or .Values.triggerer.logGroomerSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }}
apiVersion: apps/v1
kind: {{ if $persistence }}StatefulSet{{ else }}Deployment{{ end }}
metadata:
name: {{ include "airflow.fullname" . }}-triggerer
labels:
tier: airflow
component: triggerer
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if .Values.triggerer.annotations }}
annotations: {{- toYaml .Values.triggerer.annotations | nindent 4 }}
{{- end }}
spec:
{{- if $persistence }}
serviceName: {{ .Release.Name }}-triggerer
{{- end }}
{{- if not $keda }}
replicas: {{ .Values.triggerer.replicas }}
{{- end }}
{{- if $revisionHistoryLimit }}
revisionHistoryLimit: {{ $revisionHistoryLimit }}
{{- end }}
selector:
matchLabels:
tier: airflow
component: triggerer
release: {{ .Release.Name }}
{{- if and $persistence .Values.triggerer.updateStrategy }}
updateStrategy: {{- toYaml .Values.triggerer.updateStrategy | nindent 4 }}
{{- end }}
{{- if and (not $persistence) (.Values.triggerer.strategy) }}
strategy: {{- toYaml .Values.triggerer.strategy | nindent 4 }}
{{- end }}
template:
metadata:
labels:
tier: airflow
component: triggerer
release: {{ .Release.Name }}
{{- if or (.Values.labels) (.Values.triggerer.labels) }}
{{- mustMerge .Values.triggerer.labels .Values.labels | toYaml | nindent 8 }}
{{- end }}
annotations:
checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }}
checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }}
checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }}
checksum/extra-configmaps: {{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }}
checksum/extra-secrets: {{ include (print $.Template.BasePath "/secrets/extra-secrets.yaml") . | sha256sum }}
{{- if .Values.triggerer.safeToEvict }}
cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
{{- end }}
{{- if .Values.airflowPodAnnotations }}
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
{{- end }}
{{- if .Values.triggerer.podAnnotations }}
{{- toYaml .Values.triggerer.podAnnotations | nindent 8 }}
{{- end }}
spec:
{{- if .Values.triggerer.priorityClassName }}
priorityClassName: {{ .Values.triggerer.priorityClassName }}
{{- end }}
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
affinity:
{{- if $affinity }}
{{- toYaml $affinity | nindent 8 }}
{{- else }}
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchLabels:
component: triggerer
topologyKey: kubernetes.io/hostname
weight: 100
{{- end }}
tolerations: {{- toYaml $tolerations | nindent 8 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
terminationGracePeriodSeconds: {{ .Values.triggerer.terminationGracePeriodSeconds }}
restartPolicy: Always
serviceAccountName: {{ include "triggerer.serviceAccountName" . }}
securityContext: {{ $securityContext | nindent 8 }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
initContainers:
{{- if .Values.triggerer.waitForMigrations.enabled }}
- name: wait-for-airflow-migrations
resources:
{{- toYaml .Values.triggerer.resources | nindent 12 }}
image: {{ template "airflow_image_for_migrations" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContextWaitForMigrations | nindent 12 }}
{{- if $containerLifecycleHooksWaitForMigrations }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooksWaitForMigrations) . | nindent 12 }}
{{- end }}
volumeMounts:
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.triggerer.extraVolumeMounts }}
{{- tpl (toYaml .Values.triggerer.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
args: {{- include "wait-for-migrations-command" . | indent 10 }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- if .Values.triggerer.waitForMigrations.env }}
{{- tpl (toYaml .Values.triggerer.waitForMigrations.env) $ | nindent 12 }}
{{- end }}
{{- end }}
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
{{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }}
{{- end }}
{{- if .Values.triggerer.extraInitContainers }}
{{- toYaml .Values.triggerer.extraInitContainers | nindent 8 }}
{{- end }}
containers:
- name: triggerer
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 12 }}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
{{- end }}
{{- if .Values.triggerer.command }}
command: {{ tpl (toYaml .Values.triggerer.command) . | nindent 12 }}
{{- end }}
{{- if .Values.triggerer.args }}
args: {{ tpl (toYaml .Values.triggerer.args) . | nindent 12 }}
{{- end }}
resources: {{- toYaml .Values.triggerer.resources | nindent 12 }}
volumeMounts:
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.triggerer.extraVolumeMounts }}
{{- tpl (toYaml .Values.triggerer.extraVolumeMounts) . | nindent 12 }}
{{- end }}
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
{{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }}
{{- include "airflow_dags_mount" . | nindent 12 }}
{{- end }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- include "container_extra_envs" (list . .Values.triggerer.env) | nindent 10 }}
livenessProbe:
initialDelaySeconds: {{ .Values.triggerer.livenessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.triggerer.livenessProbe.timeoutSeconds }}
failureThreshold: {{ .Values.triggerer.livenessProbe.failureThreshold }}
periodSeconds: {{ .Values.triggerer.livenessProbe.periodSeconds }}
exec:
command:
{{- if .Values.triggerer.livenessProbe.command }}
{{- toYaml .Values.triggerer.livenessProbe.command | nindent 16 }}
{{- else }}
{{- include "triggerer_liveness_check_command" . | indent 14 }}
{{- end }}
{{- /* Airflow version 2.6.0 is when triggerer logs serve introduced */ -}}
{{- if semverCompare ">=2.6.0" .Values.airflowVersion }}
ports:
- name: triggerer-logs
containerPort: {{ .Values.ports.triggererLogs }}
{{- end }}
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
{{- include "git_sync_container" . | nindent 8 }}
{{- end }}
{{- if .Values.triggerer.logGroomerSidecar.enabled }}
- name: triggerer-log-groomer
resources: {{- toYaml .Values.triggerer.logGroomerSidecar.resources | nindent 12 }}
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContextLogGroomer | nindent 12 }}
{{- if $containerLifecycleHooksLogGroomerSidecar }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooksLogGroomerSidecar) . | nindent 12 }}
{{- end }}
{{- if .Values.triggerer.logGroomerSidecar.command }}
command: {{ tpl (toYaml .Values.triggerer.logGroomerSidecar.command) . | nindent 12 }}
{{- end }}
{{- if .Values.triggerer.logGroomerSidecar.args }}
args: {{- tpl (toYaml .Values.triggerer.logGroomerSidecar.args) . | nindent 12 }}
{{- end }}
{{- if .Values.triggerer.logGroomerSidecar.retentionDays }}
env:
- name: AIRFLOW__LOG_RETENTION_DAYS
value: "{{ .Values.triggerer.logGroomerSidecar.retentionDays }}"
{{- end }}
volumeMounts:
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.triggerer.extraVolumeMounts }}
{{- tpl (toYaml .Values.triggerer.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
{{- end }}
{{- if .Values.triggerer.extraContainers }}
{{- toYaml .Values.triggerer.extraContainers | nindent 8 }}
{{- end }}
volumes:
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
- name: webserver-config
configMap:
name: {{ template "airflow_webserver_config_configmap_name" . }}
{{- end }}
{{- if .Values.dags.persistence.enabled }}
- name: dags
persistentVolumeClaim:
claimName: {{ template "airflow_dags_volume_claim" . }}
{{- else if .Values.dags.gitSync.enabled }}
- name: dags
emptyDir: {}
{{- if .Values.dags.gitSync.sshKeySecret }}
{{- include "git_sync_ssh_key_volume" . | nindent 8 }}
{{- end }}
{{- end }}
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 8 }}
{{- end }}
{{- if .Values.triggerer.extraVolumes }}
{{- tpl (toYaml .Values.triggerer.extraVolumes) . | nindent 8 }}
{{- end }}
{{- if .Values.logs.persistence.enabled }}
- name: logs
persistentVolumeClaim:
claimName: {{ template "airflow_logs_volume_claim" . }}
{{- else if not $persistence }}
- name: logs
emptyDir: {}
{{- else }}
volumeClaimTemplates:
- metadata:
name: logs
{{- if .Values.triggerer.persistence.annotations }}
annotations: {{- toYaml .Values.triggerer.persistence.annotations | nindent 10 }}
{{- end }}
spec:
{{- if .Values.triggerer.persistence.storageClassName }}
storageClassName: {{ .Values.triggerer.persistence.storageClassName }}
{{- end }}
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: {{ .Values.triggerer.persistence.size }}
{{- end }}
{{- end }}
{{- end }}

57
helm/airflow/templates/triggerer/triggerer-kedaautoscaler.yaml Normal file
View File

@@ -0,0 +1,57 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Triggerer KEDA Scaler
#################################
{{- if semverCompare ">=2.2.0" .Values.airflowVersion }}
{{- if and .Values.triggerer.enabled .Values.triggerer.keda.enabled }}
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
name: {{ .Release.Name }}-triggerer
labels:
tier: airflow
component: triggerer-horizontalpodautoscaler
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
deploymentName: {{ .Release.Name }}-triggerer
{{- if or (.Values.labels) (.Values.triggerer.labels) }}
{{- mustMerge .Values.triggerer.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
scaleTargetRef:
kind: {{ ternary "StatefulSet" "Deployment" .Values.triggerer.persistence.enabled }}
name: {{ .Release.Name }}-triggerer
pollingInterval: {{ .Values.triggerer.keda.pollingInterval }}
cooldownPeriod: {{ .Values.triggerer.keda.cooldownPeriod }}
minReplicaCount: {{ .Values.triggerer.keda.minReplicaCount }}
maxReplicaCount: {{ .Values.triggerer.keda.maxReplicaCount }}
{{- if .Values.triggerer.keda.advanced }}
advanced: {{- toYaml .Values.triggerer.keda.advanced | nindent 4 }}
{{- end }}
triggers:
- type: postgresql
metadata:
targetQueryValue: "1"
connectionFromEnv: AIRFLOW_CONN_AIRFLOW_DB
query: {{ tpl .Values.triggerer.keda.query . | quote }}
{{- end }}
{{- end }}

60
helm/airflow/templates/triggerer/triggerer-networkpolicy.yaml Normal file
View File

@@ -0,0 +1,60 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
##################################
## Airflow triggerer NetworkPolicy
##################################
{{- /* Airflow version 2.6.0 is when triggerer logs serve introduced */ -}}
{{- if semverCompare ">=2.6.0" .Values.airflowVersion }}
{{- if .Values.networkPolicies.enabled }}
{{- if .Values.triggerer.enabled }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: {{ include "airflow.fullname" . }}-triggerer-policy
labels:
tier: airflow
component: airflow-triggerer-policy
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.triggerer.labels) }}
{{- mustMerge .Values.triggerer.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
podSelector:
matchLabels:
tier: airflow
component: triggerer
release: {{ .Release.Name }}
policyTypes:
- Ingress
ingress:
- from:
- podSelector:
matchLabels:
tier: airflow
release: {{ .Release.Name }}
component: webserver
ports:
- protocol: TCP
port: {{ .Values.ports.triggererLogs }}
{{- end }}
{{- end }}
{{- end }}

51
helm/airflow/templates/triggerer/triggerer-service.yaml Normal file
View File

@@ -0,0 +1,51 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow triggerer Service
#################################
{{- /* Airflow version 2.6.0 is when triggerer logs serve introduced */ -}}
{{- if semverCompare ">=2.6.0" .Values.airflowVersion }}
{{- if .Values.triggerer.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "airflow.fullname" . }}-triggerer
labels:
tier: airflow
component: triggerer
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.triggerer.labels) }}
{{- mustMerge .Values.triggerer.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
clusterIP: None
selector:
tier: airflow
component: triggerer
release: {{ .Release.Name }}
ports:
- name: triggerer-logs
protocol: TCP
port: {{ .Values.ports.triggererLogs }}
targetPort: {{ .Values.ports.triggererLogs }}
{{- end }}
{{- end }}

43
helm/airflow/templates/triggerer/triggerer-serviceaccount.yaml Normal file
View File

@@ -0,0 +1,43 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Triggerer ServiceAccount
#################################
{{- if semverCompare ">=2.2.0" .Values.airflowVersion }}
{{- if and .Values.triggerer.serviceAccount.create .Values.triggerer.enabled }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{ .Values.triggerer.serviceAccount.automountServiceAccountToken }}
metadata:
name: {{ include "triggerer.serviceAccountName" . }}
labels:
tier: airflow
component: triggerer
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.triggerer.labels) }}
{{- mustMerge .Values.triggerer.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.triggerer.serviceAccount.annotations}}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
{{- end }}

297
helm/airflow/templates/webserver/webserver-deployment.yaml Normal file
View File

@@ -0,0 +1,297 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Webserver Deployment
#################################
{{- $nodeSelector := or .Values.webserver.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.webserver.affinity .Values.affinity }}
{{- $tolerations := or .Values.webserver.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.webserver.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $revisionHistoryLimit := or .Values.webserver.revisionHistoryLimit .Values.revisionHistoryLimit }}
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.webserver) }}
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.webserver) }}
{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list . .Values.webserver.waitForMigrations) }}
{{- $containerLifecycleHooks := or .Values.webserver.containerLifecycleHooks .Values.containerLifecycleHooks }}
{{- $containerLifecycleHooksWaitForMigrations := or .Values.webserver.waitForMigrations.containerLifecycleHooks .Values.containerLifecycleHooks }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "airflow.fullname" . }}-webserver
labels:
tier: airflow
component: webserver
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if .Values.webserver.annotations }}
annotations: {{- toYaml .Values.webserver.annotations | nindent 4 }}
{{- end }}
spec:
replicas: {{ .Values.webserver.replicas }}
{{- if $revisionHistoryLimit }}
revisionHistoryLimit: {{ $revisionHistoryLimit }}
{{- end }}
strategy:
{{- if .Values.webserver.strategy }}
{{- toYaml .Values.webserver.strategy | nindent 4 }}
{{- else }}
{{- if semverCompare ">=2.0.0" .Values.airflowVersion }}
# Here we define the rolling update strategy
# - maxSurge define how many pod we can add at a time
# - maxUnavailable define how many pod can be unavailable
# during the rolling update
# Setting maxUnavailable to 0 would make sure we have the appropriate
# capacity during the rolling update.
# You can also use percentage based value instead of integer.
type: RollingUpdate
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
{{- else }}
type: Recreate
{{- end }}
{{- end }}
selector:
matchLabels:
tier: airflow
component: webserver
release: {{ .Release.Name }}
template:
metadata:
labels:
tier: airflow
component: webserver
release: {{ .Release.Name }}
{{- if or (.Values.labels) (.Values.webserver.labels) }}
{{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 8 }}
{{- end }}
annotations:
checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }}
checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }}
checksum/webserver-secret-key: {{ include (print $.Template.BasePath "/secrets/webserver-secret-key-secret.yaml") . | sha256sum }}
checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }}
checksum/webserver-config: {{ include (print $.Template.BasePath "/configmaps/webserver-configmap.yaml") . | sha256sum }}
checksum/extra-configmaps: {{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }}
checksum/extra-secrets: {{ include (print $.Template.BasePath "/secrets/extra-secrets.yaml") . | sha256sum }}
{{- if .Values.airflowPodAnnotations }}
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
{{- end }}
{{- if .Values.webserver.podAnnotations }}
{{- toYaml .Values.webserver.podAnnotations | nindent 8 }}
{{- end }}
spec:
{{- if .Values.webserver.hostAliases }}
hostAliases: {{- toYaml .Values.webserver.hostAliases | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "webserver.serviceAccountName" . }}
{{- if .Values.webserver.priorityClassName }}
priorityClassName: {{ .Values.webserver.priorityClassName }}
{{- end }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
affinity:
{{- if $affinity }}
{{- toYaml $affinity | nindent 8 }}
{{- else }}
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchLabels:
component: webserver
topologyKey: kubernetes.io/hostname
weight: 100
{{- end }}
tolerations: {{- toYaml $tolerations | nindent 8 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
restartPolicy: Always
securityContext: {{ $securityContext | nindent 8 }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
initContainers:
{{- if .Values.webserver.waitForMigrations.enabled }}
- name: wait-for-airflow-migrations
resources: {{- toYaml .Values.webserver.resources | nindent 12 }}
image: {{ template "airflow_image_for_migrations" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContextWaitForMigrations | nindent 12 }}
{{- if $containerLifecycleHooksWaitForMigrations }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooksWaitForMigrations) . | nindent 12 }}
{{- end }}
volumeMounts:
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.webserver.extraVolumeMounts }}
{{- tpl (toYaml .Values.webserver.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
args: {{- include "wait-for-migrations-command" . | indent 10 }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- if .Values.webserver.waitForMigrations.env }}
{{- tpl (toYaml .Values.webserver.waitForMigrations.env) $ | nindent 12 }}
{{- end }}
{{- end }}
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) (semverCompare "<2.0.0" .Values.airflowVersion) }}
{{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }}
{{- end }}
{{- if .Values.webserver.extraInitContainers }}
{{- toYaml .Values.webserver.extraInitContainers | nindent 8 }}
{{- end }}
containers:
- name: webserver
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ or $containerSecurityContext .Values.webserver.securityContexts.container .Values.securityContexts.container | nindent 12 }}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
{{- end }}
{{- if .Values.webserver.command }}
command: {{ tpl (toYaml .Values.webserver.command) . | nindent 12 }}
{{- end }}
{{- if .Values.webserver.args }}
args: {{- tpl (toYaml .Values.webserver.args) . | nindent 12 }}
{{- end }}
resources: {{- toYaml .Values.webserver.resources | nindent 12 }}
volumeMounts:
{{- if semverCompare ">=1.10.12" .Values.airflowVersion }}
- name: config
mountPath: {{ include "airflow_pod_template_file" . }}/pod_template_file.yaml
subPath: pod_template_file.yaml
readOnly: true
{{- end }}
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
{{- if and (semverCompare "<2.0.0" .Values.airflowVersion) (or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled) }}
{{- include "airflow_dags_mount" . | nindent 12 }}
{{- end }}
{{- if .Values.logs.persistence.enabled }}
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- end }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.webserver.extraVolumeMounts }}
{{- tpl (toYaml .Values.webserver.extraVolumeMounts) . | nindent 12 }}
{{- end }}
ports:
- name: airflow-ui
containerPort: {{ .Values.ports.airflowUI }}
livenessProbe:
httpGet:
path: {{ if .Values.config.webserver.base_url }}{{- with urlParse (tpl .Values.config.webserver.base_url .) }}{{ .path }}{{ end }}{{ end }}/health
port: {{ .Values.ports.airflowUI }}
{{- if .Values.config.webserver.base_url}}
httpHeaders:
- name: Host
value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values.config.webserver.base_url .)).host "" }}
{{- end }}
scheme: {{ .Values.webserver.livenessProbe.scheme | default "http" }}
initialDelaySeconds: {{ .Values.webserver.livenessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.webserver.livenessProbe.timeoutSeconds }}
failureThreshold: {{ .Values.webserver.livenessProbe.failureThreshold }}
periodSeconds: {{ .Values.webserver.livenessProbe.periodSeconds }}
readinessProbe:
httpGet:
path: {{ if .Values.config.webserver.base_url }}{{- with urlParse (tpl .Values.config.webserver.base_url .) }}{{ .path }}{{ end }}{{ end }}/health
port: {{ .Values.ports.airflowUI }}
{{- if .Values.config.webserver.base_url }}
httpHeaders:
- name: Host
value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values.config.webserver.base_url .)).host "" }}
{{- end }}
scheme: {{ .Values.webserver.readinessProbe.scheme | default "http" }}
initialDelaySeconds: {{ .Values.webserver.readinessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.webserver.readinessProbe.timeoutSeconds }}
failureThreshold: {{ .Values.webserver.readinessProbe.failureThreshold }}
periodSeconds: {{ .Values.webserver.readinessProbe.periodSeconds }}
startupProbe:
httpGet:
path: {{ if .Values.config.webserver.base_url }}{{- with urlParse (tpl .Values.config.webserver.base_url .) }}{{ .path }}{{ end }}{{ end }}/health
port: {{ .Values.ports.airflowUI }}
{{- if .Values.config.webserver.base_url}}
httpHeaders:
- name: Host
value: {{ regexReplaceAll ":\\d+$" (urlParse (tpl .Values.config.webserver.base_url .)).host "" }}
{{- end }}
scheme: {{ .Values.webserver.startupProbe.scheme | default "http" }}
timeoutSeconds: {{ .Values.webserver.startupProbe.timeoutSeconds }}
failureThreshold: {{ .Values.webserver.startupProbe.failureThreshold }}
periodSeconds: {{ .Values.webserver.startupProbe.periodSeconds }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- include "container_extra_envs" (list . .Values.webserver.env) | indent 10 }}
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) (semverCompare "<2.0.0" .Values.airflowVersion) }}
{{- include "git_sync_container" . | nindent 8 }}
{{- end }}
{{- if .Values.webserver.extraContainers }}
{{- toYaml .Values.webserver.extraContainers | nindent 8 }}
{{- end }}
volumes:
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
- name: webserver-config
configMap:
name: {{ template "airflow_webserver_config_configmap_name" . }}
{{- end }}
{{- if (semverCompare "<2.0.0" .Values.airflowVersion) }}
{{- if .Values.dags.persistence.enabled }}
- name: dags
persistentVolumeClaim:
claimName: {{ template "airflow_dags_volume_claim" . }}
{{- else if .Values.dags.gitSync.enabled }}
- name: dags
emptyDir: {}
{{- if .Values.dags.gitSync.sshKeySecret }}
{{- include "git_sync_ssh_key_volume" . | indent 8 }}
{{- end }}
{{- end }}
{{- end }}
{{- if .Values.logs.persistence.enabled }}
- name: logs
persistentVolumeClaim:
claimName: {{ template "airflow_logs_volume_claim" . }}
{{- end }}
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 8 }}
{{- end }}
{{- if .Values.webserver.extraVolumes }}
{{- tpl (toYaml .Values.webserver.extraVolumes) . | nindent 8 }}
{{- end }}

111
helm/airflow/templates/webserver/webserver-ingress.yaml Normal file
View File

@@ -0,0 +1,111 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Webserver Ingress
#################################
{{- if or .Values.ingress.web.enabled .Values.ingress.enabled }}
{{- $fullname := (include "airflow.fullname" .) }}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ $fullname }}-ingress
labels:
tier: airflow
component: airflow-ingress
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.webserver.labels) }}
{{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.ingress.web.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if and .Values.ingress.web.hosts (.Values.ingress.web.hosts | first | kindIs "string" | not) }}
{{- $anyTlsHosts := false -}}
{{- range .Values.ingress.web.hosts }}
{{- if .tls }}
{{- if .tls.enabled }}
{{- $anyTlsHosts = true -}}
{{- end }}
{{- end }}
{{- end }}
{{- if $anyTlsHosts }}
tls:
{{- range .Values.ingress.web.hosts }}
{{- if .tls }}
{{- if .tls.enabled }}
- hosts:
- {{ .name | quote }}
secretName: {{ .tls.secretName }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- else if .Values.ingress.web.tls.enabled }}
tls:
- hosts:
{{- .Values.ingress.web.hosts | default (list .Values.ingress.web.host) | toYaml | nindent 8 }}
secretName: {{ .Values.ingress.web.tls.secretName }}
{{- end }}
rules:
{{- range .Values.ingress.web.hosts | default (list .Values.ingress.web.host) }}
- http:
paths:
{{- range $.Values.ingress.web.precedingPaths }}
- path: {{ .path }}
pathType: {{ .pathType }}
backend:
service:
name: {{ .serviceName }}
port:
name: {{ .servicePort }}
{{- end }}
- backend:
service:
name: {{ $fullname }}-webserver
port:
name: airflow-ui
{{- if $.Values.ingress.web.path }}
path: {{ $.Values.ingress.web.path }}
pathType: {{ $.Values.ingress.web.pathType }}
{{- end }}
{{- range $.Values.ingress.web.succeedingPaths }}
- path: {{ .path }}
pathType: {{ .pathType }}
backend:
service:
name: {{ .serviceName }}
port:
name: {{ .servicePort }}
{{- end }}
{{- $hostname := . -}}
{{- if . | kindIs "string" | not }}
{{- $hostname = .name -}}
{{- end }}
{{- if $hostname }}
host: {{ tpl $hostname $ | quote }}
{{- end }}
{{- end }}
{{- if .Values.ingress.web.ingressClassName }}
ingressClassName: {{ .Values.ingress.web.ingressClassName }}
{{- end }}
{{- end }}

57
helm/airflow/templates/webserver/webserver-networkpolicy.yaml Normal file
View File

@@ -0,0 +1,57 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Webserver NetworkPolicy
#################################
{{- if .Values.networkPolicies.enabled }}
{{- $from := or .Values.webserver.networkPolicy.ingress.from .Values.webserver.extraNetworkPolicies }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: {{ include "airflow.fullname" . }}-webserver-policy
labels:
tier: airflow
component: airflow-webserver-policy
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.webserver.labels) }}
{{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
podSelector:
matchLabels:
tier: airflow
component: webserver
release: {{ .Release.Name }}
policyTypes:
- Ingress
{{- if $from }}
ingress:
- from: {{- toYaml $from | nindent 6 }}
ports:
{{ range .Values.webserver.networkPolicy.ingress.ports }}
-
{{- range $key, $val := . }}
{{ $key }}: {{ tpl (toString $val) $ }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}

44
helm/airflow/templates/webserver/webserver-poddisruptionbudget.yaml Normal file
View File

@@ -0,0 +1,44 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Webserver PodDisruptionBudget
#################################
{{- if .Values.webserver.podDisruptionBudget.enabled }}
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: {{ include "airflow.fullname" . }}-webserver-pdb
labels:
tier: airflow
component: webserver
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.webserver.labels) }}
{{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
selector:
matchLabels:
tier: airflow
component: webserver
release: {{ .Release.Name }}
{{- toYaml .Values.webserver.podDisruptionBudget.config | nindent 2 }}
{{- end }}

56
helm/airflow/templates/webserver/webserver-service.yaml Normal file
View File

@@ -0,0 +1,56 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Webserver Service
#################################
apiVersion: v1
kind: Service
metadata:
name: {{ include "airflow.fullname" . }}-webserver
labels:
tier: airflow
component: webserver
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.webserver.labels) }}
{{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.webserver.service.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
spec:
type: {{ .Values.webserver.service.type }}
selector:
tier: airflow
component: webserver
release: {{ .Release.Name }}
ports:
{{- range .Values.webserver.service.ports }}
- {{ range $key, $val := . }}
{{- $key }}: {{ tpl (toString $val) $ }}
{{ end }}
{{- end }}
{{- if .Values.webserver.service.loadBalancerIP }}
loadBalancerIP: {{ .Values.webserver.service.loadBalancerIP }}
{{- end }}
{{- if .Values.webserver.service.loadBalancerSourceRanges }}
loadBalancerSourceRanges: {{- toYaml .Values.webserver.service.loadBalancerSourceRanges | nindent 4 }}
{{- end }}

41
helm/airflow/templates/webserver/webserver-serviceaccount.yaml Normal file
View File

@@ -0,0 +1,41 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
######################################
## Airflow Webserver ServiceAccount
######################################
{{- if .Values.webserver.serviceAccount.create }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{ .Values.webserver.serviceAccount.automountServiceAccountToken }}
metadata:
name: {{ include "webserver.serviceAccountName" . }}
labels:
tier: airflow
component: webserver
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.webserver.labels) }}
{{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with .Values.webserver.serviceAccount.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}

412
helm/airflow/templates/workers/worker-deployment.yaml Normal file
View File

@@ -0,0 +1,412 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Worker Deployment
#################################
{{- $persistence := .Values.workers.persistence.enabled }}
{{- $keda := .Values.workers.keda.enabled }}
{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }}
{{- $nodeSelector := or .Values.workers.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.workers.affinity .Values.affinity }}
{{- $tolerations := or .Values.workers.tolerations .Values.tolerations }}
{{- $topologySpreadConstraints := or .Values.workers.topologySpreadConstraints .Values.topologySpreadConstraints }}
{{- $revisionHistoryLimit := or .Values.workers.revisionHistoryLimit .Values.revisionHistoryLimit }}
{{- $securityContext := include "airflowPodSecurityContext" (list . .Values.workers) }}
{{- $containerSecurityContext := include "containerSecurityContext" (list . .Values.workers) }}
{{- $containerSecurityContextPersistence := include "containerSecurityContext" (list . .Values.workers.persistence) }}
{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list . .Values.workers.waitForMigrations) }}
{{- $containerSecurityContextLogGroomerSidecar := include "containerSecurityContext" (list . .Values.workers.logGroomerSidecar) }}
{{- $containerSecurityContextKerberosSidecar := include "containerSecurityContext" (list . .Values.workers.kerberosSidecar) }}
{{- $containerLifecycleHooks := or .Values.workers.containerLifecycleHooks .Values.containerLifecycleHooks }}
{{- $containerLifecycleHooksPersistence := or .Values.workers.persistence.containerLifecycleHooks .Values.containerLifecycleHooks }}
{{- $containerLifecycleHooksWaitForMigrations := or .Values.workers.waitForMigrations.containerLifecycleHooks .Values.containerLifecycleHooks }}
{{- $containerLifecycleHooksLogGroomerSidecar := or .Values.workers.logGroomerSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }}
{{- $containerLifecycleHooksKerberosSidecar := or .Values.workers.kerberosSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }}
apiVersion: apps/v1
kind: {{ if $persistence }}StatefulSet{{ else }}Deployment{{ end }}
metadata:
name: {{ include "airflow.fullname" . }}-worker
labels:
tier: airflow
component: worker
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if .Values.workers.annotations }}
annotations: {{- toYaml .Values.workers.annotations | nindent 4 }}
{{- end }}
spec:
{{- if $persistence }}
serviceName: {{ include "airflow.fullname" . }}-worker
{{- end }}
{{- if not $keda }}
replicas: {{ .Values.workers.replicas }}
{{- end }}
{{- if $revisionHistoryLimit }}
revisionHistoryLimit: {{ $revisionHistoryLimit }}
{{- end }}
selector:
matchLabels:
tier: airflow
component: worker
release: {{ .Release.Name }}
{{- if and $persistence .Values.workers.updateStrategy }}
updateStrategy: {{- toYaml .Values.workers.updateStrategy | nindent 4 }}
{{- end }}
{{- if and (not $persistence) (.Values.workers.strategy) }}
strategy: {{- toYaml .Values.workers.strategy | nindent 4 }}
{{- end }}
template:
metadata:
labels:
tier: airflow
component: worker
release: {{ .Release.Name }}
{{- if or (.Values.labels) (.Values.workers.labels) }}
{{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 8 }}
{{- end }}
annotations:
checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }}
checksum/result-backend-secret: {{ include (print $.Template.BasePath "/secrets/result-backend-connection-secret.yaml") . | sha256sum }}
checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }}
checksum/webserver-secret-key: {{ include (print $.Template.BasePath "/secrets/webserver-secret-key-secret.yaml") . | sha256sum }}
checksum/kerberos-keytab: {{ include (print $.Template.BasePath "/secrets/kerberos-keytab-secret.yaml") . | sha256sum }}
checksum/airflow-config: {{ include (print $.Template.BasePath "/configmaps/configmap.yaml") . | sha256sum }}
checksum/extra-configmaps: {{ include (print $.Template.BasePath "/configmaps/extra-configmaps.yaml") . | sha256sum }}
checksum/extra-secrets: {{ include (print $.Template.BasePath "/secrets/extra-secrets.yaml") . | sha256sum }}
{{- if .Values.workers.safeToEvict }}
cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
{{- end }}
{{- if .Values.airflowPodAnnotations }}
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
{{- end }}
{{- if .Values.workers.podAnnotations }}
{{- toYaml .Values.workers.podAnnotations | nindent 8 }}
{{- end }}
spec:
{{- if .Values.workers.runtimeClassName }}
runtimeClassName: {{ .Values.workers.runtimeClassName }}
{{- end }}
{{- if .Values.workers.priorityClassName }}
priorityClassName: {{ .Values.workers.priorityClassName }}
{{- end }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
{{- end }}
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
affinity:
{{- if $affinity }}
{{- toYaml $affinity | nindent 8 }}
{{- else }}
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchLabels:
component: worker
topologyKey: kubernetes.io/hostname
weight: 100
{{- end }}
tolerations: {{- toYaml $tolerations | nindent 8 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 8 }}
{{- if .Values.workers.hostAliases }}
hostAliases: {{- toYaml .Values.workers.hostAliases | nindent 8 }}
{{- end }}
terminationGracePeriodSeconds: {{ .Values.workers.terminationGracePeriodSeconds }}
restartPolicy: Always
serviceAccountName: {{ include "worker.serviceAccountName" . }}
securityContext: {{ $securityContext | nindent 8 }}
{{- if or .Values.registry.secretName .Values.registry.connection }}
imagePullSecrets:
- name: {{ template "registry_secret" . }}
{{- end }}
initContainers:
{{- if and $persistence .Values.workers.persistence.fixPermissions }}
- name: volume-permissions
resources: {{- toYaml .Values.workers.resources | nindent 12 }}
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
command:
- chown
- -R
- "{{ include "airflowPodSecurityContextsIds" (list . .Values.workers) }}"
- {{ template "airflow_logs" . }}
securityContext: {{ $containerSecurityContextPersistence | nindent 12 }}
{{- if $containerLifecycleHooksPersistence }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooksPersistence) . | nindent 12 }}
{{- end }}
volumeMounts:
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- end }}
{{- if .Values.workers.waitForMigrations.enabled }}
- name: wait-for-airflow-migrations
resources: {{- toYaml .Values.workers.resources | nindent 12 }}
image: {{ template "airflow_image_for_migrations" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContextWaitForMigrations | nindent 12 }}
{{- if $containerLifecycleHooksWaitForMigrations }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooksWaitForMigrations) . | nindent 12 }}
{{- end }}
volumeMounts:
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.workers.extraVolumeMounts }}
{{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
args: {{- include "wait-for-migrations-command" . | indent 10 }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- if .Values.workers.waitForMigrations.env }}
{{- tpl (toYaml .Values.workers.waitForMigrations.env) $ | nindent 12 }}
{{- end }}
{{- end }}
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
{{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }}
{{- end }}
{{- if .Values.workers.extraInitContainers }}
{{- toYaml .Values.workers.extraInitContainers | nindent 8 }}
{{- end }}
containers:
- name: worker
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContext | nindent 12 }}
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12 }}
{{- end }}
{{- if .Values.workers.command }}
command: {{ tpl (toYaml .Values.workers.command) . | nindent 12 }}
{{- end }}
{{- if .Values.workers.args }}
args: {{ tpl (toYaml .Values.workers.args) . | nindent 12 }}
{{- end }}
resources: {{- toYaml .Values.workers.resources | nindent 12 }}
{{- if .Values.workers.livenessProbe.enabled }}
livenessProbe:
initialDelaySeconds: {{ .Values.workers.livenessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.workers.livenessProbe.timeoutSeconds }}
failureThreshold: {{ .Values.workers.livenessProbe.failureThreshold }}
periodSeconds: {{ .Values.workers.livenessProbe.periodSeconds }}
exec:
command:
{{- if .Values.workers.livenessProbe.command }}
{{- toYaml .Values.workers.livenessProbe.command | nindent 16 }}
{{- else }}
- sh
- -c
- CONNECTION_CHECK_MAX_COUNT=0 exec /entrypoint python -m celery --app {{ include "celery_executor_namespace" . }} inspect ping -d celery@$(hostname)
{{- end }}
{{- end }}
ports:
- name: worker-logs
containerPort: {{ .Values.ports.workerLogs }}
volumeMounts:
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.workers.extraVolumeMounts }}
{{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 12 }}
{{- end }}
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if .Values.kerberos.enabled }}
- name: kerberos-keytab
subPath: "kerberos.keytab"
mountPath: {{ .Values.kerberos.keytabPath | quote }}
readOnly: true
- name: config
mountPath: {{ .Values.kerberos.configPath | quote }}
subPath: krb5.conf
readOnly: true
- name: kerberos-ccache
mountPath: {{ .Values.kerberos.ccacheMountPath | quote }}
readOnly: true
{{- end }}
{{- if or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled }}
{{- include "airflow_dags_mount" . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
# Only signal the main process, not the process group, to make Warm Shutdown work properly
- name: DUMB_INIT_SETSID
value: "0"
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- include "container_extra_envs" (list . .Values.workers.env) | indent 10 }}
{{- if .Values.workers.kerberosSidecar.enabled }}
- name: KRB5_CONFIG
value: {{ .Values.kerberos.configPath | quote }}
- name: KRB5CCNAME
value: {{ include "kerberos_ccache_path" . | quote }}
{{- end }}
{{- if and (.Values.dags.gitSync.enabled) (not .Values.dags.persistence.enabled) }}
{{- include "git_sync_container" . | nindent 8 }}
{{- end }}
{{- if and $persistence .Values.workers.logGroomerSidecar.enabled }}
- name: worker-log-groomer
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContextLogGroomerSidecar | nindent 12 }}
{{- if $containerLifecycleHooksLogGroomerSidecar }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooksLogGroomerSidecar) . | nindent 12 }}
{{- end }}
{{- if .Values.workers.logGroomerSidecar.command }}
command: {{ tpl (toYaml .Values.workers.logGroomerSidecar.command) . | nindent 12 }}
{{- end }}
{{- if .Values.workers.logGroomerSidecar.args }}
args: {{ tpl (toYaml .Values.workers.logGroomerSidecar.args) . | nindent 12 }}
{{- end }}
{{- if .Values.workers.logGroomerSidecar.retentionDays }}
env:
- name: AIRFLOW__LOG_RETENTION_DAYS
value: "{{ .Values.workers.logGroomerSidecar.retentionDays }}"
{{- end }}
resources: {{- toYaml .Values.workers.logGroomerSidecar.resources | nindent 12 }}
volumeMounts:
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.workers.extraVolumeMounts }}
{{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
{{- end }}
{{- if .Values.workers.kerberosSidecar.enabled }}
- name: worker-kerberos
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContextKerberosSidecar | nindent 12 }}
{{- if $containerLifecycleHooksKerberosSidecar }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooksKerberosSidecar) . | nindent 12 }}
{{- end }}
args: ["kerberos"]
resources: {{- toYaml .Values.workers.kerberosSidecar.resources | nindent 12 }}
volumeMounts:
- name: logs
mountPath: {{ template "airflow_logs" . }}
{{- include "airflow_config_mount" . | nindent 12 }}
- name: config
mountPath: {{ .Values.kerberos.configPath | quote }}
subPath: krb5.conf
readOnly: true
- name: kerberos-keytab
subPath: "kerberos.keytab"
mountPath: {{ .Values.kerberos.keytabPath | quote }}
readOnly: true
- name: kerberos-ccache
mountPath: {{ .Values.kerberos.ccacheMountPath | quote }}
readOnly: false
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
{{- if .Values.workers.extraVolumeMounts }}
{{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 12 }}
{{- end }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
{{- include "airflow_webserver_config_mount" . | nindent 12 }}
{{- end }}
envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }}
env:
- name: KRB5_CONFIG
value: {{ .Values.kerberos.configPath | quote }}
- name: KRB5CCNAME
value: {{ include "kerberos_ccache_path" . | quote }}
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
{{- end }}
{{- if .Values.workers.extraContainers }}
{{- toYaml .Values.workers.extraContainers | nindent 8 }}
{{- end }}
volumes:
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 8 }}
{{- end }}
{{- if .Values.workers.extraVolumes }}
{{- tpl (toYaml .Values.workers.extraVolumes) . | nindent 8 }}
{{- end }}
- name: config
configMap:
name: {{ template "airflow_config" . }}
{{- if or .Values.webserver.webserverConfig .Values.webserver.webserverConfigConfigMapName }}
- name: webserver-config
configMap:
name: {{ template "airflow_webserver_config_configmap_name" . }}
{{- end }}
{{- if .Values.kerberos.enabled }}
- name: kerberos-keytab
secret:
secretName: {{ include "kerberos_keytab_secret" . | quote }}
- name: kerberos-ccache
emptyDir: {}
{{- end }}
{{- if .Values.dags.persistence.enabled }}
- name: dags
persistentVolumeClaim:
claimName: {{ template "airflow_dags_volume_claim" . }}
{{- else if .Values.dags.gitSync.enabled }}
- name: dags
emptyDir: {}
{{- if .Values.dags.gitSync.sshKeySecret }}
{{- include "git_sync_ssh_key_volume" . | indent 8 }}
{{- end }}
{{- end }}
{{- if .Values.logs.persistence.enabled }}
- name: logs
persistentVolumeClaim:
claimName: {{ template "airflow_logs_volume_claim" . }}
{{- else if not $persistence }}
- name: logs
emptyDir: {}
{{- else }}
volumeClaimTemplates:
- metadata:
name: logs
{{- if .Values.workers.persistence.annotations }}
annotations: {{- toYaml .Values.workers.persistence.annotations | nindent 10 }}
{{- end }}
spec:
{{- if .Values.workers.persistence.storageClassName }}
storageClassName: {{ .Values.workers.persistence.storageClassName }}
{{- end }}
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: {{ .Values.workers.persistence.size }}
{{- end }}
{{- end }}

59
helm/airflow/templates/workers/worker-kedaautoscaler.yaml Normal file
View File

@@ -0,0 +1,59 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Worker KEDA Scaler
#################################
{{- if and .Values.workers.keda.enabled (has .Values.executor (list "CeleryExecutor" "CeleryKubernetesExecutor")) }}
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
name: {{ include "airflow.fullname" . }}-worker
labels:
tier: airflow
component: worker-horizontalpodautoscaler
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
deploymentName: {{ .Release.Name }}-worker
{{- if or (.Values.labels) (.Values.workers.labels) }}
{{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
scaleTargetRef:
kind: {{ ternary "StatefulSet" "Deployment" .Values.workers.persistence.enabled }}
name: {{ include "airflow.fullname" . }}-worker
pollingInterval: {{ .Values.workers.keda.pollingInterval }}
cooldownPeriod: {{ .Values.workers.keda.cooldownPeriod }}
minReplicaCount: {{ .Values.workers.keda.minReplicaCount }}
maxReplicaCount: {{ .Values.workers.keda.maxReplicaCount }}
{{- if .Values.workers.keda.advanced }}
advanced: {{- toYaml .Values.workers.keda.advanced | nindent 4 }}
{{- end }}
triggers:
- type: postgresql
metadata:
targetQueryValue: "1"
{{- if and .Values.pgbouncer.enabled (not .Values.workers.keda.usePgbouncer) }}
connectionFromEnv: KEDA_DB_CONN
{{- else }}
connectionFromEnv: AIRFLOW_CONN_AIRFLOW_DB
{{- end }}
query: {{ tpl .Values.workers.keda.query . | quote }}
{{- end }}

55
helm/airflow/templates/workers/worker-networkpolicy.yaml Normal file
View File

@@ -0,0 +1,55 @@
{{/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/}}
################################
## Airflow Worker NetworkPolicy
#################################
{{- if and .Values.networkPolicies.enabled (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")) }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: {{ include "airflow.fullname" . }}-worker-policy
labels:
tier: airflow
component: airflow-worker-policy
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
{{- if or (.Values.labels) (.Values.workers.labels) }}
{{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }}
{{- end }}
spec:
podSelector:
matchLabels:
tier: airflow
component: worker
release: {{ .Release.Name }}
policyTypes:
- Ingress
ingress:
- from:
- podSelector:
matchLabels:
tier: airflow
release: {{ .Release.Name }}
component: webserver
ports:
- protocol: TCP
port: {{ .Values.ports.workerLogs }}
{{- end }}

Some files were not shown because too many files have changed in this diff Show More