Add Production Helm chart support (#8777)
Co-authored-by: Ash Berlin-Taylor <ash_github@firemirror.com>
This commit is contained in:
Greg Neiheisel
GitHub
Родитель
6a9c436f6f
Коммит
66e738296a
|
|
@ -166,7 +166,7 @@ metastore_browser/templates/.*\\.html$|.*\\.jinja2"
|
|||
name: Check yaml files with yamllint
|
||||
entry: yamllint -c yamllint-config.yml
|
||||
types: [yaml]
|
||||
exclude: ^.*init_git_sync\.template\.yaml$|^.*airflow\.template\.yaml$
|
||||
exclude: ^.*init_git_sync\.template\.yaml$|^.*airflow\.template\.yaml$|^chart/templates/.*\.yaml$
|
||||
- repo: https://github.com/timothycrosley/isort
|
||||
rev: 4.3.21-2
|
||||
hooks:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,9 @@
|
|||
# User overrides
|
||||
config.yaml
|
||||
config.yml
|
||||
|
||||
# Build dir
|
||||
repository
|
||||
|
||||
# Chart dependencies
|
||||
**/charts/*.tgz
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Patterns to ignore when building packages.
|
||||
# This supports shell glob matching, relative path matching, and
|
||||
# negation (prefixed with !). Only one pattern per line.
|
||||
.DS_Store
|
||||
# Common VCS dirs
|
||||
.git/
|
||||
.gitignore
|
||||
.bzr/
|
||||
.bzrignore
|
||||
.hg/
|
||||
.hgignore
|
||||
.svn/
|
||||
# Common backup files
|
||||
*.swp
|
||||
*.bak
|
||||
*.tmp
|
||||
*~
|
||||
# Various IDEs
|
||||
.project
|
||||
.idea/
|
||||
*.tmproj
|
||||
bin
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# apiVersion v1 is Helm 2
|
||||
apiVersion: v1
|
||||
name: airflow
|
||||
version: 1.0.0
|
||||
description: Helm chart to deploy Apache Airflow
|
||||
icon: https://www.astronomer.io/static/airflowNewA.png
|
||||
keywords:
|
||||
- airflow
|
||||
|
|
@ -0,0 +1,269 @@
|
|||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
# Helm Chart for Apache Airflow
|
||||
|
||||
[Apache Airflow](https://airflow.apache.org/) is a platform to programmatically author, schedule and monitor workflows.
|
||||
|
||||
## Introduction
|
||||
|
||||
This chart will bootstrap an [Airflow](https://airflow.apache.org) deployment on a [Kubernetes](http://kubernetes.io)
|
||||
cluster using the [Helm](https://helm.sh) package manager.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Kubernetes 1.12+
|
||||
- Helm 2.11+ or Helm 3.0+
|
||||
- PV provisioner support in the underlying infrastructure
|
||||
|
||||
## Installing the Chart
|
||||
|
||||
To install this repository from source (using helm 3)
|
||||
```bash
|
||||
kubectl create namespace airflow
|
||||
helm repo add stable https://kubernetes-charts.storage.googleapis.com
|
||||
helm dep update
|
||||
helm install airflow . --namespace airflow
|
||||
```
|
||||
|
||||
The command deploys Airflow on the Kubernetes cluster in the default configuration. The [Parameters](#parameters)
|
||||
section lists the parameters that can be configured during installation.
|
||||
|
||||
> **Tip**: List all releases using `helm list`
|
||||
|
||||
## Upgrading the Chart
|
||||
To upgrade the chart with the release name `airflow`:
|
||||
|
||||
```bash
|
||||
helm upgrade airflow . --namespace airflow
|
||||
```
|
||||
|
||||
## Uninstalling the Chart
|
||||
|
||||
To uninstall/delete the `airflow` deployment:
|
||||
|
||||
```bash
|
||||
helm delete airflow --namespace airflow
|
||||
```
|
||||
|
||||
The command removes all the Kubernetes components associated with the chart and deletes the release.
|
||||
|
||||
## Updating DAGs
|
||||
|
||||
The recommended way to update your DAGs with this chart is to build a new docker image with the latest code (`docker build -t my-company/airflow:8a0da78 .`), push it to an accessible registry (`docker push my-company/airflow:8a0da78`), then update the Airflow pods with that image:
|
||||
|
||||
```bash
|
||||
helm upgrade airflow . \
|
||||
--set images.airflow.repository=my-company/airflow \
|
||||
--set images.airflow.tag=8a0da78
|
||||
```
|
||||
|
||||
## Parameters
|
||||
|
||||
The following tables lists the configurable parameters of the Airflow chart and their default values.
|
||||
|
||||
| Parameter | Description | Default |
|
||||
| ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | ------------------------------------------------- |
|
||||
| `uid` | UID to run airflow pods under | `50000` |
|
||||
| `gid` | GID to run airflow pods under | `50000` |
|
||||
| `nodeSelector` | Node labels for pod assignment | `{}` |
|
||||
| `affinity` | Affinity labels for pod assignment | `{}` |
|
||||
| `tolerations` | Toleration labels for pod assignment | `[]` |
|
||||
| `labels` | Common labels to add to all objects defined in this chart | `{}` |
|
||||
| `privateRegistry.enabled` | Enable usage of a private registry for Airflow base image | `false` |
|
||||
| `privateRegistry.repository` | Repository where base image lives (eg: quay.io) | `~` |
|
||||
| `networkPolicies.enabled` | Enable Network Policies to restrict traffic | `true` |
|
||||
| `airflowHome` | Location of airflow home directory | `/opt/airflow` |
|
||||
| `rbacEnabled` | Deploy pods with Kubernets RBAC enabled | `true` |
|
||||
| `airflowVersion` | Default Airflow image version | `1.10.5` |
|
||||
| `executor` | Airflow executor (eg SequentialExecutor, LocalExecutor, CeleryExecutor, KubernetesExecutor) | `KubernetesExecutor` |
|
||||
| `allowPodLaunching` | Allow airflow pods to talk to Kubernetes API to launch more pods | `true` |
|
||||
| `defaultAirflowRepository` | Fallback docker repository to pull airflow image from | `apache/airflow` |
|
||||
| `defaultAirflowTag` | Fallback docker image tag to deploy | `1.10.10.1-alpha2-python3.6` |
|
||||
| `images.airflow.repository` | Docker repository to pull image from. Update this to deploy a custom image | `~` |
|
||||
| `images.airflow.tag` | Docker image tag to pull image from. Update this to deploy a new custom image tag | `~` |
|
||||
| `images.airflow.pullPolicy` | PullPolicy for airflow image | `IfNotPresent` |
|
||||
| `images.flower.repository` | Docker repository to pull image from. Update this to deploy a custom image | `~` |
|
||||
| `images.flower.tag` | Docker image tag to pull image from. Update this to deploy a new custom image tag | `~` |
|
||||
| `images.flower.pullPolicy` | PullPolicy for flower image | `IfNotPresent` |
|
||||
| `images.statsd.repository` | Docker repository to pull image from. Update this to deploy a custom image | `astronomerinc/ap-statsd-exporter` |
|
||||
| `images.statsd.tag` | Docker image tag to pull image from. Update this to deploy a new custom image tag | `~` |
|
||||
| `images.statsd.pullPolicy` | PullPolicy for statsd-exporter image | `IfNotPresent` |
|
||||
| `images.redis.repository` | Docker repository to pull image from. Update this to deploy a custom image | `redis` |
|
||||
| `images.redis.tag` | Docker image tag to pull image from. Update this to deploy a new custom image tag | `6-buster` |
|
||||
| `images.redis.pullPolicy` | PullPolicy for redis image | `IfNotPresent` |
|
||||
| `images.pgbouncer.repository` | Docker repository to pull image from. Update this to deploy a custom image | `astronomerinc/ap-pgbouncer` |
|
||||
| `images.pgbouncer.tag` | Docker image tag to pull image from. Update this to deploy a new custom image tag | `~` |
|
||||
| `images.pgbouncer.pullPolicy` | PullPolicy for pgbouncer image | `IfNotPresent` |
|
||||
| `images.pgbouncerExporter.repository` | Docker repository to pull image from. Update this to deploy a custom image | `astronomerinc/ap-pgbouncer-exporter` |
|
||||
| `images.pgbouncerExporter.tag` | Docker image tag to pull image from. Update this to deploy a new custom image tag | `~` |
|
||||
| `images.pgbouncerExporter.pullPolicy` | PullPolicy for pgbouncer-exporter image | `IfNotPresent` |
|
||||
| `env` | Environment variables key/values to mount into Airflow pods | `[]` |
|
||||
| `secret` | Secret name/key pairs to mount into Airflow pods | `[]` |
|
||||
| `data.metadataSecretName` | Secret name to mount Airflow connection string from | `~` |
|
||||
| `data.resultBackendSecretName` | Secret name to mount Celery result backend connection string from | `~` |
|
||||
| `data.metadataConection` | Field separated connection data (alternative to secret name) | `{}` |
|
||||
| `data.resultBakcnedConnection` | Field separated connection data (alternative to secret name) | `{}` |
|
||||
| `fernetKey` | String representing an Airflow fernet key | `~` |
|
||||
| `fernetKeySecretName` | Secret name for Airlow fernet key | `~` |
|
||||
| `workers.replicas` | Replica count for Celery workers (if applicable) | `1` |
|
||||
| `workers.keda.enabled` | Enable KEDA autoscaling features | `false` |
|
||||
| `workers.keda.pollingInverval` | How often KEDA should poll the backend database for metrics in seconds | `5` |
|
||||
| `workers.keda.cooldownPeriod` | How often KEDA should wait before scaling down in seconds | `30` |
|
||||
| `workers.keda.maxReplicaCount` | Maximum number of Celery workers KEDA can scale to | `10` |
|
||||
| `workers.persistence.enabled` | Enable log persistence in workers via StatefulSet | `false` |
|
||||
| `workers.persistence.size` | Size of worker volumes if enabled | `100Gi` |
|
||||
| `workers.persistence.storageClassName` | StorageClass worker volumes should use if enabled | `default` |
|
||||
| `workers.resources.limits.cpu` | CPU Limit of workers | `~` |
|
||||
| `workers.resources.limits.memory` | Memory Limit of workers | `~` |
|
||||
| `workers.resources.requests.cpu` | CPU Request of workers | `~` |
|
||||
| `workers.resources.requests.memory` | Memory Request of workers | `~` |
|
||||
| `workers.terminationGracePeriodSeconds` | How long Kubernetes should wait for Celery workers to gracefully drain before force killing | `600` |
|
||||
| `workers.safeToEvict` | Allow Kubernetes to evict worker pods if needed (node downscaling) | `true` |
|
||||
| `scheduler.podDisruptionBudget.enabled` | Enable PDB on Airflow scheduler | `false` |
|
||||
| `scheduler.podDisruptionBudget.config.maxUnavailable` | MaxUnavailable pods for scheduler | `1` |
|
||||
| `scheduler.resources.limits.cpu` | CPU Limit of scheduler | `~` |
|
||||
| `scheduler.resources.limits.memory` | Memory Limit of scheduler | `~` |
|
||||
| `scheduler.resources.requests.cpu` | CPU Request of scheduler | `~` |
|
||||
| `scheduler.resources.requests.memory` | Memory Request of scheduler | `~` |
|
||||
| `scheduler.airflowLocalSettings` | Custom Airflow local settings python file | `~` |
|
||||
| `scheduler.safeToEvict` | Allow Kubernetes to evict scheduler pods if needed (node downscaling) | `true` |
|
||||
| `webserver.livenessProbe.initialDelaySeconds` | Webserver LivenessProbe initial delay | `15` |
|
||||
| `webserver.livenessProbe.timeoutSeconds` | Webserver LivenessProbe timeout seconds | `30` |
|
||||
| `webserver.livenessProbe.failureThreshold` | Webserver LivenessProbe failure threshold | `20` |
|
||||
| `webserver.livenessProbe.periodSeconds` | Webserver LivenessProbe period seconds | `5` |
|
||||
| `webserver.readinessProbe.initialDelaySeconds` | Webserver ReadinessProbe initial delay | `15` |
|
||||
| `webserver.readinessProbe.timeoutSeconds` | Webserver ReadinessProbe timeout seconds | `30` |
|
||||
| `webserver.readinessProbe.failureThreshold` | Webserver ReadinessProbe failure threshold | `20` |
|
||||
| `webserver.readinessProbe.periodSeconds` | Webserver ReadinessProbe period seconds | `5` |
|
||||
| `webserver.replicas` | How many Airflow webserver replicas should run | `1` |
|
||||
| `webserver.resources.limits.cpu` | CPU Limit of webserver | `~` |
|
||||
| `webserver.resources.limits.memory` | Memory Limit of webserver | `~` |
|
||||
| `webserver.resources.requests.cpu` | CPU Request of webserver | `~` |
|
||||
| `webserver.resources.requests.memory` | Memory Request of webserver | `~` |
|
||||
| `webserver.jwtSigningCertificateSecretName` | Name of secret to mount Airflow Webserver JWT singing certificate from | `~` |
|
||||
| `webserver.defaultUser` | Optional default airflow user information | `{}` |
|
||||
|
||||
|
||||
Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example,
|
||||
|
||||
```bash
|
||||
helm install --name my-release \
|
||||
--set executor=CeleryExecutor \
|
||||
--set enablePodLaunching=false .
|
||||
```
|
||||
|
||||
## Autoscaling with KEDA
|
||||
|
||||
KEDA stands for Kubernetes Event Driven Autoscaling. [KEDA](https://github.com/kedacore/keda) is a custom controller that allows users to create custom bindings
|
||||
to the Kubernetes [Horizontal Pod Autoscaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/).
|
||||
We've built an experimental scaler that allows users to create scalers based on postgreSQL queries. For the moment this exists
|
||||
on a seperate branch, but will be merged upstream soon. To install our custom version of KEDA on your cluster, please run
|
||||
|
||||
```bash
|
||||
helm repo add kedacore https://kedacore.github.io/charts
|
||||
|
||||
helm repo update
|
||||
|
||||
helm install \
|
||||
--set image.keda=docker.io/kedacore/keda:1.2.0 \
|
||||
--set image.metricsAdapter=docker.io/kedacore/keda-metrics-adapter:1.2.0 \
|
||||
--namespace keda --name keda kedacore/keda
|
||||
```
|
||||
|
||||
Once KEDA is installed (which should be pretty quick since there is only one pod). You can try out KEDA autoscaling
|
||||
on this chart by setting `workers.keda.enabled=true` your helm command or in the `values.yaml`.
|
||||
(Note: KEDA does not support StatefulSets so you need to set `worker.persistence.enabled` to `false`)
|
||||
|
||||
```bash
|
||||
kubectl create namespace airflow
|
||||
|
||||
helm install airflow . \
|
||||
--namespace airflow \
|
||||
--set executor=CeleryExecutor \
|
||||
--set workers.keda.enabled=true \
|
||||
--set workers.persistence.enabled=false
|
||||
```
|
||||
|
||||
## Walkthrough using kind
|
||||
|
||||
**Install kind, and create a cluster:**
|
||||
|
||||
We recommend testing with Kubernetes 1.15, as this image doesn't support Kubernetes 1.16+ for CeleryExecutor presently.
|
||||
|
||||
```
|
||||
kind create cluster \
|
||||
--image kindest/node:v1.15.7@sha256:e2df133f80ef633c53c0200114fce2ed5e1f6947477dbc83261a6a921169488d
|
||||
```
|
||||
|
||||
Confirm it's up:
|
||||
|
||||
```
|
||||
kubectl cluster-info --context kind-kind
|
||||
```
|
||||
|
||||
**Add Astronomer's Helm repo:**
|
||||
|
||||
```
|
||||
helm repo add astronomer https://helm.astronomer.io
|
||||
helm repo update
|
||||
```
|
||||
|
||||
**Create namespace + install the chart:**
|
||||
|
||||
```
|
||||
kubectl create namespace airflow
|
||||
helm install airflow --n airflow astronomer/airflow
|
||||
```
|
||||
|
||||
It may take a few minutes. Confirm the pods are up:
|
||||
|
||||
```
|
||||
kubectl get pods --all-namespaces
|
||||
helm list -n airflow
|
||||
```
|
||||
|
||||
Run `kubectl port-forward svc/airflow-webserver 8080:8080 -n airflow`
|
||||
to port-forward the Airflow UI to http://localhost:8080/ to cofirm Airflow is working.
|
||||
|
||||
**Build a Docker image from your DAGs:**
|
||||
|
||||
1. Start a project using [astro-cli](https://github.com/astronomer/astro-cli), which will generate a Dockerfile, and load your DAGs in. You can test locally before pushing to kind with `astro airflow start`.
|
||||
|
||||
mkdir my-airflow-project && cd my-airflow-project
|
||||
astro dev init
|
||||
|
||||
2. Then build the image:
|
||||
|
||||
docker build -t my-dags:0.0.1 .
|
||||
|
||||
3. Load the image into kind:
|
||||
|
||||
kind load docker-image my-dags:0.0.1
|
||||
|
||||
4. Upgrade Helm deployment:
|
||||
|
||||
helm upgrade airflow -n airflow \
|
||||
--set images.airflow.repository=my-dags \
|
||||
--set images.airflow.tag=0.0.1 \
|
||||
astronomer/airflow
|
||||
|
||||
## Contributing
|
||||
|
||||
Check out [our contributing guide!](CONTRIBUTING.md)
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
dependencies:
|
||||
- name: postgresql
|
||||
repository: https://kubernetes-charts.storage.googleapis.com
|
||||
version: 6.3.12
|
||||
digest: sha256:1750ddcc948f15716d157d6a854b59e37571473b2a3390a3673b224b71a56308
|
||||
generated: "2019-10-24T16:03:26.569269284-04:00"
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
dependencies:
|
||||
- name: postgresql
|
||||
version: 6.3.12
|
||||
repository: "@stable"
|
||||
condition: postgresql.enabled
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
{{/*
|
||||
Licensed to the Apache Software Foundation (ASF) under one
|
||||
or more contributor license agreements. See the NOTICE file
|
||||
distributed with this work for additional information
|
||||
regarding copyright ownership. The ASF licenses this file
|
||||
to you under the Apache License, Version 2.0 (the
|
||||
"License"); you may not use this file except in compliance
|
||||
with the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing,
|
||||
software distributed under the License is distributed on an
|
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
KIND, either express or implied. See the License for the
|
||||
specific language governing permissions and limitations
|
||||
under the License.
|
||||
|
||||
*/}}Thank you for installing Airflow!
|
||||
|
||||
Your release is named {{ .Release.Name }}.
|
||||
|
||||
You can now access your dashboard(s) by executing the following command(s) and visiting the corresponding port at localhost in your browser:
|
||||
|
||||
Airflow dashboard: kubectl port-forward svc/{{ .Release.Name }}-webserver {{ .Values.ports.airflowUI }}:{{ .Values.ports.airflowUI }}
|
||||
{{- if eq .Values.executor "CeleryExecutor"}}
|
||||
Flower dashboard: kubectl port-forward svc/{{ .Release.Name }}-flower {{ .Values.ports.flowerUI }}:{{ .Values.ports.flowerUI }}
|
||||
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,260 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
{{/* Standard Airflow environment variables */}}
|
||||
{{- define "standard_airflow_environment" }}
|
||||
# Hard Coded Airflow Envs
|
||||
- name: AIRFLOW__CORE__FERNET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ template "fernet_key_secret" . }}
|
||||
key: fernet-key
|
||||
- name: AIRFLOW__CORE__SQL_ALCHEMY_CONN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ template "airflow_metadata_secret" . }}
|
||||
key: connection
|
||||
- name: AIRFLOW_CONN_AIRFLOW_DB
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ template "airflow_metadata_secret" . }}
|
||||
key: connection
|
||||
{{- if eq .Values.executor "CeleryExecutor" }}
|
||||
- name: AIRFLOW__CELERY__CELERY_RESULT_BACKEND
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ template "airflow_result_backend_secret" . }}
|
||||
key: connection
|
||||
- name: AIRFLOW__CELERY__RESULT_BACKEND
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ template "airflow_result_backend_secret" . }}
|
||||
key: connection
|
||||
- name: AIRFLOW__CELERY__BROKER_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ .Release.Name }}-broker-url
|
||||
key: connection
|
||||
{{- end }}
|
||||
{{- if .Values.elasticsearch.enabled }}
|
||||
# The elasticsearch variables were updated to the shorter names in v1.10.4
|
||||
- name: AIRFLOW__ELASTICSEARCH__HOST
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ template "elasticsearch_secret" . }}
|
||||
key: connection
|
||||
# This is the older format for these variable names, kept here for backward compatibility
|
||||
- name: AIRFLOW__ELASTICSEARCH__ELASTICSEARCH_HOST
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ template "elasticsearch_secret" . }}
|
||||
key: connection
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/* User defined Airflow environment variables */}}
|
||||
{{- define "custom_airflow_environment" }}
|
||||
# Dynamically created environment variables
|
||||
{{- range $i, $config := .Values.env }}
|
||||
- name: {{ $config.name }}
|
||||
value: {{ $config.value | quote }}
|
||||
{{- if eq $.Values.executor "KubernetesExecutor" }}
|
||||
- name: AIRFLOW__KUBERNETES_ENVIRONMENT_VARIABLES__{{ $config.name }}
|
||||
value: {{ $config.value | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
# Dynamically created secret envs
|
||||
{{- range $i, $config := .Values.secret }}
|
||||
- name: {{ $config.envName }}
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ $config.secretName }}
|
||||
key: {{ default "value" $config.secretKey }}
|
||||
{{- end }}
|
||||
{{- if eq .Values.executor "KubernetesExecutor" }}
|
||||
{{- range $i, $config := .Values.secret }}
|
||||
- name: AIRFLOW__KUBERNETES_SECRETS__{{ $config.envName }}
|
||||
value: {{ printf "%s=%s" $config.secretName $config.secretKey }}
|
||||
{{- end }}
|
||||
{{ end }}
|
||||
{{- end }}
|
||||
|
||||
# This helper will change when customers deploy a new image.
|
||||
{{ define "airflow_image" -}}
|
||||
{{ printf "%s:%s" (.Values.images.airflow.repository | default .Values.defaultAirflowRepository) (.Values.images.airflow.tag | default .Values.defaultAirflowTag) }}
|
||||
{{- end }}
|
||||
|
||||
# This helper is used for airflow containers that do not need the users code.
|
||||
{{ define "default_airflow_image" -}}
|
||||
{{ printf "%s:%s" .Values.defaultAirflowRepository .Values.defaultAirflowTag }}
|
||||
{{- end }}
|
||||
|
||||
{{ define "flower_image" -}}
|
||||
{{ printf "%s:%s" (.Values.images.airflow.repository | default .Values.defaultAirflowRepository) (.Values.images.airflow.tag | default .Values.defaultAirflowTag) }}
|
||||
{{- end }}
|
||||
|
||||
{{ define "statsd_image" -}}
|
||||
{{ printf "%s:%s" .Values.images.statsd.repository .Values.images.statsd.tag }}
|
||||
{{- end }}
|
||||
|
||||
{{ define "redis_image" -}}
|
||||
{{ printf "%s:%s" .Values.images.redis.repository .Values.images.redis.tag }}
|
||||
{{- end }}
|
||||
|
||||
{{ define "pgbouncer_image" -}}
|
||||
{{ printf "%s:%s" .Values.images.pgbouncer.repository .Values.images.pgbouncer.tag }}
|
||||
{{- end }}
|
||||
|
||||
{{ define "pgbouncer_exporter_image" -}}
|
||||
{{ printf "%s:%s" .Values.images.pgbouncerExporter.repository .Values.images.pgbouncerExporter.tag }}
|
||||
{{- end }}
|
||||
|
||||
{{ define "fernet_key_secret" -}}
|
||||
{{ default (printf "%s-fernet-key" .Release.Name) .Values.fernetKeySecretName }}
|
||||
{{- end }}
|
||||
|
||||
{{ define "redis_password_secret" -}}
|
||||
{{ default (printf "%s-redis-password" .Release.Name) .Values.redis.passwordSecretName }}
|
||||
{{- end }}
|
||||
|
||||
{{ define "airflow_metadata_secret" -}}
|
||||
{{ default (printf "%s-airflow-metadata" .Release.Name) .Values.data.metadataSecretName }}
|
||||
{{- end }}
|
||||
|
||||
{{ define "airflow_result_backend_secret" -}}
|
||||
{{ default (printf "%s-airflow-result-backend" .Release.Name) .Values.data.resultBackendSecretName }}
|
||||
{{- end }}
|
||||
|
||||
{{ define "pgbouncer_config_secret" -}}
|
||||
{{ .Release.Name }}-pgbouncer-config
|
||||
{{- end }}
|
||||
|
||||
{{ define "pgbouncer_stats_secret" -}}
|
||||
{{ .Release.Name }}-pgbouncer-stats
|
||||
{{- end }}
|
||||
|
||||
{{ define "registry_secret" -}}
|
||||
{{ default (printf "%s-registry" .Release.Name) .Values.registry.secretName }}
|
||||
{{- end }}
|
||||
|
||||
{{ define "elasticsearch_secret" -}}
|
||||
{{ default (printf "%s-elasticsearch" .Release.Name) .Values.elasticsearch.secretName }}
|
||||
{{- end }}
|
||||
|
||||
{{ define "pgbouncer_config" }}
|
||||
{{- $pgMetadataHost := .Values.data.metadataConnection.host | default (printf "%s-%s.%s.svc.cluster.local" .Release.Name "postgresql" .Release.Namespace) }}
|
||||
{{- $pgResultBackendHost := .Values.data.resultBackendConnection.host | default (printf "%s-%s.%s.svc.cluster.local" .Release.Name "postgresql" .Release.Namespace) }}
|
||||
[databases]
|
||||
{{ .Release.Name }}-metadata = host={{ $pgMetadataHost }} dbname={{ .Values.data.metadataConnection.db }} port={{ .Values.data.metadataConnection.port }} pool_size={{ .Values.pgbouncer.metadataPoolSize }}
|
||||
{{ .Release.Name }}-result-backend = host={{ $pgResultBackendHost }} dbname={{ .Values.data.resultBackendConnection.db }} port={{ .Values.data.resultBackendConnection.port }} pool_size={{ .Values.pgbouncer.resultBackendPoolSize }}
|
||||
|
||||
[pgbouncer]
|
||||
pool_mode = transaction
|
||||
listen_port = {{ .Values.ports.pgbouncer }}
|
||||
listen_addr = *
|
||||
auth_type = md5
|
||||
auth_file = /etc/pgbouncer/users.txt
|
||||
stats_users = {{ .Values.data.metadataConnection.user }}
|
||||
ignore_startup_parameters = extra_float_digits
|
||||
max_client_conn = {{ .Values.pgbouncer.maxClientConn }}
|
||||
verbose = {{ .Values.pgbouncer.verbose }}
|
||||
log_disconnections = {{ .Values.pgbouncer.logDisconnections }}
|
||||
log_connections = {{ .Values.pgbouncer.logConnections }}
|
||||
{{- end }}
|
||||
|
||||
{{ define "pgbouncer_users" }}
|
||||
{{ .Values.data.metadataConnection.user | quote }} {{ .Values.data.metadataConnection.pass | quote }}
|
||||
{{ .Values.data.resultBackendConnection.user | quote }} {{ .Values.data.resultBackendConnection.pass | quote }}
|
||||
{{- end }}
|
||||
|
||||
{{ define "airflow_logs" -}}
|
||||
{{ (printf "%s/logs" .Values.airflowHome) | quote }}
|
||||
{{- end }}
|
||||
|
||||
{{ define "airflow_config_path" -}}
|
||||
{{ (printf "%s/airflow.cfg" .Values.airflowHome) | quote }}
|
||||
{{- end }}
|
||||
{{ define "airflow_webserver_config_path" -}}
|
||||
{{ (printf "%s/webserver_config.py" .Values.airflowHome) | quote }}
|
||||
{{- end }}
|
||||
|
||||
{{ define "airflow_local_setting_path" -}}
|
||||
{{ (printf "%s/config/airflow_local_settings.py" .Values.airflowHome) | quote }}
|
||||
{{- end }}
|
||||
|
||||
{{ define "airflow_config" -}}
|
||||
{{ (printf "%s-airflow-config" .Release.Name) }}
|
||||
{{- end }}
|
||||
|
||||
{{ define "wait-for-migrations-command" }}
|
||||
{{/* From Airflow 2.0.0 this can become [airflow, db, check-migrations] */}}
|
||||
- python
|
||||
- -c
|
||||
- |
|
||||
import importlib
|
||||
import os
|
||||
import time
|
||||
|
||||
from alembic.config import Config
|
||||
from alembic.runtime.migration import MigrationContext
|
||||
from alembic.script import ScriptDirectory
|
||||
|
||||
from airflow import settings
|
||||
|
||||
package_dir = os.path.dirname(importlib.util.find_spec('airflow').origin)
|
||||
directory = os.path.join(package_dir, 'migrations')
|
||||
config = Config(os.path.join(package_dir, 'alembic.ini'))
|
||||
config.set_main_option('script_location', directory)
|
||||
config.set_main_option('sqlalchemy.url', settings.SQL_ALCHEMY_CONN)
|
||||
script_ = ScriptDirectory.from_config(config)
|
||||
|
||||
timeout=60
|
||||
|
||||
with settings.engine.connect() as connection:
|
||||
context = MigrationContext.configure(connection)
|
||||
ticker = 0
|
||||
while True:
|
||||
source_heads = set(script_.get_heads())
|
||||
|
||||
db_heads = set(context.get_current_heads())
|
||||
if source_heads == db_heads:
|
||||
break
|
||||
|
||||
if ticker >= timeout:
|
||||
raise TimeoutError("There are still unapplied migrations after {} seconds.".format(ticker))
|
||||
ticker += 1
|
||||
time.sleep(1)
|
||||
log.info('Waiting for migrations... %s second(s)', ticker)
|
||||
{{- end }}
|
||||
|
||||
{{ define "registry_docker_config" -}}
|
||||
{{- $host := .Values.registry.connection.host }}
|
||||
{{- $email := .Values.registry.connection.email }}
|
||||
{{- $user := .Values.registry.connection.user -}}
|
||||
{{- $pass := .Values.registry.connection.pass -}}
|
||||
|
||||
{{- $config := dict "auths" -}}
|
||||
{{- $auth := dict -}}
|
||||
{{- $data := dict -}}
|
||||
{{- $_ := set $data "username" $user -}}
|
||||
{{- $_ := set $data "password" $pass -}}
|
||||
{{- $_ := set $data "email" $email -}}
|
||||
{{- $_ := set $data "auth" (printf "%v:%v" $user $pass | b64enc) -}}
|
||||
{{- $_ := set $auth $host $data -}}
|
||||
{{- $_ := set $config "auths" $auth -}}
|
||||
{{ $config | toJson | print }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Cleanup Pods CronJob
|
||||
#################################
|
||||
{{- if .Values.cleanup.enabled }}
|
||||
apiVersion: batch/v1beta1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-cleanup
|
||||
labels:
|
||||
tier: airflow
|
||||
component: airflow-cleanup-pods
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
schedule: "{{ .Values.cleanup.schedule }}"
|
||||
# The cron job does not allow concurrent runs; if it is time for a new job run and the previous job run hasn’t finished yet, the cron job skips the new job run
|
||||
concurrencyPolicy: Forbid
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 1
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
sidecar.istio.io/inject: "false"
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
nodeSelector:
|
||||
{{ toYaml .Values.nodeSelector | indent 12 }}
|
||||
affinity:
|
||||
{{ toYaml .Values.affinity | indent 12 }}
|
||||
tolerations:
|
||||
{{ toYaml .Values.tolerations | indent 12 }}
|
||||
serviceAccountName: {{ .Release.Name }}-cleanup-serviceaccount
|
||||
{{- if or .Values.registry.secretName .Values.registry.connection }}
|
||||
imagePullSecrets:
|
||||
- name: {{ template "registry_secret" . }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: airflow-cleanup-pods
|
||||
image: {{ template "default_airflow_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
# Don't use entry point here, we don't need to wait on pg-bouncer etc being available.
|
||||
command: ["airflow-cleanup-pods", "--namespace={{ .Release.Namespace }}"]
|
||||
env:
|
||||
{{- include "standard_airflow_environment" . | indent 12 }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Cleanup ServiceAccount
|
||||
#################################
|
||||
{{- if and .Values.rbacEnabled .Values.cleanup.enabled }}
|
||||
kind: ServiceAccount
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-cleanup-serviceaccount
|
||||
labels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,117 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow ConfigMap
|
||||
#################################
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ template "airflow_config" . }}
|
||||
labels:
|
||||
tier: airflow
|
||||
component: config
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
data:
|
||||
# These are system-specified config overrides.
|
||||
airflow.cfg: |
|
||||
[core]
|
||||
load_examples = False
|
||||
colored_console_log = False
|
||||
executor = {{ .Values.executor }}
|
||||
{{- if .Values.elasticsearch.enabled }}
|
||||
remote_logging = True
|
||||
{{- end }}
|
||||
|
||||
[webserver]
|
||||
enable_proxy_fix = True
|
||||
expose_config = True
|
||||
rbac = True
|
||||
|
||||
{{- if eq .Values.executor "CeleryExecutor" }}
|
||||
[celery]
|
||||
default_queue = celery
|
||||
{{- end }}
|
||||
|
||||
[scheduler]
|
||||
scheduler_heartbeat_sec = 5
|
||||
{{- if .Values.statsd.enabled }}
|
||||
statsd_on = True
|
||||
statsd_port = 9125
|
||||
statsd_prefix = airflow
|
||||
statsd_host = {{ printf "%s-statsd" .Release.Name }}
|
||||
{{- end }}
|
||||
# Restart Scheduler every 41460 seconds (11 hours 31 minutes)
|
||||
# The odd time is chosen so it is not always restarting on the same "hour" boundary
|
||||
run_duration = 41460
|
||||
|
||||
{{- if .Values.elasticsearch.enabled }}
|
||||
[elasticsearch]
|
||||
# The elasticsearch variables were updated to the shorter names in v1.10.4
|
||||
write_stdout = True
|
||||
json_format = True
|
||||
log_id_template = {dag_id}_{task_id}_{execution_date}_{try_number}
|
||||
|
||||
[elasticsearch_configs]
|
||||
max_retries = 3
|
||||
timeout = 30
|
||||
retry_timeout = True
|
||||
{{- end }}
|
||||
|
||||
{{- if eq .Values.executor "KubernetesExecutor" }}
|
||||
[kubernetes]
|
||||
namespace = {{ .Release.Namespace }}
|
||||
airflow_configmap = {{ include "airflow_config" . }}
|
||||
airflow_local_settings_configmap = {{ include "airflow_config" . }}
|
||||
worker_container_repository = {{ .Values.images.airflow.repository }}
|
||||
worker_container_tag = {{ .Values.images.airflow.tag }}
|
||||
worker_container_image_pull_policy = {{ .Values.images.airflow.pullPolicy }}
|
||||
worker_service_account_name = {{ .Release.Name }}-worker-serviceaccount
|
||||
image_pull_secrets = {{ template "registry_secret" . }}
|
||||
dags_in_image = True
|
||||
delete_worker_pods = True
|
||||
|
||||
[kubernetes_secrets]
|
||||
AIRFLOW__CORE__SQL_ALCHEMY_CONN = {{ printf "%s=connection" (include "airflow_metadata_secret" .) }}
|
||||
AIRFLOW__CORE__FERNET_KEY = {{ printf "%s=fernet-key" (include "fernet_key_secret" .) }}
|
||||
{{- if .Values.elasticsearch.enabled }}
|
||||
AIRFLOW__ELASTICSEARCH__ELASTICSEARCH_HOST = {{ printf "%s=connection" (include "elasticsearch_secret" .) }}
|
||||
{{- end }}
|
||||
|
||||
[kubernetes_labels]
|
||||
tier = airflow
|
||||
component = worker
|
||||
release = {{ .Release.Name }}
|
||||
{{- range $key, $value := .Values.labels }}
|
||||
{{ printf "%s = %s" $key $value }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- if .Values.webserver.webserverConfig }}
|
||||
webserver_config.py: |
|
||||
{{- .Values.webserver.webserverConfig | nindent 4 }}
|
||||
{{- end }}
|
||||
|
||||
{{- if .Values.scheduler.airflowLocalSettings }}
|
||||
airflow_local_settings.py: |
|
||||
{{ .Values.scheduler.airflowLocalSettings | nindent 4 }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Create User Job
|
||||
#################################
|
||||
{{- if .Values.webserver.defaultUser.enabled }}
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-create-user
|
||||
labels:
|
||||
tier: airflow
|
||||
component: create-user-job
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
"helm.sh/hook": post-install
|
||||
"helm.sh/hook-delete-policy": hook-succeeded
|
||||
spec:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
tier: airflow
|
||||
component: create-user-job
|
||||
release: {{ .Release.Name }}
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
nodeSelector:
|
||||
{{ toYaml .Values.nodeSelector | indent 8 }}
|
||||
affinity:
|
||||
{{ toYaml .Values.affinity | indent 8 }}
|
||||
tolerations:
|
||||
{{ toYaml .Values.tolerations | indent 8 }}
|
||||
containers:
|
||||
- name: create-user
|
||||
image: {{ template "airflow_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
args:
|
||||
- "bash"
|
||||
- "-c"
|
||||
# Support running against 1.10.x and 2.0.0dev/master
|
||||
- 'airflow create_user "$@" || airflow users create "$@"'
|
||||
- --
|
||||
- "-r"
|
||||
- {{ .Values.webserver.defaultUser.role }}
|
||||
- "-u"
|
||||
- {{ .Values.webserver.defaultUser.username }}
|
||||
- "-e"
|
||||
- {{ .Values.webserver.defaultUser.email }}
|
||||
- "-f"
|
||||
- {{ .Values.webserver.defaultUser.firstName }}
|
||||
- "-l"
|
||||
- {{ .Values.webserver.defaultUser.lastName }}
|
||||
- "-p"
|
||||
- {{ .Values.webserver.defaultUser.password }}
|
||||
env:
|
||||
{{- include "custom_airflow_environment" . | indent 10 }}
|
||||
{{- include "standard_airflow_environment" . | indent 10 }}
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: {{ template "airflow_config_path" . }}
|
||||
subPath: airflow.cfg
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: {{ template "airflow_config" . }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,102 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Flower Deployment
|
||||
#################################
|
||||
{{- if eq .Values.executor "CeleryExecutor" }}
|
||||
kind: Deployment
|
||||
apiVersion: apps/v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-flower
|
||||
labels:
|
||||
tier: airflow
|
||||
component: flower
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
checksum/airflow-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: flower
|
||||
release: {{ .Release.Name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
tier: airflow
|
||||
component: flower
|
||||
release: {{ .Release.Name }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
nodeSelector:
|
||||
{{ toYaml .Values.nodeSelector | indent 8 }}
|
||||
affinity:
|
||||
{{ toYaml .Values.affinity | indent 8 }}
|
||||
tolerations:
|
||||
{{ toYaml .Values.tolerations | indent 8 }}
|
||||
restartPolicy: Always
|
||||
securityContext:
|
||||
runAsUser: {{ .Values.uid }}
|
||||
{{- if or .Values.registry.secretName .Values.registry.connection }}
|
||||
imagePullSecrets:
|
||||
- name: {{ template "registry_secret" . }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: flower
|
||||
image: {{ template "flower_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.flower.pullPolicy }}
|
||||
args: ["flower"]
|
||||
resources:
|
||||
{{ toYaml .Values.flower.resources | indent 12 }}
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: {{ template "airflow_config_path" . }}
|
||||
subPath: airflow.cfg
|
||||
readOnly: true
|
||||
ports:
|
||||
- name: flower-ui
|
||||
containerPort: {{ .Values.ports.flowerUI }}
|
||||
livenessProbe:
|
||||
failureThreshold: 10
|
||||
httpGet:
|
||||
path: /
|
||||
port: {{ .Values.ports.flowerUI }}
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 5
|
||||
readinessProbe:
|
||||
failureThreshold: 10
|
||||
httpGet:
|
||||
path: /
|
||||
port: {{ .Values.ports.flowerUI }}
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 5
|
||||
env:
|
||||
{{- include "standard_airflow_environment" . | indent 10 }}
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: {{ template "airflow_config" . }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Flower NetworkPolicy
|
||||
#################################
|
||||
{{- if (and .Values.networkPolicies.enabled (eq .Values.executor "CeleryExecutor")) }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-flower-policy
|
||||
labels:
|
||||
tier: airflow
|
||||
component: airflow-flower-policy
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: flower
|
||||
release: {{ .Release.Name }}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
{{- if .Values.flower.extraNetworkPolicies }}
|
||||
{{ toYaml .Values.flower.extraNetworkPolicies | indent 4 }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: {{ .Values.ports.flowerUI }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Flower Service Component
|
||||
#################################
|
||||
{{- if eq .Values.executor "CeleryExecutor" }}
|
||||
kind: Service
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-flower
|
||||
labels:
|
||||
tier: airflow
|
||||
component: flower
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
type: {{ .Values.flower.service.type }}
|
||||
selector:
|
||||
tier: airflow
|
||||
component: flower
|
||||
release: {{ .Release.Name }}
|
||||
ports:
|
||||
- name: flower-ui
|
||||
protocol: TCP
|
||||
port: {{ .Values.ports.flowerUI }}
|
||||
targetPort: {{ .Values.ports.flowerUI }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Namespace LimitRange
|
||||
#################################
|
||||
{{- if .Values.limits }}
|
||||
apiVersion: v1
|
||||
kind: LimitRange
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-limit-range
|
||||
labels:
|
||||
tier: resources
|
||||
component: limitrange
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
limits:
|
||||
{{ toYaml .Values.limits | indent 4 }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,128 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Pgbouncer Deployment
|
||||
#################################
|
||||
{{- if .Values.pgbouncer.enabled }}
|
||||
kind: Deployment
|
||||
apiVersion: apps/v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-pgbouncer
|
||||
labels:
|
||||
tier: airflow
|
||||
component: pgbouncer
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 0
|
||||
selector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: pgbouncer
|
||||
release: {{ .Release.Name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
tier: airflow
|
||||
component: pgbouncer
|
||||
release: {{ .Release.Name }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }}
|
||||
spec:
|
||||
nodeSelector:
|
||||
{{ toYaml .Values.nodeSelector | indent 8 }}
|
||||
affinity:
|
||||
{{ toYaml .Values.affinity | indent 8 }}
|
||||
tolerations:
|
||||
{{ toYaml .Values.tolerations | indent 8 }}
|
||||
restartPolicy: Always
|
||||
containers:
|
||||
- name: pgbouncer
|
||||
image: {{ template "pgbouncer_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.pgbouncer.pullPolicy }}
|
||||
command:
|
||||
- pgbouncer
|
||||
- -u
|
||||
- pgbouncer
|
||||
- /etc/pgbouncer/pgbouncer.ini
|
||||
resources:
|
||||
{{ toYaml .Values.pgbouncer.resources | indent 12 }}
|
||||
ports:
|
||||
- name: pgbouncer
|
||||
containerPort: {{ .Values.ports.pgbouncer }}
|
||||
livenessProbe:
|
||||
tcpSocket:
|
||||
port: {{ .Values.ports.pgbouncer }}
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: {{ .Values.ports.pgbouncer }}
|
||||
volumeMounts:
|
||||
- name: pgbouncer-config
|
||||
subPath: pgbouncer.ini
|
||||
mountPath: /etc/pgbouncer/pgbouncer.ini
|
||||
readOnly: true
|
||||
- name: pgbouncer-config
|
||||
subPath: users.txt
|
||||
mountPath: /etc/pgbouncer/users.txt
|
||||
readOnly: true
|
||||
lifecycle:
|
||||
preStop:
|
||||
exec:
|
||||
# Allow existing queries clients to complete within 120 seconds
|
||||
command: ["/bin/sh", "-c", "killall -INT pgbouncer && sleep 120"]
|
||||
- name: metrics-exporter
|
||||
image: {{ template "pgbouncer_exporter_image" . }}
|
||||
env:
|
||||
- name: DATABASE_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ template "pgbouncer_stats_secret" . }}
|
||||
key: connection
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: {{ .Values.ports.pgbouncerScrape }}
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- pgbouncer_exporter
|
||||
- health
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- pgbouncer_exporter
|
||||
- health
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
volumes:
|
||||
- name: pgbouncer-config
|
||||
secret:
|
||||
secretName: {{ template "pgbouncer_config_secret" . }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Pgbouncer NetworkPolicy
|
||||
#################################
|
||||
{{- if and .Values.pgbouncer.enabled .Values.networkPolicies.enabled }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-pgbouncer-policy
|
||||
labels:
|
||||
tier: airflow
|
||||
component: airflow-pgbouncer-policy
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: pgbouncer
|
||||
release: {{ .Release.Name }}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
{{- if .Values.workers.keda.enabled }}
|
||||
{{- if .Values.workers.keda.namespaceLabels }}
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
{{ toYaml .Values.workers.keda.namespaceLabels | indent 10}}
|
||||
podSelector:
|
||||
{{- else }}
|
||||
- podSelector:
|
||||
{{- end }}
|
||||
matchLabels:
|
||||
app: keda-operator
|
||||
{{- end}}
|
||||
{{- if .Values.pgbouncer.extraNetworkPolicies}}
|
||||
{{ toYaml .Values.pgbouncer.extraNetworkPolicies | indent 4}}
|
||||
{{- end}}
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: {{ .Values.ports.pgbouncer }}
|
||||
- protocol: TCP
|
||||
port: {{ .Values.ports.pgbouncerScrape }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Pgbouncer PodDisruptionBudget
|
||||
#################################
|
||||
{{- if and .Values.pgbouncer.enabled .Values.pgbouncer.podDisruptionBudget.enabled }}
|
||||
kind: PodDisruptionBudget
|
||||
apiVersion: policy/v1beta1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-pgbouncer-pdb
|
||||
labels:
|
||||
tier: airflow
|
||||
component: pgbouncer
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: pgbouncer
|
||||
release: {{ .Release.Name }}
|
||||
{{ toYaml .Values.pgbouncer.podDisruptionBudget.config | indent 2 }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Pgbouncer Service
|
||||
#################################
|
||||
{{- if .Values.pgbouncer.enabled }}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-pgbouncer
|
||||
labels:
|
||||
tier: airflow
|
||||
component: pgbouncer
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: {{ .Values.ports.pgbouncerScrape | quote }}
|
||||
{{- if .Values.pgbouncer.service.extraAnnotations }}
|
||||
{{- toYaml .Values.pgbouncer.service.extraAnnotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
tier: airflow
|
||||
component: pgbouncer
|
||||
release: {{ .Release.Name }}
|
||||
ports:
|
||||
- name: pgbouncer
|
||||
protocol: TCP
|
||||
port: {{ .Values.ports.pgbouncer }}
|
||||
targetPort: {{ .Values.ports.pgbouncer }}
|
||||
- name: pgbouncer-metrics
|
||||
protocol: TCP
|
||||
port: {{ .Values.ports.pgbouncerScrape }}
|
||||
targetPort: {{ .Values.ports.pgbouncerScrape }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Cleanup Role
|
||||
#################################
|
||||
{{- if and .Values.rbacEnabled .Values.cleanup.enabled }}
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-cleanup-role
|
||||
labels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- "pods"
|
||||
verbs:
|
||||
- "list"
|
||||
- "delete"
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Cleanup Role Binding
|
||||
#################################
|
||||
{{- if and .Values.rbacEnabled .Values.cleanup.enabled }}
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-cleanup-rolebinding
|
||||
labels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: {{ .Release.Name }}-cleanup-role
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ .Release.Name }}-cleanup-serviceaccount
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Pod Launcher Role
|
||||
#################################
|
||||
{{- if and .Values.rbacEnabled .Values.allowPodLaunching }}
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-pod-launcher-role
|
||||
labels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- "pods"
|
||||
verbs:
|
||||
- "create"
|
||||
- "list"
|
||||
- "get"
|
||||
- "watch"
|
||||
- "delete"
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- "pods/log"
|
||||
verbs:
|
||||
- "get"
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- "pods/exec"
|
||||
verbs:
|
||||
- "create"
|
||||
- "get"
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Pod Launcher Role Binding
|
||||
#################################
|
||||
{{- if and .Values.rbacEnabled .Values.allowPodLaunching }}
|
||||
{{- $grantScheduler := or (eq .Values.executor "LocalExecutor") (eq .Values.executor "SequentialExecutor") (eq .Values.executor "KubernetesExecutor") }}
|
||||
{{- $grantWorker := or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "KubernetesExecutor") }}
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-pod-launcher-rolebinding
|
||||
labels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: {{ .Release.Name }}-pod-launcher-role
|
||||
subjects:
|
||||
{{- if $grantScheduler }}
|
||||
- kind: ServiceAccount
|
||||
name: {{ .Release.Name }}-scheduler-serviceaccount
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- end }}
|
||||
{{- if $grantWorker }}
|
||||
- kind: ServiceAccount
|
||||
name: {{ .Release.Name }}-worker-serviceaccount
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Redis NetworkPolicy
|
||||
#################################
|
||||
{{- if (and .Values.networkPolicies.enabled (eq .Values.executor "CeleryExecutor")) }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-redis-policy
|
||||
labels:
|
||||
tier: airflow
|
||||
component: redis-policy
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: redis
|
||||
release: {{ .Release.Name }}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: worker
|
||||
release: {{ .Release.Name }}
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: scheduler
|
||||
release: {{ .Release.Name }}
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: flower
|
||||
release: {{ .Release.Name }}
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: {{ .Values.ports.redisDB }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Redis Service
|
||||
#################################
|
||||
{{- if eq .Values.executor "CeleryExecutor" }}
|
||||
kind: Service
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-redis
|
||||
labels:
|
||||
tier: airflow
|
||||
component: redis
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
tier: airflow
|
||||
component: redis
|
||||
release: {{ .Release.Name }}
|
||||
ports:
|
||||
- name: redis-db
|
||||
protocol: TCP
|
||||
port: {{ .Values.ports.redisDB }}
|
||||
targetPort: {{ .Values.ports.redisDB }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Redis StatefulSet
|
||||
#################################
|
||||
{{- if eq .Values.executor "CeleryExecutor" }}
|
||||
kind: StatefulSet
|
||||
apiVersion: apps/v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-redis
|
||||
labels:
|
||||
tier: airflow
|
||||
component: redis
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
serviceName: {{ .Release.Name }}-redis
|
||||
selector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: redis
|
||||
release: {{ .Release.Name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
tier: airflow
|
||||
component: redis
|
||||
release: {{ .Release.Name }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
{{- if .Values.redis.safeToEvict }}
|
||||
cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
|
||||
{{- end }}
|
||||
spec:
|
||||
nodeSelector:
|
||||
{{ toYaml .Values.nodeSelector | indent 8 }}
|
||||
affinity:
|
||||
{{ toYaml .Values.affinity | indent 8 }}
|
||||
tolerations:
|
||||
{{ toYaml .Values.tolerations | indent 8 }}
|
||||
containers:
|
||||
- name: redis
|
||||
image: {{- include "redis_image" . | indent 1 }}
|
||||
imagePullPolicy: {{ .Values.images.redis.pullPolicy }}
|
||||
command: ["/bin/sh"]
|
||||
resources:
|
||||
{{ toYaml .Values.redis.resources | indent 12 }}
|
||||
args: ["-c", "redis-server --requirepass ${REDIS_PASSWORD}"]
|
||||
ports:
|
||||
- name: redis-db
|
||||
containerPort: {{ .Values.ports.redisDB }}
|
||||
volumeMounts:
|
||||
- name: redis-db
|
||||
mountPath: /data
|
||||
env:
|
||||
- name: REDIS_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ template "redis_password_secret" . }}
|
||||
key: password
|
||||
{{- if not .Values.redis.persistence.enabled }}
|
||||
volumes:
|
||||
- name: redis-db
|
||||
emptyDir: {}
|
||||
{{- else }}
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: redis-db
|
||||
spec:
|
||||
{{- if .Values.redis.persistence.storageClassName }}
|
||||
storageClassName: {{ .Values.redis.persistence.storageClassName }}
|
||||
{{- end }}
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.redis.persistence.size }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Namespace ResourceQuota
|
||||
#################################
|
||||
{{- if .Values.quotas }}
|
||||
apiVersion: v1
|
||||
kind: ResourceQuota
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-resource-quota
|
||||
labels:
|
||||
tier: resources
|
||||
component: resourcequota
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
hard:
|
||||
{{ toYaml .Values.quotas | indent 4 }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,195 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Scheduler Deployment/StatefulSet
|
||||
#################################
|
||||
|
||||
# Are we using a local/sequenial executor?
|
||||
{{- $local := or (eq .Values.executor "LocalExecutor") (eq .Values.executor "SequentialExecutor") }}
|
||||
# Are we using the kubernetes executor?
|
||||
{{- $kube := eq .Values.executor "KubernetesExecutor" }}
|
||||
# Is persistence enabled on the _workers_?
|
||||
# This is important because in $local mode, the scheduler assumes the role of the worker
|
||||
{{- $persistence := .Values.workers.persistence.enabled }}
|
||||
# If we're using a StatefulSet
|
||||
{{- $stateful := and $local $persistence }}
|
||||
# If we're using elasticsearch logging
|
||||
{{- $elasticsearch := .Values.elasticsearch.enabled }}
|
||||
|
||||
kind: {{ if $stateful }}StatefulSet{{ else }}Deployment{{ end }}
|
||||
apiVersion: apps/v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-scheduler
|
||||
labels:
|
||||
tier: airflow
|
||||
component: scheduler
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if $stateful }}
|
||||
serviceName: {{ .Release.Name }}-scheduler
|
||||
{{- end }}
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: scheduler
|
||||
release: {{ .Release.Name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
tier: airflow
|
||||
component: scheduler
|
||||
release: {{ .Release.Name }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }}
|
||||
checksum/result-backend-secret: {{ include (print $.Template.BasePath "/secrets/result-backend-connection-secret.yaml") . | sha256sum }}
|
||||
checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }}
|
||||
checksum/airflow-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
||||
{{- if .Values.scheduler.safeToEvict }}
|
||||
cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
|
||||
{{- end }}
|
||||
{{- if .Values.airflowPodAnnotations }}
|
||||
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
nodeSelector:
|
||||
{{ toYaml .Values.nodeSelector | indent 8 }}
|
||||
affinity:
|
||||
{{ toYaml .Values.affinity | indent 8 }}
|
||||
tolerations:
|
||||
{{ toYaml .Values.tolerations | indent 8 }}
|
||||
restartPolicy: Always
|
||||
terminationGracePeriodSeconds: 10
|
||||
serviceAccountName: {{ .Release.Name }}-scheduler-serviceaccount
|
||||
securityContext:
|
||||
runAsUser: {{ .Values.uid }}
|
||||
fsGroup: {{ .Values.gid }}
|
||||
{{- if or .Values.registry.secretName .Values.registry.connection }}
|
||||
imagePullSecrets:
|
||||
- name: {{ template "registry_secret" . }}
|
||||
{{- end }}
|
||||
initContainers:
|
||||
- name: run-airflow-migrations
|
||||
image: {{ template "airflow_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
# Support running against 1.10.x and 2.0.0dev/master
|
||||
args: ["bash", "-c", "airflow upgradedb || airfow db upgrade"]
|
||||
env:
|
||||
{{- include "custom_airflow_environment" . | indent 10 }}
|
||||
{{- include "standard_airflow_environment" . | indent 10 }}
|
||||
containers:
|
||||
# Always run the main scheduler container.
|
||||
- name: scheduler
|
||||
image: {{ template "airflow_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
args: ["scheduler"]
|
||||
env:
|
||||
{{- include "custom_airflow_environment" . | indent 10 }}
|
||||
{{- include "standard_airflow_environment" . | indent 10 }}
|
||||
# If the scheduler stops heartbeating for 5 minutes (10*30s) kill the
|
||||
# scheduler and let Kubernetes restart it
|
||||
livenessProbe:
|
||||
failureThreshold: 10
|
||||
periodSeconds: 30
|
||||
exec:
|
||||
command:
|
||||
- python
|
||||
- -Wignore
|
||||
- -c
|
||||
- |
|
||||
import os
|
||||
os.environ['AIRFLOW__CORE__LOGGING_LEVEL'] = 'ERROR'
|
||||
os.environ['AIRFLOW__LOGGING__LOGGING_LEVEL'] = 'ERROR'
|
||||
|
||||
from airflow.jobs.scheduler_job import SchedulerJob
|
||||
from airflow.utils.net import get_hostname
|
||||
import sys
|
||||
|
||||
job = SchedulerJob.most_recent_job()
|
||||
sys.exit(0 if job.is_alive() and job.hostname == get_hostname() else 1)
|
||||
resources:
|
||||
{{ toYaml .Values.scheduler.resources | indent 12 }}
|
||||
volumeMounts:
|
||||
- name: logs
|
||||
mountPath: {{ template "airflow_logs" . }}
|
||||
- name: config
|
||||
mountPath: {{ template "airflow_config_path" . }}
|
||||
subPath: airflow.cfg
|
||||
readOnly: true
|
||||
{{- if .Values.scheduler.airflowLocalSettings }}
|
||||
- name: config
|
||||
mountPath: {{ template "airflow_local_setting_path" . }}
|
||||
subPath: airflow_local_settings.py
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
# Always start the garbage collector sidecar.
|
||||
- name: scheduler-gc
|
||||
image: {{ template "airflow_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
args: ["bash", "/clean-logs"]
|
||||
volumeMounts:
|
||||
- name: logs
|
||||
mountPath: {{ template "airflow_logs" . }}
|
||||
{{- if and $local (not $elasticsearch) }}
|
||||
# Start the sidecar log server if we're in local mode and
|
||||
# we don't have elasticsearch enabled.
|
||||
- name: scheduler-logs
|
||||
image: {{ template "airflow_image" . }}
|
||||
args: ["airflow", "serve_logs"]
|
||||
ports:
|
||||
- name: worker-logs
|
||||
containerPort: {{ .Values.ports.workerLogs }}
|
||||
volumeMounts:
|
||||
- name: logs
|
||||
mountPath: {{ template "airflow_logs" . }}
|
||||
- name: config
|
||||
mountPath: {{ template "airflow_config_path" . }}
|
||||
subPath: airflow.cfg
|
||||
readOnly: true
|
||||
env:
|
||||
{{- include "custom_airflow_environment" . | indent 10 }}
|
||||
{{- include "standard_airflow_environment" . | indent 10 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: {{ template "airflow_config" . }}
|
||||
{{- if not $stateful }}
|
||||
- name: logs
|
||||
emptyDir: {}
|
||||
{{- else }}
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: logs
|
||||
spec:
|
||||
{{- if .Values.workers.persistence.storageClassName }}
|
||||
storageClassName: {{ .Values.workers.persistence.storageClassName }}
|
||||
{{- end }}
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.workers.persistence.size }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Scheduler NetworkPolicy
|
||||
#################################
|
||||
{{- if .Values.networkPolicies.enabled }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-scheduler-policy
|
||||
labels:
|
||||
tier: airflow
|
||||
component: airflow-scheduler-policy
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: scheduler
|
||||
release: {{ .Release.Name }}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
{{- if (or (eq .Values.executor "LocalExecutor") (eq .Values.executor "SequentialExecutor")) }}
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: webserver
|
||||
release: {{ .Release.Name }}
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: {{ .Values.ports.workerLogs }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Scheduler PodDisruptionBudget
|
||||
#################################
|
||||
{{- if .Values.scheduler.podDisruptionBudget.enabled }}
|
||||
kind: PodDisruptionBudget
|
||||
apiVersion: policy/v1beta1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-scheduler-pdb
|
||||
labels:
|
||||
tier: airflow
|
||||
component: scheduler
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: scheduler
|
||||
release: {{ .Release.Name }}
|
||||
{{ toYaml .Values.scheduler.podDisruptionBudget.config | indent 2 }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Scheduler Service
|
||||
#################################
|
||||
{{- if (or (eq .Values.executor "LocalExecutor") (eq .Values.executor "SequentialExecutor")) }}
|
||||
kind: Service
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-scheduler
|
||||
labels:
|
||||
tier: airflow
|
||||
component: scheduler
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
clusterIP: None
|
||||
selector:
|
||||
tier: airflow
|
||||
component: scheduler
|
||||
release: {{ .Release.Name }}
|
||||
ports:
|
||||
- name: task-logs
|
||||
protocol: TCP
|
||||
port: {{ .Values.ports.workerLogs }}
|
||||
targetPort: {{ .Values.ports.workerLogs }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Scheduler ServiceAccount
|
||||
#################################
|
||||
{{- if .Values.rbacEnabled }}
|
||||
kind: ServiceAccount
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-scheduler-serviceaccount
|
||||
labels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Elasticsearch Secret
|
||||
#################################
|
||||
{{- if (and .Values.elasticsearch.connection (not .Values.elasticsearch.secretName)) }}
|
||||
kind: Secret
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-elasticsearch
|
||||
labels:
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
type: Opaque
|
||||
data:
|
||||
connection: {{ (printf "http://%s:%s@%s:%s" .Values.elasticsearch.connection.user .Values.elasticsearch.connection.pass .Values.elasticsearch.connection.host (.Values.elasticsearch.connection.port | toString)) | b64enc | quote }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Fernet Key Secret
|
||||
#################################
|
||||
{{- if not .Values.fernetKeySecretName }}
|
||||
{{ $generated_fernet_key := (randAlphaNum 32 | b64enc) }}
|
||||
kind: Secret
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-fernet-key
|
||||
labels:
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
"helm.sh/hook": "pre-install"
|
||||
"helm.sh/hook-delete-policy": "before-hook-creation"
|
||||
"helm.sh/hook-weight": "0"
|
||||
type: Opaque
|
||||
data:
|
||||
fernet-key: {{ (default $generated_fernet_key .Values.fernetKey) | b64enc | quote }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Metadata Secret
|
||||
#################################
|
||||
{{- if (and .Values.data.metadataConnection (not .Values.data.metadataSecretName)) }}
|
||||
{{- $postgresHost := .Values.data.metadataConnection.host | default (printf "%s-%s.%s.svc.cluster.local" .Release.Name "postgresql" .Release.Namespace) }}
|
||||
{{- $pgbouncerHost := (printf "%s-%s.%s.svc.cluster.local" .Release.Name "pgbouncer" .Release.Namespace) }}
|
||||
{{- $host := ternary $pgbouncerHost $postgresHost .Values.pgbouncer.enabled }}
|
||||
{{- $port := ((ternary .Values.ports.pgbouncer .Values.data.metadataConnection.port .Values.pgbouncer.enabled) | toString) }}
|
||||
{{- $database := (ternary (printf "%s-%s" .Release.Name "metadata") .Values.data.metadataConnection.db .Values.pgbouncer.enabled) }}
|
||||
|
||||
kind: Secret
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-airflow-metadata
|
||||
labels:
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
type: Opaque
|
||||
data:
|
||||
connection: {{ (printf "postgresql://%s:%s@%s:%s/%s?sslmode=%s" .Values.data.metadataConnection.user .Values.data.metadataConnection.pass $host $port $database .Values.data.metadataConnection.sslmode) | b64enc | quote }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Pgbouncer Config Secret
|
||||
#################################
|
||||
{{- if .Values.pgbouncer.enabled }}
|
||||
kind: Secret
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: {{ template "pgbouncer_config_secret" . }}
|
||||
labels:
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
type: Opaque
|
||||
data:
|
||||
pgbouncer.ini: {{ include "pgbouncer_config" . | b64enc }}
|
||||
users.txt: {{ include "pgbouncer_users" . | b64enc }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Pgbouncer Stats Secret
|
||||
#################################
|
||||
{{- if .Values.pgbouncer.enabled }}
|
||||
kind: Secret
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: {{ template "pgbouncer_stats_secret" . }}
|
||||
labels:
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
type: Opaque
|
||||
data:
|
||||
connection: {{ (printf "postgresql://%s:%s@127.0.0.1:%s/pgbouncer?sslmode=disable" .Values.data.metadataConnection.user .Values.data.metadataConnection.pass (.Values.ports.pgbouncer | toString)) | b64enc | quote }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Redis Password Secret
|
||||
#################################
|
||||
# If both of these secret names are not set, we will either use the set password, or generate one
|
||||
{{- if not (and .Values.redis.passwordSecretName .Values.redis.brokerURLSecretName) }}
|
||||
{{ $random_redis_password := randAlphaNum 10 }}
|
||||
kind: Secret
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-redis-password
|
||||
labels:
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
"helm.sh/hook": "pre-install"
|
||||
"helm.sh/hook-delete-policy": "before-hook-creation"
|
||||
"helm.sh/hook-weight": "0"
|
||||
type: Opaque
|
||||
data:
|
||||
password: {{ (default $random_redis_password .Values.redis.password) | b64enc | quote }}
|
||||
---
|
||||
################################
|
||||
## Airflow Redis Connection Secret
|
||||
#################################
|
||||
kind: Secret
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-broker-url
|
||||
labels:
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
annotations:
|
||||
"helm.sh/hook": "pre-install"
|
||||
"helm.sh/hook-delete-policy": "before-hook-creation"
|
||||
"helm.sh/hook-weight": "0"
|
||||
type: Opaque
|
||||
data:
|
||||
connection: {{ (printf "redis://:%s@%s-redis:6379/0" (default $random_redis_password .Values.redis.password) .Release.Name) | b64enc | quote }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Registry Secret
|
||||
#################################
|
||||
{{- if (and .Values.registry.connection (not .Values.registry.secretName)) }}
|
||||
kind: Secret
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-registry
|
||||
labels:
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
type: kubernetes.io/dockerconfigjson
|
||||
data:
|
||||
.dockerconfigjson: {{ include "registry_docker_config" . | b64enc }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Result Backend Secret
|
||||
#################################
|
||||
{{- if (and .Values.data.resultBackendConnection (not .Values.data.resultBackendSecretName)) }}
|
||||
{{- $host := .Values.data.resultBackendConnection.host | default (printf "%s-%s" .Release.Name "postgresql") }}
|
||||
kind: Secret
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-airflow-result-backend
|
||||
labels:
|
||||
release: {{ .Release.Name }}
|
||||
chart: {{ .Chart.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
type: Opaque
|
||||
data:
|
||||
connection: {{ (printf "db+postgresql://%s:%s@%s:%s/%s?sslmode=%s" .Values.data.resultBackendConnection.user .Values.data.resultBackendConnection.pass (ternary (printf "%s-%s" .Release.Name "pgbouncer") $host .Values.pgbouncer.enabled) ((ternary .Values.ports.pgbouncer .Values.data.resultBackendConnection.port .Values.pgbouncer.enabled) | toString) (ternary (printf "%s-%s" .Release.Name "result-backend") .Values.data.resultBackendConnection.db .Values.pgbouncer.enabled) .Values.data.resultBackendConnection.sslmode) | b64enc | quote }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow StatsD Deployment
|
||||
#################################
|
||||
{{- if .Values.statsd.enabled }}
|
||||
kind: Deployment
|
||||
apiVersion: apps/v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-statsd
|
||||
labels:
|
||||
tier: airflow
|
||||
component: statsd
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: statsd
|
||||
release: {{ .Release.Name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
tier: airflow
|
||||
component: statsd
|
||||
release: {{ .Release.Name }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
nodeSelector:
|
||||
{{ toYaml .Values.nodeSelector | indent 8 }}
|
||||
affinity:
|
||||
{{ toYaml .Values.affinity | indent 8 }}
|
||||
tolerations:
|
||||
{{ toYaml .Values.tolerations | indent 8 }}
|
||||
restartPolicy: Always
|
||||
containers:
|
||||
- name: statsd
|
||||
image: {{- include "statsd_image" . | indent 1 }}
|
||||
imagePullPolicy: {{ .Values.images.statsd.pullPolicy }}
|
||||
args:
|
||||
- "-statsd.mapping-config=/etc/statsd-exporter/mappings.yml"
|
||||
resources:
|
||||
{{ toYaml .Values.statsd.resources | indent 12 }}
|
||||
ports:
|
||||
- name: statsd-ingest
|
||||
protocol: UDP
|
||||
containerPort: {{ .Values.ports.statsdIngest }}
|
||||
- name: statsd-scrape
|
||||
containerPort: {{ .Values.ports.statsdScrape }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /metrics
|
||||
port: {{ .Values.ports.statsdScrape }}
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /metrics
|
||||
port: {{ .Values.ports.statsdScrape }}
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow StatsD NetworkPolicy
|
||||
#################################
|
||||
{{- if and .Values.networkPolicies.enabled .Values.statsd.enabled }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-statsd-policy
|
||||
labels:
|
||||
tier: airflow
|
||||
component: statsd-policy
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: statsd
|
||||
release: {{ .Release.Name }}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
{{- if .Values.statsd.extraNetworkPolicies }}
|
||||
{{ toYaml .Values.statsd.extraNetworkPolicies | indent 4 }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- protocol: UDP
|
||||
port: {{ .Values.ports.statsdIngest }}
|
||||
- protocol: TCP
|
||||
port: {{ .Values.ports.statsdScrape }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow StatsD Service
|
||||
#################################
|
||||
{{- if .Values.statsd.enabled }}
|
||||
kind: Service
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-statsd
|
||||
labels:
|
||||
tier: airflow
|
||||
component: statsd
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: {{ .Values.ports.statsdScrape | quote }}
|
||||
{{- if .Values.statsd.service.extraAnnotations }}
|
||||
{{- toYaml .Values.statsd.service.extraAnnotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
tier: airflow
|
||||
component: statsd
|
||||
release: {{ .Release.Name }}
|
||||
ports:
|
||||
- name: statsd-ingest
|
||||
protocol: UDP
|
||||
port: {{ .Values.ports.statsdIngest }}
|
||||
targetPort: {{ .Values.ports.statsdIngest }}
|
||||
- name: statsd-scrape
|
||||
protocol: TCP
|
||||
port: {{ .Values.ports.statsdScrape }}
|
||||
targetPort: {{ .Values.ports.statsdScrape }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,139 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Webserver Deployment
|
||||
#################################
|
||||
kind: Deployment
|
||||
apiVersion: apps/v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-webserver
|
||||
labels:
|
||||
tier: airflow
|
||||
component: webserver
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
replicas: {{ .Values.webserver.replicas }}
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxUnavailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: webserver
|
||||
release: {{ .Release.Name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
tier: airflow
|
||||
component: webserver
|
||||
release: {{ .Release.Name }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }}
|
||||
checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }}
|
||||
checksum/airflow-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
||||
{{- if .Values.airflowPodAnnotations }}
|
||||
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
nodeSelector:
|
||||
{{ toYaml .Values.nodeSelector | indent 8 }}
|
||||
affinity:
|
||||
{{ toYaml .Values.affinity | indent 8 }}
|
||||
tolerations:
|
||||
{{ toYaml .Values.tolerations | indent 8 }}
|
||||
restartPolicy: Always
|
||||
securityContext:
|
||||
runAsUser: {{ .Values.uid }}
|
||||
{{- if or .Values.registry.secretName .Values.registry.connection }}
|
||||
imagePullSecrets:
|
||||
- name: {{ template "registry_secret" . }}
|
||||
{{- end }}
|
||||
initContainers:
|
||||
- name: wait-for-airflow-migrations
|
||||
image: {{ template "airflow_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
args:
|
||||
{{- include "wait-for-migrations-command" . | indent 10 }}
|
||||
env:
|
||||
{{- include "custom_airflow_environment" . | indent 10 }}
|
||||
{{- include "standard_airflow_environment" . | indent 10 }}
|
||||
containers:
|
||||
- name: webserver
|
||||
image: {{ template "airflow_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
args: ["webserver"]
|
||||
resources:
|
||||
{{ toYaml .Values.webserver.resources | indent 12 }}
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: {{ template "airflow_config_path" . }}
|
||||
subPath: airflow.cfg
|
||||
readOnly: true
|
||||
{{- if .Values.webserver.webserverConfig }}
|
||||
- name: config
|
||||
mountPath: {{ template "airflow_webserver_config_path" . }}
|
||||
subPath: webserver_config.py
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if .Values.scheduler.airflowLocalSettings }}
|
||||
- name: config
|
||||
mountPath: {{ template "airflow_local_setting_path" . }}
|
||||
subPath: airflow_local_settings.py
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if .Values.webserver.extraVolumeMounts }}
|
||||
{{ toYaml .Values.webserver.extraVolumeMounts | indent 12 }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- name: webserver-ui
|
||||
containerPort: {{ .Values.ports.airflowUI }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: {{ .Values.ports.airflowUI }}
|
||||
initialDelaySeconds: {{ .Values.webserver.livenessProbe.initialDelaySeconds | default 15 }}
|
||||
timeoutSeconds: {{ .Values.webserver.livenessProbe.timeoutSeconds | default 30 }}
|
||||
failureThreshold: {{ .Values.webserver.livenessProbe.failureThreshold | default 20 }}
|
||||
periodSeconds: {{ .Values.webserver.livenessProbe.periodSeconds | default 5 }}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: {{ .Values.ports.airflowUI }}
|
||||
initialDelaySeconds: {{ .Values.webserver.readinessProbe.initialDelaySeconds | default 15 }}
|
||||
timeoutSeconds: {{ .Values.webserver.readinessProbe.timeoutSeconds | default 30 }}
|
||||
failureThreshold: {{ .Values.webserver.readinessProbe.failureThreshold | default 20 }}
|
||||
periodSeconds: {{ .Values.webserver.readinessProbe.periodSeconds | default 5 }}
|
||||
env:
|
||||
{{- include "custom_airflow_environment" . | indent 10 }}
|
||||
{{- include "standard_airflow_environment" . | indent 10 }}
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: {{ template "airflow_config" . }}
|
||||
{{- if .Values.webserver.extraVolumes }}
|
||||
{{ toYaml .Values.webserver.extraVolumes | indent 8 }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Webserver NetworkPolicy
|
||||
#################################
|
||||
{{- if .Values.networkPolicies.enabled }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-webserver-policy
|
||||
labels:
|
||||
tier: airflow
|
||||
component: airflow-webserver-policy
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: webserver
|
||||
release: {{ .Release.Name }}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
{{- if .Values.webserver.extraNetworkPolicies }}
|
||||
{{ toYaml .Values.webserver.extraNetworkPolicies | indent 4 }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: {{ .Values.ports.airflowUI }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Webserver Service
|
||||
#################################
|
||||
kind: Service
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-webserver
|
||||
labels:
|
||||
tier: airflow
|
||||
component: webserver
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
type: {{ .Values.webserver.service.type }}
|
||||
selector:
|
||||
tier: airflow
|
||||
component: webserver
|
||||
release: {{ .Release.Name }}
|
||||
ports:
|
||||
- name: airflow-ui
|
||||
protocol: TCP
|
||||
port: {{ .Values.ports.airflowUI }}
|
||||
targetPort: {{ .Values.ports.airflowUI }}
|
||||
|
|
@ -0,0 +1,161 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Worker Deployment
|
||||
#################################
|
||||
{{- $persistence := .Values.workers.persistence.enabled }}
|
||||
{{- if eq .Values.executor "CeleryExecutor" }}
|
||||
kind: {{ if $persistence }}StatefulSet{{ else }}Deployment{{ end }}
|
||||
apiVersion: apps/v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-worker
|
||||
labels:
|
||||
tier: airflow
|
||||
component: worker
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if $persistence }}
|
||||
serviceName: {{ .Release.Name }}-worker
|
||||
{{- end }}
|
||||
replicas: {{ .Values.workers.replicas }}
|
||||
selector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: worker
|
||||
release: {{ .Release.Name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
tier: airflow
|
||||
component: worker
|
||||
release: {{ .Release.Name }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }}
|
||||
checksum/result-backend-secret: {{ include (print $.Template.BasePath "/secrets/result-backend-connection-secret.yaml") . | sha256sum }}
|
||||
checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }}
|
||||
checksum/airflow-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
||||
{{- if .Values.workers.safeToEvict }}
|
||||
cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
|
||||
{{- end }}
|
||||
{{- if .Values.airflowPodAnnotations }}
|
||||
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
nodeSelector:
|
||||
{{ toYaml .Values.nodeSelector | indent 8 }}
|
||||
affinity:
|
||||
{{ toYaml .Values.affinity | indent 8 }}
|
||||
tolerations:
|
||||
{{ toYaml .Values.tolerations | indent 8 }}
|
||||
terminationGracePeriodSeconds: {{ .Values.workers.terminationGracePeriodSeconds }}
|
||||
restartPolicy: Always
|
||||
serviceAccountName: {{ .Release.Name }}-worker-serviceaccount
|
||||
securityContext:
|
||||
runAsUser: {{ .Values.uid }}
|
||||
fsGroup: {{ .Values.gid }}
|
||||
{{- if or .Values.registry.secretName .Values.registry.connection }}
|
||||
imagePullSecrets:
|
||||
- name: {{ template "registry_secret" . }}
|
||||
{{- end }}
|
||||
initContainers:
|
||||
{{- if and $persistence .Values.workers.persistence.fixPermissions }}
|
||||
- name: volume-permissions
|
||||
image: {{ template "airflow_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
command:
|
||||
- chown
|
||||
- -R
|
||||
- "{{ .Values.uid }}:{{ .Values.gid }}"
|
||||
- {{ template "airflow_logs" . }}
|
||||
securityContext:
|
||||
runAsUser: 0
|
||||
volumeMounts:
|
||||
- name: logs
|
||||
mountPath: {{ template "airflow_logs" . }}
|
||||
{{- end }}
|
||||
- name: wait-for-airflow-migrations
|
||||
image: {{ template "airflow_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
args:
|
||||
{{- include "wait-for-migrations-command" . | indent 10 }}
|
||||
env:
|
||||
{{- include "custom_airflow_environment" . | indent 10 }}
|
||||
{{- include "standard_airflow_environment" . | indent 10 }}
|
||||
containers:
|
||||
- name: worker
|
||||
image: {{ template "airflow_image" . }}
|
||||
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
|
||||
args: ["worker"]
|
||||
resources:
|
||||
{{ toYaml .Values.workers.resources | indent 12 }}
|
||||
ports:
|
||||
- name: worker-logs
|
||||
containerPort: {{ .Values.ports.workerLogs }}
|
||||
volumeMounts:
|
||||
- name: logs
|
||||
mountPath: {{ template "airflow_logs" . }}
|
||||
- name: config
|
||||
mountPath: {{ template "airflow_config_path" . }}
|
||||
subPath: airflow.cfg
|
||||
readOnly: true
|
||||
{{- if .Values.scheduler.airflowLocalSettings }}
|
||||
- name: config
|
||||
mountPath: {{ template "airflow_local_setting_path" . }}
|
||||
subPath: airflow_local_settings.py
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
env:
|
||||
{{- include "custom_airflow_environment" . | indent 10 }}
|
||||
{{- include "standard_airflow_environment" . | indent 10 }}
|
||||
{{- if $persistence }}
|
||||
- name: worker-gc
|
||||
image: {{ template "airflow_image" . }}
|
||||
args: ["bash", "/clean-logs"]
|
||||
volumeMounts:
|
||||
- name: logs
|
||||
mountPath: {{ template "airflow_logs" . }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: {{ template "airflow_config" . }}
|
||||
{{- if not $persistence }}
|
||||
- name: logs
|
||||
emptyDir: {}
|
||||
{{- else }}
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: logs
|
||||
spec:
|
||||
{{- if .Values.workers.persistence.storageClassName }}
|
||||
storageClassName: {{ .Values.workers.persistence.storageClassName }}
|
||||
{{- end }}
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.workers.persistence.size }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Worker KEDA Scaler
|
||||
#################################
|
||||
{{- if (and .Values.workers.keda.enabled (eq .Values.executor "CeleryExecutor")) }}
|
||||
apiVersion: keda.k8s.io/v1alpha1
|
||||
kind: ScaledObject
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-worker
|
||||
labels:
|
||||
tier: airflow
|
||||
component: worker-horizontalpodautoscaler
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
deploymentName: {{ .Release.Name }}-worker
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
deploymentName: {{ .Release.Name }}-worker
|
||||
pollingInterval: {{ .Values.workers.keda.pollingInterval }} # Optional. Default: 30 seconds
|
||||
cooldownPeriod: {{ .Values.workers.keda.cooldownPeriod }} # Optional. Default: 300 seconds
|
||||
maxReplicaCount: {{ .Values.workers.keda.maxReplicaCount }} # Optional. Default: 100
|
||||
triggers:
|
||||
- type: postgresql
|
||||
metadata:
|
||||
connection: AIRFLOW_CONN_AIRFLOW_DB
|
||||
query: "SELECT ceil(COUNT(*)::decimal / 16) FROM task_instance WHERE state='running' OR state='queued'"
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Worker NetworkPolicy
|
||||
#################################
|
||||
{{- if (and .Values.networkPolicies.enabled (eq .Values.executor "CeleryExecutor")) }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-worker-policy
|
||||
labels:
|
||||
tier: airflow
|
||||
component: airflow-worker-policy
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
component: worker
|
||||
release: {{ .Release.Name }}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
component: webserver
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: {{ .Values.ports.workerLogs }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Worker Service
|
||||
#################################
|
||||
{{- if eq .Values.executor "CeleryExecutor" }}
|
||||
kind: Service
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-worker
|
||||
labels:
|
||||
tier: airflow
|
||||
component: worker
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
clusterIP: None
|
||||
selector:
|
||||
tier: airflow
|
||||
component: worker
|
||||
release: {{ .Release.Name }}
|
||||
ports:
|
||||
- name: worker-logs
|
||||
protocol: TCP
|
||||
port: {{ .Values.ports.workerLogs }}
|
||||
targetPort: {{ .Values.ports.workerLogs }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
################################
|
||||
## Airflow Worker ServiceAccount
|
||||
#################################
|
||||
{{- if .Values.rbacEnabled }}
|
||||
kind: ServiceAccount
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: {{ .Release.Name }}-worker-serviceaccount
|
||||
labels:
|
||||
tier: airflow
|
||||
release: {{ .Release.Name }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- with .Values.labels }}
|
||||
{{ toYaml . | indent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,436 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Default values for airflow.
|
||||
# This is a YAML-formatted file.
|
||||
# Declare variables to be passed into your templates.
|
||||
|
||||
# User and group of airflow user
|
||||
uid: 50000
|
||||
gid: 50000
|
||||
|
||||
# Airflow home directory
|
||||
# Used for mount paths
|
||||
airflowHome: "/opt/airflow"
|
||||
|
||||
# Default airflow repository -- overrides all the specific images below
|
||||
defaultAirflowRepository: apache/airflow
|
||||
|
||||
# Default airflow tag to deploy
|
||||
defaultAirflowTag: 1.10.10.1-alpha2-python3.6
|
||||
|
||||
|
||||
# Select certain nodes for airflow pods.
|
||||
nodeSelector: {}
|
||||
affinity: {}
|
||||
tolerations: []
|
||||
|
||||
# Add common labels to all objects and pods defined in this chart.
|
||||
labels: {}
|
||||
|
||||
# Network policy configuration
|
||||
networkPolicies:
|
||||
# Enabled network policies
|
||||
enabled: false
|
||||
|
||||
|
||||
# Extra annotations to apply to all
|
||||
# Airflow pods
|
||||
airflowPodAnnotations: {}
|
||||
|
||||
# Enable RBAC (default on most clusters these days)
|
||||
rbacEnabled: true
|
||||
|
||||
# Airflow executor
|
||||
# Options: SequentialExecutor, LocalExecutor, CeleryExecutor, KubernetesExecutor
|
||||
executor: "KubernetesExecutor"
|
||||
|
||||
# If this is true and using LocalExecutor/SequentialExecutor/KubernetesExecutor, the scheudler's
|
||||
# service account will have access to communicate with the api-server and launch pods.
|
||||
# If this is true and using the CeleryExecutor, the workers will be able to launch pods.
|
||||
allowPodLaunching: true
|
||||
|
||||
# Images
|
||||
images:
|
||||
airflow:
|
||||
repository: ~
|
||||
tag: ~
|
||||
pullPolicy: IfNotPresent
|
||||
flower:
|
||||
repository: ~
|
||||
tag: ~
|
||||
pullPolicy: IfNotPresent
|
||||
statsd:
|
||||
repository: astronomerinc/ap-statsd-exporter
|
||||
tag: 0.11.0
|
||||
pullPolicy: IfNotPresent
|
||||
redis:
|
||||
repository: redis
|
||||
tag: 6-buster
|
||||
pullPolicy: IfNotPresent
|
||||
pgbouncer:
|
||||
repository: astronomerinc/ap-pgbouncer
|
||||
tag: 1.8.1
|
||||
pullPolicy: IfNotPresent
|
||||
pgbouncerExporter:
|
||||
repository: astronomerinc/ap-pgbouncer-exporter
|
||||
tag: 0.5.0-1
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
# Environment variables for all airflow containers
|
||||
env: []
|
||||
# - name: ""
|
||||
# value: ""
|
||||
|
||||
# Secrets for all airflow containers
|
||||
secret: []
|
||||
# - envName: ""
|
||||
# secretName: ""
|
||||
# secretKey: ""
|
||||
|
||||
# Airflow database config
|
||||
data:
|
||||
# If secret names are provided, use those secrets
|
||||
metadataSecretName: ~
|
||||
resultBackendSecretName: ~
|
||||
|
||||
# Otherwise pass connection values in
|
||||
metadataConnection:
|
||||
user: postgres
|
||||
pass: postgres
|
||||
host: ~
|
||||
port: 5432
|
||||
db: postgres
|
||||
sslmode: disable
|
||||
resultBackendConnection:
|
||||
user: postgres
|
||||
pass: postgres
|
||||
host: ~
|
||||
port: 5432
|
||||
db: postgres
|
||||
sslmode: disable
|
||||
|
||||
# Fernet key settings
|
||||
fernetKey: ~
|
||||
fernetKeySecretName: ~
|
||||
|
||||
# Airflow Worker Config
|
||||
workers:
|
||||
# Number of airflow celery workers in StatefulSet
|
||||
replicas: 1
|
||||
|
||||
# Allow KEDA autoscaling.
|
||||
# Persistence.enabled must be set to false to use KEDA.
|
||||
keda:
|
||||
enabled: false
|
||||
namespaceLabels: {}
|
||||
|
||||
# How often KEDA polls the airflow DB to report new scale requests to the HPA
|
||||
pollingInterval: 5
|
||||
|
||||
# How many seconds KEDA will wait before scaling to zero.
|
||||
# Note that HPA has a seperate cooldwon period for scale-downs
|
||||
cooldownPeriod: 30
|
||||
|
||||
# Maximum number of workers created by keda
|
||||
maxReplicaCount: 10
|
||||
|
||||
persistence:
|
||||
# Enable persistent volumes
|
||||
enabled: true
|
||||
# Volume size for worker StatefulSet
|
||||
size: 100Gi
|
||||
# If using a custom storageClass, pass name ref to all statefulSets here
|
||||
storageClassName:
|
||||
# Execute init container to chown log directory.
|
||||
# This is currently only needed in KinD, due to usage
|
||||
# of local-path provisioner.
|
||||
fixPermissions: false
|
||||
|
||||
resources: {}
|
||||
# limits:
|
||||
# cpu: 100m
|
||||
# memory: 128Mi
|
||||
# requests:
|
||||
# cpu: 100m
|
||||
# memory: 128Mi
|
||||
|
||||
# Grace period for tasks to finish after SIGTERM is sent from kubernetes
|
||||
terminationGracePeriodSeconds: 600
|
||||
|
||||
# This setting tells kubernetes that its ok to evict
|
||||
# when it wants to scale a node down.
|
||||
safeToEvict: true
|
||||
|
||||
# Airflow scheduler settings
|
||||
scheduler:
|
||||
# Scheduler pod disruption budget
|
||||
podDisruptionBudget:
|
||||
enabled: false
|
||||
|
||||
# PDB configuration
|
||||
config:
|
||||
maxUnavailable: 1
|
||||
|
||||
resources: {}
|
||||
# limits:
|
||||
# cpu: 100m
|
||||
# memory: 128Mi
|
||||
# requests:
|
||||
# cpu: 100m
|
||||
# memory: 128Mi
|
||||
|
||||
# This setting can overwrite
|
||||
# podMutation settings.
|
||||
airflowLocalSettings: ~
|
||||
|
||||
# This setting tells kubernetes that its ok to evict
|
||||
# when it wants to scale a node down.
|
||||
safeToEvict: true
|
||||
|
||||
# Airflow webserver settings
|
||||
webserver:
|
||||
livenessProbe:
|
||||
initialDelaySeconds: 15
|
||||
timeoutSeconds: 30
|
||||
failureThreshold: 20
|
||||
periodSeconds: 5
|
||||
|
||||
readinessProbe:
|
||||
initialDelaySeconds: 15
|
||||
timeoutSeconds: 30
|
||||
failureThreshold: 20
|
||||
periodSeconds: 5
|
||||
|
||||
# Number of webservers
|
||||
replicas: 1
|
||||
|
||||
# Additional network policies as needed
|
||||
extraNetworkPolicies: []
|
||||
|
||||
resources: {}
|
||||
# limits:
|
||||
# cpu: 100m
|
||||
# memory: 128Mi
|
||||
# requests:
|
||||
# cpu: 100m
|
||||
# memory: 128Mi
|
||||
|
||||
# Create initial user.
|
||||
defaultUser:
|
||||
enabled: true
|
||||
role: Admin
|
||||
username: admin
|
||||
email: admin@example.com
|
||||
firstName: admin
|
||||
lastName: user
|
||||
password: admin
|
||||
|
||||
# Mount additional volumes into webserver.
|
||||
extraVolumes: []
|
||||
extraVolumeMounts: []
|
||||
|
||||
# This will be mounted into the Airflow Webserver as a custom
|
||||
# webserver_config.py. You can bake a webserver_config.py in to your image
|
||||
# instead
|
||||
webserverConfig: ~
|
||||
# webserverConfig: |
|
||||
# from airflow import configuration as conf
|
||||
|
||||
# # The SQLAlchemy connection string.
|
||||
# SQLALCHEMY_DATABASE_URI = conf.get('core', 'SQL_ALCHEMY_CONN')
|
||||
|
||||
# # Flask-WTF flag for CSRF
|
||||
# CSRF_ENABLED = True
|
||||
|
||||
service:
|
||||
type: ClusterIP
|
||||
|
||||
# Flower settings
|
||||
flower:
|
||||
# Additional network policies as needed
|
||||
extraNetworkPolicies: []
|
||||
resources: {}
|
||||
# limits:
|
||||
# cpu: 100m
|
||||
# memory: 128Mi
|
||||
# requests:
|
||||
# cpu: 100m
|
||||
# memory: 128Mi
|
||||
|
||||
service:
|
||||
type: ClusterIP
|
||||
|
||||
# Statsd settings
|
||||
statsd:
|
||||
enabled: true
|
||||
# Additional network policies as needed
|
||||
extraNetworkPolicies: []
|
||||
resources: {}
|
||||
# limits:
|
||||
# cpu: 100m
|
||||
# memory: 128Mi
|
||||
# requests:
|
||||
# cpu: 100m
|
||||
# memory: 128Mi
|
||||
|
||||
service:
|
||||
extraAnnotations: {}
|
||||
|
||||
# Pgbouncer settings
|
||||
pgbouncer:
|
||||
# Enable pgbouncer
|
||||
enabled: false
|
||||
# Additional network policies as needed
|
||||
extraNetworkPolicies: []
|
||||
|
||||
# Pool sizes
|
||||
metadataPoolSize: 10
|
||||
resultBackendPoolSize: 5
|
||||
|
||||
# Maximum clients that can connect to pgbouncer (higher = more file descriptors)
|
||||
maxClientConn: 100
|
||||
|
||||
# Pgbouner pod disruption budget
|
||||
podDisruptionBudget:
|
||||
enabled: false
|
||||
|
||||
# PDB configuration
|
||||
config:
|
||||
maxUnavailable: 1
|
||||
|
||||
resources: {}
|
||||
# limits:
|
||||
# cpu: 100m
|
||||
# memory: 128Mi
|
||||
# requests:
|
||||
# cpu: 100m
|
||||
# memory: 128Mi
|
||||
|
||||
service:
|
||||
extraAnnotations: {}
|
||||
|
||||
# https://www.pgbouncer.org/config.html
|
||||
verbose: 0
|
||||
logDisconnections: 0
|
||||
logConnections: 0
|
||||
|
||||
redis:
|
||||
terminationGracePeriodSeconds: 600
|
||||
|
||||
persistence:
|
||||
# Enable persistent volumes
|
||||
enabled: true
|
||||
# Volume size for worker StatefulSet
|
||||
size: 1Gi
|
||||
# If using a custom storageClass, pass name ref to all statefulSets here
|
||||
storageClassName:
|
||||
|
||||
resources: {}
|
||||
# limits:
|
||||
# cpu: 100m
|
||||
# memory: 128Mi
|
||||
# requests:
|
||||
# cpu: 100m
|
||||
# memory: 128Mi
|
||||
|
||||
# If set use as redis secret
|
||||
passwordSecretName: ~
|
||||
brokerURLSecretName: ~
|
||||
|
||||
# Else, if password is set, create secret with it,
|
||||
# else generate a new one on install
|
||||
password: ~
|
||||
|
||||
# This setting tells kubernetes that its ok to evict
|
||||
# when it wants to scale a node down.
|
||||
safeToEvict: true
|
||||
|
||||
# Auth secret for a private registry
|
||||
# This is used if pulling airflow images from a private registry
|
||||
registry:
|
||||
secretName: ~
|
||||
|
||||
connection: {}
|
||||
# user: ~
|
||||
# pass: ~
|
||||
# host: ~
|
||||
# email: ~
|
||||
|
||||
# Elasticsearch logging configuration
|
||||
elasticsearch:
|
||||
# Enable elasticsearch task logging
|
||||
enabled: false
|
||||
# A secret containing the connection
|
||||
secretName: ~
|
||||
# Or an object representing the connection
|
||||
connection: {}
|
||||
# user: ~
|
||||
# pass: ~
|
||||
# host: ~
|
||||
# port: ~
|
||||
|
||||
# All ports used by chart
|
||||
ports:
|
||||
flowerUI: 5555
|
||||
airflowUI: 8080
|
||||
workerLogs: 8793
|
||||
redisDB: 6379
|
||||
statsdIngest: 9125
|
||||
statsdScrape: 9102
|
||||
pgbouncer: 6543
|
||||
pgbouncerScrape: 9127
|
||||
|
||||
# Define any ResourceQuotas for namespace
|
||||
quotas: {}
|
||||
|
||||
# Define default/max/min values for pods and containers in namespace
|
||||
limits: []
|
||||
|
||||
# Config Settings for pod_mutation_hook
|
||||
podMutation:
|
||||
# Tolerations provided here would be applied using pod_mutation_hook
|
||||
# So any pods spun up using KubernetesExecutor or KubernetesPodOperator will contain these tolerations.
|
||||
tolerations: []
|
||||
# - key: "dynamic-pods"
|
||||
# operator: "Equal"
|
||||
# value: "true"
|
||||
# effect: "NoSchedule"
|
||||
|
||||
# Pods spun up would land in the node that matches the affinity
|
||||
affinity: {}
|
||||
# nodeAffinity:
|
||||
# requiredDuringSchedulingIgnoredDuringExecution:
|
||||
# nodeSelectorTerms:
|
||||
# - matchExpressions:
|
||||
# - key: "company.io/dynamic-pods"
|
||||
# operator: In
|
||||
# values:
|
||||
# - "true"
|
||||
|
||||
# This runs as a CronJob to cleanup old pods.
|
||||
cleanup:
|
||||
enabled: false
|
||||
# Run every 15 minutes
|
||||
schedule: "*/15 * * * *"
|
||||
|
||||
# Configuration for postgresql subchart
|
||||
# Not recommended for production
|
||||
postgresql:
|
||||
enabled: true
|
||||
postgresqlPassword: postgres
|
||||
postgresqlUsername: postgres
|
||||
Загрузка…
Ссылка в новой задаче