diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index befff35c08..55c01437ca 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -166,7 +166,7 @@ metastore_browser/templates/.*\\.html$|.*\\.jinja2" name: Check yaml files with yamllint entry: yamllint -c yamllint-config.yml types: [yaml] - exclude: ^.*init_git_sync\.template\.yaml$|^.*airflow\.template\.yaml$ + exclude: ^.*init_git_sync\.template\.yaml$|^.*airflow\.template\.yaml$|^chart/templates/.*\.yaml$ - repo: https://github.com/timothycrosley/isort rev: 4.3.21-2 hooks: diff --git a/chart/.gitignore b/chart/.gitignore new file mode 100644 index 0000000000..6cd3d919cf --- /dev/null +++ b/chart/.gitignore @@ -0,0 +1,9 @@ +# User overrides +config.yaml +config.yml + +# Build dir +repository + +# Chart dependencies +**/charts/*.tgz diff --git a/chart/.helmignore b/chart/.helmignore new file mode 100644 index 0000000000..a188e174de --- /dev/null +++ b/chart/.helmignore @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +bin diff --git a/chart/Chart.yaml b/chart/Chart.yaml new file mode 100644 index 0000000000..927b412320 --- /dev/null +++ b/chart/Chart.yaml @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# apiVersion v1 is Helm 2 +apiVersion: v1 +name: airflow +version: 1.0.0 +description: Helm chart to deploy Apache Airflow +icon: https://www.astronomer.io/static/airflowNewA.png +keywords: + - airflow diff --git a/chart/README.md b/chart/README.md new file mode 100644 index 0000000000..8657eeeb9e --- /dev/null +++ b/chart/README.md @@ -0,0 +1,269 @@ + + +# Helm Chart for Apache Airflow + +[Apache Airflow](https://airflow.apache.org/) is a platform to programmatically author, schedule and monitor workflows. + +## Introduction + +This chart will bootstrap an [Airflow](https://airflow.apache.org) deployment on a [Kubernetes](http://kubernetes.io) +cluster using the [Helm](https://helm.sh) package manager. + +## Prerequisites + +- Kubernetes 1.12+ +- Helm 2.11+ or Helm 3.0+ +- PV provisioner support in the underlying infrastructure + +## Installing the Chart + +To install this repository from source (using helm 3) +```bash +kubectl create namespace airflow +helm repo add stable https://kubernetes-charts.storage.googleapis.com +helm dep update +helm install airflow . --namespace airflow +``` + +The command deploys Airflow on the Kubernetes cluster in the default configuration. The [Parameters](#parameters) +section lists the parameters that can be configured during installation. + +> **Tip**: List all releases using `helm list` + +## Upgrading the Chart +To upgrade the chart with the release name `airflow`: + +```bash +helm upgrade airflow . --namespace airflow +``` + +## Uninstalling the Chart + +To uninstall/delete the `airflow` deployment: + +```bash +helm delete airflow --namespace airflow +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Updating DAGs + +The recommended way to update your DAGs with this chart is to build a new docker image with the latest code (`docker build -t my-company/airflow:8a0da78 .`), push it to an accessible registry (`docker push my-company/airflow:8a0da78`), then update the Airflow pods with that image: + +```bash +helm upgrade airflow . \ + --set images.airflow.repository=my-company/airflow \ + --set images.airflow.tag=8a0da78 +``` + +## Parameters + +The following tables lists the configurable parameters of the Airflow chart and their default values. + +| Parameter | Description | Default | +| ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | ------------------------------------------------- | +| `uid` | UID to run airflow pods under | `50000` | +| `gid` | GID to run airflow pods under | `50000` | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `affinity` | Affinity labels for pod assignment | `{}` | +| `tolerations` | Toleration labels for pod assignment | `[]` | +| `labels` | Common labels to add to all objects defined in this chart | `{}` | +| `privateRegistry.enabled` | Enable usage of a private registry for Airflow base image | `false` | +| `privateRegistry.repository` | Repository where base image lives (eg: quay.io) | `~` | +| `networkPolicies.enabled` | Enable Network Policies to restrict traffic | `true` | +| `airflowHome` | Location of airflow home directory | `/opt/airflow` | +| `rbacEnabled` | Deploy pods with Kubernets RBAC enabled | `true` | +| `airflowVersion` | Default Airflow image version | `1.10.5` | +| `executor` | Airflow executor (eg SequentialExecutor, LocalExecutor, CeleryExecutor, KubernetesExecutor) | `KubernetesExecutor` | +| `allowPodLaunching` | Allow airflow pods to talk to Kubernetes API to launch more pods | `true` | +| `defaultAirflowRepository` | Fallback docker repository to pull airflow image from | `apache/airflow` | +| `defaultAirflowTag` | Fallback docker image tag to deploy | `1.10.10.1-alpha2-python3.6` | +| `images.airflow.repository` | Docker repository to pull image from. Update this to deploy a custom image | `~` | +| `images.airflow.tag` | Docker image tag to pull image from. Update this to deploy a new custom image tag | `~` | +| `images.airflow.pullPolicy` | PullPolicy for airflow image | `IfNotPresent` | +| `images.flower.repository` | Docker repository to pull image from. Update this to deploy a custom image | `~` | +| `images.flower.tag` | Docker image tag to pull image from. Update this to deploy a new custom image tag | `~` | +| `images.flower.pullPolicy` | PullPolicy for flower image | `IfNotPresent` | +| `images.statsd.repository` | Docker repository to pull image from. Update this to deploy a custom image | `astronomerinc/ap-statsd-exporter` | +| `images.statsd.tag` | Docker image tag to pull image from. Update this to deploy a new custom image tag | `~` | +| `images.statsd.pullPolicy` | PullPolicy for statsd-exporter image | `IfNotPresent` | +| `images.redis.repository` | Docker repository to pull image from. Update this to deploy a custom image | `redis` | +| `images.redis.tag` | Docker image tag to pull image from. Update this to deploy a new custom image tag | `6-buster` | +| `images.redis.pullPolicy` | PullPolicy for redis image | `IfNotPresent` | +| `images.pgbouncer.repository` | Docker repository to pull image from. Update this to deploy a custom image | `astronomerinc/ap-pgbouncer` | +| `images.pgbouncer.tag` | Docker image tag to pull image from. Update this to deploy a new custom image tag | `~` | +| `images.pgbouncer.pullPolicy` | PullPolicy for pgbouncer image | `IfNotPresent` | +| `images.pgbouncerExporter.repository` | Docker repository to pull image from. Update this to deploy a custom image | `astronomerinc/ap-pgbouncer-exporter` | +| `images.pgbouncerExporter.tag` | Docker image tag to pull image from. Update this to deploy a new custom image tag | `~` | +| `images.pgbouncerExporter.pullPolicy` | PullPolicy for pgbouncer-exporter image | `IfNotPresent` | +| `env` | Environment variables key/values to mount into Airflow pods | `[]` | +| `secret` | Secret name/key pairs to mount into Airflow pods | `[]` | +| `data.metadataSecretName` | Secret name to mount Airflow connection string from | `~` | +| `data.resultBackendSecretName` | Secret name to mount Celery result backend connection string from | `~` | +| `data.metadataConection` | Field separated connection data (alternative to secret name) | `{}` | +| `data.resultBakcnedConnection` | Field separated connection data (alternative to secret name) | `{}` | +| `fernetKey` | String representing an Airflow fernet key | `~` | +| `fernetKeySecretName` | Secret name for Airlow fernet key | `~` | +| `workers.replicas` | Replica count for Celery workers (if applicable) | `1` | +| `workers.keda.enabled` | Enable KEDA autoscaling features | `false` | +| `workers.keda.pollingInverval` | How often KEDA should poll the backend database for metrics in seconds | `5` | +| `workers.keda.cooldownPeriod` | How often KEDA should wait before scaling down in seconds | `30` | +| `workers.keda.maxReplicaCount` | Maximum number of Celery workers KEDA can scale to | `10` | +| `workers.persistence.enabled` | Enable log persistence in workers via StatefulSet | `false` | +| `workers.persistence.size` | Size of worker volumes if enabled | `100Gi` | +| `workers.persistence.storageClassName` | StorageClass worker volumes should use if enabled | `default` | +| `workers.resources.limits.cpu` | CPU Limit of workers | `~` | +| `workers.resources.limits.memory` | Memory Limit of workers | `~` | +| `workers.resources.requests.cpu` | CPU Request of workers | `~` | +| `workers.resources.requests.memory` | Memory Request of workers | `~` | +| `workers.terminationGracePeriodSeconds` | How long Kubernetes should wait for Celery workers to gracefully drain before force killing | `600` | +| `workers.safeToEvict` | Allow Kubernetes to evict worker pods if needed (node downscaling) | `true` | +| `scheduler.podDisruptionBudget.enabled` | Enable PDB on Airflow scheduler | `false` | +| `scheduler.podDisruptionBudget.config.maxUnavailable` | MaxUnavailable pods for scheduler | `1` | +| `scheduler.resources.limits.cpu` | CPU Limit of scheduler | `~` | +| `scheduler.resources.limits.memory` | Memory Limit of scheduler | `~` | +| `scheduler.resources.requests.cpu` | CPU Request of scheduler | `~` | +| `scheduler.resources.requests.memory` | Memory Request of scheduler | `~` | +| `scheduler.airflowLocalSettings` | Custom Airflow local settings python file | `~` | +| `scheduler.safeToEvict` | Allow Kubernetes to evict scheduler pods if needed (node downscaling) | `true` | +| `webserver.livenessProbe.initialDelaySeconds` | Webserver LivenessProbe initial delay | `15` | +| `webserver.livenessProbe.timeoutSeconds` | Webserver LivenessProbe timeout seconds | `30` | +| `webserver.livenessProbe.failureThreshold` | Webserver LivenessProbe failure threshold | `20` | +| `webserver.livenessProbe.periodSeconds` | Webserver LivenessProbe period seconds | `5` | +| `webserver.readinessProbe.initialDelaySeconds` | Webserver ReadinessProbe initial delay | `15` | +| `webserver.readinessProbe.timeoutSeconds` | Webserver ReadinessProbe timeout seconds | `30` | +| `webserver.readinessProbe.failureThreshold` | Webserver ReadinessProbe failure threshold | `20` | +| `webserver.readinessProbe.periodSeconds` | Webserver ReadinessProbe period seconds | `5` | +| `webserver.replicas` | How many Airflow webserver replicas should run | `1` | +| `webserver.resources.limits.cpu` | CPU Limit of webserver | `~` | +| `webserver.resources.limits.memory` | Memory Limit of webserver | `~` | +| `webserver.resources.requests.cpu` | CPU Request of webserver | `~` | +| `webserver.resources.requests.memory` | Memory Request of webserver | `~` | +| `webserver.jwtSigningCertificateSecretName` | Name of secret to mount Airflow Webserver JWT singing certificate from | `~` | +| `webserver.defaultUser` | Optional default airflow user information | `{}` | + + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +```bash +helm install --name my-release \ + --set executor=CeleryExecutor \ + --set enablePodLaunching=false . +``` + +## Autoscaling with KEDA + +KEDA stands for Kubernetes Event Driven Autoscaling. [KEDA](https://github.com/kedacore/keda) is a custom controller that allows users to create custom bindings +to the Kubernetes [Horizontal Pod Autoscaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/). +We've built an experimental scaler that allows users to create scalers based on postgreSQL queries. For the moment this exists +on a seperate branch, but will be merged upstream soon. To install our custom version of KEDA on your cluster, please run + +```bash +helm repo add kedacore https://kedacore.github.io/charts + +helm repo update + +helm install \ + --set image.keda=docker.io/kedacore/keda:1.2.0 \ + --set image.metricsAdapter=docker.io/kedacore/keda-metrics-adapter:1.2.0 \ + --namespace keda --name keda kedacore/keda +``` + +Once KEDA is installed (which should be pretty quick since there is only one pod). You can try out KEDA autoscaling +on this chart by setting `workers.keda.enabled=true` your helm command or in the `values.yaml`. +(Note: KEDA does not support StatefulSets so you need to set `worker.persistence.enabled` to `false`) + +```bash +kubectl create namespace airflow + +helm install airflow . \ + --namespace airflow \ + --set executor=CeleryExecutor \ + --set workers.keda.enabled=true \ + --set workers.persistence.enabled=false +``` + +## Walkthrough using kind + +**Install kind, and create a cluster:** + +We recommend testing with Kubernetes 1.15, as this image doesn't support Kubernetes 1.16+ for CeleryExecutor presently. + +``` +kind create cluster \ + --image kindest/node:v1.15.7@sha256:e2df133f80ef633c53c0200114fce2ed5e1f6947477dbc83261a6a921169488d +``` + +Confirm it's up: + +``` +kubectl cluster-info --context kind-kind +``` + +**Add Astronomer's Helm repo:** + +``` +helm repo add astronomer https://helm.astronomer.io +helm repo update +``` + +**Create namespace + install the chart:** + +``` +kubectl create namespace airflow +helm install airflow --n airflow astronomer/airflow +``` + +It may take a few minutes. Confirm the pods are up: + +``` +kubectl get pods --all-namespaces +helm list -n airflow +``` + +Run `kubectl port-forward svc/airflow-webserver 8080:8080 -n airflow` +to port-forward the Airflow UI to http://localhost:8080/ to cofirm Airflow is working. + +**Build a Docker image from your DAGs:** + +1. Start a project using [astro-cli](https://github.com/astronomer/astro-cli), which will generate a Dockerfile, and load your DAGs in. You can test locally before pushing to kind with `astro airflow start`. + + mkdir my-airflow-project && cd my-airflow-project + astro dev init + +2. Then build the image: + + docker build -t my-dags:0.0.1 . + +3. Load the image into kind: + + kind load docker-image my-dags:0.0.1 + +4. Upgrade Helm deployment: + + helm upgrade airflow -n airflow \ + --set images.airflow.repository=my-dags \ + --set images.airflow.tag=0.0.1 \ + astronomer/airflow + +## Contributing + +Check out [our contributing guide!](CONTRIBUTING.md) diff --git a/chart/requirements.lock b/chart/requirements.lock new file mode 100644 index 0000000000..2eff289e26 --- /dev/null +++ b/chart/requirements.lock @@ -0,0 +1,6 @@ +dependencies: +- name: postgresql + repository: https://kubernetes-charts.storage.googleapis.com + version: 6.3.12 +digest: sha256:1750ddcc948f15716d157d6a854b59e37571473b2a3390a3673b224b71a56308 +generated: "2019-10-24T16:03:26.569269284-04:00" diff --git a/chart/requirements.yaml b/chart/requirements.yaml new file mode 100644 index 0000000000..56f29ee32a --- /dev/null +++ b/chart/requirements.yaml @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +dependencies: + - name: postgresql + version: 6.3.12 + repository: "@stable" + condition: postgresql.enabled diff --git a/chart/templates/NOTES.txt b/chart/templates/NOTES.txt new file mode 100644 index 0000000000..3df1e3bc05 --- /dev/null +++ b/chart/templates/NOTES.txt @@ -0,0 +1,29 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +*/}}Thank you for installing Airflow! + +Your release is named {{ .Release.Name }}. + +You can now access your dashboard(s) by executing the following command(s) and visiting the corresponding port at localhost in your browser: + +Airflow dashboard: kubectl port-forward svc/{{ .Release.Name }}-webserver {{ .Values.ports.airflowUI }}:{{ .Values.ports.airflowUI }} +{{- if eq .Values.executor "CeleryExecutor"}} +Flower dashboard: kubectl port-forward svc/{{ .Release.Name }}-flower {{ .Values.ports.flowerUI }}:{{ .Values.ports.flowerUI }} + +{{- end }} diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml new file mode 100644 index 0000000000..66d1850777 --- /dev/null +++ b/chart/templates/_helpers.yaml @@ -0,0 +1,260 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{{/* Standard Airflow environment variables */}} +{{- define "standard_airflow_environment" }} + # Hard Coded Airflow Envs + - name: AIRFLOW__CORE__FERNET_KEY + valueFrom: + secretKeyRef: + name: {{ template "fernet_key_secret" . }} + key: fernet-key + - name: AIRFLOW__CORE__SQL_ALCHEMY_CONN + valueFrom: + secretKeyRef: + name: {{ template "airflow_metadata_secret" . }} + key: connection + - name: AIRFLOW_CONN_AIRFLOW_DB + valueFrom: + secretKeyRef: + name: {{ template "airflow_metadata_secret" . }} + key: connection + {{- if eq .Values.executor "CeleryExecutor" }} + - name: AIRFLOW__CELERY__CELERY_RESULT_BACKEND + valueFrom: + secretKeyRef: + name: {{ template "airflow_result_backend_secret" . }} + key: connection + - name: AIRFLOW__CELERY__RESULT_BACKEND + valueFrom: + secretKeyRef: + name: {{ template "airflow_result_backend_secret" . }} + key: connection + - name: AIRFLOW__CELERY__BROKER_URL + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-broker-url + key: connection + {{- end }} + {{- if .Values.elasticsearch.enabled }} + # The elasticsearch variables were updated to the shorter names in v1.10.4 + - name: AIRFLOW__ELASTICSEARCH__HOST + valueFrom: + secretKeyRef: + name: {{ template "elasticsearch_secret" . }} + key: connection + # This is the older format for these variable names, kept here for backward compatibility + - name: AIRFLOW__ELASTICSEARCH__ELASTICSEARCH_HOST + valueFrom: + secretKeyRef: + name: {{ template "elasticsearch_secret" . }} + key: connection + {{- end }} +{{- end }} + +{{/* User defined Airflow environment variables */}} +{{- define "custom_airflow_environment" }} + # Dynamically created environment variables + {{- range $i, $config := .Values.env }} + - name: {{ $config.name }} + value: {{ $config.value | quote }} + {{- if eq $.Values.executor "KubernetesExecutor" }} + - name: AIRFLOW__KUBERNETES_ENVIRONMENT_VARIABLES__{{ $config.name }} + value: {{ $config.value | quote }} + {{- end }} + {{- end }} + # Dynamically created secret envs + {{- range $i, $config := .Values.secret }} + - name: {{ $config.envName }} + valueFrom: + secretKeyRef: + name: {{ $config.secretName }} + key: {{ default "value" $config.secretKey }} + {{- end }} + {{- if eq .Values.executor "KubernetesExecutor" }} + {{- range $i, $config := .Values.secret }} + - name: AIRFLOW__KUBERNETES_SECRETS__{{ $config.envName }} + value: {{ printf "%s=%s" $config.secretName $config.secretKey }} + {{- end }} + {{ end }} +{{- end }} + +# This helper will change when customers deploy a new image. +{{ define "airflow_image" -}} +{{ printf "%s:%s" (.Values.images.airflow.repository | default .Values.defaultAirflowRepository) (.Values.images.airflow.tag | default .Values.defaultAirflowTag) }} +{{- end }} + +# This helper is used for airflow containers that do not need the users code. +{{ define "default_airflow_image" -}} +{{ printf "%s:%s" .Values.defaultAirflowRepository .Values.defaultAirflowTag }} +{{- end }} + +{{ define "flower_image" -}} +{{ printf "%s:%s" (.Values.images.airflow.repository | default .Values.defaultAirflowRepository) (.Values.images.airflow.tag | default .Values.defaultAirflowTag) }} +{{- end }} + +{{ define "statsd_image" -}} +{{ printf "%s:%s" .Values.images.statsd.repository .Values.images.statsd.tag }} +{{- end }} + +{{ define "redis_image" -}} +{{ printf "%s:%s" .Values.images.redis.repository .Values.images.redis.tag }} +{{- end }} + +{{ define "pgbouncer_image" -}} +{{ printf "%s:%s" .Values.images.pgbouncer.repository .Values.images.pgbouncer.tag }} +{{- end }} + +{{ define "pgbouncer_exporter_image" -}} +{{ printf "%s:%s" .Values.images.pgbouncerExporter.repository .Values.images.pgbouncerExporter.tag }} +{{- end }} + +{{ define "fernet_key_secret" -}} +{{ default (printf "%s-fernet-key" .Release.Name) .Values.fernetKeySecretName }} +{{- end }} + +{{ define "redis_password_secret" -}} +{{ default (printf "%s-redis-password" .Release.Name) .Values.redis.passwordSecretName }} +{{- end }} + +{{ define "airflow_metadata_secret" -}} +{{ default (printf "%s-airflow-metadata" .Release.Name) .Values.data.metadataSecretName }} +{{- end }} + +{{ define "airflow_result_backend_secret" -}} +{{ default (printf "%s-airflow-result-backend" .Release.Name) .Values.data.resultBackendSecretName }} +{{- end }} + +{{ define "pgbouncer_config_secret" -}} +{{ .Release.Name }}-pgbouncer-config +{{- end }} + +{{ define "pgbouncer_stats_secret" -}} +{{ .Release.Name }}-pgbouncer-stats +{{- end }} + +{{ define "registry_secret" -}} +{{ default (printf "%s-registry" .Release.Name) .Values.registry.secretName }} +{{- end }} + +{{ define "elasticsearch_secret" -}} +{{ default (printf "%s-elasticsearch" .Release.Name) .Values.elasticsearch.secretName }} +{{- end }} + +{{ define "pgbouncer_config" }} +{{- $pgMetadataHost := .Values.data.metadataConnection.host | default (printf "%s-%s.%s.svc.cluster.local" .Release.Name "postgresql" .Release.Namespace) }} +{{- $pgResultBackendHost := .Values.data.resultBackendConnection.host | default (printf "%s-%s.%s.svc.cluster.local" .Release.Name "postgresql" .Release.Namespace) }} +[databases] +{{ .Release.Name }}-metadata = host={{ $pgMetadataHost }} dbname={{ .Values.data.metadataConnection.db }} port={{ .Values.data.metadataConnection.port }} pool_size={{ .Values.pgbouncer.metadataPoolSize }} +{{ .Release.Name }}-result-backend = host={{ $pgResultBackendHost }} dbname={{ .Values.data.resultBackendConnection.db }} port={{ .Values.data.resultBackendConnection.port }} pool_size={{ .Values.pgbouncer.resultBackendPoolSize }} + +[pgbouncer] +pool_mode = transaction +listen_port = {{ .Values.ports.pgbouncer }} +listen_addr = * +auth_type = md5 +auth_file = /etc/pgbouncer/users.txt +stats_users = {{ .Values.data.metadataConnection.user }} +ignore_startup_parameters = extra_float_digits +max_client_conn = {{ .Values.pgbouncer.maxClientConn }} +verbose = {{ .Values.pgbouncer.verbose }} +log_disconnections = {{ .Values.pgbouncer.logDisconnections }} +log_connections = {{ .Values.pgbouncer.logConnections }} +{{- end }} + +{{ define "pgbouncer_users" }} +{{ .Values.data.metadataConnection.user | quote }} {{ .Values.data.metadataConnection.pass | quote }} +{{ .Values.data.resultBackendConnection.user | quote }} {{ .Values.data.resultBackendConnection.pass | quote }} +{{- end }} + +{{ define "airflow_logs" -}} +{{ (printf "%s/logs" .Values.airflowHome) | quote }} +{{- end }} + +{{ define "airflow_config_path" -}} +{{ (printf "%s/airflow.cfg" .Values.airflowHome) | quote }} +{{- end }} +{{ define "airflow_webserver_config_path" -}} +{{ (printf "%s/webserver_config.py" .Values.airflowHome) | quote }} +{{- end }} + +{{ define "airflow_local_setting_path" -}} +{{ (printf "%s/config/airflow_local_settings.py" .Values.airflowHome) | quote }} +{{- end }} + +{{ define "airflow_config" -}} +{{ (printf "%s-airflow-config" .Release.Name) }} +{{- end }} + +{{ define "wait-for-migrations-command" }} + {{/* From Airflow 2.0.0 this can become [airflow, db, check-migrations] */}} + - python + - -c + - | + import importlib + import os + import time + + from alembic.config import Config + from alembic.runtime.migration import MigrationContext + from alembic.script import ScriptDirectory + + from airflow import settings + + package_dir = os.path.dirname(importlib.util.find_spec('airflow').origin) + directory = os.path.join(package_dir, 'migrations') + config = Config(os.path.join(package_dir, 'alembic.ini')) + config.set_main_option('script_location', directory) + config.set_main_option('sqlalchemy.url', settings.SQL_ALCHEMY_CONN) + script_ = ScriptDirectory.from_config(config) + + timeout=60 + + with settings.engine.connect() as connection: + context = MigrationContext.configure(connection) + ticker = 0 + while True: + source_heads = set(script_.get_heads()) + + db_heads = set(context.get_current_heads()) + if source_heads == db_heads: + break + + if ticker >= timeout: + raise TimeoutError("There are still unapplied migrations after {} seconds.".format(ticker)) + ticker += 1 + time.sleep(1) + log.info('Waiting for migrations... %s second(s)', ticker) +{{- end }} + +{{ define "registry_docker_config" -}} + {{- $host := .Values.registry.connection.host }} + {{- $email := .Values.registry.connection.email }} + {{- $user := .Values.registry.connection.user -}} + {{- $pass := .Values.registry.connection.pass -}} + + {{- $config := dict "auths" -}} + {{- $auth := dict -}} + {{- $data := dict -}} + {{- $_ := set $data "username" $user -}} + {{- $_ := set $data "password" $pass -}} + {{- $_ := set $data "email" $email -}} + {{- $_ := set $data "auth" (printf "%v:%v" $user $pass | b64enc) -}} + {{- $_ := set $auth $host $data -}} + {{- $_ := set $config "auths" $auth -}} + {{ $config | toJson | print }} +{{- end }} diff --git a/chart/templates/cleanup/cleanup-cronjob.yaml b/chart/templates/cleanup/cleanup-cronjob.yaml new file mode 100644 index 0000000000..3f44976566 --- /dev/null +++ b/chart/templates/cleanup/cleanup-cronjob.yaml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Cleanup Pods CronJob +################################# +{{- if .Values.cleanup.enabled }} +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: {{ .Release.Name }}-cleanup + labels: + tier: airflow + component: airflow-cleanup-pods + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + schedule: "{{ .Values.cleanup.schedule }}" + # The cron job does not allow concurrent runs; if it is time for a new job run and the previous job run hasn’t finished yet, the cron job skips the new job run + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 1 + template: + metadata: + annotations: + sidecar.istio.io/inject: "false" + spec: + restartPolicy: Never + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 12 }} + affinity: +{{ toYaml .Values.affinity | indent 12 }} + tolerations: +{{ toYaml .Values.tolerations | indent 12 }} + serviceAccountName: {{ .Release.Name }}-cleanup-serviceaccount + {{- if or .Values.registry.secretName .Values.registry.connection }} + imagePullSecrets: + - name: {{ template "registry_secret" . }} + {{- end }} + containers: + - name: airflow-cleanup-pods + image: {{ template "default_airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + # Don't use entry point here, we don't need to wait on pg-bouncer etc being available. + command: ["airflow-cleanup-pods", "--namespace={{ .Release.Namespace }}"] + env: + {{- include "standard_airflow_environment" . | indent 12 }} +{{- end }} diff --git a/chart/templates/cleanup/cleanup-serviceaccount.yaml b/chart/templates/cleanup/cleanup-serviceaccount.yaml new file mode 100644 index 0000000000..769cbdcfa0 --- /dev/null +++ b/chart/templates/cleanup/cleanup-serviceaccount.yaml @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Cleanup ServiceAccount +################################# +{{- if and .Values.rbacEnabled .Values.cleanup.enabled }} +kind: ServiceAccount +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-cleanup-serviceaccount + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +{{- end }} diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml new file mode 100644 index 0000000000..c0718e7ac2 --- /dev/null +++ b/chart/templates/configmap.yaml @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow ConfigMap +################################# +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "airflow_config" . }} + labels: + tier: airflow + component: config + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +data: + # These are system-specified config overrides. + airflow.cfg: | + [core] + load_examples = False + colored_console_log = False + executor = {{ .Values.executor }} +{{- if .Values.elasticsearch.enabled }} + remote_logging = True +{{- end }} + + [webserver] + enable_proxy_fix = True + expose_config = True + rbac = True + +{{- if eq .Values.executor "CeleryExecutor" }} + [celery] + default_queue = celery +{{- end }} + + [scheduler] + scheduler_heartbeat_sec = 5 +{{- if .Values.statsd.enabled }} + statsd_on = True + statsd_port = 9125 + statsd_prefix = airflow + statsd_host = {{ printf "%s-statsd" .Release.Name }} +{{- end }} + # Restart Scheduler every 41460 seconds (11 hours 31 minutes) + # The odd time is chosen so it is not always restarting on the same "hour" boundary + run_duration = 41460 + +{{- if .Values.elasticsearch.enabled }} + [elasticsearch] + # The elasticsearch variables were updated to the shorter names in v1.10.4 + write_stdout = True + json_format = True + log_id_template = {dag_id}_{task_id}_{execution_date}_{try_number} + + [elasticsearch_configs] + max_retries = 3 + timeout = 30 + retry_timeout = True +{{- end }} + +{{- if eq .Values.executor "KubernetesExecutor" }} + [kubernetes] + namespace = {{ .Release.Namespace }} + airflow_configmap = {{ include "airflow_config" . }} + airflow_local_settings_configmap = {{ include "airflow_config" . }} + worker_container_repository = {{ .Values.images.airflow.repository }} + worker_container_tag = {{ .Values.images.airflow.tag }} + worker_container_image_pull_policy = {{ .Values.images.airflow.pullPolicy }} + worker_service_account_name = {{ .Release.Name }}-worker-serviceaccount + image_pull_secrets = {{ template "registry_secret" . }} + dags_in_image = True + delete_worker_pods = True + + [kubernetes_secrets] + AIRFLOW__CORE__SQL_ALCHEMY_CONN = {{ printf "%s=connection" (include "airflow_metadata_secret" .) }} + AIRFLOW__CORE__FERNET_KEY = {{ printf "%s=fernet-key" (include "fernet_key_secret" .) }} +{{- if .Values.elasticsearch.enabled }} + AIRFLOW__ELASTICSEARCH__ELASTICSEARCH_HOST = {{ printf "%s=connection" (include "elasticsearch_secret" .) }} +{{- end }} + + [kubernetes_labels] + tier = airflow + component = worker + release = {{ .Release.Name }} + {{- range $key, $value := .Values.labels }} + {{ printf "%s = %s" $key $value }} + {{- end }} +{{- end }} + +{{- if .Values.webserver.webserverConfig }} + webserver_config.py: | + {{- .Values.webserver.webserverConfig | nindent 4 }} +{{- end }} + +{{- if .Values.scheduler.airflowLocalSettings }} + airflow_local_settings.py: | + {{ .Values.scheduler.airflowLocalSettings | nindent 4 }} +{{- end }} diff --git a/chart/templates/create-user-job.yaml b/chart/templates/create-user-job.yaml new file mode 100644 index 0000000000..f21d4b4d55 --- /dev/null +++ b/chart/templates/create-user-job.yaml @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Create User Job +################################# +{{- if .Values.webserver.defaultUser.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-create-user + labels: + tier: airflow + component: create-user-job + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} + annotations: + "helm.sh/hook": post-install + "helm.sh/hook-delete-policy": hook-succeeded +spec: + template: + metadata: + labels: + tier: airflow + component: create-user-job + release: {{ .Release.Name }} + spec: + restartPolicy: OnFailure + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + tolerations: +{{ toYaml .Values.tolerations | indent 8 }} + containers: + - name: create-user + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + args: + - "bash" + - "-c" + # Support running against 1.10.x and 2.0.0dev/master + - 'airflow create_user "$@" || airflow users create "$@"' + - -- + - "-r" + - {{ .Values.webserver.defaultUser.role }} + - "-u" + - {{ .Values.webserver.defaultUser.username }} + - "-e" + - {{ .Values.webserver.defaultUser.email }} + - "-f" + - {{ .Values.webserver.defaultUser.firstName }} + - "-l" + - {{ .Values.webserver.defaultUser.lastName }} + - "-p" + - {{ .Values.webserver.defaultUser.password }} + env: + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" . | indent 10 }} + volumeMounts: + - name: config + mountPath: {{ template "airflow_config_path" . }} + subPath: airflow.cfg + readOnly: true + volumes: + - name: config + configMap: + name: {{ template "airflow_config" . }} +{{- end }} diff --git a/chart/templates/flower/flower-deployment.yaml b/chart/templates/flower/flower-deployment.yaml new file mode 100644 index 0000000000..81325d06d0 --- /dev/null +++ b/chart/templates/flower/flower-deployment.yaml @@ -0,0 +1,102 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Flower Deployment +################################# +{{- if eq .Values.executor "CeleryExecutor" }} +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ .Release.Name }}-flower + labels: + tier: airflow + component: flower + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} + annotations: + checksum/airflow-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} +spec: + replicas: 1 + selector: + matchLabels: + tier: airflow + component: flower + release: {{ .Release.Name }} + template: + metadata: + labels: + tier: airflow + component: flower + release: {{ .Release.Name }} +{{- with .Values.labels }} +{{ toYaml . | indent 8 }} +{{- end }} + spec: + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + tolerations: +{{ toYaml .Values.tolerations | indent 8 }} + restartPolicy: Always + securityContext: + runAsUser: {{ .Values.uid }} + {{- if or .Values.registry.secretName .Values.registry.connection }} + imagePullSecrets: + - name: {{ template "registry_secret" . }} + {{- end }} + containers: + - name: flower + image: {{ template "flower_image" . }} + imagePullPolicy: {{ .Values.images.flower.pullPolicy }} + args: ["flower"] + resources: +{{ toYaml .Values.flower.resources | indent 12 }} + volumeMounts: + - name: config + mountPath: {{ template "airflow_config_path" . }} + subPath: airflow.cfg + readOnly: true + ports: + - name: flower-ui + containerPort: {{ .Values.ports.flowerUI }} + livenessProbe: + failureThreshold: 10 + httpGet: + path: / + port: {{ .Values.ports.flowerUI }} + initialDelaySeconds: 10 + periodSeconds: 5 + readinessProbe: + failureThreshold: 10 + httpGet: + path: / + port: {{ .Values.ports.flowerUI }} + initialDelaySeconds: 10 + periodSeconds: 5 + env: + {{- include "standard_airflow_environment" . | indent 10 }} + volumes: + - name: config + configMap: + name: {{ template "airflow_config" . }} +{{- end }} diff --git a/chart/templates/flower/flower-networkpolicy.yaml b/chart/templates/flower/flower-networkpolicy.yaml new file mode 100644 index 0000000000..5e897bbf3c --- /dev/null +++ b/chart/templates/flower/flower-networkpolicy.yaml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Flower NetworkPolicy +################################# +{{- if (and .Values.networkPolicies.enabled (eq .Values.executor "CeleryExecutor")) }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ .Release.Name }}-flower-policy + labels: + tier: airflow + component: airflow-flower-policy + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + podSelector: + matchLabels: + tier: airflow + component: flower + release: {{ .Release.Name }} + policyTypes: + - Ingress + ingress: + - from: +{{- if .Values.flower.extraNetworkPolicies }} +{{ toYaml .Values.flower.extraNetworkPolicies | indent 4 }} +{{- end }} + ports: + - protocol: TCP + port: {{ .Values.ports.flowerUI }} +{{- end }} diff --git a/chart/templates/flower/flower-service.yaml b/chart/templates/flower/flower-service.yaml new file mode 100644 index 0000000000..187046b8fc --- /dev/null +++ b/chart/templates/flower/flower-service.yaml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Flower Service Component +################################# +{{- if eq .Values.executor "CeleryExecutor" }} +kind: Service +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-flower + labels: + tier: airflow + component: flower + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + type: {{ .Values.flower.service.type }} + selector: + tier: airflow + component: flower + release: {{ .Release.Name }} + ports: + - name: flower-ui + protocol: TCP + port: {{ .Values.ports.flowerUI }} + targetPort: {{ .Values.ports.flowerUI }} +{{- end }} diff --git a/chart/templates/limitrange.yaml b/chart/templates/limitrange.yaml new file mode 100644 index 0000000000..7bf099af26 --- /dev/null +++ b/chart/templates/limitrange.yaml @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Namespace LimitRange +################################# +{{- if .Values.limits }} +apiVersion: v1 +kind: LimitRange +metadata: + name: {{ .Release.Name }}-limit-range + labels: + tier: resources + component: limitrange + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + limits: +{{ toYaml .Values.limits | indent 4 }} +{{- end }} diff --git a/chart/templates/pgbouncer/pgbouncer-deployment.yaml b/chart/templates/pgbouncer/pgbouncer-deployment.yaml new file mode 100644 index 0000000000..08712a7ed1 --- /dev/null +++ b/chart/templates/pgbouncer/pgbouncer-deployment.yaml @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Pgbouncer Deployment +################################# +{{- if .Values.pgbouncer.enabled }} +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ .Release.Name }}-pgbouncer + labels: + tier: airflow + component: pgbouncer + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + replicas: 1 + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + tier: airflow + component: pgbouncer + release: {{ .Release.Name }} + template: + metadata: + labels: + tier: airflow + component: pgbouncer + release: {{ .Release.Name }} +{{- with .Values.labels }} +{{ toYaml . | indent 8 }} +{{- end }} + annotations: + checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }} + spec: + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + tolerations: +{{ toYaml .Values.tolerations | indent 8 }} + restartPolicy: Always + containers: + - name: pgbouncer + image: {{ template "pgbouncer_image" . }} + imagePullPolicy: {{ .Values.images.pgbouncer.pullPolicy }} + command: + - pgbouncer + - -u + - pgbouncer + - /etc/pgbouncer/pgbouncer.ini + resources: +{{ toYaml .Values.pgbouncer.resources | indent 12 }} + ports: + - name: pgbouncer + containerPort: {{ .Values.ports.pgbouncer }} + livenessProbe: + tcpSocket: + port: {{ .Values.ports.pgbouncer }} + readinessProbe: + tcpSocket: + port: {{ .Values.ports.pgbouncer }} + volumeMounts: + - name: pgbouncer-config + subPath: pgbouncer.ini + mountPath: /etc/pgbouncer/pgbouncer.ini + readOnly: true + - name: pgbouncer-config + subPath: users.txt + mountPath: /etc/pgbouncer/users.txt + readOnly: true + lifecycle: + preStop: + exec: + # Allow existing queries clients to complete within 120 seconds + command: ["/bin/sh", "-c", "killall -INT pgbouncer && sleep 120"] + - name: metrics-exporter + image: {{ template "pgbouncer_exporter_image" . }} + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ template "pgbouncer_stats_secret" . }} + key: connection + ports: + - name: metrics + containerPort: {{ .Values.ports.pgbouncerScrape }} + livenessProbe: + exec: + command: + - pgbouncer_exporter + - health + initialDelaySeconds: 10 + periodSeconds: 10 + readinessProbe: + exec: + command: + - pgbouncer_exporter + - health + initialDelaySeconds: 10 + periodSeconds: 10 + volumes: + - name: pgbouncer-config + secret: + secretName: {{ template "pgbouncer_config_secret" . }} +{{- end }} diff --git a/chart/templates/pgbouncer/pgbouncer-networkpolicy.yaml b/chart/templates/pgbouncer/pgbouncer-networkpolicy.yaml new file mode 100644 index 0000000000..aa27f553ad --- /dev/null +++ b/chart/templates/pgbouncer/pgbouncer-networkpolicy.yaml @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Pgbouncer NetworkPolicy +################################# +{{- if and .Values.pgbouncer.enabled .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ .Release.Name }}-pgbouncer-policy + labels: + tier: airflow + component: airflow-pgbouncer-policy + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + podSelector: + matchLabels: + tier: airflow + component: pgbouncer + release: {{ .Release.Name }} + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + tier: airflow + release: {{ .Release.Name }} +{{- if .Values.workers.keda.enabled }} +{{- if .Values.workers.keda.namespaceLabels }} + - namespaceSelector: + matchLabels: +{{ toYaml .Values.workers.keda.namespaceLabels | indent 10}} + podSelector: +{{- else }} + - podSelector: +{{- end }} + matchLabels: + app: keda-operator +{{- end}} +{{- if .Values.pgbouncer.extraNetworkPolicies}} +{{ toYaml .Values.pgbouncer.extraNetworkPolicies | indent 4}} +{{- end}} + ports: + - protocol: TCP + port: {{ .Values.ports.pgbouncer }} + - protocol: TCP + port: {{ .Values.ports.pgbouncerScrape }} +{{- end }} diff --git a/chart/templates/pgbouncer/pgbouncer-poddisruptionbudget.yaml b/chart/templates/pgbouncer/pgbouncer-poddisruptionbudget.yaml new file mode 100644 index 0000000000..37c64c27f8 --- /dev/null +++ b/chart/templates/pgbouncer/pgbouncer-poddisruptionbudget.yaml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Pgbouncer PodDisruptionBudget +################################# +{{- if and .Values.pgbouncer.enabled .Values.pgbouncer.podDisruptionBudget.enabled }} +kind: PodDisruptionBudget +apiVersion: policy/v1beta1 +metadata: + name: {{ .Release.Name }}-pgbouncer-pdb + labels: + tier: airflow + component: pgbouncer + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + selector: + matchLabels: + tier: airflow + component: pgbouncer + release: {{ .Release.Name }} +{{ toYaml .Values.pgbouncer.podDisruptionBudget.config | indent 2 }} +{{- end }} diff --git a/chart/templates/pgbouncer/pgbouncer-service.yaml b/chart/templates/pgbouncer/pgbouncer-service.yaml new file mode 100644 index 0000000000..6913a4766a --- /dev/null +++ b/chart/templates/pgbouncer/pgbouncer-service.yaml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Pgbouncer Service +################################# +{{- if .Values.pgbouncer.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-pgbouncer + labels: + tier: airflow + component: pgbouncer + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: {{ .Values.ports.pgbouncerScrape | quote }} +{{- if .Values.pgbouncer.service.extraAnnotations }} +{{- toYaml .Values.pgbouncer.service.extraAnnotations | nindent 4 }} +{{- end }} +spec: + type: ClusterIP + selector: + tier: airflow + component: pgbouncer + release: {{ .Release.Name }} + ports: + - name: pgbouncer + protocol: TCP + port: {{ .Values.ports.pgbouncer }} + targetPort: {{ .Values.ports.pgbouncer }} + - name: pgbouncer-metrics + protocol: TCP + port: {{ .Values.ports.pgbouncerScrape }} + targetPort: {{ .Values.ports.pgbouncerScrape }} +{{- end }} diff --git a/chart/templates/rbac/pod-cleanup-role.yaml b/chart/templates/rbac/pod-cleanup-role.yaml new file mode 100644 index 0000000000..58f76e6045 --- /dev/null +++ b/chart/templates/rbac/pod-cleanup-role.yaml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Cleanup Role +################################# +{{- if and .Values.rbacEnabled .Values.cleanup.enabled }} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Name }}-cleanup-role + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +rules: + - apiGroups: + - "" + resources: + - "pods" + verbs: + - "list" + - "delete" +{{- end }} diff --git a/chart/templates/rbac/pod-cleanup-rolebinding.yaml b/chart/templates/rbac/pod-cleanup-rolebinding.yaml new file mode 100644 index 0000000000..4c2dd259b2 --- /dev/null +++ b/chart/templates/rbac/pod-cleanup-rolebinding.yaml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Cleanup Role Binding +################################# +{{- if and .Values.rbacEnabled .Values.cleanup.enabled }} +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Name }}-cleanup-rolebinding + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-cleanup-role +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-cleanup-serviceaccount + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/chart/templates/rbac/pod-launcher-role.yaml b/chart/templates/rbac/pod-launcher-role.yaml new file mode 100644 index 0000000000..b3d68188b3 --- /dev/null +++ b/chart/templates/rbac/pod-launcher-role.yaml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Pod Launcher Role +################################# +{{- if and .Values.rbacEnabled .Values.allowPodLaunching }} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Name }}-pod-launcher-role + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +rules: + - apiGroups: + - "" + resources: + - "pods" + verbs: + - "create" + - "list" + - "get" + - "watch" + - "delete" + - apiGroups: + - "" + resources: + - "pods/log" + verbs: + - "get" + - apiGroups: + - "" + resources: + - "pods/exec" + verbs: + - "create" + - "get" +{{- end }} diff --git a/chart/templates/rbac/pod-launcher-rolebinding.yaml b/chart/templates/rbac/pod-launcher-rolebinding.yaml new file mode 100644 index 0000000000..9b8aadfc17 --- /dev/null +++ b/chart/templates/rbac/pod-launcher-rolebinding.yaml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Pod Launcher Role Binding +################################# +{{- if and .Values.rbacEnabled .Values.allowPodLaunching }} +{{- $grantScheduler := or (eq .Values.executor "LocalExecutor") (eq .Values.executor "SequentialExecutor") (eq .Values.executor "KubernetesExecutor") }} +{{- $grantWorker := or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "KubernetesExecutor") }} +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Release.Name }}-pod-launcher-rolebinding + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-pod-launcher-role +subjects: +{{- if $grantScheduler }} + - kind: ServiceAccount + name: {{ .Release.Name }}-scheduler-serviceaccount + namespace: {{ .Release.Namespace }} +{{- end }} +{{- if $grantWorker }} + - kind: ServiceAccount + name: {{ .Release.Name }}-worker-serviceaccount + namespace: {{ .Release.Namespace }} +{{- end }} +{{- end }} diff --git a/chart/templates/redis/redis-networkpolicy.yaml b/chart/templates/redis/redis-networkpolicy.yaml new file mode 100644 index 0000000000..b0a9e495cd --- /dev/null +++ b/chart/templates/redis/redis-networkpolicy.yaml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Redis NetworkPolicy +################################# +{{- if (and .Values.networkPolicies.enabled (eq .Values.executor "CeleryExecutor")) }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ .Release.Name }}-redis-policy + labels: + tier: airflow + component: redis-policy + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + podSelector: + matchLabels: + tier: airflow + component: redis + release: {{ .Release.Name }} + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + tier: airflow + component: worker + release: {{ .Release.Name }} + - podSelector: + matchLabels: + tier: airflow + component: scheduler + release: {{ .Release.Name }} + - podSelector: + matchLabels: + tier: airflow + component: flower + release: {{ .Release.Name }} + ports: + - protocol: TCP + port: {{ .Values.ports.redisDB }} +{{- end }} diff --git a/chart/templates/redis/redis-service.yaml b/chart/templates/redis/redis-service.yaml new file mode 100644 index 0000000000..ac4bb21a82 --- /dev/null +++ b/chart/templates/redis/redis-service.yaml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Redis Service +################################# +{{- if eq .Values.executor "CeleryExecutor" }} +kind: Service +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-redis + labels: + tier: airflow + component: redis + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + type: ClusterIP + selector: + tier: airflow + component: redis + release: {{ .Release.Name }} + ports: + - name: redis-db + protocol: TCP + port: {{ .Values.ports.redisDB }} + targetPort: {{ .Values.ports.redisDB }} +{{- end }} diff --git a/chart/templates/redis/redis-statefulset.yaml b/chart/templates/redis/redis-statefulset.yaml new file mode 100644 index 0000000000..e1eff0cceb --- /dev/null +++ b/chart/templates/redis/redis-statefulset.yaml @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Redis StatefulSet +################################# +{{- if eq .Values.executor "CeleryExecutor" }} +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: {{ .Release.Name }}-redis + labels: + tier: airflow + component: redis + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + serviceName: {{ .Release.Name }}-redis + selector: + matchLabels: + tier: airflow + component: redis + release: {{ .Release.Name }} + template: + metadata: + labels: + tier: airflow + component: redis + release: {{ .Release.Name }} +{{- with .Values.labels }} +{{ toYaml . | indent 8 }} +{{- end }} + annotations: + {{- if .Values.redis.safeToEvict }} + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + {{- end }} + spec: + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + tolerations: +{{ toYaml .Values.tolerations | indent 8 }} + containers: + - name: redis + image: {{- include "redis_image" . | indent 1 }} + imagePullPolicy: {{ .Values.images.redis.pullPolicy }} + command: ["/bin/sh"] + resources: +{{ toYaml .Values.redis.resources | indent 12 }} + args: ["-c", "redis-server --requirepass ${REDIS_PASSWORD}"] + ports: + - name: redis-db + containerPort: {{ .Values.ports.redisDB }} + volumeMounts: + - name: redis-db + mountPath: /data + env: + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "redis_password_secret" . }} + key: password +{{- if not .Values.redis.persistence.enabled }} + volumes: + - name: redis-db + emptyDir: {} +{{- else }} + volumeClaimTemplates: + - metadata: + name: redis-db + spec: + {{- if .Values.redis.persistence.storageClassName }} + storageClassName: {{ .Values.redis.persistence.storageClassName }} + {{- end }} + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.redis.persistence.size }} +{{- end }} +{{- end }} diff --git a/chart/templates/resourcequota.yaml b/chart/templates/resourcequota.yaml new file mode 100644 index 0000000000..04d59290da --- /dev/null +++ b/chart/templates/resourcequota.yaml @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Namespace ResourceQuota +################################# +{{- if .Values.quotas }} +apiVersion: v1 +kind: ResourceQuota +metadata: + name: {{ .Release.Name }}-resource-quota + labels: + tier: resources + component: resourcequota + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + hard: +{{ toYaml .Values.quotas | indent 4 }} +{{- end }} diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml new file mode 100644 index 0000000000..1b46f6a62b --- /dev/null +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -0,0 +1,195 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Scheduler Deployment/StatefulSet +################################# + +# Are we using a local/sequenial executor? +{{- $local := or (eq .Values.executor "LocalExecutor") (eq .Values.executor "SequentialExecutor") }} +# Are we using the kubernetes executor? +{{- $kube := eq .Values.executor "KubernetesExecutor" }} +# Is persistence enabled on the _workers_? +# This is important because in $local mode, the scheduler assumes the role of the worker +{{- $persistence := .Values.workers.persistence.enabled }} +# If we're using a StatefulSet +{{- $stateful := and $local $persistence }} +# If we're using elasticsearch logging +{{- $elasticsearch := .Values.elasticsearch.enabled }} + +kind: {{ if $stateful }}StatefulSet{{ else }}Deployment{{ end }} +apiVersion: apps/v1 +metadata: + name: {{ .Release.Name }}-scheduler + labels: + tier: airflow + component: scheduler + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: +{{- if $stateful }} + serviceName: {{ .Release.Name }}-scheduler +{{- end }} + replicas: 1 + selector: + matchLabels: + tier: airflow + component: scheduler + release: {{ .Release.Name }} + template: + metadata: + labels: + tier: airflow + component: scheduler + release: {{ .Release.Name }} +{{- with .Values.labels }} +{{ toYaml . | indent 8 }} +{{- end }} + annotations: + checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }} + checksum/result-backend-secret: {{ include (print $.Template.BasePath "/secrets/result-backend-connection-secret.yaml") . | sha256sum }} + checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }} + checksum/airflow-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- if .Values.scheduler.safeToEvict }} + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + {{- end }} + {{- if .Values.airflowPodAnnotations }} + {{- toYaml .Values.airflowPodAnnotations | nindent 8 }} + {{- end }} + spec: + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + tolerations: +{{ toYaml .Values.tolerations | indent 8 }} + restartPolicy: Always + terminationGracePeriodSeconds: 10 + serviceAccountName: {{ .Release.Name }}-scheduler-serviceaccount + securityContext: + runAsUser: {{ .Values.uid }} + fsGroup: {{ .Values.gid }} + {{- if or .Values.registry.secretName .Values.registry.connection }} + imagePullSecrets: + - name: {{ template "registry_secret" . }} + {{- end }} + initContainers: + - name: run-airflow-migrations + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + # Support running against 1.10.x and 2.0.0dev/master + args: ["bash", "-c", "airflow upgradedb || airfow db upgrade"] + env: + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" . | indent 10 }} + containers: + # Always run the main scheduler container. + - name: scheduler + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + args: ["scheduler"] + env: + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" . | indent 10 }} + # If the scheduler stops heartbeating for 5 minutes (10*30s) kill the + # scheduler and let Kubernetes restart it + livenessProbe: + failureThreshold: 10 + periodSeconds: 30 + exec: + command: + - python + - -Wignore + - -c + - | + import os + os.environ['AIRFLOW__CORE__LOGGING_LEVEL'] = 'ERROR' + os.environ['AIRFLOW__LOGGING__LOGGING_LEVEL'] = 'ERROR' + + from airflow.jobs.scheduler_job import SchedulerJob + from airflow.utils.net import get_hostname + import sys + + job = SchedulerJob.most_recent_job() + sys.exit(0 if job.is_alive() and job.hostname == get_hostname() else 1) + resources: +{{ toYaml .Values.scheduler.resources | indent 12 }} + volumeMounts: + - name: logs + mountPath: {{ template "airflow_logs" . }} + - name: config + mountPath: {{ template "airflow_config_path" . }} + subPath: airflow.cfg + readOnly: true +{{- if .Values.scheduler.airflowLocalSettings }} + - name: config + mountPath: {{ template "airflow_local_setting_path" . }} + subPath: airflow_local_settings.py + readOnly: true +{{- end }} + # Always start the garbage collector sidecar. + - name: scheduler-gc + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + args: ["bash", "/clean-logs"] + volumeMounts: + - name: logs + mountPath: {{ template "airflow_logs" . }} +{{- if and $local (not $elasticsearch) }} + # Start the sidecar log server if we're in local mode and + # we don't have elasticsearch enabled. + - name: scheduler-logs + image: {{ template "airflow_image" . }} + args: ["airflow", "serve_logs"] + ports: + - name: worker-logs + containerPort: {{ .Values.ports.workerLogs }} + volumeMounts: + - name: logs + mountPath: {{ template "airflow_logs" . }} + - name: config + mountPath: {{ template "airflow_config_path" . }} + subPath: airflow.cfg + readOnly: true + env: + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" . | indent 10 }} +{{- end }} + volumes: + - name: config + configMap: + name: {{ template "airflow_config" . }} +{{- if not $stateful }} + - name: logs + emptyDir: {} +{{- else }} + volumeClaimTemplates: + - metadata: + name: logs + spec: + {{- if .Values.workers.persistence.storageClassName }} + storageClassName: {{ .Values.workers.persistence.storageClassName }} + {{- end }} + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.workers.persistence.size }} +{{- end }} diff --git a/chart/templates/scheduler/scheduler-networkpolicy.yaml b/chart/templates/scheduler/scheduler-networkpolicy.yaml new file mode 100644 index 0000000000..de36ce7788 --- /dev/null +++ b/chart/templates/scheduler/scheduler-networkpolicy.yaml @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Scheduler NetworkPolicy +################################# +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ .Release.Name }}-scheduler-policy + labels: + tier: airflow + component: airflow-scheduler-policy + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + podSelector: + matchLabels: + tier: airflow + component: scheduler + release: {{ .Release.Name }} + policyTypes: + - Ingress +{{- if (or (eq .Values.executor "LocalExecutor") (eq .Values.executor "SequentialExecutor")) }} + ingress: + - from: + - podSelector: + matchLabels: + tier: airflow + component: webserver + release: {{ .Release.Name }} + ports: + - protocol: TCP + port: {{ .Values.ports.workerLogs }} +{{- end }} +{{- end }} diff --git a/chart/templates/scheduler/scheduler-poddisruptionbudget.yaml b/chart/templates/scheduler/scheduler-poddisruptionbudget.yaml new file mode 100644 index 0000000000..967c2cf041 --- /dev/null +++ b/chart/templates/scheduler/scheduler-poddisruptionbudget.yaml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Scheduler PodDisruptionBudget +################################# +{{- if .Values.scheduler.podDisruptionBudget.enabled }} +kind: PodDisruptionBudget +apiVersion: policy/v1beta1 +metadata: + name: {{ .Release.Name }}-scheduler-pdb + labels: + tier: airflow + component: scheduler + release: {{ .Release.Name }} + chart: {{ .Chart.Name }} + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + selector: + matchLabels: + tier: airflow + component: scheduler + release: {{ .Release.Name }} +{{ toYaml .Values.scheduler.podDisruptionBudget.config | indent 2 }} +{{- end }} diff --git a/chart/templates/scheduler/scheduler-service.yaml b/chart/templates/scheduler/scheduler-service.yaml new file mode 100644 index 0000000000..c8d13a5cf8 --- /dev/null +++ b/chart/templates/scheduler/scheduler-service.yaml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Scheduler Service +################################# +{{- if (or (eq .Values.executor "LocalExecutor") (eq .Values.executor "SequentialExecutor")) }} +kind: Service +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-scheduler + labels: + tier: airflow + component: scheduler + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + clusterIP: None + selector: + tier: airflow + component: scheduler + release: {{ .Release.Name }} + ports: + - name: task-logs + protocol: TCP + port: {{ .Values.ports.workerLogs }} + targetPort: {{ .Values.ports.workerLogs }} +{{- end }} diff --git a/chart/templates/scheduler/scheduler-serviceaccount.yaml b/chart/templates/scheduler/scheduler-serviceaccount.yaml new file mode 100644 index 0000000000..bfd9d3badb --- /dev/null +++ b/chart/templates/scheduler/scheduler-serviceaccount.yaml @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Scheduler ServiceAccount +################################# +{{- if .Values.rbacEnabled }} +kind: ServiceAccount +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-scheduler-serviceaccount + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +{{- end }} diff --git a/chart/templates/secrets/elasticsearch-secret.yaml b/chart/templates/secrets/elasticsearch-secret.yaml new file mode 100644 index 0000000000..cae81581fc --- /dev/null +++ b/chart/templates/secrets/elasticsearch-secret.yaml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Elasticsearch Secret +################################# +{{- if (and .Values.elasticsearch.connection (not .Values.elasticsearch.secretName)) }} +kind: Secret +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-elasticsearch + labels: + release: {{ .Release.Name }} + chart: {{ .Chart.Name }} + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +type: Opaque +data: + connection: {{ (printf "http://%s:%s@%s:%s" .Values.elasticsearch.connection.user .Values.elasticsearch.connection.pass .Values.elasticsearch.connection.host (.Values.elasticsearch.connection.port | toString)) | b64enc | quote }} +{{- end }} diff --git a/chart/templates/secrets/fernetkey-secret.yaml b/chart/templates/secrets/fernetkey-secret.yaml new file mode 100644 index 0000000000..6881af7ccf --- /dev/null +++ b/chart/templates/secrets/fernetkey-secret.yaml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Fernet Key Secret +################################# +{{- if not .Values.fernetKeySecretName }} +{{ $generated_fernet_key := (randAlphaNum 32 | b64enc) }} +kind: Secret +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-fernet-key + labels: + release: {{ .Release.Name }} + chart: {{ .Chart.Name }} + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} + annotations: + "helm.sh/hook": "pre-install" + "helm.sh/hook-delete-policy": "before-hook-creation" + "helm.sh/hook-weight": "0" +type: Opaque +data: + fernet-key: {{ (default $generated_fernet_key .Values.fernetKey) | b64enc | quote }} +{{- end }} diff --git a/chart/templates/secrets/metadata-connection-secret.yaml b/chart/templates/secrets/metadata-connection-secret.yaml new file mode 100644 index 0000000000..031d52735d --- /dev/null +++ b/chart/templates/secrets/metadata-connection-secret.yaml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Metadata Secret +################################# +{{- if (and .Values.data.metadataConnection (not .Values.data.metadataSecretName)) }} +{{- $postgresHost := .Values.data.metadataConnection.host | default (printf "%s-%s.%s.svc.cluster.local" .Release.Name "postgresql" .Release.Namespace) }} +{{- $pgbouncerHost := (printf "%s-%s.%s.svc.cluster.local" .Release.Name "pgbouncer" .Release.Namespace) }} +{{- $host := ternary $pgbouncerHost $postgresHost .Values.pgbouncer.enabled }} +{{- $port := ((ternary .Values.ports.pgbouncer .Values.data.metadataConnection.port .Values.pgbouncer.enabled) | toString) }} +{{- $database := (ternary (printf "%s-%s" .Release.Name "metadata") .Values.data.metadataConnection.db .Values.pgbouncer.enabled) }} + +kind: Secret +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-airflow-metadata + labels: + release: {{ .Release.Name }} + chart: {{ .Chart.Name }} + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +type: Opaque +data: + connection: {{ (printf "postgresql://%s:%s@%s:%s/%s?sslmode=%s" .Values.data.metadataConnection.user .Values.data.metadataConnection.pass $host $port $database .Values.data.metadataConnection.sslmode) | b64enc | quote }} +{{- end }} diff --git a/chart/templates/secrets/pgbouncer-config-secret.yaml b/chart/templates/secrets/pgbouncer-config-secret.yaml new file mode 100644 index 0000000000..0eb3ad8702 --- /dev/null +++ b/chart/templates/secrets/pgbouncer-config-secret.yaml @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Pgbouncer Config Secret +################################# +{{- if .Values.pgbouncer.enabled }} +kind: Secret +apiVersion: v1 +metadata: + name: {{ template "pgbouncer_config_secret" . }} + labels: + release: {{ .Release.Name }} + chart: {{ .Chart.Name }} + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +type: Opaque +data: + pgbouncer.ini: {{ include "pgbouncer_config" . | b64enc }} + users.txt: {{ include "pgbouncer_users" . | b64enc }} +{{- end }} diff --git a/chart/templates/secrets/pgbouncer-stats-secret.yaml b/chart/templates/secrets/pgbouncer-stats-secret.yaml new file mode 100644 index 0000000000..ba81ea6509 --- /dev/null +++ b/chart/templates/secrets/pgbouncer-stats-secret.yaml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Pgbouncer Stats Secret +################################# +{{- if .Values.pgbouncer.enabled }} +kind: Secret +apiVersion: v1 +metadata: + name: {{ template "pgbouncer_stats_secret" . }} + labels: + release: {{ .Release.Name }} + chart: {{ .Chart.Name }} + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +type: Opaque +data: + connection: {{ (printf "postgresql://%s:%s@127.0.0.1:%s/pgbouncer?sslmode=disable" .Values.data.metadataConnection.user .Values.data.metadataConnection.pass (.Values.ports.pgbouncer | toString)) | b64enc | quote }} +{{- end }} diff --git a/chart/templates/secrets/redis-secrets.yaml b/chart/templates/secrets/redis-secrets.yaml new file mode 100644 index 0000000000..7c9fe26ecf --- /dev/null +++ b/chart/templates/secrets/redis-secrets.yaml @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Redis Password Secret +################################# +# If both of these secret names are not set, we will either use the set password, or generate one +{{- if not (and .Values.redis.passwordSecretName .Values.redis.brokerURLSecretName) }} +{{ $random_redis_password := randAlphaNum 10 }} +kind: Secret +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-redis-password + labels: + release: {{ .Release.Name }} + chart: {{ .Chart.Name }} + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} + annotations: + "helm.sh/hook": "pre-install" + "helm.sh/hook-delete-policy": "before-hook-creation" + "helm.sh/hook-weight": "0" +type: Opaque +data: + password: {{ (default $random_redis_password .Values.redis.password) | b64enc | quote }} +--- +################################ +## Airflow Redis Connection Secret +################################# +kind: Secret +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-broker-url + labels: + release: {{ .Release.Name }} + chart: {{ .Chart.Name }} + heritage: {{ .Release.Service }} + annotations: + "helm.sh/hook": "pre-install" + "helm.sh/hook-delete-policy": "before-hook-creation" + "helm.sh/hook-weight": "0" +type: Opaque +data: + connection: {{ (printf "redis://:%s@%s-redis:6379/0" (default $random_redis_password .Values.redis.password) .Release.Name) | b64enc | quote }} +{{- end }} diff --git a/chart/templates/secrets/registry-secret.yaml b/chart/templates/secrets/registry-secret.yaml new file mode 100644 index 0000000000..7a3d7bb06e --- /dev/null +++ b/chart/templates/secrets/registry-secret.yaml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Registry Secret +################################# +{{- if (and .Values.registry.connection (not .Values.registry.secretName)) }} +kind: Secret +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-registry + labels: + release: {{ .Release.Name }} + chart: {{ .Chart.Name }} + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: {{ include "registry_docker_config" . | b64enc }} +{{- end }} diff --git a/chart/templates/secrets/result-backend-connection-secret.yaml b/chart/templates/secrets/result-backend-connection-secret.yaml new file mode 100644 index 0000000000..693404d185 --- /dev/null +++ b/chart/templates/secrets/result-backend-connection-secret.yaml @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Result Backend Secret +################################# +{{- if (and .Values.data.resultBackendConnection (not .Values.data.resultBackendSecretName)) }} +{{- $host := .Values.data.resultBackendConnection.host | default (printf "%s-%s" .Release.Name "postgresql") }} +kind: Secret +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-airflow-result-backend + labels: + release: {{ .Release.Name }} + chart: {{ .Chart.Name }} + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +type: Opaque +data: + connection: {{ (printf "db+postgresql://%s:%s@%s:%s/%s?sslmode=%s" .Values.data.resultBackendConnection.user .Values.data.resultBackendConnection.pass (ternary (printf "%s-%s" .Release.Name "pgbouncer") $host .Values.pgbouncer.enabled) ((ternary .Values.ports.pgbouncer .Values.data.resultBackendConnection.port .Values.pgbouncer.enabled) | toString) (ternary (printf "%s-%s" .Release.Name "result-backend") .Values.data.resultBackendConnection.db .Values.pgbouncer.enabled) .Values.data.resultBackendConnection.sslmode) | b64enc | quote }} +{{- end }} diff --git a/chart/templates/statsd/statsd-deployment.yaml b/chart/templates/statsd/statsd-deployment.yaml new file mode 100644 index 0000000000..d7f546443a --- /dev/null +++ b/chart/templates/statsd/statsd-deployment.yaml @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow StatsD Deployment +################################# +{{- if .Values.statsd.enabled }} +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ .Release.Name }}-statsd + labels: + tier: airflow + component: statsd + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + replicas: 1 + selector: + matchLabels: + tier: airflow + component: statsd + release: {{ .Release.Name }} + template: + metadata: + labels: + tier: airflow + component: statsd + release: {{ .Release.Name }} +{{- with .Values.labels }} +{{ toYaml . | indent 8 }} +{{- end }} + spec: + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + tolerations: +{{ toYaml .Values.tolerations | indent 8 }} + restartPolicy: Always + containers: + - name: statsd + image: {{- include "statsd_image" . | indent 1 }} + imagePullPolicy: {{ .Values.images.statsd.pullPolicy }} + args: + - "-statsd.mapping-config=/etc/statsd-exporter/mappings.yml" + resources: +{{ toYaml .Values.statsd.resources | indent 12 }} + ports: + - name: statsd-ingest + protocol: UDP + containerPort: {{ .Values.ports.statsdIngest }} + - name: statsd-scrape + containerPort: {{ .Values.ports.statsdScrape }} + livenessProbe: + httpGet: + path: /metrics + port: {{ .Values.ports.statsdScrape }} + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /metrics + port: {{ .Values.ports.statsdScrape }} + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 +{{- end }} diff --git a/chart/templates/statsd/statsd-networkpolicy.yaml b/chart/templates/statsd/statsd-networkpolicy.yaml new file mode 100644 index 0000000000..6c42a45ede --- /dev/null +++ b/chart/templates/statsd/statsd-networkpolicy.yaml @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow StatsD NetworkPolicy +################################# +{{- if and .Values.networkPolicies.enabled .Values.statsd.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ .Release.Name }}-statsd-policy + labels: + tier: airflow + component: statsd-policy + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + podSelector: + matchLabels: + tier: airflow + component: statsd + release: {{ .Release.Name }} + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + tier: airflow + release: {{ .Release.Name }} +{{- if .Values.statsd.extraNetworkPolicies }} +{{ toYaml .Values.statsd.extraNetworkPolicies | indent 4 }} +{{- end }} + ports: + - protocol: UDP + port: {{ .Values.ports.statsdIngest }} + - protocol: TCP + port: {{ .Values.ports.statsdScrape }} +{{- end }} diff --git a/chart/templates/statsd/statsd-service.yaml b/chart/templates/statsd/statsd-service.yaml new file mode 100644 index 0000000000..561dcf9fff --- /dev/null +++ b/chart/templates/statsd/statsd-service.yaml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow StatsD Service +################################# +{{- if .Values.statsd.enabled }} +kind: Service +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-statsd + labels: + tier: airflow + component: statsd + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: {{ .Values.ports.statsdScrape | quote }} +{{- if .Values.statsd.service.extraAnnotations }} +{{- toYaml .Values.statsd.service.extraAnnotations | nindent 4 }} +{{- end }} +spec: + type: ClusterIP + selector: + tier: airflow + component: statsd + release: {{ .Release.Name }} + ports: + - name: statsd-ingest + protocol: UDP + port: {{ .Values.ports.statsdIngest }} + targetPort: {{ .Values.ports.statsdIngest }} + - name: statsd-scrape + protocol: TCP + port: {{ .Values.ports.statsdScrape }} + targetPort: {{ .Values.ports.statsdScrape }} +{{- end }} diff --git a/chart/templates/webserver/webserver-deployment.yaml b/chart/templates/webserver/webserver-deployment.yaml new file mode 100644 index 0000000000..b4c9714bec --- /dev/null +++ b/chart/templates/webserver/webserver-deployment.yaml @@ -0,0 +1,139 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Webserver Deployment +################################# +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ .Release.Name }}-webserver + labels: + tier: airflow + component: webserver + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + replicas: {{ .Values.webserver.replicas }} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + tier: airflow + component: webserver + release: {{ .Release.Name }} + template: + metadata: + labels: + tier: airflow + component: webserver + release: {{ .Release.Name }} +{{- with .Values.labels }} +{{ toYaml . | indent 8 }} +{{- end }} + annotations: + checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }} + checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }} + checksum/airflow-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- if .Values.airflowPodAnnotations }} + {{- toYaml .Values.airflowPodAnnotations | nindent 8 }} + {{- end }} + spec: + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + tolerations: +{{ toYaml .Values.tolerations | indent 8 }} + restartPolicy: Always + securityContext: + runAsUser: {{ .Values.uid }} + {{- if or .Values.registry.secretName .Values.registry.connection }} + imagePullSecrets: + - name: {{ template "registry_secret" . }} + {{- end }} + initContainers: + - name: wait-for-airflow-migrations + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + args: + {{- include "wait-for-migrations-command" . | indent 10 }} + env: + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" . | indent 10 }} + containers: + - name: webserver + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + args: ["webserver"] + resources: +{{ toYaml .Values.webserver.resources | indent 12 }} + volumeMounts: + - name: config + mountPath: {{ template "airflow_config_path" . }} + subPath: airflow.cfg + readOnly: true +{{- if .Values.webserver.webserverConfig }} + - name: config + mountPath: {{ template "airflow_webserver_config_path" . }} + subPath: webserver_config.py + readOnly: true +{{- end }} +{{- if .Values.scheduler.airflowLocalSettings }} + - name: config + mountPath: {{ template "airflow_local_setting_path" . }} + subPath: airflow_local_settings.py + readOnly: true +{{- end }} +{{- if .Values.webserver.extraVolumeMounts }} +{{ toYaml .Values.webserver.extraVolumeMounts | indent 12 }} +{{- end }} + ports: + - name: webserver-ui + containerPort: {{ .Values.ports.airflowUI }} + livenessProbe: + httpGet: + path: /health + port: {{ .Values.ports.airflowUI }} + initialDelaySeconds: {{ .Values.webserver.livenessProbe.initialDelaySeconds | default 15 }} + timeoutSeconds: {{ .Values.webserver.livenessProbe.timeoutSeconds | default 30 }} + failureThreshold: {{ .Values.webserver.livenessProbe.failureThreshold | default 20 }} + periodSeconds: {{ .Values.webserver.livenessProbe.periodSeconds | default 5 }} + readinessProbe: + httpGet: + path: /health + port: {{ .Values.ports.airflowUI }} + initialDelaySeconds: {{ .Values.webserver.readinessProbe.initialDelaySeconds | default 15 }} + timeoutSeconds: {{ .Values.webserver.readinessProbe.timeoutSeconds | default 30 }} + failureThreshold: {{ .Values.webserver.readinessProbe.failureThreshold | default 20 }} + periodSeconds: {{ .Values.webserver.readinessProbe.periodSeconds | default 5 }} + env: + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" . | indent 10 }} + volumes: + - name: config + configMap: + name: {{ template "airflow_config" . }} +{{- if .Values.webserver.extraVolumes }} +{{ toYaml .Values.webserver.extraVolumes | indent 8 }} +{{- end }} diff --git a/chart/templates/webserver/webserver-networkpolicy.yaml b/chart/templates/webserver/webserver-networkpolicy.yaml new file mode 100644 index 0000000000..3b2eaa75e4 --- /dev/null +++ b/chart/templates/webserver/webserver-networkpolicy.yaml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Webserver NetworkPolicy +################################# +{{- if .Values.networkPolicies.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ .Release.Name }}-webserver-policy + labels: + tier: airflow + component: airflow-webserver-policy + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + podSelector: + matchLabels: + tier: airflow + component: webserver + release: {{ .Release.Name }} + policyTypes: + - Ingress + ingress: + - from: +{{- if .Values.webserver.extraNetworkPolicies }} +{{ toYaml .Values.webserver.extraNetworkPolicies | indent 4 }} +{{- end }} + ports: + - protocol: TCP + port: {{ .Values.ports.airflowUI }} +{{- end }} diff --git a/chart/templates/webserver/webserver-service.yaml b/chart/templates/webserver/webserver-service.yaml new file mode 100644 index 0000000000..77f5995dfe --- /dev/null +++ b/chart/templates/webserver/webserver-service.yaml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Webserver Service +################################# +kind: Service +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-webserver + labels: + tier: airflow + component: webserver + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + type: {{ .Values.webserver.service.type }} + selector: + tier: airflow + component: webserver + release: {{ .Release.Name }} + ports: + - name: airflow-ui + protocol: TCP + port: {{ .Values.ports.airflowUI }} + targetPort: {{ .Values.ports.airflowUI }} diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml new file mode 100644 index 0000000000..d5043a7735 --- /dev/null +++ b/chart/templates/workers/worker-deployment.yaml @@ -0,0 +1,161 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Worker Deployment +################################# +{{- $persistence := .Values.workers.persistence.enabled }} +{{- if eq .Values.executor "CeleryExecutor" }} +kind: {{ if $persistence }}StatefulSet{{ else }}Deployment{{ end }} +apiVersion: apps/v1 +metadata: + name: {{ .Release.Name }}-worker + labels: + tier: airflow + component: worker + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: +{{- if $persistence }} + serviceName: {{ .Release.Name }}-worker +{{- end }} + replicas: {{ .Values.workers.replicas }} + selector: + matchLabels: + tier: airflow + component: worker + release: {{ .Release.Name }} + template: + metadata: + labels: + tier: airflow + component: worker + release: {{ .Release.Name }} +{{- with .Values.labels }} +{{ toYaml . | indent 8 }} +{{- end }} + annotations: + checksum/metadata-secret: {{ include (print $.Template.BasePath "/secrets/metadata-connection-secret.yaml") . | sha256sum }} + checksum/result-backend-secret: {{ include (print $.Template.BasePath "/secrets/result-backend-connection-secret.yaml") . | sha256sum }} + checksum/pgbouncer-config-secret: {{ include (print $.Template.BasePath "/secrets/pgbouncer-config-secret.yaml") . | sha256sum }} + checksum/airflow-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- if .Values.workers.safeToEvict }} + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + {{- end }} + {{- if .Values.airflowPodAnnotations }} + {{- toYaml .Values.airflowPodAnnotations | nindent 8 }} + {{- end }} + spec: + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + tolerations: +{{ toYaml .Values.tolerations | indent 8 }} + terminationGracePeriodSeconds: {{ .Values.workers.terminationGracePeriodSeconds }} + restartPolicy: Always + serviceAccountName: {{ .Release.Name }}-worker-serviceaccount + securityContext: + runAsUser: {{ .Values.uid }} + fsGroup: {{ .Values.gid }} + {{- if or .Values.registry.secretName .Values.registry.connection }} + imagePullSecrets: + - name: {{ template "registry_secret" . }} + {{- end }} + initContainers: + {{- if and $persistence .Values.workers.persistence.fixPermissions }} + - name: volume-permissions + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + command: + - chown + - -R + - "{{ .Values.uid }}:{{ .Values.gid }}" + - {{ template "airflow_logs" . }} + securityContext: + runAsUser: 0 + volumeMounts: + - name: logs + mountPath: {{ template "airflow_logs" . }} + {{- end }} + - name: wait-for-airflow-migrations + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + args: + {{- include "wait-for-migrations-command" . | indent 10 }} + env: + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" . | indent 10 }} + containers: + - name: worker + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + args: ["worker"] + resources: +{{ toYaml .Values.workers.resources | indent 12 }} + ports: + - name: worker-logs + containerPort: {{ .Values.ports.workerLogs }} + volumeMounts: + - name: logs + mountPath: {{ template "airflow_logs" . }} + - name: config + mountPath: {{ template "airflow_config_path" . }} + subPath: airflow.cfg + readOnly: true +{{- if .Values.scheduler.airflowLocalSettings }} + - name: config + mountPath: {{ template "airflow_local_setting_path" . }} + subPath: airflow_local_settings.py + readOnly: true +{{- end }} + env: + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" . | indent 10 }} +{{- if $persistence }} + - name: worker-gc + image: {{ template "airflow_image" . }} + args: ["bash", "/clean-logs"] + volumeMounts: + - name: logs + mountPath: {{ template "airflow_logs" . }} +{{- end }} + volumes: + - name: config + configMap: + name: {{ template "airflow_config" . }} +{{- if not $persistence }} + - name: logs + emptyDir: {} +{{- else }} + volumeClaimTemplates: + - metadata: + name: logs + spec: + {{- if .Values.workers.persistence.storageClassName }} + storageClassName: {{ .Values.workers.persistence.storageClassName }} + {{- end }} + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.workers.persistence.size }} +{{- end }} +{{- end }} diff --git a/chart/templates/workers/worker-kedaautoscaler.yaml b/chart/templates/workers/worker-kedaautoscaler.yaml new file mode 100644 index 0000000000..891afbf788 --- /dev/null +++ b/chart/templates/workers/worker-kedaautoscaler.yaml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Worker KEDA Scaler +################################# +{{- if (and .Values.workers.keda.enabled (eq .Values.executor "CeleryExecutor")) }} +apiVersion: keda.k8s.io/v1alpha1 +kind: ScaledObject +metadata: + name: {{ .Release.Name }}-worker + labels: + tier: airflow + component: worker-horizontalpodautoscaler + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + deploymentName: {{ .Release.Name }}-worker +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + scaleTargetRef: + deploymentName: {{ .Release.Name }}-worker + pollingInterval: {{ .Values.workers.keda.pollingInterval }} # Optional. Default: 30 seconds + cooldownPeriod: {{ .Values.workers.keda.cooldownPeriod }} # Optional. Default: 300 seconds + maxReplicaCount: {{ .Values.workers.keda.maxReplicaCount }} # Optional. Default: 100 + triggers: + - type: postgresql + metadata: + connection: AIRFLOW_CONN_AIRFLOW_DB + query: "SELECT ceil(COUNT(*)::decimal / 16) FROM task_instance WHERE state='running' OR state='queued'" +{{- end }} diff --git a/chart/templates/workers/worker-networkpolicy.yaml b/chart/templates/workers/worker-networkpolicy.yaml new file mode 100644 index 0000000000..f6420e3797 --- /dev/null +++ b/chart/templates/workers/worker-networkpolicy.yaml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Worker NetworkPolicy +################################# +{{- if (and .Values.networkPolicies.enabled (eq .Values.executor "CeleryExecutor")) }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ .Release.Name }}-worker-policy + labels: + tier: airflow + component: airflow-worker-policy + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + podSelector: + matchLabels: + tier: airflow + component: worker + release: {{ .Release.Name }} + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + tier: airflow + release: {{ .Release.Name }} + component: webserver + ports: + - protocol: TCP + port: {{ .Values.ports.workerLogs }} +{{- end }} diff --git a/chart/templates/workers/worker-service.yaml b/chart/templates/workers/worker-service.yaml new file mode 100644 index 0000000000..2c7768fbf5 --- /dev/null +++ b/chart/templates/workers/worker-service.yaml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Worker Service +################################# +{{- if eq .Values.executor "CeleryExecutor" }} +kind: Service +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-worker + labels: + tier: airflow + component: worker + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + clusterIP: None + selector: + tier: airflow + component: worker + release: {{ .Release.Name }} + ports: + - name: worker-logs + protocol: TCP + port: {{ .Values.ports.workerLogs }} + targetPort: {{ .Values.ports.workerLogs }} +{{- end }} diff --git a/chart/templates/workers/worker-serviceaccount.yaml b/chart/templates/workers/worker-serviceaccount.yaml new file mode 100644 index 0000000000..87350975bd --- /dev/null +++ b/chart/templates/workers/worker-serviceaccount.yaml @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################ +## Airflow Worker ServiceAccount +################################# +{{- if .Values.rbacEnabled }} +kind: ServiceAccount +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-worker-serviceaccount + labels: + tier: airflow + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} +{{- with .Values.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +{{- end }} diff --git a/chart/values.yaml b/chart/values.yaml new file mode 100644 index 0000000000..19184198a5 --- /dev/null +++ b/chart/values.yaml @@ -0,0 +1,436 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Default values for airflow. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# User and group of airflow user +uid: 50000 +gid: 50000 + +# Airflow home directory +# Used for mount paths +airflowHome: "/opt/airflow" + +# Default airflow repository -- overrides all the specific images below +defaultAirflowRepository: apache/airflow + +# Default airflow tag to deploy +defaultAirflowTag: 1.10.10.1-alpha2-python3.6 + + +# Select certain nodes for airflow pods. +nodeSelector: {} +affinity: {} +tolerations: [] + +# Add common labels to all objects and pods defined in this chart. +labels: {} + +# Network policy configuration +networkPolicies: + # Enabled network policies + enabled: false + + +# Extra annotations to apply to all +# Airflow pods +airflowPodAnnotations: {} + +# Enable RBAC (default on most clusters these days) +rbacEnabled: true + +# Airflow executor +# Options: SequentialExecutor, LocalExecutor, CeleryExecutor, KubernetesExecutor +executor: "KubernetesExecutor" + +# If this is true and using LocalExecutor/SequentialExecutor/KubernetesExecutor, the scheudler's +# service account will have access to communicate with the api-server and launch pods. +# If this is true and using the CeleryExecutor, the workers will be able to launch pods. +allowPodLaunching: true + +# Images +images: + airflow: + repository: ~ + tag: ~ + pullPolicy: IfNotPresent + flower: + repository: ~ + tag: ~ + pullPolicy: IfNotPresent + statsd: + repository: astronomerinc/ap-statsd-exporter + tag: 0.11.0 + pullPolicy: IfNotPresent + redis: + repository: redis + tag: 6-buster + pullPolicy: IfNotPresent + pgbouncer: + repository: astronomerinc/ap-pgbouncer + tag: 1.8.1 + pullPolicy: IfNotPresent + pgbouncerExporter: + repository: astronomerinc/ap-pgbouncer-exporter + tag: 0.5.0-1 + pullPolicy: IfNotPresent + +# Environment variables for all airflow containers +env: [] +# - name: "" +# value: "" + +# Secrets for all airflow containers +secret: [] +# - envName: "" +# secretName: "" +# secretKey: "" + +# Airflow database config +data: + # If secret names are provided, use those secrets + metadataSecretName: ~ + resultBackendSecretName: ~ + + # Otherwise pass connection values in + metadataConnection: + user: postgres + pass: postgres + host: ~ + port: 5432 + db: postgres + sslmode: disable + resultBackendConnection: + user: postgres + pass: postgres + host: ~ + port: 5432 + db: postgres + sslmode: disable + +# Fernet key settings +fernetKey: ~ +fernetKeySecretName: ~ + +# Airflow Worker Config +workers: + # Number of airflow celery workers in StatefulSet + replicas: 1 + + # Allow KEDA autoscaling. + # Persistence.enabled must be set to false to use KEDA. + keda: + enabled: false + namespaceLabels: {} + + # How often KEDA polls the airflow DB to report new scale requests to the HPA + pollingInterval: 5 + + # How many seconds KEDA will wait before scaling to zero. + # Note that HPA has a seperate cooldwon period for scale-downs + cooldownPeriod: 30 + + # Maximum number of workers created by keda + maxReplicaCount: 10 + + persistence: + # Enable persistent volumes + enabled: true + # Volume size for worker StatefulSet + size: 100Gi + # If using a custom storageClass, pass name ref to all statefulSets here + storageClassName: + # Execute init container to chown log directory. + # This is currently only needed in KinD, due to usage + # of local-path provisioner. + fixPermissions: false + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Grace period for tasks to finish after SIGTERM is sent from kubernetes + terminationGracePeriodSeconds: 600 + + # This setting tells kubernetes that its ok to evict + # when it wants to scale a node down. + safeToEvict: true + +# Airflow scheduler settings +scheduler: + # Scheduler pod disruption budget + podDisruptionBudget: + enabled: false + + # PDB configuration + config: + maxUnavailable: 1 + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # This setting can overwrite + # podMutation settings. + airflowLocalSettings: ~ + + # This setting tells kubernetes that its ok to evict + # when it wants to scale a node down. + safeToEvict: true + +# Airflow webserver settings +webserver: + livenessProbe: + initialDelaySeconds: 15 + timeoutSeconds: 30 + failureThreshold: 20 + periodSeconds: 5 + + readinessProbe: + initialDelaySeconds: 15 + timeoutSeconds: 30 + failureThreshold: 20 + periodSeconds: 5 + + # Number of webservers + replicas: 1 + + # Additional network policies as needed + extraNetworkPolicies: [] + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Create initial user. + defaultUser: + enabled: true + role: Admin + username: admin + email: admin@example.com + firstName: admin + lastName: user + password: admin + + # Mount additional volumes into webserver. + extraVolumes: [] + extraVolumeMounts: [] + + # This will be mounted into the Airflow Webserver as a custom + # webserver_config.py. You can bake a webserver_config.py in to your image + # instead + webserverConfig: ~ + # webserverConfig: | + # from airflow import configuration as conf + + # # The SQLAlchemy connection string. + # SQLALCHEMY_DATABASE_URI = conf.get('core', 'SQL_ALCHEMY_CONN') + + # # Flask-WTF flag for CSRF + # CSRF_ENABLED = True + + service: + type: ClusterIP + +# Flower settings +flower: + # Additional network policies as needed + extraNetworkPolicies: [] + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + service: + type: ClusterIP + +# Statsd settings +statsd: + enabled: true + # Additional network policies as needed + extraNetworkPolicies: [] + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + service: + extraAnnotations: {} + +# Pgbouncer settings +pgbouncer: + # Enable pgbouncer + enabled: false + # Additional network policies as needed + extraNetworkPolicies: [] + + # Pool sizes + metadataPoolSize: 10 + resultBackendPoolSize: 5 + + # Maximum clients that can connect to pgbouncer (higher = more file descriptors) + maxClientConn: 100 + + # Pgbouner pod disruption budget + podDisruptionBudget: + enabled: false + + # PDB configuration + config: + maxUnavailable: 1 + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + service: + extraAnnotations: {} + + # https://www.pgbouncer.org/config.html + verbose: 0 + logDisconnections: 0 + logConnections: 0 + +redis: + terminationGracePeriodSeconds: 600 + + persistence: + # Enable persistent volumes + enabled: true + # Volume size for worker StatefulSet + size: 1Gi + # If using a custom storageClass, pass name ref to all statefulSets here + storageClassName: + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # If set use as redis secret + passwordSecretName: ~ + brokerURLSecretName: ~ + + # Else, if password is set, create secret with it, + # else generate a new one on install + password: ~ + + # This setting tells kubernetes that its ok to evict + # when it wants to scale a node down. + safeToEvict: true + +# Auth secret for a private registry +# This is used if pulling airflow images from a private registry +registry: + secretName: ~ + + connection: {} + # user: ~ + # pass: ~ + # host: ~ + # email: ~ + +# Elasticsearch logging configuration +elasticsearch: + # Enable elasticsearch task logging + enabled: false + # A secret containing the connection + secretName: ~ + # Or an object representing the connection + connection: {} + # user: ~ + # pass: ~ + # host: ~ + # port: ~ + +# All ports used by chart +ports: + flowerUI: 5555 + airflowUI: 8080 + workerLogs: 8793 + redisDB: 6379 + statsdIngest: 9125 + statsdScrape: 9102 + pgbouncer: 6543 + pgbouncerScrape: 9127 + +# Define any ResourceQuotas for namespace +quotas: {} + +# Define default/max/min values for pods and containers in namespace +limits: [] + +# Config Settings for pod_mutation_hook +podMutation: + # Tolerations provided here would be applied using pod_mutation_hook + # So any pods spun up using KubernetesExecutor or KubernetesPodOperator will contain these tolerations. + tolerations: [] + # - key: "dynamic-pods" + # operator: "Equal" + # value: "true" + # effect: "NoSchedule" + + # Pods spun up would land in the node that matches the affinity + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: "company.io/dynamic-pods" + # operator: In + # values: + # - "true" + +# This runs as a CronJob to cleanup old pods. +cleanup: + enabled: false + # Run every 15 minutes + schedule: "*/15 * * * *" + +# Configuration for postgresql subchart +# Not recommended for production +postgresql: + enabled: true + postgresqlPassword: postgres + postgresqlUsername: postgres