642 строки
17 KiB
YAML
642 строки
17 KiB
YAML
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
---
|
|
# Default values for airflow.
|
|
# This is a YAML-formatted file.
|
|
# Declare variables to be passed into your templates.
|
|
|
|
# User and group of airflow user
|
|
uid: 50000
|
|
gid: 50000
|
|
|
|
# Airflow home directory
|
|
# Used for mount paths
|
|
airflowHome: "/opt/airflow"
|
|
|
|
# Default airflow repository -- overrides all the specific images below
|
|
defaultAirflowRepository: apache/airflow
|
|
|
|
# Default airflow tag to deploy
|
|
defaultAirflowTag: 1.10.12
|
|
|
|
|
|
# Select certain nodes for airflow pods.
|
|
nodeSelector: {}
|
|
affinity: {}
|
|
tolerations: []
|
|
|
|
# Add common labels to all objects and pods defined in this chart.
|
|
labels: {}
|
|
|
|
# Ingress configuration
|
|
ingress:
|
|
# Enable ingress resource
|
|
enabled: false
|
|
|
|
# Configs for the Ingress of the web Service
|
|
web:
|
|
# Annotations for the web Ingress
|
|
annotations: {}
|
|
|
|
# The path for the web Ingress
|
|
path: ""
|
|
|
|
# The hostname for the web Ingress
|
|
host: ""
|
|
|
|
# configs for web Ingress TLS
|
|
tls:
|
|
# Enable TLS termination for the web Ingress
|
|
enabled: false
|
|
# the name of a pre-created Secret containing a TLS private key and certificate
|
|
secretName: ""
|
|
|
|
# HTTP paths to add to the web Ingress before the default path
|
|
precedingPaths: []
|
|
|
|
# Http paths to add to the web Ingress after the default path
|
|
succeedingPaths: []
|
|
|
|
# Configs for the Ingress of the flower Service
|
|
flower:
|
|
# Annotations for the flower Ingress
|
|
annotations: {}
|
|
|
|
# The path for the flower Ingress
|
|
path: ""
|
|
|
|
# The hostname for the flower Ingress
|
|
host: ""
|
|
|
|
# configs for web Ingress TLS
|
|
tls:
|
|
# Enable TLS termination for the flower Ingress
|
|
enabled: false
|
|
# the name of a pre-created Secret containing a TLS private key and certificate
|
|
secretName: ""
|
|
|
|
# HTTP paths to add to the flower Ingress before the default path
|
|
precedingPaths: []
|
|
|
|
# Http paths to add to the flower Ingress after the default path
|
|
succeedingPaths: []
|
|
|
|
# Network policy configuration
|
|
networkPolicies:
|
|
# Enabled network policies
|
|
enabled: false
|
|
|
|
|
|
# Extra annotations to apply to all
|
|
# Airflow pods
|
|
airflowPodAnnotations: {}
|
|
|
|
# Enable RBAC (default on most clusters these days)
|
|
rbacEnabled: true
|
|
|
|
# Airflow executor
|
|
# Options: SequentialExecutor, LocalExecutor, CeleryExecutor, KubernetesExecutor
|
|
executor: "KubernetesExecutor"
|
|
|
|
# If this is true and using LocalExecutor/SequentialExecutor/KubernetesExecutor, the scheudler's
|
|
# service account will have access to communicate with the api-server and launch pods.
|
|
# If this is true and using the CeleryExecutor, the workers will be able to launch pods.
|
|
allowPodLaunching: true
|
|
|
|
# Images
|
|
images:
|
|
airflow:
|
|
repository: ~
|
|
tag: ~
|
|
pullPolicy: IfNotPresent
|
|
flower:
|
|
repository: ~
|
|
tag: ~
|
|
pullPolicy: IfNotPresent
|
|
statsd:
|
|
repository: apache/airflow
|
|
tag: airflow-statsd-exporter-2020.07.10-v0.17.0
|
|
pullPolicy: IfNotPresent
|
|
redis:
|
|
repository: redis
|
|
tag: 6-buster
|
|
pullPolicy: IfNotPresent
|
|
pgbouncer:
|
|
repository: apache/airflow
|
|
tag: airflow-pgbouncer-2020.07.10-1.14.0
|
|
pullPolicy: IfNotPresent
|
|
pgbouncerExporter:
|
|
repository: apache/airflow
|
|
tag: airflow-pgbouncer-exporter-2020.07.10-0.5.0
|
|
pullPolicy: IfNotPresent
|
|
|
|
# Environment variables for all airflow containers
|
|
env: []
|
|
# - name: ""
|
|
# value: ""
|
|
|
|
# Secrets for all airflow containers
|
|
secret: []
|
|
# - envName: ""
|
|
# secretName: ""
|
|
# secretKey: ""
|
|
|
|
# Airflow database config
|
|
data:
|
|
# If secret names are provided, use those secrets
|
|
metadataSecretName: ~
|
|
resultBackendSecretName: ~
|
|
|
|
# Otherwise pass connection values in
|
|
metadataConnection:
|
|
user: postgres
|
|
pass: postgres
|
|
host: ~
|
|
port: 5432
|
|
db: postgres
|
|
sslmode: disable
|
|
resultBackendConnection:
|
|
user: postgres
|
|
pass: postgres
|
|
host: ~
|
|
port: 5432
|
|
db: postgres
|
|
sslmode: disable
|
|
|
|
# Fernet key settings
|
|
fernetKey: ~
|
|
fernetKeySecretName: ~
|
|
|
|
# Airflow Worker Config
|
|
workers:
|
|
# Number of airflow celery workers in StatefulSet
|
|
replicas: 1
|
|
|
|
# Allow KEDA autoscaling.
|
|
# Persistence.enabled must be set to false to use KEDA.
|
|
keda:
|
|
enabled: false
|
|
namespaceLabels: {}
|
|
|
|
# How often KEDA polls the airflow DB to report new scale requests to the HPA
|
|
pollingInterval: 5
|
|
|
|
# How many seconds KEDA will wait before scaling to zero.
|
|
# Note that HPA has a separate cooldown period for scale-downs
|
|
cooldownPeriod: 30
|
|
|
|
# Maximum number of workers created by keda
|
|
maxReplicaCount: 10
|
|
|
|
persistence:
|
|
# Enable persistent volumes
|
|
enabled: true
|
|
# Volume size for worker StatefulSet
|
|
size: 100Gi
|
|
# If using a custom storageClass, pass name ref to all statefulSets here
|
|
storageClassName:
|
|
# Execute init container to chown log directory.
|
|
# This is currently only needed in KinD, due to usage
|
|
# of local-path provisioner.
|
|
fixPermissions: false
|
|
|
|
resources: {}
|
|
# limits:
|
|
# cpu: 100m
|
|
# memory: 128Mi
|
|
# requests:
|
|
# cpu: 100m
|
|
# memory: 128Mi
|
|
|
|
# Grace period for tasks to finish after SIGTERM is sent from kubernetes
|
|
terminationGracePeriodSeconds: 600
|
|
|
|
# This setting tells kubernetes that its ok to evict
|
|
# when it wants to scale a node down.
|
|
safeToEvict: true
|
|
|
|
# Airflow scheduler settings
|
|
scheduler:
|
|
# Scheduler pod disruption budget
|
|
podDisruptionBudget:
|
|
enabled: false
|
|
|
|
# PDB configuration
|
|
config:
|
|
maxUnavailable: 1
|
|
|
|
resources: {}
|
|
# limits:
|
|
# cpu: 100m
|
|
# memory: 128Mi
|
|
# requests:
|
|
# cpu: 100m
|
|
# memory: 128Mi
|
|
|
|
# This setting can overwrite
|
|
# podMutation settings.
|
|
airflowLocalSettings: ~
|
|
|
|
# This setting tells kubernetes that its ok to evict
|
|
# when it wants to scale a node down.
|
|
safeToEvict: true
|
|
|
|
# Airflow webserver settings
|
|
webserver:
|
|
livenessProbe:
|
|
initialDelaySeconds: 15
|
|
timeoutSeconds: 30
|
|
failureThreshold: 20
|
|
periodSeconds: 5
|
|
|
|
readinessProbe:
|
|
initialDelaySeconds: 15
|
|
timeoutSeconds: 30
|
|
failureThreshold: 20
|
|
periodSeconds: 5
|
|
|
|
# Number of webservers
|
|
replicas: 1
|
|
|
|
# Additional network policies as needed
|
|
extraNetworkPolicies: []
|
|
|
|
resources: {}
|
|
# limits:
|
|
# cpu: 100m
|
|
# memory: 128Mi
|
|
# requests:
|
|
# cpu: 100m
|
|
# memory: 128Mi
|
|
|
|
# Create initial user.
|
|
defaultUser:
|
|
enabled: true
|
|
role: Admin
|
|
username: admin
|
|
email: admin@example.com
|
|
firstName: admin
|
|
lastName: user
|
|
password: admin
|
|
|
|
# Mount additional volumes into webserver.
|
|
extraVolumes: []
|
|
extraVolumeMounts: []
|
|
|
|
# This will be mounted into the Airflow Webserver as a custom
|
|
# webserver_config.py. You can bake a webserver_config.py in to your image
|
|
# instead
|
|
webserverConfig: ~
|
|
# webserverConfig: |
|
|
# from airflow import configuration as conf
|
|
|
|
# # The SQLAlchemy connection string.
|
|
# SQLALCHEMY_DATABASE_URI = conf.get('core', 'SQL_ALCHEMY_CONN')
|
|
|
|
# # Flask-WTF flag for CSRF
|
|
# CSRF_ENABLED = True
|
|
|
|
service:
|
|
type: ClusterIP
|
|
## service annotations
|
|
annotations: {}
|
|
|
|
# Flower settings
|
|
flower:
|
|
# Additional network policies as needed
|
|
extraNetworkPolicies: []
|
|
resources: {}
|
|
# limits:
|
|
# cpu: 100m
|
|
# memory: 128Mi
|
|
# requests:
|
|
# cpu: 100m
|
|
# memory: 128Mi
|
|
|
|
service:
|
|
type: ClusterIP
|
|
|
|
# Statsd settings
|
|
statsd:
|
|
enabled: true
|
|
# Additional network policies as needed
|
|
extraNetworkPolicies: []
|
|
resources: {}
|
|
# limits:
|
|
# cpu: 100m
|
|
# memory: 128Mi
|
|
# requests:
|
|
# cpu: 100m
|
|
# memory: 128Mi
|
|
|
|
service:
|
|
extraAnnotations: {}
|
|
|
|
# Pgbouncer settings
|
|
pgbouncer:
|
|
# Enable pgbouncer
|
|
enabled: false
|
|
# Additional network policies as needed
|
|
extraNetworkPolicies: []
|
|
|
|
# Pool sizes
|
|
metadataPoolSize: 10
|
|
resultBackendPoolSize: 5
|
|
|
|
# Maximum clients that can connect to pgbouncer (higher = more file descriptors)
|
|
maxClientConn: 100
|
|
|
|
# Pgbouner pod disruption budget
|
|
podDisruptionBudget:
|
|
enabled: false
|
|
|
|
# PDB configuration
|
|
config:
|
|
maxUnavailable: 1
|
|
|
|
resources: {}
|
|
# limits:
|
|
# cpu: 100m
|
|
# memory: 128Mi
|
|
# requests:
|
|
# cpu: 100m
|
|
# memory: 128Mi
|
|
|
|
service:
|
|
extraAnnotations: {}
|
|
|
|
# https://www.pgbouncer.org/config.html
|
|
verbose: 0
|
|
logDisconnections: 0
|
|
logConnections: 0
|
|
|
|
redis:
|
|
terminationGracePeriodSeconds: 600
|
|
|
|
persistence:
|
|
# Enable persistent volumes
|
|
enabled: true
|
|
# Volume size for worker StatefulSet
|
|
size: 1Gi
|
|
# If using a custom storageClass, pass name ref to all statefulSets here
|
|
storageClassName:
|
|
|
|
resources: {}
|
|
# limits:
|
|
# cpu: 100m
|
|
# memory: 128Mi
|
|
# requests:
|
|
# cpu: 100m
|
|
# memory: 128Mi
|
|
|
|
# If set use as redis secret
|
|
passwordSecretName: ~
|
|
brokerURLSecretName: ~
|
|
|
|
# Else, if password is set, create secret with it,
|
|
# else generate a new one on install
|
|
password: ~
|
|
|
|
# This setting tells kubernetes that its ok to evict
|
|
# when it wants to scale a node down.
|
|
safeToEvict: true
|
|
|
|
# Auth secret for a private registry
|
|
# This is used if pulling airflow images from a private registry
|
|
registry:
|
|
secretName: ~
|
|
|
|
connection: {}
|
|
# user: ~
|
|
# pass: ~
|
|
# host: ~
|
|
# email: ~
|
|
|
|
# Elasticsearch logging configuration
|
|
elasticsearch:
|
|
# Enable elasticsearch task logging
|
|
enabled: false
|
|
# A secret containing the connection
|
|
secretName: ~
|
|
# Or an object representing the connection
|
|
connection: {}
|
|
# user: ~
|
|
# pass: ~
|
|
# host: ~
|
|
# port: ~
|
|
|
|
# All ports used by chart
|
|
ports:
|
|
flowerUI: 5555
|
|
airflowUI: 8080
|
|
workerLogs: 8793
|
|
redisDB: 6379
|
|
statsdIngest: 9125
|
|
statsdScrape: 9102
|
|
pgbouncer: 6543
|
|
pgbouncerScrape: 9127
|
|
|
|
# Define any ResourceQuotas for namespace
|
|
quotas: {}
|
|
|
|
# Define default/max/min values for pods and containers in namespace
|
|
limits: []
|
|
|
|
# Config Settings for pod_mutation_hook
|
|
podMutation:
|
|
# Tolerations provided here would be applied using pod_mutation_hook
|
|
# So any pods spun up using KubernetesExecutor or KubernetesPodOperator will contain these tolerations.
|
|
tolerations: []
|
|
# - key: "dynamic-pods"
|
|
# operator: "Equal"
|
|
# value: "true"
|
|
# effect: "NoSchedule"
|
|
|
|
# Pods spun up would land in the node that matches the affinity
|
|
affinity: {}
|
|
# nodeAffinity:
|
|
# requiredDuringSchedulingIgnoredDuringExecution:
|
|
# nodeSelectorTerms:
|
|
# - matchExpressions:
|
|
# - key: "company.io/dynamic-pods"
|
|
# operator: In
|
|
# values:
|
|
# - "true"
|
|
|
|
# This runs as a CronJob to cleanup old pods.
|
|
cleanup:
|
|
enabled: false
|
|
# Run every 15 minutes
|
|
schedule: "*/15 * * * *"
|
|
|
|
# Configuration for postgresql subchart
|
|
# Not recommended for production
|
|
postgresql:
|
|
enabled: true
|
|
postgresqlPassword: postgres
|
|
postgresqlUsername: postgres
|
|
|
|
# Config settings to go into the mounted airflow.cfg
|
|
#
|
|
# Please note that these values are passed through the `tpl` function, so are
|
|
# all subject to being rendered as go templates. If you need to include a
|
|
# litera `{{` in a value, it must be expessed like this:
|
|
#
|
|
# a: '{{ "{{ not a template }}" }}'
|
|
#
|
|
# yamllint disable rule:line-length
|
|
config:
|
|
core:
|
|
dags_folder: '{{ include "airflow_dags" . }}'
|
|
load_examples: 'False'
|
|
colored_console_log: 'False'
|
|
executor: '{{ .Values.executor }}'
|
|
remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}'
|
|
# Authentication backend used for the experimental API
|
|
api:
|
|
auth_backend: airflow.api.auth.backend.deny_all
|
|
logging:
|
|
logging_level: DEBUG
|
|
webserver:
|
|
enable_proxy_fix: 'True'
|
|
expose_config: 'True'
|
|
rbac: 'True'
|
|
|
|
celery:
|
|
default_queue: celery
|
|
|
|
scheduler:
|
|
scheduler_heartbeat_sec: 5
|
|
statsd_on: '{{ ternary "True" "False" .Values.statsd.enabled }}'
|
|
statsd_port: 9125
|
|
statsd_prefix: airflow
|
|
statsd_host: '{{ printf "%s-statsd" .Release.Name }}'
|
|
|
|
# Restart Scheduler every 41460 seconds (11 hours 31 minutes)
|
|
# The odd time is chosen so it is not always restarting on the same "hour" boundary
|
|
run_duration: 41460
|
|
elasticsearch:
|
|
json_format: 'True'
|
|
log_id_template: "{dag_id}_{task_id}_{execution_date}_{try_number}"
|
|
elasticsearch_configs:
|
|
max_retries: 3
|
|
timeout: 30
|
|
retry_timeout: 'True'
|
|
|
|
kubernetes:
|
|
namespace: '{{ .Release.Namespace }}'
|
|
airflow_configmap: '{{ include "airflow_config" . }}'
|
|
airflow_local_settings_configmap: '{{ include "airflow_config" . }}'
|
|
worker_container_repository: '{{ .Values.images.airflow.repository | default .Values.defaultAirflowRepository }}'
|
|
worker_container_tag: '{{ .Values.images.airflow.tag | default .Values.defaultAirflowTag }}'
|
|
worker_container_image_pull_policy: '{{ .Values.images.airflow.pullPolicy }}'
|
|
worker_service_account_name: '{{ .Release.Name }}-worker-serviceaccount'
|
|
image_pull_secrets: '{{ template "registry_secret" . }}'
|
|
dags_in_image: '{{ ternary "False" "True" (or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled) }}'
|
|
delete_worker_pods: 'True'
|
|
run_as_user: '{{ .Values.uid }}'
|
|
fs_group: '{{ .Values.gid }}'
|
|
git_dags_folder_mount_point: '{{- if or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled }}{{ include "airflow_dags_mount_path" . }}{{end}}'
|
|
dags_volume_mount_point: '{{- if or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled }}{{ include "airflow_dags_mount_path" . }}{{ end }}'
|
|
dags_volume_claim: '{{- if .Values.dags.persistence.enabled }}{{ include "airflow_dags_volume_claim" . }}{{ end }}'
|
|
dags_volume_subpath: '{{- if .Values.dags.persistence.enabled }}{{.Values.dags.gitSync.dest }}/{{ .Values.dags.gitSync.subPath }}{{ end }}'
|
|
git_repo: '{{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }}{{ .Values.dags.gitSync.repo }}{{ end }}'
|
|
git_branch: '{{ .Values.dags.gitSync.branch }}'
|
|
git_sync_rev: '{{ .Values.dags.gitSync.rev }}'
|
|
git_sync_depth: '{{ .Values.dags.gitSync.depth }}'
|
|
git_sync_root: '{{ .Values.dags.gitSync.root }}'
|
|
git_sync_dest: '{{ .Values.dags.gitSync.dest }}'
|
|
git_sync_container_repository: '{{ .Values.dags.gitSync.containerRepository }}'
|
|
git_sync_container_tag: '{{ .Values.dags.gitSync.containerTag }}'
|
|
git_sync_init_container_name: '{{ .Values.dags.gitSync.containerName }}'
|
|
git_sync_run_as_user: '{{ .Values.uid }}'
|
|
git_ssh_known_hosts_configmap_name: '{{- if .Values.dags.gitSync.knownHosts }}{{ include "airflow_config" . }}{{ end }}'
|
|
git_ssh_key_secret_name: '{{- if .Values.dags.gitSync.sshKeySecret }}{{ .Values.dags.gitSync.sshKeySecret }}{{ end }}'
|
|
git_sync_credentials_secret: '{{- if .Values.dags.gitSync.credentialsSecret }}{{ .Values.dags.gitSync.credentialsSecret }}{{ end }}'
|
|
kubernetes_secrets:
|
|
AIRFLOW__CORE__SQL_ALCHEMY_CONN: '{{ printf "%s=connection" (include "airflow_metadata_secret" .) }}'
|
|
AIRFLOW__CORE__FERNET_KEY: '{{ printf "%s=fernet-key" (include "fernet_key_secret" .) }}'
|
|
# yamllint enable rule:line-length
|
|
|
|
# Git sync
|
|
dags:
|
|
persistence:
|
|
# Enable persistent volume for storing dags
|
|
enabled: false
|
|
# Volume size for dags
|
|
size: 1Gi
|
|
# If using a custom storageClass, pass name here
|
|
storageClassName:
|
|
# access mode of the persistent volume
|
|
accessMode: ReadWriteMany
|
|
## the name of an existing PVC to use
|
|
existingClaim: ~
|
|
gitSync:
|
|
enabled: false
|
|
# git repo clone url
|
|
# ssh examples ssh://git@github.com/apache/airflow.git
|
|
# git@github.com:apache/airflow.git
|
|
# https example: https://github.com/apache/airflow.git
|
|
repo: https://github.com/apache/airflow.git
|
|
branch: v1-10-stable
|
|
rev: HEAD
|
|
root: "/git"
|
|
dest: "repo"
|
|
depth: 1
|
|
# the number of consecutive failures allowed before aborting
|
|
maxFailures: 0
|
|
# subpath within the repo where dags are located
|
|
# should be "" if dags are at repo root
|
|
subPath: "tests/dags"
|
|
# if your repo needs a user name password
|
|
# you can load them to a k8s secret like the one below
|
|
# ---
|
|
# apiVersion: v1
|
|
# kind: Secret
|
|
# metadata:
|
|
# name: git-credentials
|
|
# data:
|
|
# GIT_SYNC_USERNAME: <base64_encoded_git_username>
|
|
# GIT_SYNC_PASSWORD: <base64_encoded_git_password>
|
|
# and specify the name of the secret below
|
|
#credentialsSecret: git-credentials
|
|
#
|
|
#
|
|
# If you are using an ssh clone url, you can load
|
|
# the ssh private key to a k8s secret like the one below
|
|
# ---
|
|
# apiVersion: v1
|
|
# kind: Secret
|
|
# metadata:
|
|
# name: airflow-ssh-secret
|
|
# data:
|
|
# # key needs to be gitSshKey
|
|
# gitSshKey: <base64_encoded_data>
|
|
# and specify the name of the secret below
|
|
#sshKeySecret: airflow-ssh-secret
|
|
# If you are using an ssh private key, you can additionally
|
|
# specify the content of your known_hosts file, example:
|
|
#knownHosts: |
|
|
# <host1>,<ip1> <key1>
|
|
# <host2>,<ip2> <key2>
|
|
# interval between git sync attempts in seconds
|
|
wait: 60
|
|
# git sync image details
|
|
containerRepository: k8s.gcr.io/git-sync
|
|
containerTag: v3.1.6
|
|
containerName: git-sync
|