Add schema validation for config.yml (#13025)
* Adding schema validation for config.yml * Update airflow/config_templates/config.yml.schema.json Co-authored-by: Kamil Breguła <mik-laj@users.noreply.github.com> Co-authored-by: Kamil Breguła <mik-laj@users.noreply.github.com>
This commit is contained in:
Sam Wheating
GitHub
Родитель
f6448b4e48
Коммит
1dc36d8704
|
|
@ -499,6 +499,7 @@ repos:
|
|||
files: >
|
||||
(?x)
|
||||
^airflow/provider.yaml.schema.json$|
|
||||
^airflow/config_templates/config.yml.schema.json$|
|
||||
^airflow/serialization/schema.json$
|
||||
require_serial: true
|
||||
additional_dependencies: ['jsonschema==3.2.0', 'PyYAML==5.3.1', 'requests==2.25.0']
|
||||
|
|
@ -535,6 +536,17 @@ repos:
|
|||
files: chart/values.yaml$
|
||||
require_serial: true
|
||||
additional_dependencies: ['jsonschema==3.2.0', 'PyYAML==5.3.1', 'requests==2.25.0']
|
||||
- id: json-schema
|
||||
name: Lint airflow/config_templates/config.yml file with JSON Schema
|
||||
entry: ./scripts/ci/pre_commit/pre_commit_json_schema.py
|
||||
args:
|
||||
- --spec-file
|
||||
- airflow/config_templates/config.yml.schema.json
|
||||
language: python
|
||||
pass_filenames: true
|
||||
files: airflow/config_templates/config.yml$
|
||||
require_serial: true
|
||||
additional_dependencies: ['jsonschema==3.2.0', 'PyYAML==5.3.1', 'requests==2.25.0']
|
||||
## ADD MOST PRE-COMMITS ABOVE THAT LINE
|
||||
# The below pre-commits are those requiring CI image to be built
|
||||
- id: build
|
||||
|
|
|
|||
|
|
@ -409,7 +409,7 @@
|
|||
Not all transactions will be retried as it can cause undesired state.
|
||||
Currently it is only used in ``DagFileProcessor.process_file`` to retry ``dagbag.sync_to_db``.
|
||||
version_added: ~
|
||||
type: int
|
||||
type: integer
|
||||
example: ~
|
||||
default: "3"
|
||||
|
||||
|
|
@ -739,12 +739,14 @@
|
|||
If no limit is supplied, the OpenApi spec default is used.
|
||||
type: integer
|
||||
example: ~
|
||||
version_added: ~
|
||||
default: "100"
|
||||
- name: google_oauth2_audience
|
||||
description: The intended audience for JWT token credentials used for authorization.
|
||||
This value must match on the client and server sides.
|
||||
If empty, audience will not be tested.
|
||||
type: string
|
||||
version_added: ~
|
||||
example: project-id-random-value.apps.googleusercontent.com
|
||||
default: ""
|
||||
- name: google_key_path
|
||||
|
|
@ -754,6 +756,7 @@
|
|||
<https://cloud.google.com/docs/authentication/production#finding_credentials_automatically>`__ will
|
||||
be used.
|
||||
type: string
|
||||
version_added: ~
|
||||
example: /files/service-account-json
|
||||
default: ""
|
||||
- name: lineage
|
||||
|
|
@ -1054,21 +1057,21 @@
|
|||
description: |
|
||||
Time interval (in secs) to wait before next log fetching.
|
||||
version_added: 1.10.8
|
||||
type: int
|
||||
type: integer
|
||||
example: ~
|
||||
default: "2"
|
||||
- name: log_auto_tailing_offset
|
||||
description: |
|
||||
Distance away from page bottom to enable auto tailing.
|
||||
version_added: 1.10.8
|
||||
type: int
|
||||
type: integer
|
||||
example: ~
|
||||
default: "30"
|
||||
- name: log_animation_speed
|
||||
description: |
|
||||
Animation speed for auto tailing log display.
|
||||
version_added: 1.10.8
|
||||
type: int
|
||||
type: integer
|
||||
example: ~
|
||||
default: "1000"
|
||||
- name: hide_paused_dags_by_default
|
||||
|
|
@ -1206,7 +1209,7 @@
|
|||
The UI cookie lifetime in minutes. User will be logged out from UI after
|
||||
``session_lifetime_minutes`` of non-activity
|
||||
version_added: 1.10.13
|
||||
type: int
|
||||
type: integer
|
||||
example: ~
|
||||
default: "43200"
|
||||
|
||||
|
|
@ -1288,13 +1291,13 @@
|
|||
- name: smtp_timeout
|
||||
description: ~
|
||||
version_added: ~
|
||||
type: int
|
||||
type: integer
|
||||
example: ~
|
||||
default: "30"
|
||||
- name: smtp_retry_limit
|
||||
description: ~
|
||||
version_added: ~
|
||||
type: int
|
||||
type: integer
|
||||
example: ~
|
||||
default: "5"
|
||||
- name: sentry
|
||||
|
|
@ -1375,7 +1378,7 @@
|
|||
claimed blocked tasks.
|
||||
https://docs.celeryproject.org/en/stable/userguide/optimizing.html#prefetch-limits
|
||||
version_added: ~
|
||||
type: int
|
||||
type: integer
|
||||
example: "1"
|
||||
default: ~
|
||||
- name: worker_log_server_port
|
||||
|
|
@ -1530,7 +1533,7 @@
|
|||
Time in seconds after which Adopted tasks are cleared by CeleryExecutor. This is helpful to clear
|
||||
stalled tasks.
|
||||
version_added: 2.0.0
|
||||
type: int
|
||||
type: integer
|
||||
example: ~
|
||||
default: "600"
|
||||
- name: task_publish_max_retries
|
||||
|
|
@ -1538,7 +1541,7 @@
|
|||
The Maximum number of retries for publishing task messages to the broker when failing
|
||||
due to ``AirflowTaskTimeout`` error before giving up and marking Task as failed.
|
||||
version_added: 2.0.0
|
||||
type: int
|
||||
type: integer
|
||||
example: ~
|
||||
default: "3"
|
||||
- name: worker_precheck
|
||||
|
|
@ -2045,7 +2048,7 @@
|
|||
When the `enable_tcp_keepalive` option is enabled, TCP probes a connection that has
|
||||
been idle for `tcp_keep_idle` seconds.
|
||||
version_added: ~
|
||||
type: int
|
||||
type: integer
|
||||
example: ~
|
||||
default: "120"
|
||||
- name: tcp_keep_intvl
|
||||
|
|
@ -2053,7 +2056,7 @@
|
|||
When the `enable_tcp_keepalive` option is enabled, if Kubernetes API does not respond
|
||||
to a keepalive probe, TCP retransmits the probe after `tcp_keep_intvl` seconds.
|
||||
version_added: ~
|
||||
type: int
|
||||
type: integer
|
||||
example: ~
|
||||
default: "30"
|
||||
- name: tcp_keep_cnt
|
||||
|
|
@ -2062,7 +2065,7 @@
|
|||
to a keepalive probe, TCP retransmits the probe `tcp_keep_cnt number` of times before
|
||||
a connection is considered to be broken.
|
||||
version_added: ~
|
||||
type: int
|
||||
type: integer
|
||||
example: ~
|
||||
default: "6"
|
||||
- name: smart_sensor
|
||||
|
|
@ -2081,14 +2084,14 @@
|
|||
`shard_code_upper_limit` is the upper limit of `shard_code` value. The `shard_code` is generated
|
||||
by `hashcode % shard_code_upper_limit`.
|
||||
version_added: ~
|
||||
type: int
|
||||
type: integer
|
||||
example: ~
|
||||
default: "10000"
|
||||
- name: shards
|
||||
description: |
|
||||
The number of running smart sensor processes for each service.
|
||||
version_added: ~
|
||||
type: int
|
||||
type: integer
|
||||
example: ~
|
||||
default: "5"
|
||||
- name: sensors_enabled
|
||||
|
|
|
|||
|
|
@ -0,0 +1,61 @@
|
|||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": {
|
||||
"type": ["string", "null"]
|
||||
},
|
||||
"options": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": {
|
||||
"type": ["string", "null"]
|
||||
},
|
||||
"version_added": {
|
||||
"type": ["string", "null"]
|
||||
},
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": ["string", "boolean", "integer", "float"]
|
||||
},
|
||||
"example": {
|
||||
"type": ["string", "null", "number"]
|
||||
},
|
||||
"default": {
|
||||
"type": ["string", "null", "number"]
|
||||
},
|
||||
"sensitive": {
|
||||
"type": "boolean",
|
||||
"description": "When true, this option is sensitive and can be specified using AIRFLOW__{section}___{name}__SECRET or AIRFLOW__{section}___{name}__CMD environment variables. See: airflow.configuration.AirflowConfigParser.sensitive_config_values"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"description",
|
||||
"version_added",
|
||||
"type",
|
||||
"example",
|
||||
"default"
|
||||
],
|
||||
"additional_properties": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"description",
|
||||
"options"
|
||||
],
|
||||
"additional_properties": false
|
||||
}
|
||||
}
|
||||
Загрузка…
Ссылка в новой задаче