476 строки
17 KiB
YAML
476 строки
17 KiB
YAML
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
---
|
|
default_stages: [commit, push]
|
|
default_language_version:
|
|
# force all unspecified python hooks to run python3
|
|
python: python3
|
|
minimum_pre_commit_version: "1.20.0"
|
|
repos:
|
|
- repo: https://github.com/Lucas-C/pre-commit-hooks
|
|
rev: v1.1.9
|
|
hooks:
|
|
- id: forbid-tabs
|
|
exclude: ^docs/Makefile$|^clients/gen/go.sh
|
|
- id: insert-license
|
|
name: Add license for all SQL files
|
|
files: \.sql$
|
|
exclude: ^\.github/.*$
|
|
args:
|
|
- --comment-style
|
|
- "/*||*/"
|
|
- --license-filepath
|
|
- license-templates/LICENSE.txt
|
|
- --fuzzy-match-generates-todo
|
|
- id: insert-license
|
|
name: Add license for all other files
|
|
exclude: ^\.github/.*$
|
|
args:
|
|
- --comment-style
|
|
- "|#|"
|
|
- --license-filepath
|
|
- license-templates/LICENSE.txt
|
|
- --fuzzy-match-generates-todo
|
|
files: >
|
|
\.properties$|\.cfg$|\.conf$|\.ini$|\.ldif$|\.readthedocs$|\.service$|\.tf$|^Dockerfile.*$
|
|
- id: insert-license
|
|
name: Add license for all rst files
|
|
exclude: ^\.github/.*$
|
|
args:
|
|
- --comment-style
|
|
- "||"
|
|
- --license-filepath
|
|
- license-templates/LICENSE.rst
|
|
- --fuzzy-match-generates-todo
|
|
files: \.rst$
|
|
- id: insert-license
|
|
name: Add license for all JS/CSS/PUML files
|
|
files: \.(js|css|puml)$
|
|
exclude: ^\.github/.*$
|
|
args:
|
|
- --comment-style
|
|
- "/*!| *| */"
|
|
- --license-filepath
|
|
- license-templates/LICENSE.txt
|
|
- --fuzzy-match-generates-todo
|
|
- id: insert-license
|
|
name: Add license for all JINJA template files
|
|
files: "^airflow/www/templates/.*\\.html$|^docs/templates/.*\\.html$.*\\.jinja2"
|
|
exclude: ^\.github/.*$
|
|
args:
|
|
- --comment-style
|
|
- "{#||#}"
|
|
- --license-filepath
|
|
- license-templates/LICENSE.txt
|
|
- --fuzzy-match-generates-todo
|
|
- id: insert-license
|
|
name: Add license for all shell files
|
|
exclude: ^\.github/.*$
|
|
files: ^breeze$|^breeze-complete$|\.sh$|\.bash$|\.bats$
|
|
args:
|
|
- --comment-style
|
|
- "|#|"
|
|
- --license-filepath
|
|
- license-templates/LICENSE.txt
|
|
- --fuzzy-match-generates-todo
|
|
- id: insert-license
|
|
name: Add license for all python files
|
|
exclude: ^\.github/.*$
|
|
types: [python]
|
|
args:
|
|
- --comment-style
|
|
- "|#|"
|
|
- --license-filepath
|
|
- license-templates/LICENSE.txt
|
|
- --fuzzy-match-generates-todo
|
|
- id: insert-license
|
|
name: Add license for all XML files
|
|
exclude: ^\.github/.*$
|
|
files: \.xml$
|
|
args:
|
|
- --comment-style
|
|
- "<!--||-->"
|
|
- --license-filepath
|
|
- license-templates/LICENSE.txt
|
|
- --fuzzy-match-generates-todo
|
|
- id: insert-license
|
|
name: Add license for all yaml files
|
|
exclude: ^\.github/.*$
|
|
types: [yaml]
|
|
files: \.yml$|\.yaml$
|
|
args:
|
|
- --comment-style
|
|
- "|#|"
|
|
- --license-filepath
|
|
- license-templates/LICENSE.txt
|
|
- --fuzzy-match-generates-todo
|
|
- id: insert-license
|
|
name: Add license for all md files
|
|
files: \.md$
|
|
exclude: ^\.github/.*$|PROVIDER_CHANGES.*\.md
|
|
args:
|
|
- --comment-style
|
|
- "<!--|| -->"
|
|
- --license-filepath
|
|
- license-templates/LICENSE.txt
|
|
- --fuzzy-match-generates-todo
|
|
- id: insert-license
|
|
name: Add license for all mermaid files
|
|
args:
|
|
- --comment-style
|
|
- "|%%|"
|
|
- --license-filepath
|
|
- license-templates/LICENSE.txt
|
|
- --fuzzy-match-generates-todo
|
|
files: \.mermaid$
|
|
- repo: https://github.com/thlorenz/doctoc.git
|
|
rev: v1.4.0
|
|
hooks:
|
|
- id: doctoc
|
|
name: Add TOC for md files
|
|
files: ^README\.md$|^CONTRIBUTING\.md$|^UPDATING.md$|^dev/README\.md$|^dev/PROVIDER_PACKAGES.md$
|
|
args:
|
|
- "--maxlevel"
|
|
- "2"
|
|
- repo: meta
|
|
hooks:
|
|
- id: check-hooks-apply
|
|
- repo: https://github.com/psf/black
|
|
rev: 20.8b1
|
|
hooks:
|
|
- id: black
|
|
files: api_connexion/.*\.py|.*providers.*\.py
|
|
exclude: .*kubernetes_pod\.py|.*google/common/hooks/base_google\.py$
|
|
args: [--config=./pyproject.toml]
|
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
rev: v3.3.0
|
|
hooks:
|
|
- id: check-merge-conflict
|
|
- id: debug-statements
|
|
- id: check-builtin-literals
|
|
- id: detect-private-key
|
|
- id: end-of-file-fixer
|
|
- id: mixed-line-ending
|
|
- id: check-executables-have-shebangs
|
|
- id: check-xml
|
|
- id: trailing-whitespace
|
|
- id: fix-encoding-pragma
|
|
args:
|
|
- --remove
|
|
- repo: https://github.com/pre-commit/pygrep-hooks
|
|
rev: v1.6.0
|
|
hooks:
|
|
- id: rst-backticks
|
|
- id: python-no-log-warn
|
|
- repo: https://github.com/adrienverge/yamllint
|
|
rev: v1.25.0
|
|
hooks:
|
|
- id: yamllint
|
|
name: Check yaml files with yamllint
|
|
entry: yamllint -c yamllint-config.yml --strict
|
|
types: [yaml]
|
|
exclude:
|
|
^.*init_git_sync\.template\.yaml$|^.*airflow\.template\.yaml$|^chart/(?:templates|files)/.*\.yaml
|
|
- repo: https://github.com/timothycrosley/isort
|
|
rev: 5.6.4
|
|
hooks:
|
|
- id: isort
|
|
name: Run isort to sort imports
|
|
types: [python]
|
|
# To keep consistent with the global isort skip config defined in setup.cfg
|
|
exclude: ^build/.*$|^.tox/.*$|^venv/.*$|.*api_connexion/.*\.py|.*providers.*\.py
|
|
- repo: https://github.com/pycqa/pydocstyle
|
|
rev: 5.1.1
|
|
hooks:
|
|
- id: pydocstyle
|
|
name: Run pydocstyle
|
|
args:
|
|
- --convention=pep257
|
|
- --add-ignore=D100,D102,D104,D105,D107,D205,D400,D401
|
|
exclude: ^tests/.*\.py$|^scripts/.*\.py$|^dev|^provider_packages|^kubernetes_tests|.*example_dags/.*
|
|
- repo: local
|
|
hooks:
|
|
- id: lint-openapi
|
|
name: Lint OpenAPI using spectral
|
|
language: docker_image
|
|
entry: stoplight/spectral lint -r ./scripts/ci/spectral_rules/connexion.yml
|
|
files: ^airflow/api_connexion/openapi/
|
|
- id: lint-openapi
|
|
name: Lint OpenAPI using openapi-spec-validator
|
|
entry: openapi-spec-validator --schema 3.0.0
|
|
language: python
|
|
additional_dependencies: ['openapi-spec-validator']
|
|
files: ^airflow/api_connexion/openapi/
|
|
- id: lint-dockerfile
|
|
name: Lint dockerfile
|
|
language: system
|
|
entry: "./scripts/ci/pre_commit/pre_commit_lint_dockerfile.sh"
|
|
files: Dockerfile.*$
|
|
pass_filenames: true
|
|
- id: setup-order
|
|
name: Checks for an order of dependencies in setup.py
|
|
language: python
|
|
files: ^setup.py$
|
|
pass_filenames: false
|
|
entry: ./scripts/ci/pre_commit/pre_commit_check_order_setup.py
|
|
- id: update-breeze-file
|
|
name: Update output of breeze command in BREEZE.rst
|
|
entry: "./scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh"
|
|
language: system
|
|
files: ^BREEZE.rst$|^breeze$|^breeze-complete$
|
|
pass_filenames: false
|
|
- id: update-local-yml-file
|
|
name: Update mounts in the local yml file
|
|
entry: "./scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh"
|
|
language: system
|
|
files: ^scripts/ci/libraries/_local_mounts.sh$|s^scripts/ci/docker_compose/local.yml"
|
|
pass_filenames: false
|
|
- id: update-setup-cfg-file
|
|
name: Update setup.cfg file with all licenses
|
|
entry: "./scripts/ci/pre_commit/pre_commit_setup_cfg_file.sh"
|
|
language: system
|
|
files: ^setup.cfg$
|
|
pass_filenames: false
|
|
- id: build-providers-dependencies
|
|
name: Build cross-dependencies for providers packages
|
|
entry: "./scripts/ci/pre_commit/pre_commit_build_providers_dependencies.sh"
|
|
language: system
|
|
files: ^airflow/providers/.*\.py$|^tests/providers/.*\.py$
|
|
pass_filenames: false
|
|
- id: update-extras
|
|
name: Update extras in documentation
|
|
entry: ./scripts/ci/pre_commit/pre_commit_insert_extras.py
|
|
language: python
|
|
files: ^setup.py$|^INSTALL$|^CONTRIBUTING.rst$
|
|
pass_filenames: false
|
|
- id: pydevd
|
|
language: pygrep
|
|
name: Check for pydevd debug statements accidentally left
|
|
entry: "pydevd.*settrace\\("
|
|
pass_filenames: true
|
|
files: \.py$
|
|
- id: dont-use-safe-filter
|
|
language: pygrep
|
|
name: Don't use safe in templates
|
|
description: the Safe filter is error-prone, use Markup() in code instead
|
|
entry: "\\|\\s*safe"
|
|
files: \.html$
|
|
pass_filenames: true
|
|
- id: no-relative-imports
|
|
language: pygrep
|
|
name: No relative imports
|
|
description: Airflow style is to use absolute imports only
|
|
entry: "^\\s*from\\s+\\."
|
|
pass_filenames: true
|
|
files: \.py$
|
|
exclude: ^tests/
|
|
- id: language-matters
|
|
language: pygrep
|
|
name: Check for language that we do not accept as community
|
|
description: Please use "deny_list" or "allow_list" instead.
|
|
entry: "(?i)(black|white)[_-]?list"
|
|
pass_filenames: true
|
|
exclude: >
|
|
(?x)
|
|
^airflow/providers/apache/cassandra/hooks/cassandra.py$|
|
|
^airflow/providers/apache/hive/operators/hive_stats.py$|
|
|
^airflow/providers/apache/hive/.*PROVIDER_CHANGES_*|
|
|
^airflow/providers/apache/hive/.*README.md$|
|
|
^tests/providers/apache/cassandra/hooks/test_cassandra.py$|
|
|
^docs/howto/connection/cassandra.rst$|
|
|
^CHANGELOG.txt$
|
|
- id: consistent-pylint
|
|
language: pygrep
|
|
name: Check for inconsitent pylint disable/enable without space
|
|
entry: "pylint:disable|pylint:enable"
|
|
pass_filenames: true
|
|
files: \.py$
|
|
- id: base-operator
|
|
language: pygrep
|
|
name: Make sure BaseOperator[Link] is imported from airflow.models.baseoperator in core
|
|
entry: "from airflow\\.models import.* BaseOperator"
|
|
files: \.py$
|
|
pass_filenames: true
|
|
exclude: >
|
|
(?x)
|
|
^airflow/gcp/.*$|
|
|
^airflow/hooks/.*$|
|
|
^airflow/operators/.*$|
|
|
^airflow/sensors/.*$|
|
|
^airflow/providers/.*$|
|
|
^provider_packages/.*$
|
|
- id: base-operator
|
|
language: pygrep
|
|
name: Make sure BaseOperator[Link] is imported from airflow.models outside of core
|
|
entry: "from airflow\\.models\\.baseoperator import.* BaseOperator"
|
|
pass_filenames: true
|
|
files: >
|
|
(?x)
|
|
^airflow/gcp/.*$|
|
|
^airflow/hooks/.*$|
|
|
^airflow/operators/.*$|
|
|
^airflow/sensors/.*$|
|
|
^airflow/providers/.*\.py$
|
|
- id: provide-create-sessions
|
|
language: pygrep
|
|
name: To avoid import cycles make sure provide_session and create_session are imported from
|
|
airflow.utils.session
|
|
entry: "from airflow\\.utils\\.db import.* (provide_session|create_session)"
|
|
files: \.py$
|
|
pass_filenames: true
|
|
- id: incorrect-use-of-LoggingMixin
|
|
language: pygrep
|
|
name: Make sure LoggingMixin is not used alone
|
|
entry: "LoggingMixin\\(\\)"
|
|
files: \.py$
|
|
pass_filenames: true
|
|
- id: daysago-import-check
|
|
language: pygrep
|
|
name: Make sure days_ago is imported from airflow.utils.dates
|
|
entry: "(airflow\\.){0,1}utils\\.dates\\.days_ago"
|
|
files: \.py$
|
|
pass_filenames: true
|
|
- id: restrict-start_date
|
|
language: pygrep
|
|
name: "'start_date' should not be defined in default_args in example_dags"
|
|
entry: "default_args\\s*=\\s*{\\s*(\"|')start_date(\"|')|(\"|')start_date(\"|'):"
|
|
files: \.*example_dags.*.py$
|
|
pass_filenames: true
|
|
- id: check-integrations
|
|
name: Check if integration list is aligned
|
|
entry: ./scripts/ci/pre_commit/pre_commit_check_integrations.sh
|
|
language: system
|
|
pass_filenames: false
|
|
files: ^common/_common_values.sh$|^breeze-complete$
|
|
- id: check-apache-license
|
|
name: Check if licenses are OK for Apache
|
|
entry: "./scripts/ci/pre_commit/pre_commit_check_license.sh"
|
|
language: system
|
|
files: ^.*LICENSE.*$|^.*LICENCE.*$
|
|
pass_filenames: false
|
|
- id: airflow-config-yaml
|
|
name: Checks for consistency between config.yml and default_config.cfg
|
|
language: python
|
|
entry: ./scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py
|
|
files: "config.yml$|default_airflow.cfg$|default.cfg$"
|
|
pass_filenames: false
|
|
require_serial: true
|
|
additional_dependencies: ['pyyaml']
|
|
- id: pre-commit-descriptions
|
|
name: Check if pre-commits are described
|
|
entry: ./scripts/ci/pre_commit/pre_commit_check_pre_commits.sh
|
|
language: system
|
|
files: ^.pre-commit-config.yaml$|^STATIC_CODE_CHECKS.rst|^breeze-complete$
|
|
require_serial: true
|
|
- id: sort-in-the-wild
|
|
name: Sort INTHEWILD.md alphabetically
|
|
entry: ./scripts/ci/pre_commit/pre_commit_sort_in_the_wild.sh
|
|
language: system
|
|
files: ^.pre-commit-config.yaml$|^INTHEWILD.md$
|
|
require_serial: true
|
|
- id: helm-lint
|
|
name: Lint Helm Chart
|
|
entry: ./scripts/ci/pre_commit/pre_commit_helm_lint.sh
|
|
language: system
|
|
pass_filenames: false
|
|
files: ^chart
|
|
require_serial: true
|
|
- id: shellcheck
|
|
name: Check Shell scripts syntax correctness
|
|
language: docker_image
|
|
entry: koalaman/shellcheck:stable -x -a
|
|
files: ^breeze$|^breeze-complete$|\.sh$|^hooks/build$|^hooks/push$|\.bash$|\.bats$
|
|
- id: bats-tests
|
|
name: Run BATS bash tests for changed Breeze bash files
|
|
language: system
|
|
entry: "./scripts/ci/pre_commit/pre_commit_bat_tests.sh tests/bats/breeze/"
|
|
files: ^breeze$|^breeze-complete$|^tests/bats/breeze
|
|
pass_filenames: false
|
|
- id: bats-tests
|
|
name: Run BATS bash tests for changed bash files
|
|
language: system
|
|
entry: "./scripts/ci/pre_commit/pre_commit_bat_tests.sh"
|
|
files: \.sh$|\.bash$|\.bats$
|
|
exclude: ^tests/bats/in_container|^scripts/in_container|^tests/bats/breeze
|
|
pass_filenames: true
|
|
require_serial: true
|
|
- id: stylelint
|
|
name: stylelint
|
|
entry: "stylelint"
|
|
language: node
|
|
files: ^airflow/www/.*\.(css|scss|sass)$
|
|
# Keep dependency versions in sync w/ airflow/www/package.json
|
|
additional_dependencies: ['stylelint@13.3.1', 'stylelint-config-standard@20.0.0']
|
|
- id: mermaid
|
|
name: Generate mermaid images
|
|
entry: ./scripts/ci/pre_commit/pre_commit_mermaid.sh
|
|
language: system
|
|
files: \.mermaid$
|
|
require_serial: true
|
|
- id: pre-commit-descriptions
|
|
name: Check if pre-commits are described
|
|
entry: ./scripts/ci/pre_commit/pre_commit_check_pre_commits.sh
|
|
language: system
|
|
files: ^.pre-commit-config.yaml$|^STATIC_CODE_CHECKS.rst|^breeze-complete$
|
|
require_serial: true
|
|
- id: sort-in-the-wild
|
|
name: Sort INTHEWILD.md alphabetically
|
|
entry: ./scripts/ci/pre_commit/pre_commit_sort_in_the_wild.sh
|
|
language: system
|
|
files: ^.pre-commit-config.yaml$|^INTHEWILD.md$
|
|
require_serial: true
|
|
- id: helm-lint
|
|
name: Lint Helm Chart
|
|
entry: ./scripts/ci/pre_commit/pre_commit_helm_lint.sh
|
|
language: system
|
|
pass_filenames: false
|
|
files: ^chart
|
|
require_serial: true
|
|
- id: build
|
|
name: Check if image build is needed
|
|
entry: ./scripts/ci/pre_commit/pre_commit_ci_build.sh 3.6 false
|
|
language: system
|
|
always_run: true
|
|
pass_filenames: false
|
|
- id: mypy
|
|
name: Run mypy
|
|
language: system
|
|
entry: "./scripts/ci/pre_commit/pre_commit_mypy.sh"
|
|
files: \.py$
|
|
exclude: ^dev|^provider_packages
|
|
require_serial: true
|
|
- id: pylint
|
|
name: Run pylint
|
|
language: system
|
|
entry: "./scripts/ci/pre_commit/pre_commit_pylint.sh"
|
|
files: \.py$
|
|
exclude: ^scripts/.*\.py$|^dev|^provider_packages
|
|
pass_filenames: true
|
|
require_serial: true
|
|
- id: flake8
|
|
name: Run flake8
|
|
language: system
|
|
entry: "./scripts/ci/pre_commit/pre_commit_flake8.sh"
|
|
files: \.py$
|
|
pass_filenames: true
|
|
- id: bats-in-container-tests
|
|
name: Run in container bats tests
|
|
language: system
|
|
entry: "./scripts/ci/pre_commit/pre_commit_in_container_bats_test.sh"
|
|
files: ^tests/bats/in_container/.*.bats$|^scripts/in_container/.*sh
|
|
pass_filenames: false
|