diff --git a/Dockerfile b/Dockerfile index 1bd7ae53..3b595e4c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM apache/airflow:slim-2.3.3-python3.10 +FROM apache/airflow:slim-2.5.3-python3.10 ARG PROJECT_DIR="/opt/airflow" diff --git a/constraints.txt b/constraints.txt index d507d22b..af8b8d9e 100644 --- a/constraints.txt +++ b/constraints.txt @@ -1,6 +1,6 @@ # -# This constraints file was automatically generated on 2022-07-05T21:11:24Z -# via "eager-upgrade" mechanism of PIP. For the "v2-3-test" branch of Airflow. +# This constraints file was automatically generated on 2023-03-29T20:31:31Z +# via "eager-upgrade" mechanism of PIP. For the "v2-5-test" branch of Airflow. # This variant of constraints install uses the HEAD of the branch version for 'apache-airflow' but installs # the providers from PIP-released packages at the moment of the constraint generation. # @@ -8,188 +8,195 @@ # We also use those constraints after "apache-airflow" is released and the constraints are tagged with # "constraints-X.Y.Z" tag to build the production image for that version. # -# Editable install with no version control (apache-airflow==2.3.3) -APScheduler==3.6.3 -Authlib==0.15.5 -Babel==2.10.3 +Authlib==1.2.0 +Babel==2.12.1 +ConfigUpdater==3.1.1 Deprecated==1.2.13 -Flask-AppBuilder==4.1.2 +Flask-AppBuilder==4.1.4 Flask-Babel==2.0.0 Flask-Bcrypt==1.0.1 -Flask-Caching==2.0.0 -Flask-JWT-Extended==4.4.2 -Flask-Login==0.6.1 +Flask-Caching==2.0.2 +Flask-JWT-Extended==4.4.4 +Flask-Login==0.6.2 Flask-SQLAlchemy==2.5.1 Flask-Session==0.4.0 -Flask-WTF==0.15.1 -Flask==2.1.2 -GitPython==3.1.27 +Flask-WTF==1.1.1 +Flask==2.2.3 +GitPython==3.1.31 HeapDict==1.0.1 -JPype1==1.4.0 +JPype1==1.4.1 JayDeBeApi==1.2.3 Jinja2==3.1.2 -Mako==1.2.1 -Markdown==3.3.7 -MarkupSafe==2.1.1 -PyGithub==1.55 +Mako==1.2.4 +Markdown==3.4.3 +MarkupSafe==2.1.2 +PyGithub==1.58.1 PyHive==0.6.5 -PyJWT==2.4.0 +PyJWT==2.6.0 PyNaCl==1.5.0 PyYAML==6.0 -Pygments==2.12.0 -Rx==3.2.0 -SQLAlchemy-JSONField==1.0.0 -SQLAlchemy-Utils==0.38.2 -SQLAlchemy==1.4.27 -SecretStorage==3.3.2 -Sphinx==5.0.2 -Unidecode==1.3.4 -WTForms==2.3.3 -Werkzeug==2.1.2 +Pygments==2.14.0 +SQLAlchemy-JSONField==1.0.1.post0 +SQLAlchemy-Utils==0.40.0 +SQLAlchemy==1.4.47 +SecretStorage==3.3.3 +Sphinx==5.3.0 +Unidecode==1.3.6 +WTForms==3.0.1 +Werkzeug==2.2.3 adal==1.2.7 -aiohttp==3.8.1 -aiosignal==1.2.0 -alabaster==0.7.12 -alembic==1.8.0 +aiofiles==22.1.0 +aiohttp==3.8.4 +aiosignal==1.3.1 +alabaster==0.7.13 +alembic==1.10.2 aliyun-python-sdk-core==2.13.36 -aliyun-python-sdk-kms==2.15.0 +aliyun-python-sdk-kms==2.16.0 amqp==5.1.1 -analytics-python==1.4.0 +analytics-python==1.4.post1 ansiwrap==0.8.4 -anyio==3.6.1 -apache-airflow-providers-airbyte==3.0.0 -apache-airflow-providers-alibaba==2.0.0 -apache-airflow-providers-amazon==4.0.0 -apache-airflow-providers-apache-beam==4.0.0 -apache-airflow-providers-apache-cassandra==3.0.0 -apache-airflow-providers-apache-drill==2.0.0 -apache-airflow-providers-apache-druid==3.0.0 -apache-airflow-providers-apache-hdfs==3.0.0 -apache-airflow-providers-apache-hive==3.0.0 -apache-airflow-providers-apache-kylin==3.0.0 -apache-airflow-providers-apache-livy==3.0.0 -apache-airflow-providers-apache-pig==3.0.0 -apache-airflow-providers-apache-pinot==3.0.0 -apache-airflow-providers-apache-spark==3.0.0 -apache-airflow-providers-apache-sqoop==3.0.0 -apache-airflow-providers-arangodb==2.0.0 -apache-airflow-providers-asana==2.0.0 -apache-airflow-providers-celery==3.0.0 -apache-airflow-providers-cloudant==3.0.0 -apache-airflow-providers-cncf-kubernetes==4.1.0 -apache-airflow-providers-databricks==3.0.0 -apache-airflow-providers-datadog==3.0.0 -apache-airflow-providers-dbt-cloud==2.0.0 -apache-airflow-providers-dingding==3.0.0 -apache-airflow-providers-discord==3.0.0 -apache-airflow-providers-docker==3.0.0 -apache-airflow-providers-elasticsearch==4.0.0 -apache-airflow-providers-exasol==3.0.0 -apache-airflow-providers-facebook==3.0.0 -apache-airflow-providers-ftp==3.0.0 -apache-airflow-providers-github==2.0.0 -apache-airflow-providers-google==8.1.0 -apache-airflow-providers-grpc==3.0.0 -apache-airflow-providers-hashicorp==3.0.0 -apache-airflow-providers-http==3.0.0 -apache-airflow-providers-imap==3.0.0 -apache-airflow-providers-influxdb==2.0.0 -apache-airflow-providers-jdbc==3.0.0 -apache-airflow-providers-jenkins==3.0.0 -apache-airflow-providers-jira==3.0.0 -apache-airflow-providers-microsoft-azure==4.0.0 -apache-airflow-providers-microsoft-mssql==3.0.0 -apache-airflow-providers-microsoft-psrp==2.0.0 -apache-airflow-providers-microsoft-winrm==3.0.0 -apache-airflow-providers-mongo==3.0.0 -apache-airflow-providers-mysql==3.0.0 -apache-airflow-providers-neo4j==3.0.0 -apache-airflow-providers-odbc==3.0.0 -apache-airflow-providers-openfaas==3.0.0 -apache-airflow-providers-opsgenie==4.0.0 -apache-airflow-providers-oracle==3.1.0 -apache-airflow-providers-pagerduty==3.0.0 -apache-airflow-providers-papermill==3.0.0 -apache-airflow-providers-plexus==3.0.0 -apache-airflow-providers-postgres==5.0.0 -apache-airflow-providers-presto==3.0.0 -apache-airflow-providers-qubole==3.0.0 -apache-airflow-providers-redis==3.0.0 -apache-airflow-providers-salesforce==4.0.0 -apache-airflow-providers-samba==4.0.0 -apache-airflow-providers-segment==3.0.0 -apache-airflow-providers-sendgrid==3.0.0 -apache-airflow-providers-sftp==3.0.0 -apache-airflow-providers-singularity==3.0.0 -apache-airflow-providers-slack==5.0.0 -apache-airflow-providers-snowflake==3.0.0 -apache-airflow-providers-sqlite==3.0.0 -apache-airflow-providers-ssh==3.0.0 -apache-airflow-providers-tableau==3.0.0 -apache-airflow-providers-telegram==3.0.0 -apache-airflow-providers-trino==3.0.0 -apache-airflow-providers-vertica==3.0.0 -apache-airflow-providers-yandex==3.0.0 -apache-airflow-providers-zendesk==4.0.0 -apache-beam==2.40.0 +anyio==3.6.2 +apache-airflow-providers-airbyte==3.2.0 +apache-airflow-providers-alibaba==2.2.0 +apache-airflow-providers-amazon==7.3.0 +apache-airflow-providers-apache-beam==4.3.0 +apache-airflow-providers-apache-cassandra==3.1.1 +apache-airflow-providers-apache-drill==2.3.1 +apache-airflow-providers-apache-druid==3.3.1 +apache-airflow-providers-apache-hdfs==3.2.0 +apache-airflow-providers-apache-hive==5.1.3 +apache-airflow-providers-apache-kylin==3.1.0 +apache-airflow-providers-apache-livy==3.3.0 +apache-airflow-providers-apache-pig==4.0.0 +apache-airflow-providers-apache-pinot==4.0.1 +apache-airflow-providers-apache-spark==4.0.0 +apache-airflow-providers-apache-sqoop==3.1.1 +apache-airflow-providers-arangodb==2.1.1 +apache-airflow-providers-asana==2.1.0 +apache-airflow-providers-atlassian-jira==2.0.1 +apache-airflow-providers-celery==3.1.0 +apache-airflow-providers-cloudant==3.1.0 +apache-airflow-providers-cncf-kubernetes==5.2.2 +apache-airflow-providers-common-sql==1.3.4 +apache-airflow-providers-databricks==4.0.0 +apache-airflow-providers-datadog==3.1.0 +apache-airflow-providers-dbt-cloud==3.1.0 +apache-airflow-providers-dingding==3.1.0 +apache-airflow-providers-discord==3.1.0 +apache-airflow-providers-docker==3.5.1 +apache-airflow-providers-elasticsearch==4.4.0 +apache-airflow-providers-exasol==4.1.3 +apache-airflow-providers-facebook==3.1.0 +apache-airflow-providers-ftp==3.3.1 +apache-airflow-providers-github==2.2.1 +apache-airflow-providers-google==8.11.0 +apache-airflow-providers-grpc==3.1.0 +apache-airflow-providers-hashicorp==3.3.0 +apache-airflow-providers-http==4.2.0 +apache-airflow-providers-imap==3.1.1 +apache-airflow-providers-influxdb==2.1.0 +apache-airflow-providers-jdbc==3.3.0 +apache-airflow-providers-jenkins==3.2.0 +apache-airflow-providers-microsoft-azure==5.2.1 +apache-airflow-providers-microsoft-mssql==3.3.2 +apache-airflow-providers-microsoft-psrp==2.2.0 +apache-airflow-providers-microsoft-winrm==3.1.1 +apache-airflow-providers-mongo==3.1.1 +apache-airflow-providers-mysql==4.0.2 +apache-airflow-providers-neo4j==3.2.1 +apache-airflow-providers-odbc==3.2.1 +apache-airflow-providers-openfaas==3.1.0 +apache-airflow-providers-opsgenie==5.0.0 +apache-airflow-providers-oracle==3.6.0 +apache-airflow-providers-pagerduty==3.1.0 +apache-airflow-providers-papermill==3.1.1 +apache-airflow-providers-plexus==3.1.0 +apache-airflow-providers-postgres==5.4.0 +apache-airflow-providers-presto==4.2.2 +apache-airflow-providers-qubole==3.3.1 +apache-airflow-providers-redis==3.1.0 +apache-airflow-providers-salesforce==5.3.0 +apache-airflow-providers-samba==4.1.0 +apache-airflow-providers-segment==3.1.0 +apache-airflow-providers-sendgrid==3.1.0 +apache-airflow-providers-sftp==4.2.4 +apache-airflow-providers-singularity==3.1.0 +apache-airflow-providers-slack==7.2.0 +apache-airflow-providers-snowflake==4.0.4 +apache-airflow-providers-sqlite==3.3.1 +apache-airflow-providers-ssh==3.5.0 +apache-airflow-providers-tableau==4.1.0 +apache-airflow-providers-tabular==1.1.0 +apache-airflow-providers-telegram==4.0.0 +apache-airflow-providers-trino==4.3.2 +apache-airflow-providers-vertica==3.3.1 +apache-airflow-providers-yandex==3.3.0 +apache-airflow-providers-zendesk==4.2.0 +apache-beam==2.46.0 apispec==3.3.2 appdirs==1.4.4 -argcomplete==2.0.0 -arrow==1.2.2 -asana==1.0.0 +argcomplete==3.0.5 +arrow==1.2.3 +asana==3.2.0 +asgiref==3.6.0 asn1crypto==1.5.1 -astroid==2.11.6 -asttokens==2.0.5 +astroid==2.15.1 +asttokens==2.2.1 async-timeout==4.0.2 asynctest==0.13.0 atlasclient==1.0.0 -attrs==20.3.0 -aws-sam-translator==1.46.0 -aws-xray-sdk==2.10.0 -azure-batch==12.0.0 +atlassian-python-api==3.35.0 +attrs==22.2.0 +aws-sam-translator==1.63.0 +aws-xray-sdk==2.11.0 +azure-batch==13.0.0 azure-common==1.1.28 -azure-core==1.24.2 -azure-cosmos==4.3.0 +azure-core==1.26.3 +azure-cosmos==4.3.1 azure-datalake-store==0.0.52 -azure-identity==1.10.0 -azure-keyvault-secrets==4.4.0 +azure-identity==1.12.0 +azure-keyvault-secrets==4.7.0 azure-kusto-data==0.0.45 azure-mgmt-containerinstance==1.5.0 -azure-mgmt-core==1.3.1 +azure-mgmt-core==1.3.2 azure-mgmt-datafactory==1.1.0 azure-mgmt-datalake-nspkg==3.0.1 azure-mgmt-datalake-store==0.5.0 azure-mgmt-nspkg==3.0.2 -azure-mgmt-resource==21.1.0 +azure-mgmt-resource==23.0.0 azure-nspkg==3.0.2 -azure-servicebus==7.7.0 -azure-storage-blob==12.8.1 +azure-servicebus==7.8.3 +azure-storage-blob==12.15.0 azure-storage-common==2.1.0 +azure-storage-file-datalake==12.10.1 azure-storage-file==2.1.0 +azure-synapse-spark==0.7.0 backcall==0.2.0 backoff==1.10.0 -bcrypt==3.2.2 -beautifulsoup4==4.11.1 +bcrypt==4.0.1 +beautifulsoup4==4.12.0 billiard==3.6.4.0 -black==22.6.0 -bleach==5.0.1 -blinker==1.4 -boto3==1.24.23 +black==23.1a1 +bleach==6.0.0 +blinker==1.5 +boto3==1.26.102 boto==2.49.0 -botocore==1.27.23 +botocore==1.29.102 bowler==0.9.0 -cached-property==1.5.2 cachelib==0.9.0 -cachetools==4.2.2 -cassandra-driver==3.25.0 -cattrs==1.10.0 +cachetools==5.3.0 +cassandra-driver==3.26.0 +cattrs==22.2.0 celery==5.2.7 -certifi==2020.12.5 +certifi==2022.12.7 cffi==1.15.1 cfgv==3.3.1 -cfn-lint==0.61.1 +cfn-lint==0.76.1 cgroupspy==0.2.2 -charset-normalizer==2.0.12 +chardet==4.0.0 +charset-normalizer==2.1.1 +checksumdir==1.2.0 +ciso8601==2.3.0 click-default-group==1.2.2 click-didyoumean==0.3.0 click-plugins==1.1.1 @@ -197,419 +204,443 @@ click-repl==0.2.0 click==8.1.3 clickclick==20.10.2 cloudant==2.15.0 -cloudpickle==2.1.0 -colorama==0.4.5 +cloudpickle==2.2.1 +colorama==0.4.6 colorlog==4.8.0 -commonmark==0.9.1 -connexion==2.14.0 -coverage==6.4.1 +connexion==2.14.2 +coverage==7.2.2 crcmod==1.7 -cron-descriptor==1.2.30 -croniter==1.3.5 -cryptography==36.0.2 +cron-descriptor==1.2.35 +croniter==1.3.8 +cryptography==39.0.2 curlify==2.2.1 -cx-Oracle==8.3.0 -dask==2022.6.1 -databricks-sql-connector==2.0.2 -datadog==0.44.0 -db-dtypes==1.0.2 +dask==2023.3.2 +databricks-sql-connector==2.4.1 +datadog==0.45.0 +db-dtypes==1.1.0 decorator==5.1.1 defusedxml==0.7.1 dill==0.3.1.1 -distlib==0.3.4 -distributed==2022.6.1 -dnspython==2.2.1 -docker==5.0.3 +distlib==0.3.6 +distributed==2023.3.2 +dnspython==2.3.0 +docker==6.0.1 docopt==0.6.2 docutils==0.19 -ecdsa==0.17.0 -elasticsearch-dbapi==0.2.9 -elasticsearch-dsl==7.4.0 +ecdsa==0.18.0 +elasticsearch-dbapi==0.2.10 +elasticsearch-dsl==7.4.1 elasticsearch==7.13.4 -email-validator==1.2.1 +email-validator==1.3.1 entrypoints==0.4 -eventlet==0.33.1 +eralchemy2==1.3.7 +et-xmlfile==1.1.0 +eventlet==0.33.3 +exceptiongroup==1.1.1 execnet==1.9.0 -executing==0.8.3 -facebook-business==14.0.0 -fastavro==1.5.2 -fastjsonschema==2.15.3 -filelock==3.7.1 +executing==1.2.0 +facebook-business==16.0.2 +fastavro==1.7.3 +fasteners==0.18 +fastjsonschema==2.16.3 +filelock==3.10.7 fissix==21.11.13 flake8-colors==0.1.9 -flake8==4.0.1 -flake8_implicit_str_concat==0.3.0 +flake8==6.0.0 +flake8_implicit_str_concat==0.4.0 flaky==3.7.0 -flower==1.1.0 -freezegun==1.2.1 -frozenlist==1.3.0 -fsspec==2022.5.0 -future==0.18.2 -gcsfs==2022.5.0 +flower==1.2.0 +freezegun==1.2.2 +frozenlist==1.3.3 +fsspec==2023.3.0 +future==0.18.3 +gcloud-aio-auth==4.2.0 +gcloud-aio-bigquery==6.3.0 +gcloud-aio-storage==8.1.0 +gcsfs==2023.3.0 geomet==0.2.1.post1 -gevent==21.12.0 -gitdb==4.0.9 -google-ads==17.0.0 +gevent==22.10.2 +gitdb==4.0.10 +google-ads==18.0.0 google-api-core==2.8.2 google-api-python-client==1.12.11 google-auth-httplib2==0.1.0 -google-auth-oauthlib==0.5.2 -google-auth==2.9.0 -google-cloud-aiplatform==1.15.0 -google-cloud-appengine-logging==1.1.2 -google-cloud-audit-log==0.2.2 -google-cloud-automl==2.7.3 -google-cloud-bigquery-datatransfer==3.6.2 -google-cloud-bigquery-storage==2.13.2 +google-auth-oauthlib==0.8.0 +google-auth==2.17.0 +google-cloud-aiplatform==1.16.1 +google-cloud-appengine-logging==1.1.3 +google-cloud-audit-log==0.2.4 +google-cloud-automl==2.8.0 +google-cloud-bigquery-datatransfer==3.7.0 +google-cloud-bigquery-storage==2.14.1 google-cloud-bigquery==2.34.4 -google-cloud-bigtable==1.7.2 -google-cloud-build==3.8.3 -google-cloud-container==2.10.8 -google-cloud-core==2.3.1 -google-cloud-datacatalog==3.8.1 -google-cloud-dataplex==1.0.1 -google-cloud-dataproc-metastore==1.5.1 -google-cloud-dataproc==4.0.3 -google-cloud-dlp==1.0.2 -google-cloud-kms==2.11.2 +google-cloud-bigtable==2.11.1 +google-cloud-build==3.9.0 +google-cloud-compute==0.7.0 +google-cloud-container==2.11.1 +google-cloud-core==2.3.2 +google-cloud-datacatalog==3.9.0 +google-cloud-dataflow-client==0.5.4 +google-cloud-dataform==0.2.0 +google-cloud-dataplex==1.1.0 +google-cloud-dataproc-metastore==1.6.0 +google-cloud-dataproc==5.0.0 +google-cloud-dlp==3.7.1 +google-cloud-kms==2.12.0 google-cloud-language==1.3.2 -google-cloud-logging==3.1.2 -google-cloud-memcache==1.3.2 -google-cloud-monitoring==2.9.2 -google-cloud-orchestration-airflow==1.3.2 -google-cloud-os-login==2.6.2 -google-cloud-pubsub==2.13.0 -google-cloud-redis==2.8.1 -google-cloud-resource-manager==1.5.1 +google-cloud-logging==3.2.1 +google-cloud-memcache==1.4.1 +google-cloud-monitoring==2.11.0 +google-cloud-orchestration-airflow==1.4.1 +google-cloud-os-login==2.7.1 +google-cloud-pubsub==2.13.5 +google-cloud-redis==2.9.0 +google-cloud-resource-manager==1.6.0 google-cloud-secret-manager==1.0.2 google-cloud-spanner==1.19.3 google-cloud-speech==1.3.4 -google-cloud-storage==1.44.0 -google-cloud-tasks==2.9.1 +google-cloud-storage==2.7.0 +google-cloud-tasks==2.10.1 google-cloud-texttospeech==1.0.3 google-cloud-translate==1.7.2 google-cloud-videointelligence==1.16.3 google-cloud-vision==1.0.2 -google-cloud-workflows==1.6.3 -google-crc32c==1.3.0 -google-resumable-media==2.3.3 -googleapis-common-protos==1.56.3 -graphql-core==3.2.1 -graphviz==0.20 -greenlet==1.1.2 +google-cloud-workflows==1.7.1 +google-crc32c==1.5.0 +google-resumable-media==2.4.1 +googleapis-common-protos==1.56.4 +graphql-core==3.2.3 +graphviz==0.20.1 +greenlet==2.0.2 grpc-google-iam-v1==0.12.4 grpcio-gcp==0.2.2 -grpcio-status==1.47.0 -grpcio==1.47.0 -gssapi==1.7.3 +grpcio-status==1.48.2 +grpcio==1.53.0 +gssapi==1.8.2 gunicorn==20.1.0 -h11==0.12.0 +h11==0.14.0 hdfs==2.7.0 hmsclient==0.1.1 -httpcore==0.15.0 -httplib2==0.20.4 -httpx==0.23.0 -humanize==4.2.3 -hvac==0.11.2 -identify==2.5.1 -idna==3.3 -ijson==3.1.4 +httpcore==0.16.3 +httplib2==0.21.0 +httpx==0.23.3 +humanize==4.6.0 +hvac==1.1.0 +identify==2.5.22 +idna==3.4 +ijson==3.2.0.post0 imagesize==1.4.1 -importlib-metadata==4.12.0 -incremental==21.3.0 +importlib-metadata==6.1.0 +incremental==22.10.0 inflection==0.5.1 -influxdb-client==1.30.0 -iniconfig==1.1.1 -ipdb==0.13.9 -ipython==8.4.0 +influxdb-client==1.36.1 +iniconfig==2.0.0 +ipdb==0.13.13 +ipython==8.11.0 isodate==0.6.1 +isort==5.11.2 itsdangerous==2.1.2 -jedi==0.18.1 +jaraco.classes==3.2.3 +jedi==0.18.2 jeepney==0.8.0 -jira==3.2.0 +jira==3.5.0 jmespath==0.10.0 jschema-to-python==1.2.3 json-merge-patch==0.2 jsondiff==2.0.0 jsonpatch==1.32 jsonpath-ng==1.5.3 -jsonpickle==2.2.0 +jsonpickle==3.0.1 jsonpointer==2.3 -jsonschema==4.6.1 +jsonschema-spec==0.1.4 +jsonschema==4.17.3 junit-xml==1.9 -jupyter-client==7.3.4 -jupyter-core==4.10.0 -keyring==23.6.0 +jupyter_client==8.1.0 +jupyter_core==5.3.0 +keyring==23.13.1 kombu==5.2.4 -krb5==0.3.0 +krb5==0.5.0 +kubernetes-asyncio==24.2.2 kubernetes==23.6.0 kylinpy==2.8.4 -lazy-object-proxy==1.7.1 +lazy-object-proxy==1.9.0 ldap3==2.9.1 linkify-it-py==2.0.0 locket==1.0.0 lockfile==0.12.2 -looker-sdk==22.4.0 -lxml==4.9.1 -markdown-it-py==2.1.0 +looker-sdk==23.2.0 +lxml==4.9.2 +lz4==4.3.2 +markdown-it-py==2.2.0 marshmallow-enum==1.5.1 marshmallow-oneofschema==3.0.1 marshmallow-sqlalchemy==0.26.1 -marshmallow==3.17.0 -matplotlib-inline==0.1.3 -mccabe==0.6.1 -mdit-py-plugins==0.3.0 -mdurl==0.1.1 -mongomock==4.0.0 +marshmallow==3.19.0 +matplotlib-inline==0.1.6 +mccabe==0.7.0 +mdit-py-plugins==0.3.5 +mdurl==0.1.2 +mongomock==4.1.2 monotonic==1.6 -more-itertools==8.13.0 +more-itertools==9.1.0 moreorless==0.4.0 -moto==3.1.16 +moto==4.1.6 +mpmath==1.3.0 msal-extensions==1.0.0 -msal==1.18.0 -msgpack==1.0.4 +msal==1.21.0 +msgpack==1.0.5 msrest==0.7.1 msrestazure==0.6.4 multi-key-dict==2.0.3 -multidict==6.0.2 -mypy-boto3-rds==1.24.23 -mypy-boto3-redshift-data==1.24.11.post3 -mypy-extensions==0.4.3 -mypy==0.910 -mysql-connector-python==8.0.29 +multidict==6.0.4 +mypy-boto3-appflow==1.26.78 +mypy-boto3-rds==1.26.102 +mypy-boto3-redshift-data==1.26.88 +mypy-extensions==1.0.0 +mypy==0.971 +mysql-connector-python==8.0.32 mysqlclient==2.1.1 -nbclient==0.6.6 -nbformat==5.4.0 -neo4j==4.4.4 -nest-asyncio==1.5.5 -networkx==2.8.4 +nbclient==0.7.2 +nbformat==5.8.0 +neo4j==5.6.0 +networkx==3.0 nodeenv==1.7.0 -ntlm-auth==1.5.0 -numpy==1.22.4 -oauthlib==3.2.0 -openapi-schema-validator==0.2.3 -openapi-spec-validator==0.4.0 +numpy==1.24.2 +oauthlib==3.2.2 +objsize==0.6.1 +openapi-schema-validator==0.4.4 +openapi-spec-validator==0.5.6 +openpyxl==3.1.2 opsgenie-sdk==2.1.5 -oracledb==1.0.1 -orjson==3.7.6 +oracledb==1.2.2 +orjson==3.8.9 oscrypto==1.3.0 -oss2==2.15.0 +oss2==2.17.0 packaging==21.3 -pandas-gbq==0.17.6 -pandas==1.4.3 -papermill==2.3.4 -parameterized==0.8.1 -paramiko==2.11.0 +pandas-gbq==0.17.9 +pandas==1.5.3 +papermill==2.4.0 +parameterized==0.9.0 +paramiko==3.1.0 parso==0.8.3 -partd==1.2.0 +partd==1.3.0 +pathable==0.4.3 pathspec==0.9.0 -pbr==5.9.0 -pdpyras==4.5.0 +pbr==5.11.1 +pdpyras==4.5.2 pendulum==2.1.2 pexpect==4.8.0 pickleshare==0.7.5 -pinotdb==0.3.12 -pipdeptree==2.2.1 -pipx==1.1.0 -pkginfo==1.8.3 -platformdirs==2.5.2 +pinotdb==0.4.14 +pipdeptree==2.7.0 +pipx==1.2.0 +pkginfo==1.9.6 +platformdirs==3.2.0 pluggy==1.0.0 ply==3.11 -plyvel==1.4.0 -portalocker==2.4.0 -pre-commit==2.19.0 -presto-python-client==0.8.2 +plyvel==1.5.0 +portalocker==2.7.0 +pre-commit==3.2.1 +presto-python-client==0.8.3 prison==0.2.1 -prometheus-client==0.14.1 -prompt-toolkit==3.0.30 +prometheus-client==0.16.0 +prompt-toolkit==3.0.38 proto-plus==1.19.6 protobuf==3.20.0 -psutil==5.9.1 -psycopg2-binary==2.9.3 +psutil==5.9.4 +psycopg2-binary==2.9.5 +psycopg2==2.9.5 ptyprocess==0.7.0 pure-eval==0.2.2 pure-sasl==0.6.2 +py-partiql-parser==0.1.0 py4j==0.10.9.5 -py==1.11.0 -pyOpenSSL==22.0.0 -pyarrow==6.0.1 +pyOpenSSL==23.1.1 +pyarrow==9.0.0 pyasn1-modules==0.2.8 pyasn1==0.4.8 -pycodestyle==2.8.0 +pycodestyle==2.10.0 pycountry==22.3.5 pycparser==2.21 -pycryptodome==3.15.0 -pycryptodomex==3.15.0 -pydata-google-auth==1.4.0 +pycryptodome==3.17 +pycryptodomex==3.17 +pydantic==1.10.7 +pydata-google-auth==1.7.0 pydot==1.4.2 -pydruid==0.6.3 +pydruid==0.6.5 pyenchant==3.2.2 -pyexasol==0.24.0 -pyflakes==2.4.0 +pyexasol==0.25.2 +pyflakes==3.0.1 +pygraphviz==1.10 +pyhcl==0.4.4 pykerberos==1.2.4 -pymongo==3.12.3 -pymssql==2.2.5 -pyodbc==4.0.32 +pymongo==3.13.0 +pymssql==2.2.7 +pyodbc==4.0.35 pyparsing==3.0.9 pypsrp==0.8.1 -pyrsistent==0.18.1 -pysftp==0.2.9 -pyspark==3.3.0 -pyspnego==0.5.2 -pytest-asyncio==0.18.3 -pytest-cov==3.0.0 -pytest-forked==1.4.0 -pytest-httpx==0.21.0 +pyrsistent==0.19.3 +pyspark==3.3.2 +pyspnego==0.8.0 +pytest-asyncio==0.21.0 +pytest-capture-warnings==0.0.4 +pytest-cov==4.0.0 +pytest-httpx==0.21.3 pytest-instafail==0.4.2 -pytest-rerunfailures==9.1.1 +pytest-rerunfailures==11.1.2 pytest-timeouts==1.2.1 -pytest-xdist==2.5.0 -pytest==6.2.5 -python-arango==7.4.1 -python-daemon==2.3.0 +pytest-xdist==3.2.1 +pytest==7.2.2 +python-arango==7.5.7 +python-daemon==3.0.1 python-dateutil==2.8.2 +python-dotenv==1.0.0 python-http-client==3.3.7 python-jenkins==1.7.0 python-jose==3.3.0 -python-ldap==3.4.0 +python-ldap==3.4.3 python-nvd3==0.15.0 -python-slugify==6.1.2 -python-telegram-bot==13.13 +python-slugify==8.0.1 +python-telegram-bot==20.2 pytz-deprecation-shim==0.1.0.post0 -pytz==2022.1 +pytz==2023.3 pytzdata==2020.1 pywinrm==0.4.3 -pyzmq==23.2.0 +pyzmq==25.0.2 qds-sdk==1.16.1 -readme-renderer==35.0 +reactivex==4.0.4 +readme-renderer==37.3 redis==3.5.3 -redshift-connector==2.0.908 +redshift-connector==2.0.910 +regex==2023.3.23 requests-file==1.5.1 requests-kerberos==0.14.0 -requests-mock==1.9.3 -requests-ntlm==1.1.0 +requests-mock==1.10.0 +requests-ntlm==1.2.0 requests-oauthlib==1.3.1 -requests-toolbelt==0.9.1 -requests==2.28.0 -responses==0.21.0 +requests-toolbelt==0.10.1 +requests==2.28.2 +responses==0.23.1 +rfc3339-validator==0.1.4 rfc3986==1.5.0 -rich-click==1.5.1 -rich==12.4.4 -rsa==4.8 +rich-click==1.6.1 +rich==13.3.3 +rsa==4.9 s3transfer==0.6.0 sarif-om==1.0.4 sasl==0.3.1 -scramp==1.4.1 +scramp==1.4.4 scrapbook==0.5.0 semver==2.13.0 -sendgrid==6.9.7 +sendgrid==6.10.0 sentinels==1.0.0 -sentry-sdk==1.6.0 -setproctitle==1.2.3 -simple-salesforce==1.11.6 +sentry-sdk==1.18.0 +setproctitle==1.3.2 +simple-salesforce==1.12.3 six==1.16.0 -slack-sdk==3.17.2 -smbprotocol==1.9.0 +slack-sdk==3.20.2 +smbprotocol==1.10.1 smmap==5.0.0 snakebite-py3==3.0.5 -sniffio==1.2.0 +sniffio==1.3.0 snowballstemmer==2.2.0 -snowflake-connector-python==2.7.9 -snowflake-sqlalchemy==1.3.4 +snowflake-connector-python==3.0.2 +snowflake-sqlalchemy==1.4.7 sortedcontainers==2.4.0 -soupsieve==2.3.2.post1 -sphinx-airflow-theme==0.0.9 -sphinx-argparse==0.3.1 -sphinx-autoapi==1.8.4 -sphinx-copybutton==0.5.0 +soupsieve==2.4 +sphinx-airflow-theme==0.0.11 +sphinx-argparse==0.4.0 +sphinx-autoapi==2.1.0 +sphinx-copybutton==0.5.1 sphinx-jinja==2.0.2 -sphinx-rtd-theme==1.0.0 -sphinxcontrib-applehelp==1.0.2 +sphinx-rtd-theme==1.2.0 +sphinxcontrib-applehelp==1.0.4 sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==2.0.0 -sphinxcontrib-httpdomain==1.8.0 +sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-httpdomain==1.8.1 +sphinxcontrib-jquery==4.1 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==1.0.3 sphinxcontrib-redoc==1.6.0 sphinxcontrib-serializinghtml==1.1.5 -sphinxcontrib-spelling==7.6.0 -spython==0.2.1 -sqlalchemy-bigquery==1.4.4 +sphinxcontrib-spelling==8.0.0 +spython==0.3.0 +sqlalchemy-bigquery==1.6.1 sqlalchemy-drill==1.1.2 -sqlalchemy-redshift==0.8.9 -sqlparse==0.4.2 +sqlalchemy-redshift==0.8.13 +sqlparse==0.4.3 sshpubkeys==3.3.1 sshtunnel==0.4.0 -stack-data==0.3.0 -starkbank-ecdsa==2.0.3 -statsd==3.3.0 -swagger-ui-bundle==0.0.9 -tableauserverclient==0.19.0 -tabulate==0.8.10 +stack-data==0.6.2 +starkbank-ecdsa==2.2.0 +statsd==4.0.1 +sympy==1.11.1 +tableauserverclient==0.24 +tabulate==0.9.0 tblib==1.7.0 -tenacity==8.0.1 -termcolor==1.1.0 +tenacity==8.2.2 +termcolor==2.2.0 text-unidecode==1.3 textwrap3==0.9.2 thrift-sasl==0.4.3 thrift==0.16.0 -toml==0.10.2 +time-machine==2.9.0 tomli==2.0.1 -toolz==0.11.2 +toolz==0.12.0 tornado==6.2 -towncrier==21.9.0 -tqdm==4.64.0 -traitlets==5.3.0 -trino==0.314.0 -twine==4.0.1 -types-Deprecated==1.2.9 -types-Markdown==3.3.29 -types-PyMySQL==1.0.19 -types-PyYAML==6.0.9 -types-boto==2.49.16 +towncrier==22.12.0 +tqdm==4.65.0 +traitlets==5.9.0 +trino==0.322.0 +twine==4.0.2 +types-Deprecated==1.2.9.2 +types-Markdown==3.4.2.6 +types-PyMySQL==1.0.19.6 +types-PyYAML==6.0.12.9 +types-boto==2.49.18.7 types-certifi==2021.10.8.3 -types-croniter==1.3.0 -types-cryptography==3.3.21 -types-docutils==0.18.3 +types-croniter==1.3.2.7 +types-docutils==0.19.1.7 types-freezegun==1.1.10 -types-paramiko==2.11.2 -types-protobuf==3.19.22 -types-python-dateutil==2.8.18 -types-python-slugify==6.1.0 -types-pytz==2022.1.1 -types-redis==4.3.3 -types-requests==2.28.0 -types-setuptools==57.4.18 -types-six==1.16.17 -types-tabulate==0.8.11 -types-termcolor==1.1.5 -types-toml==0.10.7 -types-urllib3==1.26.15 -typing_extensions==4.3.0 -tzdata==2022.1 -tzlocal==4.2 -uamqp==1.5.3 +types-paramiko==3.0.0.6 +types-protobuf==4.22.0.0 +types-pyOpenSSL==23.1.0.1 +types-python-dateutil==2.8.19.11 +types-python-slugify==8.0.0.2 +types-pytz==2023.2.0.1 +types-redis==4.5.3.1 +types-requests==2.28.11.17 +types-setuptools==67.6.0.6 +types-tabulate==0.9.0.2 +types-termcolor==1.1.6.2 +types-toml==0.10.8.6 +types-urllib3==1.26.25.10 +typing_extensions==4.5.0 +tzdata==2023.3 +tzlocal==4.3 +uamqp==1.6.4 uc-micro-py==1.0.1 unicodecsv==0.14.1 uritemplate==3.0.1 -urllib3==1.26.9 +urllib3==1.26.15 userpath==1.8.0 -vertica-python==1.1.0 +vertica-python==1.3.1 vine==5.0.0 -virtualenv==20.15.1 +virtualenv==20.21.0 volatile==2.1.0 watchtower==2.0.1 -wcwidth==0.2.5 +wcwidth==0.2.6 webencodings==0.5.1 -websocket-client==1.3.3 -wrapt==1.14.1 +websocket-client==1.5.1 +wrapt==1.15.0 xmltodict==0.13.0 -yamllint==1.26.3 -yandexcloud==0.171.0 -yarl==1.7.2 -zeep==4.1.0 -zenpy==2.0.24 +yamllint==1.30.0 +yandexcloud==0.206.0 +yarl==1.8.2 +zeep==4.2.1 +zenpy==2.0.25 zict==2.2.0 -zipp==3.8.0 -zope.event==4.5.0 -zope.interface==5.4.0 +zipp==3.15.0 +zope.event==4.6 +zope.interface==6.0 +zstandard==0.20.0 diff --git a/dags/backfill.py b/dags/backfill.py index 82333e7c..c851e167 100644 --- a/dags/backfill.py +++ b/dags/backfill.py @@ -55,7 +55,7 @@ def generate_bash_command(params: dict) -> str: doc_md = """ # Backfill DAG -#### Use with caution +#### Use with caution #### Some tips/notes: @@ -79,12 +79,12 @@ doc_md = """ params={ "dag_name": Param("dag_name", type="string"), "start_date": Param( - (datetime.date.today() - datetime.timedelta(days=10)).isoformat(), + (datetime.datetime.today() - datetime.timedelta(days=10)).isoformat(), type="string", format="date-time", ), "end_date": Param( - datetime.date.today().isoformat(), type="string", format="date-time" + datetime.datetime.today().isoformat(), type="string", format="date-time" ), "clear": Param(False, type="boolean"), "dry_run": Param(True, type="boolean"), diff --git a/dags/copy_deduplicate.py b/dags/copy_deduplicate.py index 379ac1b9..674d5bc6 100644 --- a/dags/copy_deduplicate.py +++ b/dags/copy_deduplicate.py @@ -63,7 +63,6 @@ with models.DAG( default_args=default_args, tags=tags, ) as dag: - # This single task is responsible for sequentially running copy queries # over all the tables in _live datasets into _stable datasets except those # that are specifically used in another DAG. @@ -89,8 +88,8 @@ with models.DAG( "telemetry_live.first_shutdown_v4", "telemetry_live.saved_session_v4", ], - node_selectors={"nodepool": "highmem"}, - resources=resources, + node_selector={"nodepool": "highmem"}, + container_resources=resources, ) with TaskGroup("copy_deduplicate_all_external") as copy_deduplicate_all_external: diff --git a/dags/dim_active_users_aggregates.py b/dags/dim_active_users_aggregates.py index bc2ee6aa..eb2cf5c4 100644 --- a/dags/dim_active_users_aggregates.py +++ b/dags/dim_active_users_aggregates.py @@ -13,7 +13,7 @@ from collections import namedtuple from datetime import datetime, timedelta from airflow import DAG -from airflow.operators.dummy import DummyOperator +from airflow.operators.empty import EmptyOperator from airflow.sensors.external_task import ExternalTaskSensor from operators.gcp_container_operator import GKEPodOperator from utils.constants import ALLOWED_STATES, FAILED_STATES @@ -65,8 +65,7 @@ for platform, config in CONFIGS.items(): start_date=datetime(2023, 3, 20), doc_md=__doc__, ) as dag: - - run_all = DummyOperator( + run_all = EmptyOperator( task_id="run_all", ) diff --git a/dags/glam_subdags/extract.py b/dags/glam_subdags/extract.py index e3f141da..fbe3145b 100644 --- a/dags/glam_subdags/extract.py +++ b/dags/glam_subdags/extract.py @@ -1,18 +1,20 @@ -from airflow.providers.google.cloud.transfers.bigquery_to_gcs import BigQueryToGCSOperator -from airflow.providers.google.cloud.operators.gcs import GCSDeleteObjectsOperator -from airflow.operators.subdag_operator import SubDagOperator from airflow.models import DAG +from airflow.operators.subdag import SubDagOperator +from airflow.providers.google.cloud.operators.gcs import GCSDeleteObjectsOperator +from airflow.providers.google.cloud.transfers.bigquery_to_gcs import ( + BigQueryToGCSOperator, +) from utils.gcp import bigquery_etl_query - gcp_conn_id = "google_cloud_airflow_dataproc" project_id = "moz-fx-data-shared-prod" glam_bucket = "moz-fx-data-glam-prod-fca7-etl-data" + def extracts_subdag( parent_dag_name, child_dag_name, default_args, schedule_interval, dataset_id ): - dag_id = "{}.{}".format(parent_dag_name, child_dag_name) + dag_id = f"{parent_dag_name}.{child_dag_name}" dag = DAG( dag_id=dag_id, default_args=default_args, schedule_interval=schedule_interval ) @@ -21,13 +23,13 @@ def extracts_subdag( SubDagOperator( subdag=extract_channel_subdag( dag_id, - "extract_{}".format(channel), + f"extract_{channel}", default_args, schedule_interval, dataset_id, channel, ), - task_id="extract_{}".format(channel), + task_id=f"extract_{channel}", dag=dag, ) @@ -43,14 +45,14 @@ def extract_channel_subdag( channel, ): dag = DAG( - dag_id="{}.{}".format(parent_dag_name, child_dag_name), + dag_id=f"{parent_dag_name}.{child_dag_name}", default_args=default_args, schedule_interval=schedule_interval, ) - bq_extract_table = "glam_extract_firefox_{}_v1".format(channel) + bq_extract_table = f"glam_extract_firefox_{channel}_v1" etl_query = bigquery_etl_query( - task_id="glam_client_probe_counts_{}_extract".format(channel), + task_id=f"glam_client_probe_counts_{channel}_extract", destination_table=bq_extract_table, dataset_id=dataset_id, project_id=project_id, @@ -59,14 +61,14 @@ def extract_channel_subdag( sql_file_path="sql/moz-fx-data-shared-prod/{}/glam_client_probe_counts_extract_v1/query.sql".format( dataset_id ), - parameters=("channel:STRING:{}".format(channel),), + parameters=(f"channel:STRING:{channel}",), dag=dag, ) gcs_delete = GCSDeleteObjectsOperator( - task_id="glam_gcs_delete_old_{}_extracts".format(channel), + task_id=f"glam_gcs_delete_old_{channel}_extracts", bucket_name=glam_bucket, - prefix="aggs-desktop-{}".format(channel), + prefix=f"aggs-desktop-{channel}", gcp_conn_id=gcp_conn_id, dag=dag, ) @@ -75,7 +77,7 @@ def extract_channel_subdag( bucket=glam_bucket, channel=channel ) bq2gcs = BigQueryToGCSOperator( - task_id="glam_extract_{}_to_csv".format(channel), + task_id=f"glam_extract_{channel}_to_csv", source_project_dataset_table="{}.{}.{}".format( project_id, dataset_id, bq_extract_table ), @@ -98,17 +100,17 @@ def extract_user_counts( schedule_interval, dataset_id, task_prefix, - file_prefix + file_prefix, ): - bq_extract_table="glam_{}_extract_v1".format(task_prefix) + bq_extract_table = f"glam_{task_prefix}_extract_v1" dag = DAG( - dag_id="{}.{}".format(parent_dag_name, child_dag_name), + dag_id=f"{parent_dag_name}.{child_dag_name}", default_args=default_args, schedule_interval=schedule_interval, ) - + etl_query = bigquery_etl_query( - task_id="glam_{}_extract".format(task_prefix), + task_id=f"glam_{task_prefix}_extract", destination_table=bq_extract_table, dataset_id=dataset_id, project_id=project_id, @@ -117,27 +119,25 @@ def extract_user_counts( dag=dag, ) - gcs_delete = GCSDeleteObjectsOperator( - task_id="glam_gcs_delete_{}_extracts".format(task_prefix), + task_id=f"glam_gcs_delete_{task_prefix}_extracts", bucket_name=glam_bucket, - - prefix="glam-extract-firefox-{}".format(file_prefix), + prefix=f"glam-extract-firefox-{file_prefix}", gcp_conn_id=gcp_conn_id, dag=dag, ) - if file_prefix=="sample-counts": + if file_prefix == "sample-counts": gcs_destination = "gs://{}/glam-extract-firefox-{}-*.csv".format( - glam_bucket, file_prefix - ) - else: + glam_bucket, file_prefix + ) + else: gcs_destination = "gs://{}/glam-extract-firefox-{}.csv".format( - glam_bucket, file_prefix - ) + glam_bucket, file_prefix + ) bq2gcs = BigQueryToGCSOperator( - task_id="glam_extract_{}_to_csv".format(task_prefix), + task_id=f"glam_extract_{task_prefix}_to_csv", source_project_dataset_table="{}.{}.{}".format( project_id, dataset_id, bq_extract_table ), diff --git a/dags/operators/backport/README.md b/dags/operators/backport/README.md deleted file mode 100644 index 0eb27be9..00000000 --- a/dags/operators/backport/README.md +++ /dev/null @@ -1,17 +0,0 @@ -### Using kube_client.py from 1.10.2 -We used to include the airflow/contrib/kubernetes/kube_client.py from 1.10.2 -because the 1.10.7 kube_client.py has some configuration issues when -trying to push xcom from gkepodoperator. if do_push_xcom is set to False, -the upstream GkePodOperator works fine. - -### As of 1.10.12 I've removed the backported 1.10.7 gcp_container_operator, -kubernetes_pod_operator, and the 1.10.2 kube_client - - -### Fivetran operator backported from 2.0+ -Fivetran provides an [operator and sensor](https://github.com/fivetran/airflow-provider-fivetran) -for integrating with the Fivetran API for Airflow version 2.0+. This was backported for -Airflow 1.10.15, and then our backport was removed after upgrading to Airflow 2.3.3. - -### For 2.1.0 I've removed bigquery_operator_1_10_2.py, in favor of the new -google provider code. diff --git a/dags/operators/backport/__init__.py b/dags/operators/backport/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/dags/operators/bq_sensor.py b/dags/operators/bq_sensor.py index cec0b382..f011e039 100644 --- a/dags/operators/bq_sensor.py +++ b/dags/operators/bq_sensor.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -17,9 +16,9 @@ # specific language governing permissions and limitations # under the License. -from airflow.sensors.base_sensor_operator import BaseSensorOperator - from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook +from airflow.sensors.base import BaseSensorOperator + class BigQuerySQLSensorOperator(BaseSensorOperator): """ @@ -39,42 +38,40 @@ class BigQuerySQLSensorOperator(BaseSensorOperator): :type timeout: int """ - template_fields = BaseSensorOperator.template_fields + ( - 'sql', - ) + template_fields = (*BaseSensorOperator.template_fields, "sql") - def __init__(self, - sql, - gcp_conn_id='bigquery_default_conn', - use_legacy_sql=False, - timeout=60*60*24, - *args, - **kwargs): - - super(BigQuerySQLSensorOperator, self).__init__( - timeout=timeout, - *args, - **kwargs) + def __init__( + self, + sql, + gcp_conn_id="bigquery_default_conn", + use_legacy_sql=False, + timeout=60 * 60 * 24, + *args, + **kwargs + ): + super().__init__(timeout=timeout, *args, **kwargs) # noqa: B026 self.sql = sql self.gcp_conn_id = gcp_conn_id self.use_legacy_sql = use_legacy_sql self.poke_interval = 120 - self.mode = 'reschedule' - self.pool = 'DATA_ENG_EXTERNALTASKSENSOR' + self.mode = "reschedule" + self.pool = "DATA_ENG_EXTERNALTASKSENSOR" def poke(self, context): - self.log.info('Running query: %s', self.sql) + self.log.info("Running query: %s", self.sql) record = self.get_db_hook().get_first(self.sql) - self.log.info('Resulting Record: %s', record) + self.log.info("Resulting Record: %s", record) - if not record: - return False - else: - if str(record[0]).lower() in ('0', '', 'false', 'null',): - return False - else: - return True + if record: + return record[0].lower() not in ( + "0", + "", + "false", + "null", + ) + return False def get_db_hook(self): - return BigQueryHook(gcp_conn_id=self.gcp_conn_id, - use_legacy_sql=self.use_legacy_sql) + return BigQueryHook( + gcp_conn_id=self.gcp_conn_id, use_legacy_sql=self.use_legacy_sql + ) diff --git a/dags/probe_scraper.py b/dags/probe_scraper.py index afa9f0f1..94165afe 100644 --- a/dags/probe_scraper.py +++ b/dags/probe_scraper.py @@ -74,7 +74,6 @@ with DAG( schedule_interval="0 0 * * *", tags=tags, ) as dag: - airflow_gke_prod_kwargs = { "gcp_conn_id": "google_cloud_airflow_gke", "project_id": "moz-fx-data-airflow-gke-prod", @@ -112,7 +111,7 @@ with DAG( name="probe-scraper-moz-central", # Needed to scale the highmem pool from 0 -> 1, because cluster autoscaling # works on pod resource requests, instead of usage - resources={ + container_resources={ "request_memory": "13312Mi", "request_cpu": None, "limit_memory": "20480Mi", @@ -120,7 +119,7 @@ with DAG( "limit_gpu": None, }, # This python job requires 13 GB of memory, thus the highmem node pool - node_selectors={"nodepool": "highmem"}, + node_selector={"nodepool": "highmem"}, # Due to the nature of the container run, we set get_logs to False, to avoid # urllib3.exceptions.ProtocolError: 'Connection broken: IncompleteRead(0 bytes # read)' errors where the pod continues to run, but airflow loses its connection diff --git a/dags/shredder.py b/dags/shredder.py index 64b27280..1532ab5a 100644 --- a/dags/shredder.py +++ b/dags/shredder.py @@ -127,7 +127,7 @@ flat_rate = gke_command( reattach_on_restart=True, # Needed to scale the highmem pool from 0 -> 1, because cluster autoscaling # works on pod resource requests, instead of usage - resources={ + container_resources={ "request_memory": "13312Mi", "request_cpu": None, "limit_memory": "20480Mi", @@ -135,7 +135,7 @@ flat_rate = gke_command( "limit_gpu": None, }, # This job was being killed by Kubernetes for using too much memory, thus the highmem node pool - node_selectors={"nodepool": "highmem"}, + node_selector={"nodepool": "highmem"}, # Give additional time since we may need to scale up when running this job startup_timeout_seconds=360, dag=dag, diff --git a/docker-compose.yml b/docker-compose.yml index 5d9b0b04..4a93f09d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ # AIRFLOW_PROJ_DIR - Base path to which all the files will be volumed. # Default: . --- -version: '3' +version: '3.8' x-airflow-common: &airflow-common build: . @@ -30,6 +30,11 @@ x-airflow-common: AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true' AIRFLOW__CORE__LOAD_EXAMPLES: 'false' AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.default' + # yamllint disable rule:line-length + # Use simple http server on scheduler for health checks + # See https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/logging-monitoring/check-health.html#scheduler-health-check-server + # yamllint enable rule:line-length + AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true' DEPLOY_ENVIRONMENT: dev volumes: - ./dags:/opt/airflow/dags @@ -57,8 +62,9 @@ services: - postgres-db-volume:/var/lib/postgresql/data healthcheck: test: ["CMD", "pg_isready", "-U", "airflow"] - interval: 5s + interval: 10s retries: 5 + start_period: 5s restart: always redis: @@ -67,21 +73,23 @@ services: - 6379 healthcheck: test: ["CMD", "redis-cli", "ping"] - interval: 5s + interval: 10s timeout: 30s retries: 50 + start_period: 30s restart: always airflow-webserver: <<: *airflow-common command: webserver ports: - - '8080:8080' + - "8080:8080" healthcheck: test: ["CMD", "curl", "--fail", "http://localhost:8080/health"] - interval: 10s + interval: 30s timeout: 10s retries: 5 + start_period: 30s restart: always depends_on: <<: *airflow-common-depends-on @@ -92,10 +100,11 @@ services: <<: *airflow-common command: scheduler healthcheck: - test: ["CMD-SHELL", 'airflow jobs check --job-type SchedulerJob --hostname "$${HOSTNAME}"'] - interval: 10s + test: ["CMD", "curl", "--fail", "http://localhost:8974/health"] + interval: 30s timeout: 10s retries: 5 + start_period: 30s restart: always depends_on: <<: *airflow-common-depends-on @@ -109,9 +118,10 @@ services: test: - "CMD-SHELL" - 'celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"' - interval: 10s + interval: 30s timeout: 10s retries: 5 + start_period: 30s environment: <<: *airflow-common-env # Required to handle warm shutdown of the celery workers properly @@ -128,9 +138,10 @@ services: command: triggerer healthcheck: test: ["CMD-SHELL", 'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"'] - interval: 10s + interval: 30s timeout: 10s retries: 5 + start_period: 30s restart: always depends_on: <<: *airflow-common-depends-on @@ -164,7 +175,7 @@ services: echo "If you are on Linux, you SHOULD follow the instructions below to set " echo "AIRFLOW_UID environment variable, otherwise files will be owned by root." echo "For other operating systems you can get rid of the warning with manually created .env file:" - echo " See: https://airflow.apache.org/docs/apache-airflow/stable/start/docker.html#setting-the-right-airflow-user" + echo " See: https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#setting-the-right-airflow-user" echo fi one_meg=1048576 @@ -197,7 +208,7 @@ services: echo echo -e "\033[1;33mWARNING!!!: You have not enough resources to run Airflow (see above)!\e[0m" echo "Please follow the instructions to increase amount of resources available:" - echo " https://airflow.apache.org/docs/apache-airflow/stable/start/docker.html#before-you-begin" + echo " https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#before-you-begin" echo fi mkdir -p /sources/logs /sources/dags /sources/plugins @@ -213,7 +224,7 @@ services: _PIP_ADDITIONAL_REQUIREMENTS: '' user: "0:0" volumes: - - .:/sources + - ${AIRFLOW_PROJ_DIR:-.}:/sources airflow-cli: <<: *airflow-common @@ -237,12 +248,13 @@ services: profiles: - flower ports: - - 5555:5555 + - "5555:5555" healthcheck: test: ["CMD", "curl", "--fail", "http://localhost:5555/"] - interval: 10s + interval: 30s timeout: 10s retries: 5 + start_period: 30s restart: always depends_on: <<: *airflow-common-depends-on diff --git a/requirements.in b/requirements.in index 95216868..670c80fb 100644 --- a/requirements.in +++ b/requirements.in @@ -1,24 +1,21 @@ # Official Airflow constraints file # Doc: https://airflow.apache.org/docs/apache-airflow/stable/installation/installing-from-pypi.html#constraints-files -# File: https://raw.githubusercontent.com/apache/airflow/constraints-2.3.3/constraints-3.10.txt +# File: https://raw.githubusercontent.com/apache/airflow/constraints-2.5.3/constraints-3.10.txt --constraint ./constraints.txt # Airflow dependencies -apache-airflow[amazon,async,celery,cncf.kubernetes,github_enterprise,google_auth,jdbc,mysql,password,postgres,redis,statsd]==2.3.3 +apache-airflow[amazon,async,celery,cncf.kubernetes,github_enterprise,google_auth,jdbc,mysql,password,postgres,redis,statsd]==2.5.3 apache-airflow-providers-google apache-airflow-providers-http apache-airflow-providers-slack airflow-provider-fivetran==1.1.2 # Code quality -pytest==6.2.5 +pytest==7.2.2 pytest-mock==3.10.0 -black==22.6.0 +black==23.1a1 ruff==0.0.269 -# Misc -mozlogging - # Required for backfill UI flask-admin shelljob==0.5.6 diff --git a/requirements.txt b/requirements.txt index a252d8e9..447755d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,9 +4,23 @@ # # pip-compile # +aiofiles==22.1.0 + # via + # -c ./constraints.txt + # gcloud-aio-storage +aiohttp==3.8.4 + # via + # -c ./constraints.txt + # apache-airflow-providers-http + # gcloud-aio-auth + # kubernetes-asyncio +aiosignal==1.3.1 + # via + # -c ./constraints.txt + # aiohttp airflow-provider-fivetran==1.1.2 # via -r requirements.in -alembic==1.8.0 +alembic==1.10.2 # via # -c ./constraints.txt # apache-airflow @@ -14,11 +28,11 @@ amqp==5.1.1 # via # -c ./constraints.txt # kombu -anyio==3.6.1 +anyio==3.6.2 # via # -c ./constraints.txt # httpcore -apache-airflow[amazon,async,celery,cncf.kubernetes,github_enterprise,google_auth,jdbc,mysql,password,postgres,redis,statsd]==2.3.3 +apache-airflow[amazon,async,celery,cncf.kubernetes,github_enterprise,google_auth,jdbc,mysql,password,postgres,redis,statsd]==2.5.3 # via # -r requirements.in # airflow-provider-fivetran @@ -31,56 +45,67 @@ apache-airflow[amazon,async,celery,cncf.kubernetes,github_enterprise,google_auth # apache-airflow-providers-postgres # apache-airflow-providers-redis # apache-airflow-providers-slack -apache-airflow-providers-amazon==4.0.0 +apache-airflow-providers-amazon==7.3.0 # via # -c ./constraints.txt # apache-airflow -apache-airflow-providers-celery==3.0.0 +apache-airflow-providers-celery==3.1.0 # via # -c ./constraints.txt # apache-airflow -apache-airflow-providers-cncf-kubernetes==4.1.0 +apache-airflow-providers-cncf-kubernetes==5.2.2 # via # -c ./constraints.txt # apache-airflow -apache-airflow-providers-ftp==3.0.0 +apache-airflow-providers-common-sql==1.3.4 # via # -c ./constraints.txt # apache-airflow -apache-airflow-providers-google==8.1.0 + # apache-airflow-providers-amazon + # apache-airflow-providers-google + # apache-airflow-providers-jdbc + # apache-airflow-providers-mysql + # apache-airflow-providers-postgres + # apache-airflow-providers-slack + # apache-airflow-providers-sqlite +apache-airflow-providers-ftp==3.3.1 + # via + # -c ./constraints.txt + # apache-airflow +apache-airflow-providers-google==8.11.0 # via # -c ./constraints.txt # -r requirements.in -apache-airflow-providers-http==3.0.0 +apache-airflow-providers-http==4.2.0 # via # -c ./constraints.txt # -r requirements.in # apache-airflow -apache-airflow-providers-imap==3.0.0 +apache-airflow-providers-imap==3.1.1 # via # -c ./constraints.txt # apache-airflow -apache-airflow-providers-jdbc==3.0.0 +apache-airflow-providers-jdbc==3.3.0 # via # -c ./constraints.txt # apache-airflow -apache-airflow-providers-mysql==3.0.0 +apache-airflow-providers-mysql==4.0.2 # via # -c ./constraints.txt # apache-airflow -apache-airflow-providers-postgres==5.0.0 +apache-airflow-providers-postgres==5.4.0 # via # -c ./constraints.txt # apache-airflow -apache-airflow-providers-redis==3.0.0 +apache-airflow-providers-redis==3.1.0 # via # -c ./constraints.txt # apache-airflow -apache-airflow-providers-slack==5.0.0 +apache-airflow-providers-slack==7.2.0 # via # -c ./constraints.txt # -r requirements.in -apache-airflow-providers-sqlite==3.0.0 +apache-airflow-providers-sqlite==3.3.1 # via # -c ./constraints.txt # apache-airflow @@ -88,32 +113,51 @@ apispec[yaml]==3.3.2 # via # -c ./constraints.txt # flask-appbuilder -argcomplete==2.0.0 +argcomplete==3.0.5 # via # -c ./constraints.txt # apache-airflow +asgiref==3.6.0 + # via + # -c ./constraints.txt + # apache-airflow-providers-cncf-kubernetes + # apache-airflow-providers-google + # apache-airflow-providers-http asn1crypto==1.5.1 # via # -c ./constraints.txt # scramp -attrs==20.3.0 +async-timeout==4.0.2 # via # -c ./constraints.txt + # aiohttp +attrs==22.2.0 + # via + # -c ./constraints.txt + # aiohttp # apache-airflow # cattrs # jsonschema # looker-sdk # pytest -babel==2.10.3 +authlib==1.2.0 + # via + # -c ./constraints.txt + # apache-airflow +babel==2.12.1 # via # -c ./constraints.txt # flask-babel -bcrypt==3.2.2 +backoff==1.10.0 + # via + # -c ./constraints.txt + # gcloud-aio-auth +bcrypt==4.0.1 # via # -c ./constraints.txt # apache-airflow # flask-bcrypt -beautifulsoup4==4.11.1 +beautifulsoup4==4.12.0 # via # -c ./constraints.txt # redshift-connector @@ -121,21 +165,21 @@ billiard==3.6.4.0 # via # -c ./constraints.txt # celery -black==22.6.0 +black==23.1a1 # via # -c ./constraints.txt # -r requirements.in -blinker==1.4 +blinker==1.5 # via # -c ./constraints.txt # apache-airflow -boto3==1.24.23 +boto3==1.26.102 # via # -c ./constraints.txt # apache-airflow-providers-amazon # redshift-connector # watchtower -botocore==1.27.23 +botocore==1.29.102 # via # -c ./constraints.txt # boto3 @@ -146,11 +190,11 @@ cachelib==0.9.0 # -c ./constraints.txt # flask-caching # flask-session -cachetools==4.2.2 +cachetools==5.3.0 # via # -c ./constraints.txt # google-auth -cattrs==1.10.0 +cattrs==22.2.0 # via # -c ./constraints.txt # apache-airflow @@ -161,21 +205,26 @@ celery==5.2.7 # apache-airflow # apache-airflow-providers-celery # flower -certifi==2020.12.5 +certifi==2022.12.7 # via # -c ./constraints.txt # httpcore # httpx # kubernetes + # kubernetes-asyncio # requests cffi==1.15.1 # via # -c ./constraints.txt - # bcrypt # cryptography -charset-normalizer==2.0.12 +chardet==4.0.0 # via # -c ./constraints.txt + # gcloud-aio-auth +charset-normalizer==2.1.1 + # via + # -c ./constraints.txt + # aiohttp # requests click==8.1.3 # via @@ -204,7 +253,7 @@ clickclick==20.10.2 # via # -c ./constraints.txt # connexion -colorama==0.4.5 +colorama==0.4.6 # via # -c ./constraints.txt # flask-appbuilder @@ -212,29 +261,31 @@ colorlog==4.8.0 # via # -c ./constraints.txt # apache-airflow -commonmark==0.9.1 - # via - # -c ./constraints.txt - # rich -connexion[flask,swagger-ui]==2.14.0 +configupdater==3.1.1 # via # -c ./constraints.txt # apache-airflow -cron-descriptor==1.2.30 +connexion[flask]==2.14.2 # via # -c ./constraints.txt # apache-airflow -croniter==1.3.5 +cron-descriptor==1.2.35 # via # -c ./constraints.txt # apache-airflow -cryptography==36.0.2 +croniter==1.3.8 + # via + # -c ./constraints.txt + # apache-airflow +cryptography==39.0.2 # via # -c ./constraints.txt # apache-airflow # apache-airflow-providers-cncf-kubernetes + # authlib + # gcloud-aio-auth # pyopenssl -db-dtypes==1.0.2 +db-dtypes==1.1.0 # via # -c ./constraints.txt # pandas-gbq @@ -250,7 +301,7 @@ dill==0.3.1.1 # via # -c ./constraints.txt # apache-airflow -dnspython==2.2.1 +dnspython==2.3.0 # via # -c ./constraints.txt # email-validator @@ -259,15 +310,20 @@ docutils==0.19 # via # -c ./constraints.txt # python-daemon -email-validator==1.2.1 +email-validator==1.3.1 # via # -c ./constraints.txt # flask-appbuilder -eventlet==0.33.1 +eventlet==0.33.3 # via # -c ./constraints.txt # apache-airflow -flask==2.1.2 +exceptiongroup==1.1.1 + # via + # -c ./constraints.txt + # cattrs + # pytest +flask==2.2.3 # via # -c ./constraints.txt # apache-airflow @@ -284,7 +340,7 @@ flask==2.1.2 # flask-wtf flask-admin==1.6.0 # via -r requirements.in -flask-appbuilder==4.1.2 +flask-appbuilder==4.1.4 # via # -c ./constraints.txt # apache-airflow @@ -296,15 +352,15 @@ flask-bcrypt==1.0.1 # via # -c ./constraints.txt # apache-airflow -flask-caching==2.0.0 +flask-caching==2.0.2 # via # -c ./constraints.txt # apache-airflow -flask-jwt-extended==4.4.2 +flask-jwt-extended==4.4.4 # via # -c ./constraints.txt # flask-appbuilder -flask-login==0.6.1 +flask-login==0.6.2 # via # -c ./constraints.txt # apache-airflow @@ -317,25 +373,44 @@ flask-sqlalchemy==2.5.1 # via # -c ./constraints.txt # flask-appbuilder -flask-wtf==0.15.1 +flask-wtf==1.1.1 # via # -c ./constraints.txt # apache-airflow # flask-appbuilder -flower==1.1.0 +flower==1.2.0 # via # -c ./constraints.txt # apache-airflow # apache-airflow-providers-celery -future==0.18.2 +frozenlist==1.3.3 + # via + # -c ./constraints.txt + # aiohttp + # aiosignal +future==0.18.3 # via # -c ./constraints.txt # sqlalchemy-bigquery -gevent==21.12.0 +gcloud-aio-auth==4.2.0 + # via + # -c ./constraints.txt + # apache-airflow-providers-google + # gcloud-aio-bigquery + # gcloud-aio-storage +gcloud-aio-bigquery==6.3.0 + # via + # -c ./constraints.txt + # apache-airflow-providers-google +gcloud-aio-storage==8.1.0 + # via + # -c ./constraints.txt + # apache-airflow-providers-google +gevent==22.10.2 # via # -c ./constraints.txt # apache-airflow -google-ads==17.0.0 +google-ads==18.0.0 # via # -c ./constraints.txt # apache-airflow-providers-google @@ -353,9 +428,12 @@ google-api-core[grpc,grpcgcp]==2.8.2 # google-cloud-bigquery-storage # google-cloud-bigtable # google-cloud-build + # google-cloud-compute # google-cloud-container # google-cloud-core # google-cloud-datacatalog + # google-cloud-dataflow-client + # google-cloud-dataform # google-cloud-dataplex # google-cloud-dataproc # google-cloud-dataproc-metastore @@ -386,7 +464,7 @@ google-api-python-client==1.12.11 # via # -c ./constraints.txt # apache-airflow-providers-google -google-auth==2.9.0 +google-auth==2.17.0 # via # -c ./constraints.txt # apache-airflow-providers-google @@ -405,25 +483,25 @@ google-auth-httplib2==0.1.0 # -c ./constraints.txt # apache-airflow-providers-google # google-api-python-client -google-auth-oauthlib==0.5.2 +google-auth-oauthlib==0.8.0 # via # -c ./constraints.txt # google-ads # pandas-gbq # pydata-google-auth -google-cloud-aiplatform==1.15.0 +google-cloud-aiplatform==1.16.1 # via # -c ./constraints.txt # apache-airflow-providers-google -google-cloud-appengine-logging==1.1.2 +google-cloud-appengine-logging==1.1.3 # via # -c ./constraints.txt # google-cloud-logging -google-cloud-audit-log==0.2.2 +google-cloud-audit-log==0.2.4 # via # -c ./constraints.txt # google-cloud-logging -google-cloud-automl==2.7.3 +google-cloud-automl==2.8.0 # via # -c ./constraints.txt # apache-airflow-providers-google @@ -433,28 +511,32 @@ google-cloud-bigquery==2.34.4 # google-cloud-aiplatform # pandas-gbq # sqlalchemy-bigquery -google-cloud-bigquery-datatransfer==3.6.2 +google-cloud-bigquery-datatransfer==3.7.0 # via # -c ./constraints.txt # apache-airflow-providers-google -google-cloud-bigquery-storage==2.13.2 +google-cloud-bigquery-storage==2.14.1 # via # -c ./constraints.txt # pandas-gbq # sqlalchemy-bigquery -google-cloud-bigtable==1.7.2 +google-cloud-bigtable==2.11.1 # via # -c ./constraints.txt # apache-airflow-providers-google -google-cloud-build==3.8.3 +google-cloud-build==3.9.0 # via # -c ./constraints.txt # apache-airflow-providers-google -google-cloud-container==2.10.8 +google-cloud-compute==0.7.0 # via # -c ./constraints.txt # apache-airflow-providers-google -google-cloud-core==2.3.1 +google-cloud-container==2.11.1 + # via + # -c ./constraints.txt + # apache-airflow-providers-google +google-cloud-core==2.3.2 # via # -c ./constraints.txt # google-cloud-bigquery @@ -463,27 +545,35 @@ google-cloud-core==2.3.1 # google-cloud-spanner # google-cloud-storage # google-cloud-translate -google-cloud-datacatalog==3.8.1 +google-cloud-datacatalog==3.9.0 # via # -c ./constraints.txt # apache-airflow-providers-google -google-cloud-dataplex==1.0.1 +google-cloud-dataflow-client==0.5.4 # via # -c ./constraints.txt # apache-airflow-providers-google -google-cloud-dataproc==4.0.3 +google-cloud-dataform==0.2.0 # via # -c ./constraints.txt # apache-airflow-providers-google -google-cloud-dataproc-metastore==1.5.1 +google-cloud-dataplex==1.1.0 # via # -c ./constraints.txt # apache-airflow-providers-google -google-cloud-dlp==1.0.2 +google-cloud-dataproc==5.0.0 # via # -c ./constraints.txt # apache-airflow-providers-google -google-cloud-kms==2.11.2 +google-cloud-dataproc-metastore==1.6.0 + # via + # -c ./constraints.txt + # apache-airflow-providers-google +google-cloud-dlp==3.7.1 + # via + # -c ./constraints.txt + # apache-airflow-providers-google +google-cloud-kms==2.12.0 # via # -c ./constraints.txt # apache-airflow-providers-google @@ -491,35 +581,35 @@ google-cloud-language==1.3.2 # via # -c ./constraints.txt # apache-airflow-providers-google -google-cloud-logging==3.1.2 +google-cloud-logging==3.2.1 # via # -c ./constraints.txt # apache-airflow-providers-google -google-cloud-memcache==1.3.2 +google-cloud-memcache==1.4.1 # via # -c ./constraints.txt # apache-airflow-providers-google -google-cloud-monitoring==2.9.2 +google-cloud-monitoring==2.11.0 # via # -c ./constraints.txt # apache-airflow-providers-google -google-cloud-orchestration-airflow==1.3.2 +google-cloud-orchestration-airflow==1.4.1 # via # -c ./constraints.txt # apache-airflow-providers-google -google-cloud-os-login==2.6.2 +google-cloud-os-login==2.7.1 # via # -c ./constraints.txt # apache-airflow-providers-google -google-cloud-pubsub==2.13.0 +google-cloud-pubsub==2.13.5 # via # -c ./constraints.txt # apache-airflow-providers-google -google-cloud-redis==2.8.1 +google-cloud-redis==2.9.0 # via # -c ./constraints.txt # apache-airflow-providers-google -google-cloud-resource-manager==1.5.1 +google-cloud-resource-manager==1.6.0 # via # -c ./constraints.txt # google-cloud-aiplatform @@ -535,12 +625,12 @@ google-cloud-speech==1.3.4 # via # -c ./constraints.txt # apache-airflow-providers-google -google-cloud-storage==1.44.0 +google-cloud-storage==2.7.0 # via # -c ./constraints.txt # apache-airflow-providers-google # google-cloud-aiplatform -google-cloud-tasks==2.9.1 +google-cloud-tasks==2.10.1 # via # -c ./constraints.txt # apache-airflow-providers-google @@ -560,20 +650,20 @@ google-cloud-vision==1.0.2 # via # -c ./constraints.txt # apache-airflow-providers-google -google-cloud-workflows==1.6.3 +google-cloud-workflows==1.7.1 # via # -c ./constraints.txt # apache-airflow-providers-google -google-crc32c==1.3.0 +google-crc32c==1.5.0 # via # -c ./constraints.txt # google-resumable-media -google-resumable-media==2.3.3 +google-resumable-media==2.4.1 # via # -c ./constraints.txt # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.56.3 +googleapis-common-protos[grpc]==1.56.4 # via # -c ./constraints.txt # google-ads @@ -581,11 +671,11 @@ googleapis-common-protos[grpc]==1.56.3 # google-cloud-audit-log # grpc-google-iam-v1 # grpcio-status -graphviz==0.20 +graphviz==0.20.1 # via # -c ./constraints.txt # apache-airflow -greenlet==1.1.2 +greenlet==2.0.2 # via # -c ./constraints.txt # apache-airflow @@ -598,6 +688,8 @@ grpc-google-iam-v1==0.12.4 # google-cloud-bigtable # google-cloud-container # google-cloud-datacatalog + # google-cloud-dataform + # google-cloud-dataplex # google-cloud-kms # google-cloud-logging # google-cloud-pubsub @@ -605,7 +697,7 @@ grpc-google-iam-v1==0.12.4 # google-cloud-secret-manager # google-cloud-spanner # google-cloud-tasks -grpcio==1.47.0 +grpcio==1.53.0 # via # -c ./constraints.txt # google-ads @@ -620,7 +712,7 @@ grpcio-gcp==0.2.2 # via # -c ./constraints.txt # apache-airflow-providers-google -grpcio-status==1.47.0 +grpcio-status==1.48.2 # via # -c ./constraints.txt # google-ads @@ -630,40 +722,41 @@ gunicorn==20.1.0 # via # -c ./constraints.txt # apache-airflow -h11==0.12.0 +h11==0.14.0 # via # -c ./constraints.txt # httpcore -httpcore==0.15.0 +httpcore==0.16.3 # via # -c ./constraints.txt # httpx -httplib2==0.20.4 +httplib2==0.21.0 # via # -c ./constraints.txt # google-api-python-client # google-auth-httplib2 -httpx==0.23.0 +httpx==0.23.3 # via # -c ./constraints.txt # apache-airflow # apache-airflow-providers-google -humanize==4.2.3 +humanize==4.6.0 # via # -c ./constraints.txt # flower -idna==3.3 +idna==3.4 # via # -c ./constraints.txt # anyio # email-validator # requests # rfc3986 + # yarl inflection==0.5.1 # via # -c ./constraints.txt # connexion -iniconfig==1.1.1 +iniconfig==2.0.0 # via # -c ./constraints.txt # pytest @@ -685,13 +778,12 @@ jinja2==3.1.2 # flask # flask-babel # python-nvd3 - # swagger-ui-bundle jmespath==0.10.0 # via # -c ./constraints.txt # boto3 # botocore -jpype1==1.4.0 +jpype1==1.4.1 # via # -c ./constraints.txt # jaydebeapi @@ -703,7 +795,7 @@ jsonpath-ng==1.5.3 # via # -c ./constraints.txt # apache-airflow-providers-amazon -jsonschema==4.6.1 +jsonschema==4.17.3 # via # -c ./constraints.txt # apache-airflow @@ -718,7 +810,11 @@ kubernetes==23.6.0 # -c ./constraints.txt # apache-airflow # apache-airflow-providers-cncf-kubernetes -lazy-object-proxy==1.7.1 +kubernetes-asyncio==24.2.2 + # via + # -c ./constraints.txt + # apache-airflow-providers-cncf-kubernetes +lazy-object-proxy==1.9.0 # via # -c ./constraints.txt # apache-airflow @@ -731,35 +827,37 @@ lockfile==0.12.2 # -c ./constraints.txt # apache-airflow # python-daemon -looker-sdk==22.4.0 +looker-sdk==23.2.0 # via # -c ./constraints.txt # apache-airflow-providers-google -lxml==4.9.1 +lxml==4.9.2 # via # -c ./constraints.txt # redshift-connector -mako==1.2.1 +mako==1.2.4 # via # -c ./constraints.txt # alembic -markdown==3.3.7 +markdown==3.4.3 # via # -c ./constraints.txt # apache-airflow -markdown-it-py==2.1.0 +markdown-it-py==2.2.0 # via # -c ./constraints.txt # apache-airflow # mdit-py-plugins -markupsafe==2.1.1 + # rich +markupsafe==2.1.2 # via # -c ./constraints.txt # apache-airflow # jinja2 # mako + # werkzeug # wtforms -marshmallow==3.17.0 +marshmallow==3.19.0 # via # -c ./constraints.txt # flask-appbuilder @@ -778,29 +876,36 @@ marshmallow-sqlalchemy==0.26.1 # via # -c ./constraints.txt # flask-appbuilder -mdit-py-plugins==0.3.0 +mdit-py-plugins==0.3.5 # via # -c ./constraints.txt # apache-airflow -mdurl==0.1.1 +mdurl==0.1.2 # via # -c ./constraints.txt # markdown-it-py -mozlogging==0.1.0 - # via -r requirements.in -mypy-boto3-rds==1.24.23 +multidict==6.0.4 + # via + # -c ./constraints.txt + # aiohttp + # yarl +mypy-boto3-appflow==1.26.78 # via # -c ./constraints.txt # apache-airflow-providers-amazon -mypy-boto3-redshift-data==1.24.11.post3 +mypy-boto3-rds==1.26.102 # via # -c ./constraints.txt # apache-airflow-providers-amazon -mypy-extensions==0.4.3 +mypy-boto3-redshift-data==1.26.88 + # via + # -c ./constraints.txt + # apache-airflow-providers-amazon +mypy-extensions==1.0.0 # via # -c ./constraints.txt # black -mysql-connector-python==8.0.29 +mysql-connector-python==8.0.32 # via # -c ./constraints.txt # apache-airflow-providers-mysql @@ -808,14 +913,14 @@ mysqlclient==2.1.1 # via # -c ./constraints.txt # apache-airflow-providers-mysql -numpy==1.22.4 +numpy==1.24.2 # via # -c ./constraints.txt # db-dtypes # pandas # pandas-gbq # pyarrow -oauthlib==3.2.0 +oauthlib==3.2.2 # via # -c ./constraints.txt # requests-oauthlib @@ -827,18 +932,20 @@ packaging==21.3 # db-dtypes # google-cloud-aiplatform # google-cloud-bigquery + # google-cloud-compute + # jpype1 # marshmallow # pytest # redshift-connector + # sqlalchemy-bigquery # sqlalchemy-redshift -pandas==1.4.3 +pandas==1.5.3 # via # -c ./constraints.txt - # apache-airflow-providers-amazon # apache-airflow-providers-google # db-dtypes # pandas-gbq -pandas-gbq==0.17.6 +pandas-gbq==0.17.9 # via # -c ./constraints.txt # apache-airflow-providers-google @@ -851,7 +958,7 @@ pendulum==2.1.2 # via # -c ./constraints.txt # apache-airflow -platformdirs==2.5.2 +platformdirs==3.2.0 # via # -c ./constraints.txt # black @@ -868,11 +975,11 @@ prison==0.2.1 # via # -c ./constraints.txt # flask-appbuilder -prometheus-client==0.14.1 +prometheus-client==0.16.0 # via # -c ./constraints.txt # flower -prompt-toolkit==3.0.30 +prompt-toolkit==3.0.38 # via # -c ./constraints.txt # click-repl @@ -887,12 +994,17 @@ proto-plus==1.19.6 # google-cloud-bigquery # google-cloud-bigquery-datatransfer # google-cloud-bigquery-storage + # google-cloud-bigtable # google-cloud-build + # google-cloud-compute # google-cloud-container # google-cloud-datacatalog + # google-cloud-dataflow-client + # google-cloud-dataform # google-cloud-dataplex # google-cloud-dataproc # google-cloud-dataproc-metastore + # google-cloud-dlp # google-cloud-kms # google-cloud-logging # google-cloud-memcache @@ -920,6 +1032,8 @@ protobuf==3.20.0 # google-cloud-build # google-cloud-container # google-cloud-datacatalog + # google-cloud-dataflow-client + # google-cloud-dataform # google-cloud-dataplex # google-cloud-dataproc # google-cloud-dataproc-metastore @@ -937,7 +1051,6 @@ protobuf==3.20.0 # google-cloud-secret-manager # google-cloud-spanner # google-cloud-speech - # google-cloud-storage # google-cloud-tasks # google-cloud-texttospeech # google-cloud-translate @@ -948,19 +1061,15 @@ protobuf==3.20.0 # grpcio-status # mysql-connector-python # proto-plus -psutil==5.9.1 +psutil==5.9.4 # via # -c ./constraints.txt # apache-airflow -psycopg2-binary==2.9.3 +psycopg2-binary==2.9.5 # via # -c ./constraints.txt # apache-airflow-providers-postgres -py==1.11.0 - # via - # -c ./constraints.txt - # pytest -pyarrow==6.0.1 +pyarrow==9.0.0 # via # -c ./constraints.txt # db-dtypes @@ -974,27 +1083,29 @@ pyasn1==0.4.8 pyasn1-modules==0.2.8 # via # -c ./constraints.txt + # gcloud-aio-storage # google-auth pycparser==2.21 # via # -c ./constraints.txt # cffi -pydata-google-auth==1.4.0 +pydata-google-auth==1.7.0 # via # -c ./constraints.txt # pandas-gbq -pygments==2.12.0 +pygments==2.14.0 # via # -c ./constraints.txt # apache-airflow # rich -pyjwt==2.4.0 +pyjwt==2.6.0 # via # -c ./constraints.txt # apache-airflow # flask-appbuilder # flask-jwt-extended -pyopenssl==22.0.0 + # gcloud-aio-auth +pyopenssl==23.1.1 # via # -c ./constraints.txt # apache-airflow-providers-google @@ -1003,18 +1114,18 @@ pyparsing==3.0.9 # -c ./constraints.txt # httplib2 # packaging -pyrsistent==0.18.1 +pyrsistent==0.19.3 # via # -c ./constraints.txt # jsonschema -pytest==6.2.5 +pytest==7.2.2 # via # -c ./constraints.txt # -r requirements.in # pytest-mock pytest-mock==3.10.0 # via -r requirements.in -python-daemon==2.3.0 +python-daemon==3.0.1 # via # -c ./constraints.txt # apache-airflow @@ -1027,21 +1138,21 @@ python-dateutil==2.8.2 # flask-appbuilder # google-cloud-bigquery # kubernetes + # kubernetes-asyncio # pandas # pendulum python-nvd3==0.15.0 # via # -c ./constraints.txt # apache-airflow -python-slugify==6.1.2 +python-slugify==8.0.1 # via # -c ./constraints.txt # apache-airflow # python-nvd3 -pytz==2022.1 +pytz==2023.3 # via # -c ./constraints.txt - # babel # celery # flask-babel # flower @@ -1059,15 +1170,16 @@ pyyaml==6.0 # connexion # google-ads # kubernetes + # kubernetes-asyncio redis==3.5.3 # via # -c ./constraints.txt # apache-airflow-providers-redis -redshift-connector==2.0.908 +redshift-connector==2.0.910 # via # -c ./constraints.txt # apache-airflow-providers-amazon -requests==2.28.0 +requests==2.28.2 # via # -c ./constraints.txt # airflow-provider-fivetran @@ -1080,22 +1192,32 @@ requests==2.28.0 # looker-sdk # redshift-connector # requests-oauthlib + # requests-toolbelt requests-oauthlib==1.3.1 # via # -c ./constraints.txt # google-auth-oauthlib # kubernetes +requests-toolbelt==0.10.1 + # via + # -c ./constraints.txt + # apache-airflow-providers-http +rfc3339-validator==0.1.4 + # via + # -c ./constraints.txt + # apache-airflow rfc3986[idna2008]==1.5.0 # via # -c ./constraints.txt # httpx -rich==12.4.4 +rich==13.3.3 # via # -c ./constraints.txt # apache-airflow -rsa==4.8 +rsa==4.9 # via # -c ./constraints.txt + # gcloud-aio-storage # google-auth ruff==0.0.269 # via -r requirements.in @@ -1103,11 +1225,11 @@ s3transfer==0.6.0 # via # -c ./constraints.txt # boto3 -scramp==1.4.1 +scramp==1.4.4 # via # -c ./constraints.txt # redshift-connector -setproctitle==1.2.3 +setproctitle==1.3.2 # via # -c ./constraints.txt # apache-airflow @@ -1121,28 +1243,27 @@ six==1.16.0 # google-api-python-client # google-auth # google-auth-httplib2 - # google-cloud-storage - # grpcio # jsonpath-ng # kubernetes + # kubernetes-asyncio # prison # python-dateutil - # sqlalchemy-utils -slack-sdk==3.17.2 + # rfc3339-validator +slack-sdk==3.20.2 # via # -c ./constraints.txt # apache-airflow-providers-slack -sniffio==1.2.0 +sniffio==1.3.0 # via # -c ./constraints.txt # anyio # httpcore # httpx -soupsieve==2.3.2.post1 +soupsieve==2.4 # via # -c ./constraints.txt # beautifulsoup4 -sqlalchemy==1.4.27 +sqlalchemy==1.4.47 # via # -c ./constraints.txt # alembic @@ -1154,39 +1275,39 @@ sqlalchemy==1.4.27 # sqlalchemy-jsonfield # sqlalchemy-redshift # sqlalchemy-utils -sqlalchemy-bigquery==1.4.4 +sqlalchemy-bigquery==1.6.1 # via # -c ./constraints.txt # apache-airflow-providers-google -sqlalchemy-jsonfield==1.0.0 +sqlalchemy-jsonfield==1.0.1.post0 # via # -c ./constraints.txt # apache-airflow -sqlalchemy-redshift==0.8.9 +sqlalchemy-redshift==0.8.13 # via # -c ./constraints.txt # apache-airflow-providers-amazon -sqlalchemy-utils==0.38.2 +sqlalchemy-utils==0.40.0 # via # -c ./constraints.txt # flask-appbuilder -statsd==3.3.0 +sqlparse==0.4.3 + # via + # -c ./constraints.txt + # apache-airflow-providers-common-sql +statsd==4.0.1 # via # -c ./constraints.txt # apache-airflow -swagger-ui-bundle==0.0.9 - # via - # -c ./constraints.txt - # connexion -tabulate==0.8.10 +tabulate==0.9.0 # via # -c ./constraints.txt # apache-airflow -tenacity==8.0.1 +tenacity==8.2.2 # via # -c ./constraints.txt # apache-airflow -termcolor==1.1.0 +termcolor==2.2.0 # via # -c ./constraints.txt # apache-airflow @@ -1194,24 +1315,22 @@ text-unidecode==1.3 # via # -c ./constraints.txt # python-slugify -toml==0.10.2 - # via - # -c ./constraints.txt - # pytest tomli==2.0.1 # via # -c ./constraints.txt # black + # pytest tornado==6.2 # via # -c ./constraints.txt # flower -typing-extensions==4.3.0 +typing-extensions==4.5.0 # via # -c ./constraints.txt + # alembic # apache-airflow # looker-sdk - # mypy-boto3-rds + # mypy-boto3-appflow # mypy-boto3-redshift-data uc-micro-py==1.0.1 # via @@ -1225,11 +1344,12 @@ uritemplate==3.0.1 # via # -c ./constraints.txt # google-api-python-client -urllib3==1.26.9 +urllib3==1.26.15 # via # -c ./constraints.txt # botocore # kubernetes + # kubernetes-asyncio # requests vine==5.0.0 # via @@ -1241,15 +1361,15 @@ watchtower==2.0.1 # via # -c ./constraints.txt # apache-airflow-providers-amazon -wcwidth==0.2.5 +wcwidth==0.2.6 # via # -c ./constraints.txt # prompt-toolkit -websocket-client==1.3.3 +websocket-client==1.5.1 # via # -c ./constraints.txt # kubernetes -werkzeug==2.1.2 +werkzeug==2.2.3 # via # -c ./constraints.txt # apache-airflow @@ -1257,11 +1377,11 @@ werkzeug==2.1.2 # flask # flask-jwt-extended # flask-login -wrapt==1.14.1 +wrapt==1.15.0 # via # -c ./constraints.txt # deprecated -wtforms==2.3.3 +wtforms==3.0.1 # via # -c ./constraints.txt # flask-admin @@ -1271,11 +1391,15 @@ xmltodict==0.13.0 # via # -c ./constraints.txt # -r requirements.in -zope-event==4.5.0 +yarl==1.8.2 + # via + # -c ./constraints.txt + # aiohttp +zope-event==4.6 # via # -c ./constraints.txt # gevent -zope-interface==5.4.0 +zope-interface==6.0 # via # -c ./constraints.txt # gevent