From 06584fc4b1d82a2dbba98e484d0b4515a169a818 Mon Sep 17 00:00:00 2001 From: Andy Cooper Date: Tue, 24 Jul 2018 15:57:36 -0400 Subject: [PATCH] [AIRFLOW-2800] Remove low-hanging linting errors --- airflow/__init__.py | 13 ++--- airflow/contrib/auth/backends/ldap_auth.py | 4 +- airflow/contrib/hooks/aws_hook.py | 2 +- .../contrib/operators/awsbatch_operator.py | 2 +- .../operators/mlengine_prediction_summary.py | 14 +++--- airflow/jobs.py | 50 ++++++++++++------- airflow/migrations/__init__.py | 5 +- airflow/migrations/env.py | 4 +- ...add_dag_id_state_index_on_dag_run_table.py | 5 +- .../13eb55f81627_for_compatibility.py | 4 +- .../1507a7289a2f_create_is_encrypted.py | 4 +- ...e3_add_is_encrypted_column_to_variable_.py | 4 +- .../versions/1b38cef5b76e_add_dagrun.py | 4 +- .../211e584da130_add_ti_state_index.py | 4 +- .../versions/2e541a1dcfed_task_duration.py | 4 +- .../2e82aab8ef20_rename_user_table.py | 5 +- ...0f54d61_more_logging_into_task_isntance.py | 4 +- .../versions/40e67319e3a9_dagrun_config.py | 4 +- .../versions/4446e08588_dagrun_start_end.py | 4 +- ..._add_fractional_seconds_to_mysql_tables.py | 4 +- .../502898887f84_adding_extra_to_log.py | 4 +- .../versions/52d714495f0_job_id_indices.py | 4 +- ...61833c1c74b_add_password_column_to_user.py | 4 +- ...d17757c7a_add_pid_field_to_taskinstance.py | 4 +- ...de9cddf6c9_add_task_fails_journal_table.py | 4 +- .../8504051e801b_xcom_dag_task_indices.py | 4 +- .../947454bf1dff_add_ti_job_id_index.py | 4 +- airflow/migrations/versions/__init__.py | 5 +- ...6_add_a_column_to_track_the_encryption_.py | 4 +- ...dd_notification_sent_column_to_sla_miss.py | 4 +- ...6_make_xcom_value_column_a_large_binary.py | 6 +-- ...7_add_max_tries_column_to_task_instance.py | 4 +- ...ae31099d61_increase_text_size_for_mysql.py | 4 +- .../versions/e3a246e0dc1_current_schema.py | 4 +- .../f2ca10b85618_add_dag_stats_table.py | 4 +- airflow/plugins_manager.py | 6 ++- airflow/settings.py | 6 +-- airflow/utils/__init__.py | 4 +- airflow/utils/asciiart.py | 4 +- airflow/utils/cli_action_loggers.py | 4 +- airflow/utils/compression.py | 4 +- airflow/utils/dag_processing.py | 4 +- airflow/utils/dates.py | 20 ++++---- airflow/utils/db.py | 2 +- airflow/utils/decorators.py | 4 +- airflow/utils/file.py | 4 +- airflow/utils/helpers.py | 4 +- airflow/utils/json.py | 4 +- airflow/utils/log/__init__.py | 4 +- airflow/utils/log/es_task_handler.py | 4 +- airflow/utils/log/file_processor_handler.py | 4 +- airflow/utils/log/file_task_handler.py | 4 +- airflow/utils/log/logging_mixin.py | 4 +- airflow/utils/log/s3_task_handler.py | 6 +-- airflow/utils/net.py | 4 +- airflow/utils/operator_resources.py | 4 +- airflow/utils/sqlalchemy.py | 4 +- airflow/utils/state.py | 4 +- airflow/utils/tests.py | 4 +- airflow/utils/timeout.py | 4 +- airflow/utils/trigger_rule.py | 4 +- airflow/utils/weight_rule.py | 4 +- airflow/www/__init__.py | 5 +- airflow/www/api/__init__.py | 4 +- airflow/www/api/experimental/__init__.py | 4 +- airflow/www/blueprints.py | 4 +- airflow/www/gunicorn_config.py | 4 +- airflow/www/validators.py | 11 ++-- airflow/www/views.py | 44 ++++++++-------- airflow/www_rbac/__init__.py | 4 +- airflow/www_rbac/api/__init__.py | 4 +- airflow/www_rbac/api/experimental/__init__.py | 4 +- airflow/www_rbac/blueprints.py | 4 +- airflow/www_rbac/validators.py | 4 +- airflow/www_rbac/widgets.py | 4 +- 75 files changed, 226 insertions(+), 209 deletions(-) diff --git a/airflow/__init__.py b/airflow/__init__.py index f40b08aab5..bc6a7bbe19 100644 --- a/airflow/__init__.py +++ b/airflow/__init__.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -80,11 +80,12 @@ class AirflowMacroPlugin(object): def __init__(self, namespace): self.namespace = namespace -from airflow import operators + +from airflow import operators # noqa: E402 from airflow import sensors # noqa: E402 -from airflow import hooks -from airflow import executors -from airflow import macros +from airflow import hooks # noqa: E402 +from airflow import executors # noqa: E402 +from airflow import macros # noqa: E402 operators._integrate_plugins() sensors._integrate_plugins() # noqa: E402 diff --git a/airflow/contrib/auth/backends/ldap_auth.py b/airflow/contrib/auth/backends/ldap_auth.py index eefaa1263b..516e121c9b 100644 --- a/airflow/contrib/auth/backends/ldap_auth.py +++ b/airflow/contrib/auth/backends/ldap_auth.py @@ -62,7 +62,7 @@ def get_ldap_connection(dn=None, password=None): cacert = configuration.conf.get("ldap", "cacert") tls_configuration = Tls(validate=ssl.CERT_REQUIRED, ca_certs_file=cacert) use_ssl = True - except: + except Exception: pass server = Server(configuration.conf.get("ldap", "uri"), use_ssl, tls_configuration) @@ -94,7 +94,7 @@ def groups_user(conn, search_base, user_filter, user_name_att, username): search_filter = "(&({0})({1}={2}))".format(user_filter, user_name_att, username) try: memberof_attr = configuration.conf.get("ldap", "group_member_attr") - except: + except Exception: memberof_attr = "memberOf" res = conn.search(native(search_base), native(search_filter), attributes=[native(memberof_attr)]) diff --git a/airflow/contrib/hooks/aws_hook.py b/airflow/contrib/hooks/aws_hook.py index 69a1b0bed3..8ca1f3d744 100644 --- a/airflow/contrib/hooks/aws_hook.py +++ b/airflow/contrib/hooks/aws_hook.py @@ -72,7 +72,7 @@ def _parse_s3_config(config_file_name, config_format='boto', profile=None): try: access_key = config.get(cred_section, key_id_option) secret_key = config.get(cred_section, secret_key_option) - except: + except Exception: logging.warning("Option Error in parsing s3 config file") raise return access_key, secret_key diff --git a/airflow/contrib/operators/awsbatch_operator.py b/airflow/contrib/operators/awsbatch_operator.py index a5c86afce6..353fbbb0a0 100644 --- a/airflow/contrib/operators/awsbatch_operator.py +++ b/airflow/contrib/operators/awsbatch_operator.py @@ -139,7 +139,7 @@ class AWSBatchOperator(BaseOperator): if response['jobs'][-1]['status'] in ['SUCCEEDED', 'FAILED']: retry = False - sleep( 1 + pow(retries * 0.1, 2)) + sleep(1 + pow(retries * 0.1, 2)) retries += 1 def _check_success_task(self): diff --git a/airflow/contrib/operators/mlengine_prediction_summary.py b/airflow/contrib/operators/mlengine_prediction_summary.py index 17fc2c0903..4efe81e641 100644 --- a/airflow/contrib/operators/mlengine_prediction_summary.py +++ b/airflow/contrib/operators/mlengine_prediction_summary.py @@ -112,14 +112,14 @@ class JsonCoder(object): @beam.ptransform_fn def MakeSummary(pcoll, metric_fn, metric_keys): # pylint: disable=invalid-name return ( - pcoll - | "ApplyMetricFnPerInstance" >> beam.Map(metric_fn) - | "PairWith1" >> beam.Map(lambda tup: tup + (1,)) - | "SumTuple" >> beam.CombineGlobally(beam.combiners.TupleCombineFn( - *([sum] * (len(metric_keys) + 1)))) - | "AverageAndMakeDict" >> beam.Map( + pcoll | + "ApplyMetricFnPerInstance" >> beam.Map(metric_fn) | + "PairWith1" >> beam.Map(lambda tup: tup + (1,)) | + "SumTuple" >> beam.CombineGlobally(beam.combiners.TupleCombineFn( + *([sum] * (len(metric_keys) + 1)))) | + "AverageAndMakeDict" >> beam.Map( lambda tup: dict( - [(name, tup[i]/tup[-1]) for i, name in enumerate(metric_keys)] + + [(name, tup[i] / tup[-1]) for i, name in enumerate(metric_keys)] + [("count", tup[-1])]))) diff --git a/airflow/jobs.py b/airflow/jobs.py index 00ede5451d..224ff185fb 100644 --- a/airflow/jobs.py +++ b/airflow/jobs.py @@ -173,7 +173,8 @@ class BaseJob(Base, LoggingMixin): if job.latest_heartbeat: sleep_for = max( 0, - self.heartrate - (timezone.utcnow() - job.latest_heartbeat).total_seconds()) + self.heartrate - ( + timezone.utcnow() - job.latest_heartbeat).total_seconds()) sleep(sleep_for) @@ -391,7 +392,7 @@ class DagFileProcessor(AbstractDagFileProcessor, LoggingMixin): log.info( "Processing %s took %.3f seconds", file_path, end_time - start_time ) - except: + except Exception: # Log exceptions through the logging framework. log.exception("Got an exception! Propagating...") raise @@ -1089,17 +1090,18 @@ class SchedulerJob(BaseJob): session .query(TI) .filter(TI.dag_id.in_(simple_dag_bag.dag_ids)) - .outerjoin(DR, - and_(DR.dag_id == TI.dag_id, - DR.execution_date == TI.execution_date)) - .filter(or_(DR.run_id == None, + .outerjoin( + DR, + and_(DR.dag_id == TI.dag_id, DR.execution_date == TI.execution_date) + ) + .filter(or_(DR.run_id == None, # noqa E711 not_(DR.run_id.like(BackfillJob.ID_PREFIX + '%')))) - .outerjoin(DM, DM.dag_id==TI.dag_id) - .filter(or_(DM.dag_id == None, + .outerjoin(DM, DM.dag_id == TI.dag_id) + .filter(or_(DM.dag_id == None, # noqa E711 not_(DM.is_paused))) ) if None in states: - ti_query = ti_query.filter(or_(TI.state == None, TI.state.in_(states))) + ti_query = ti_query.filter(or_(TI.state == None, TI.state.in_(states))) # noqa E711 else: ti_query = ti_query.filter(TI.state.in_(states)) @@ -1121,7 +1123,8 @@ class SchedulerJob(BaseJob): for task_instance in task_instances_to_examine: pool_to_task_instances[task_instance.pool].append(task_instance) - task_concurrency_map = self.__get_task_concurrency_map(states=states_to_count_as_running, session=session) + task_concurrency_map = self.__get_task_concurrency_map( + states=states_to_count_as_running, session=session) # Go through each pool, and queue up a task for execution if there are # any open slots in the pool. @@ -1192,9 +1195,14 @@ class SchedulerJob(BaseJob): ) continue - task_concurrency = simple_dag.get_task_special_arg(task_instance.task_id, 'task_concurrency') + task_concurrency = simple_dag.get_task_special_arg( + task_instance.task_id, + 'task_concurrency') if task_concurrency is not None: - num_running = task_concurrency_map[((task_instance.dag_id, task_instance.task_id))] + num_running = task_concurrency_map[ + ((task_instance.dag_id, task_instance.task_id)) + ] + if num_running >= task_concurrency: self.log.info("Not executing %s since the task concurrency for" " this task has been reached.", task_instance) @@ -1214,7 +1222,8 @@ class SchedulerJob(BaseJob): task_instance_str = "\n\t".join( ["{}".format(x) for x in executable_tis]) - self.log.info("Setting the follow tasks to queued state:\n\t%s", task_instance_str) + self.log.info( + "Setting the follow tasks to queued state:\n\t%s", task_instance_str) # so these dont expire on commit for ti in executable_tis: copy_dag_id = ti.dag_id @@ -1256,7 +1265,9 @@ class SchedulerJob(BaseJob): .filter(or_(*filter_for_ti_state_change))) if None in acceptable_states: - ti_query = ti_query.filter(or_(TI.state == None, TI.state.in_(acceptable_states))) + ti_query = ti_query.filter( + or_(TI.state == None, TI.state.in_(acceptable_states)) # noqa E711 + ) else: ti_query = ti_query.filter(TI.state.in_(acceptable_states)) @@ -1601,7 +1612,8 @@ class SchedulerJob(BaseJob): child.terminate() # TODO: Remove magic number timeout = 5 - self.log.info("Waiting up to %s seconds for processes to exit...", timeout) + self.log.info( + "Waiting up to %s seconds for processes to exit...", timeout) try: psutil.wait_procs( child_processes, timeout=timeout, @@ -1658,7 +1670,9 @@ class SchedulerJob(BaseJob): self.log.info("Searching for files in %s", self.subdir) known_file_paths = list_py_file_paths(self.subdir) last_dag_dir_refresh_time = timezone.utcnow() - self.log.info("There are %s files in %s", len(known_file_paths), self.subdir) + self.log.info( + "There are %s files in %s", len(known_file_paths), self.subdir) + processor_manager.set_file_paths(known_file_paths) self.log.debug("Removing old import errors") @@ -1671,7 +1685,9 @@ class SchedulerJob(BaseJob): if self.using_sqlite: # For the sqlite case w/ 1 thread, wait until the processor # is finished to avoid concurrent access to the DB. - self.log.debug("Waiting for processors to finish since we're using sqlite") + self.log.debug( + "Waiting for processors to finish since we're using sqlite") + processor_manager.wait_until_finished() # Send tasks for execution if available diff --git a/airflow/migrations/__init__.py b/airflow/migrations/__init__.py index f0f8b68337..114d189da1 100644 --- a/airflow/migrations/__init__.py +++ b/airflow/migrations/__init__.py @@ -7,13 +7,12 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - diff --git a/airflow/migrations/env.py b/airflow/migrations/env.py index 828cacf0e0..97ebe4257f 100644 --- a/airflow/migrations/env.py +++ b/airflow/migrations/env.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/migrations/versions/127d2bf2dfa7_add_dag_id_state_index_on_dag_run_table.py b/airflow/migrations/versions/127d2bf2dfa7_add_dag_id_state_index_on_dag_run_table.py index 5851725694..6ee50aa94d 100644 --- a/airflow/migrations/versions/127d2bf2dfa7_add_dag_id_state_index_on_dag_run_table.py +++ b/airflow/migrations/versions/127d2bf2dfa7_add_dag_id_state_index_on_dag_run_table.py @@ -6,9 +6,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -39,4 +39,3 @@ def upgrade(): def downgrade(): op.drop_index('dag_id_state', table_name='dag_run') - diff --git a/airflow/migrations/versions/13eb55f81627_for_compatibility.py b/airflow/migrations/versions/13eb55f81627_for_compatibility.py index a14e195bbe..b9e88bd01b 100644 --- a/airflow/migrations/versions/13eb55f81627_for_compatibility.py +++ b/airflow/migrations/versions/13eb55f81627_for_compatibility.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/migrations/versions/1507a7289a2f_create_is_encrypted.py b/airflow/migrations/versions/1507a7289a2f_create_is_encrypted.py index 47bdf13d67..3db0d41190 100644 --- a/airflow/migrations/versions/1507a7289a2f_create_is_encrypted.py +++ b/airflow/migrations/versions/1507a7289a2f_create_is_encrypted.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/migrations/versions/1968acfc09e3_add_is_encrypted_column_to_variable_.py b/airflow/migrations/versions/1968acfc09e3_add_is_encrypted_column_to_variable_.py index 57ee27aeb4..aaf938a035 100644 --- a/airflow/migrations/versions/1968acfc09e3_add_is_encrypted_column_to_variable_.py +++ b/airflow/migrations/versions/1968acfc09e3_add_is_encrypted_column_to_variable_.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/migrations/versions/1b38cef5b76e_add_dagrun.py b/airflow/migrations/versions/1b38cef5b76e_add_dagrun.py index a0af7fa7ad..79fcff4541 100644 --- a/airflow/migrations/versions/1b38cef5b76e_add_dagrun.py +++ b/airflow/migrations/versions/1b38cef5b76e_add_dagrun.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/migrations/versions/211e584da130_add_ti_state_index.py b/airflow/migrations/versions/211e584da130_add_ti_state_index.py index 93347a2649..afc600d58a 100644 --- a/airflow/migrations/versions/211e584da130_add_ti_state_index.py +++ b/airflow/migrations/versions/211e584da130_add_ti_state_index.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/migrations/versions/2e541a1dcfed_task_duration.py b/airflow/migrations/versions/2e541a1dcfed_task_duration.py index 7b540430a5..6b24ef66e4 100644 --- a/airflow/migrations/versions/2e541a1dcfed_task_duration.py +++ b/airflow/migrations/versions/2e541a1dcfed_task_duration.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/migrations/versions/2e82aab8ef20_rename_user_table.py b/airflow/migrations/versions/2e82aab8ef20_rename_user_table.py index 0326667927..75db27cdb3 100644 --- a/airflow/migrations/versions/2e82aab8ef20_rename_user_table.py +++ b/airflow/migrations/versions/2e82aab8ef20_rename_user_table.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -41,4 +41,3 @@ def upgrade(): def downgrade(): op.rename_table('users', 'user') - diff --git a/airflow/migrations/versions/338e90f54d61_more_logging_into_task_isntance.py b/airflow/migrations/versions/338e90f54d61_more_logging_into_task_isntance.py index c101e4a477..4f1364b971 100644 --- a/airflow/migrations/versions/338e90f54d61_more_logging_into_task_isntance.py +++ b/airflow/migrations/versions/338e90f54d61_more_logging_into_task_isntance.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/migrations/versions/40e67319e3a9_dagrun_config.py b/airflow/migrations/versions/40e67319e3a9_dagrun_config.py index ecfe58cfc3..02ea51501c 100644 --- a/airflow/migrations/versions/40e67319e3a9_dagrun_config.py +++ b/airflow/migrations/versions/40e67319e3a9_dagrun_config.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/migrations/versions/4446e08588_dagrun_start_end.py b/airflow/migrations/versions/4446e08588_dagrun_start_end.py index dc546f8c77..101f2ad905 100644 --- a/airflow/migrations/versions/4446e08588_dagrun_start_end.py +++ b/airflow/migrations/versions/4446e08588_dagrun_start_end.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/migrations/versions/4addfa1236f1_add_fractional_seconds_to_mysql_tables.py b/airflow/migrations/versions/4addfa1236f1_add_fractional_seconds_to_mysql_tables.py index 80af578367..c3898e9596 100644 --- a/airflow/migrations/versions/4addfa1236f1_add_fractional_seconds_to_mysql_tables.py +++ b/airflow/migrations/versions/4addfa1236f1_add_fractional_seconds_to_mysql_tables.py @@ -6,9 +6,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/migrations/versions/502898887f84_adding_extra_to_log.py b/airflow/migrations/versions/502898887f84_adding_extra_to_log.py index f5cded7ea5..333b18e8d2 100644 --- a/airflow/migrations/versions/502898887f84_adding_extra_to_log.py +++ b/airflow/migrations/versions/502898887f84_adding_extra_to_log.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/migrations/versions/52d714495f0_job_id_indices.py b/airflow/migrations/versions/52d714495f0_job_id_indices.py index 43893dba8a..13f561cffb 100644 --- a/airflow/migrations/versions/52d714495f0_job_id_indices.py +++ b/airflow/migrations/versions/52d714495f0_job_id_indices.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/migrations/versions/561833c1c74b_add_password_column_to_user.py b/airflow/migrations/versions/561833c1c74b_add_password_column_to_user.py index ad82c9378b..d92db9f891 100644 --- a/airflow/migrations/versions/561833c1c74b_add_password_column_to_user.py +++ b/airflow/migrations/versions/561833c1c74b_add_password_column_to_user.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/migrations/versions/5e7d17757c7a_add_pid_field_to_taskinstance.py b/airflow/migrations/versions/5e7d17757c7a_add_pid_field_to_taskinstance.py index 5e2963f060..e23a0a8b2f 100644 --- a/airflow/migrations/versions/5e7d17757c7a_add_pid_field_to_taskinstance.py +++ b/airflow/migrations/versions/5e7d17757c7a_add_pid_field_to_taskinstance.py @@ -6,9 +6,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/migrations/versions/64de9cddf6c9_add_task_fails_journal_table.py b/airflow/migrations/versions/64de9cddf6c9_add_task_fails_journal_table.py index be74738328..6dda5df43b 100644 --- a/airflow/migrations/versions/64de9cddf6c9_add_task_fails_journal_table.py +++ b/airflow/migrations/versions/64de9cddf6c9_add_task_fails_journal_table.py @@ -6,9 +6,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/migrations/versions/8504051e801b_xcom_dag_task_indices.py b/airflow/migrations/versions/8504051e801b_xcom_dag_task_indices.py index 47473e318d..d6a4514ae2 100644 --- a/airflow/migrations/versions/8504051e801b_xcom_dag_task_indices.py +++ b/airflow/migrations/versions/8504051e801b_xcom_dag_task_indices.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/migrations/versions/947454bf1dff_add_ti_job_id_index.py b/airflow/migrations/versions/947454bf1dff_add_ti_job_id_index.py index 4c8bce9b47..b821cacedc 100644 --- a/airflow/migrations/versions/947454bf1dff_add_ti_job_id_index.py +++ b/airflow/migrations/versions/947454bf1dff_add_ti_job_id_index.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/migrations/versions/__init__.py b/airflow/migrations/versions/__init__.py index f0f8b68337..114d189da1 100644 --- a/airflow/migrations/versions/__init__.py +++ b/airflow/migrations/versions/__init__.py @@ -7,13 +7,12 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - diff --git a/airflow/migrations/versions/bba5a7cfc896_add_a_column_to_track_the_encryption_.py b/airflow/migrations/versions/bba5a7cfc896_add_a_column_to_track_the_encryption_.py index c780e5830d..a19eb58eb5 100644 --- a/airflow/migrations/versions/bba5a7cfc896_add_a_column_to_track_the_encryption_.py +++ b/airflow/migrations/versions/bba5a7cfc896_add_a_column_to_track_the_encryption_.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/migrations/versions/bbc73705a13e_add_notification_sent_column_to_sla_miss.py b/airflow/migrations/versions/bbc73705a13e_add_notification_sent_column_to_sla_miss.py index 84c54fad71..04cdc22d6c 100644 --- a/airflow/migrations/versions/bbc73705a13e_add_notification_sent_column_to_sla_miss.py +++ b/airflow/migrations/versions/bbc73705a13e_add_notification_sent_column_to_sla_miss.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/migrations/versions/bdaa763e6c56_make_xcom_value_column_a_large_binary.py b/airflow/migrations/versions/bdaa763e6c56_make_xcom_value_column_a_large_binary.py index 5e06766182..a02eea5519 100644 --- a/airflow/migrations/versions/bdaa763e6c56_make_xcom_value_column_a_large_binary.py +++ b/airflow/migrations/versions/bdaa763e6c56_make_xcom_value_column_a_large_binary.py @@ -6,9 +6,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -36,7 +36,7 @@ import sqlalchemy as sa def upgrade(): - # There can be data truncation here as LargeBinary can be smaller than the pickle + # There can be data truncation here as LargeBinary can be smaller than the pickle # type. # use batch_alter_table to support SQLite workaround diff --git a/airflow/migrations/versions/cc1e65623dc7_add_max_tries_column_to_task_instance.py b/airflow/migrations/versions/cc1e65623dc7_add_max_tries_column_to_task_instance.py index 0503d93df5..58fb58fa7d 100644 --- a/airflow/migrations/versions/cc1e65623dc7_add_max_tries_column_to_task_instance.py +++ b/airflow/migrations/versions/cc1e65623dc7_add_max_tries_column_to_task_instance.py @@ -6,9 +6,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/migrations/versions/d2ae31099d61_increase_text_size_for_mysql.py b/airflow/migrations/versions/d2ae31099d61_increase_text_size_for_mysql.py index ff67b4fb16..5ecb0d5c72 100644 --- a/airflow/migrations/versions/d2ae31099d61_increase_text_size_for_mysql.py +++ b/airflow/migrations/versions/d2ae31099d61_increase_text_size_for_mysql.py @@ -6,9 +6,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/migrations/versions/e3a246e0dc1_current_schema.py b/airflow/migrations/versions/e3a246e0dc1_current_schema.py index 6c63d0a9dd..cfa4147dd9 100644 --- a/airflow/migrations/versions/e3a246e0dc1_current_schema.py +++ b/airflow/migrations/versions/e3a246e0dc1_current_schema.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/migrations/versions/f2ca10b85618_add_dag_stats_table.py b/airflow/migrations/versions/f2ca10b85618_add_dag_stats_table.py index 7c23d50712..3e3d40abc4 100644 --- a/airflow/migrations/versions/f2ca10b85618_add_dag_stats_table.py +++ b/airflow/migrations/versions/f2ca10b85618_add_dag_stats_table.py @@ -6,9 +6,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/plugins_manager.py b/airflow/plugins_manager.py index 735f2de1e8..ad630701de 100644 --- a/airflow/plugins_manager.py +++ b/airflow/plugins_manager.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -34,6 +34,7 @@ from airflow.utils.log.logging_mixin import LoggingMixin log = LoggingMixin().log + class AirflowPluginException(Exception): pass @@ -107,6 +108,7 @@ def make_module(name, objects): module.__dict__.update((o.__name__, o) for o in objects) return module + # Plugin components to integrate as modules operators_modules = [] sensors_modules = [] diff --git a/airflow/settings.py b/airflow/settings.py index 788ecc4a2a..7d660ab5e6 100644 --- a/airflow/settings.py +++ b/airflow/settings.py @@ -47,7 +47,7 @@ try: TIMEZONE = pendulum.local_timezone() else: TIMEZONE = pendulum.timezone(tz) -except: +except Exception: pass log.info("Configured default timezone %s" % TIMEZONE) @@ -226,9 +226,9 @@ def configure_action_logging(): try: - from airflow_local_settings import * + from airflow_local_settings import * # noqa F403 F401 log.info("Loaded airflow_local_settings.") -except: +except Exception: pass configure_logging() diff --git a/airflow/utils/__init__.py b/airflow/utils/__init__.py index cc4e12be80..e8e889c59b 100644 --- a/airflow/utils/__init__.py +++ b/airflow/utils/__init__.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/utils/asciiart.py b/airflow/utils/asciiart.py index faf796c18d..952ebfe1e0 100644 --- a/airflow/utils/asciiart.py +++ b/airflow/utils/asciiart.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/utils/cli_action_loggers.py b/airflow/utils/cli_action_loggers.py index e61642d300..8b8cda538f 100644 --- a/airflow/utils/cli_action_loggers.py +++ b/airflow/utils/cli_action_loggers.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/utils/compression.py b/airflow/utils/compression.py index f6e923a156..2565299175 100644 --- a/airflow/utils/compression.py +++ b/airflow/utils/compression.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/utils/dag_processing.py b/airflow/utils/dag_processing.py index e236397da0..3796f3e950 100644 --- a/airflow/utils/dag_processing.py +++ b/airflow/utils/dag_processing.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/utils/dates.py b/airflow/utils/dates.py index c147a65430..216b2ed626 100644 --- a/airflow/utils/dates.py +++ b/airflow/utils/dates.py @@ -160,7 +160,7 @@ def round_time(dt, delta, start_date=timezone.make_aware(datetime.min)): # We first search an upper limit for i for which start_date + upper * delta # exceeds dt. upper = 1 - while start_date + upper*delta < dt: + while start_date + upper * delta < dt: # To speed up finding an upper limit we grow this exponentially by a # factor of 2 upper *= 2 @@ -177,20 +177,20 @@ def round_time(dt, delta, start_date=timezone.make_aware(datetime.min)): # Invariant: start + lower * delta < dt <= start + upper * delta # If start_date + (lower + 1)*delta exceeds dt, then either lower or # lower+1 has to be the solution we are searching for - if start_date + (lower + 1)*delta >= dt: + if start_date + (lower + 1) * delta >= dt: # Check if start_date + (lower + 1)*delta or # start_date + lower*delta is closer to dt and return the solution if ( (start_date + (lower + 1) * delta) - dt <= dt - (start_date + lower * delta)): - return start_date + (lower + 1)*delta + return start_date + (lower + 1) * delta else: return start_date + lower * delta # We intersect the interval and either replace the lower or upper # limit with the candidate candidate = lower + (upper - lower) // 2 - if start_date + candidate*delta >= dt: + if start_date + candidate * delta >= dt: upper = candidate else: lower = candidate @@ -209,11 +209,11 @@ def infer_time_unit(time_seconds_arr): if len(time_seconds_arr) == 0: return 'hours' max_time_seconds = max(time_seconds_arr) - if max_time_seconds <= 60*2: + if max_time_seconds <= 60 * 2: return 'seconds' - elif max_time_seconds <= 60*60*2: + elif max_time_seconds <= 60 * 60 * 2: return 'minutes' - elif max_time_seconds <= 24*60*60*2: + elif max_time_seconds <= 24 * 60 * 60 * 2: return 'hours' else: return 'days' @@ -224,11 +224,11 @@ def scale_time_units(time_seconds_arr, unit): Convert an array of time durations in seconds to the specified time unit. """ if unit == 'minutes': - return list(map(lambda x: x*1.0/60, time_seconds_arr)) + return list(map(lambda x: x * 1.0 / 60, time_seconds_arr)) elif unit == 'hours': - return list(map(lambda x: x*1.0/(60*60), time_seconds_arr)) + return list(map(lambda x: x * 1.0 / (60 * 60), time_seconds_arr)) elif unit == 'days': - return list(map(lambda x: x*1.0/(24*60*60), time_seconds_arr)) + return list(map(lambda x: x * 1.0 / (24 * 60 * 60), time_seconds_arr)) return time_seconds_arr diff --git a/airflow/utils/db.py b/airflow/utils/db.py index b5e0c49c61..93458c6f61 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -43,7 +43,7 @@ def create_session(): yield session session.expunge_all() session.commit() - except: + except Exception: session.rollback() raise finally: diff --git a/airflow/utils/decorators.py b/airflow/utils/decorators.py index 966d27e30a..f1f0ea9a42 100644 --- a/airflow/utils/decorators.py +++ b/airflow/utils/decorators.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/utils/file.py b/airflow/utils/file.py index 88938d422b..918375fe5e 100644 --- a/airflow/utils/file.py +++ b/airflow/utils/file.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/utils/helpers.py b/airflow/utils/helpers.py index 911890dc30..db58e650d8 100644 --- a/airflow/utils/helpers.py +++ b/airflow/utils/helpers.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/utils/json.py b/airflow/utils/json.py index 434926f9d2..1767c523d6 100644 --- a/airflow/utils/json.py +++ b/airflow/utils/json.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/utils/log/__init__.py b/airflow/utils/log/__init__.py index 4067cc78ee..114d189da1 100644 --- a/airflow/utils/log/__init__.py +++ b/airflow/utils/log/__init__.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/utils/log/es_task_handler.py b/airflow/utils/log/es_task_handler.py index 0468231865..d74aabfbae 100644 --- a/airflow/utils/log/es_task_handler.py +++ b/airflow/utils/log/es_task_handler.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/utils/log/file_processor_handler.py b/airflow/utils/log/file_processor_handler.py index 5e637440a1..f39dffe0c9 100644 --- a/airflow/utils/log/file_processor_handler.py +++ b/airflow/utils/log/file_processor_handler.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/utils/log/file_task_handler.py b/airflow/utils/log/file_task_handler.py index 627c875570..113bd254ad 100644 --- a/airflow/utils/log/file_task_handler.py +++ b/airflow/utils/log/file_task_handler.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/utils/log/logging_mixin.py b/airflow/utils/log/logging_mixin.py index 3f696931c9..85bdcaad7a 100644 --- a/airflow/utils/log/logging_mixin.py +++ b/airflow/utils/log/logging_mixin.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/utils/log/s3_task_handler.py b/airflow/utils/log/s3_task_handler.py index 07b9b3ec11..bdbf94b6bb 100644 --- a/airflow/utils/log/s3_task_handler.py +++ b/airflow/utils/log/s3_task_handler.py @@ -42,7 +42,7 @@ class S3TaskHandler(FileTaskHandler, LoggingMixin): try: from airflow.hooks.S3_hook import S3Hook return S3Hook(remote_conn_id) - except: + except Exception: self.log.error( 'Could not create an S3Hook with connection id "%s". ' 'Please make sure that airflow[s3] is installed and ' @@ -139,7 +139,7 @@ class S3TaskHandler(FileTaskHandler, LoggingMixin): """ try: return self.hook.read_key(remote_log_location) - except: + except Exception: msg = 'Could not read logs from {}'.format(remote_log_location) self.log.exception(msg) # return error if needed @@ -169,5 +169,5 @@ class S3TaskHandler(FileTaskHandler, LoggingMixin): replace=True, encrypt=configuration.conf.getboolean('core', 'ENCRYPT_S3_LOGS'), ) - except: + except Exception: self.log.exception('Could not write logs to %s', remote_log_location) diff --git a/airflow/utils/net.py b/airflow/utils/net.py index 03e8fde9c9..c435496ffc 100644 --- a/airflow/utils/net.py +++ b/airflow/utils/net.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/utils/operator_resources.py b/airflow/utils/operator_resources.py index 47ff336991..649961340e 100644 --- a/airflow/utils/operator_resources.py +++ b/airflow/utils/operator_resources.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/utils/sqlalchemy.py b/airflow/utils/sqlalchemy.py index 4dab322d4a..baddd9dcf1 100644 --- a/airflow/utils/sqlalchemy.py +++ b/airflow/utils/sqlalchemy.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/utils/state.py b/airflow/utils/state.py index 9da98510eb..e95f58c350 100644 --- a/airflow/utils/state.py +++ b/airflow/utils/state.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/utils/tests.py b/airflow/utils/tests.py index 6f29ffc25b..954ac35f78 100644 --- a/airflow/utils/tests.py +++ b/airflow/utils/tests.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/utils/timeout.py b/airflow/utils/timeout.py index c5ca5e2f17..a86b9d357b 100644 --- a/airflow/utils/timeout.py +++ b/airflow/utils/timeout.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/utils/trigger_rule.py b/airflow/utils/trigger_rule.py index 7bad8ecb78..ae51d6a301 100644 --- a/airflow/utils/trigger_rule.py +++ b/airflow/utils/trigger_rule.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/utils/weight_rule.py b/airflow/utils/weight_rule.py index 745cc76970..b920ef4022 100644 --- a/airflow/utils/weight_rule.py +++ b/airflow/utils/weight_rule.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/www/__init__.py b/airflow/www/__init__.py index f0f8b68337..114d189da1 100644 --- a/airflow/www/__init__.py +++ b/airflow/www/__init__.py @@ -7,13 +7,12 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - diff --git a/airflow/www/api/__init__.py b/airflow/www/api/__init__.py index db5ba598d7..b7f8352944 100644 --- a/airflow/www/api/__init__.py +++ b/airflow/www/api/__init__.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/www/api/experimental/__init__.py b/airflow/www/api/experimental/__init__.py index db5ba598d7..b7f8352944 100644 --- a/airflow/www/api/experimental/__init__.py +++ b/airflow/www/api/experimental/__init__.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/www/blueprints.py b/airflow/www/blueprints.py index ec719b96d2..eb92837fe3 100644 --- a/airflow/www/blueprints.py +++ b/airflow/www/blueprints.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/www/gunicorn_config.py b/airflow/www/gunicorn_config.py index 7cf12c56c7..db76b6de0c 100644 --- a/airflow/www/gunicorn_config.py +++ b/airflow/www/gunicorn_config.py @@ -8,9 +8,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/www/validators.py b/airflow/www/validators.py index 6eee76a32a..3db8b2850e 100644 --- a/airflow/www/validators.py +++ b/airflow/www/validators.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -45,8 +45,11 @@ class GreaterEqualThan(EqualTo): if field.data < other.data: d = { - 'other_label': hasattr(other, 'label') and other.label.text - or self.fieldname, + 'other_label': ( + hasattr(other, 'label') and + other.label.text or + self.fieldname + ), 'other_name': self.fieldname, } message = self.message diff --git a/airflow/www/views.py b/airflow/www/views.py index 178a8e648c..fdca4999f8 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -199,7 +199,7 @@ def nobr_f(v, c, m, p): def label_link(v, c, m, p): try: default_params = ast.literal_eval(m.default_params) - except: + except Exception: default_params = {} url = url_for( 'airflow.chart', chart_id=m.id, iteration_no=m.iteration_no, @@ -397,9 +397,9 @@ class Airflow(BaseView): # Processing templated fields try: args = ast.literal_eval(chart.default_params) - if type(args) is not type(dict()): + if not isinstance(args, dict): raise AirflowException('Not a dict') - except: + except Exception: args = {} payload['error'] += ( "Default params is not valid, string has to evaluate as " @@ -441,15 +441,15 @@ class Airflow(BaseView): if not payload['error'] and len(df) == 0: payload['error'] += "Empty result set. " elif ( - not payload['error'] and - chart.sql_layout == 'series' and - chart.chart_type != "datatable" and - len(df.columns) < 3): + not payload['error'] and + chart.sql_layout == 'series' and + chart.chart_type != "datatable" and + len(df.columns) < 3): payload['error'] += "SQL needs to return at least 3 columns. " elif ( - not payload['error'] and - chart.sql_layout == 'columns' and - len(df.columns) < 2): + not payload['error'] and + chart.sql_layout == 'columns' and + len(df.columns) < 2): payload['error'] += "SQL needs to return at least 2 columns. " elif not payload['error']: import numpy as np @@ -615,13 +615,13 @@ class Airflow(BaseView): # If no dag_run is active, return task instances from most recent dag_run. LastTI = ( session.query(TI.dag_id.label('dag_id'), TI.state.label('state')) - .join(LastDagRun, and_( + .join(LastDagRun, and_( LastDagRun.c.dag_id == TI.dag_id, LastDagRun.c.execution_date == TI.execution_date)) ) RunningTI = ( session.query(TI.dag_id.label('dag_id'), TI.state.label('state')) - .join(RunningDagRun, and_( + .join(RunningDagRun, and_( RunningDagRun.c.dag_id == TI.dag_id, RunningDagRun.c.execution_date == TI.execution_date)) ) @@ -629,7 +629,7 @@ class Airflow(BaseView): UnionTI = union_all(LastTI, RunningTI).alias('union_ti') qry = ( session.query(UnionTI.c.dag_id, UnionTI.c.state, sqla.func.count()) - .group_by(UnionTI.c.dag_id, UnionTI.c.state) + .group_by(UnionTI.c.dag_id, UnionTI.c.state) ) data = {} @@ -645,7 +645,7 @@ class Airflow(BaseView): for state in State.task_states: try: count = data[dag.dag_id][state] - except: + except Exception: count = 0 d = { 'state': state, @@ -1632,13 +1632,13 @@ class Airflow(BaseView): TF = models.TaskFail ti_fails = ( session - .query(TF) - .filter( + .query(TF) + .filter( TF.dag_id == dag.dag_id, TF.execution_date >= min_date, TF.execution_date <= base_date, TF.task_id.in_([t.task_id for t in dag.tasks])) - .all() + .all() ) fails_totals = defaultdict(int) @@ -1995,7 +1995,7 @@ class Airflow(BaseView): return self.render( 'airflow/variables/{}.html'.format(form) ) - except: + except Exception: # prevent XSS form = escape(form) return ("Error: form airflow/variables/{}.html " @@ -2555,7 +2555,7 @@ class VariableView(wwwutils.DataProfilingMixin, AirflowModelView): val = None try: val = d.decode(var.val) - except: + except Exception: val = var.val var_dict[var.key] = val @@ -2961,7 +2961,7 @@ class ConnectionModelView(wwwutils.SuperUserMixin, AirflowModelView): fk = None try: fk = conf.get('core', 'fernet_key') - except: + except Exception: pass return fk is None @@ -2973,10 +2973,10 @@ class ConnectionModelView(wwwutils.SuperUserMixin, AirflowModelView): """ is_secure = False try: - import cryptography + import cryptography # noqa F401 conf.get('core', 'fernet_key') is_secure = True - except: + except Exception: pass return is_secure diff --git a/airflow/www_rbac/__init__.py b/airflow/www_rbac/__init__.py index 4067cc78ee..114d189da1 100644 --- a/airflow/www_rbac/__init__.py +++ b/airflow/www_rbac/__init__.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/www_rbac/api/__init__.py b/airflow/www_rbac/api/__init__.py index db5ba598d7..b7f8352944 100644 --- a/airflow/www_rbac/api/__init__.py +++ b/airflow/www_rbac/api/__init__.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/www_rbac/api/experimental/__init__.py b/airflow/www_rbac/api/experimental/__init__.py index db5ba598d7..b7f8352944 100644 --- a/airflow/www_rbac/api/experimental/__init__.py +++ b/airflow/www_rbac/api/experimental/__init__.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/www_rbac/blueprints.py b/airflow/www_rbac/blueprints.py index 35a5badb6a..75bbb93aea 100644 --- a/airflow/www_rbac/blueprints.py +++ b/airflow/www_rbac/blueprints.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/www_rbac/validators.py b/airflow/www_rbac/validators.py index 7746e74338..62b7db8252 100644 --- a/airflow/www_rbac/validators.py +++ b/airflow/www_rbac/validators.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/airflow/www_rbac/widgets.py b/airflow/www_rbac/widgets.py index 3bca3869ce..ea61d3ff94 100644 --- a/airflow/www_rbac/widgets.py +++ b/airflow/www_rbac/widgets.py @@ -7,9 +7,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY