[AIRFLOW-3932] Optionally skip dag discovery heuristic. (#4746)

UPDATING.md

@@ -24,6 +24,12 @@ assists users migrating to a new version.
 
 ## Airflow Master
 
+### New `dag_discovery_safe_mode` config option
+
+If `dag_discovery_safe_mode` is enabled, only check files for DAGs if
+they contain the strings "airflow" and "DAG". For backwards
+compatibility, this option is enabled by default.
+
 ### Removed deprecated import mechanism
 
 The deprecated import mechanism has been removed so the import of modules becomes more consistent and explicit.

airflow/config_templates/default_airflow.cfg

@@ -182,6 +182,10 @@ dag_run_conf_overrides_params = False
 # Worker initialisation check to validate Metadata Database connection
 worker_precheck = False
 
+# When discovering DAGs, ignore any files that don't contain the strings `DAG` and `airflow`.
+dag_discovery_safe_mode = True
+
+
 [cli]
 # In what way should the cli access the API. The LocalClient will use the
 # database directly, while the json_client will use the api running on the

airflow/models/__init__.py

@@ -559,7 +559,10 @@ class DagBag(BaseDagBag, LoggingMixin):
         stats = []
         FileLoadStat = namedtuple(
             'FileLoadStat', "file duration dag_num task_num dags")
-        for filepath in list_py_file_paths(dag_folder, include_examples=include_examples):
+
+        safe_mode = configuration.conf.getboolean('core', 'dag_discovery_safe_mode')
+        for filepath in list_py_file_paths(dag_folder, safe_mode=safe_mode,
+                                           include_examples=include_examples):
             try:
                 ts = timezone.utcnow()
                 found_dags = self.process_file(