Skip ingestion when DATABASE_URL points to remote host (#6367)

Co-authored-by: Armen Zambrano <armenzg@users.noreply.github.com>

treeherder/config/settings.py

@@ -134,13 +134,22 @@ TEMPLATES = [{'BACKEND': 'django.template.backends.django.DjangoTemplates', 'APP
 #   'mysql://username:password@host:optional_port/database_name'
 #
 # which django-environ converts into the Django DB settings dict format.
-LOCALHOST_MYSQL_HOST = 'mysql://root@{}:3306/treeherder'.format(
-    'localhost' if IS_WINDOWS else '127.0.0.1'
-)
+MYSQL_LOCALHOST = 'localhost' if IS_WINDOWS else '127.0.0.1'
+MYSQL_TEMPLATE = 'mysql://root@{}:3306/treeherder'
 DATABASES = {
-    'default': env.db_url('DATABASE_URL', default=LOCALHOST_MYSQL_HOST),
+    'default': env.db_url('DATABASE_URL', default=MYSQL_TEMPLATE.format(MYSQL_LOCALHOST)),
 }
 
+SKIP_INGESTION = env('SKIP_INGESTION', default=False)
+# Skip data ingestion if points to a remote host
+if DATABASES['default']['HOST'] not in ['mysql', '127.0.0.1', 'localhost']:
+    SKIP_INGESTION = True
+
+# Only used when syncing local database with production replicas
+UPSTREAM_DATABASE_URL = env('UPSTREAM_DATABASE_URL', default=None)
+if UPSTREAM_DATABASE_URL:
+    DATABASES['upstream'] = env.db_url_config(UPSTREAM_DATABASE_URL)
+
 # Only used when syncing local database with production replicas
 UPSTREAM_DATABASE_URL = env('UPSTREAM_DATABASE_URL', default=None)
 if UPSTREAM_DATABASE_URL:

treeherder/etl/management/commands/pulse_listener.py

@@ -1,9 +1,11 @@
 import environ
+import logging
 from django.core.management.base import BaseCommand
-
+from treeherder.config.settings import SKIP_INGESTION
 from treeherder.services.pulse import JointConsumer, prepare_joint_consumers
 
 env = environ.Env()
+logger = logging.getLogger(__name__)
 
 
 class Command(BaseCommand):
@@ -17,6 +19,9 @@ class Command(BaseCommand):
     help = "Read tasks and pushes from a set of pulse exchanges and queue for ingestion"
 
     def handle(self, *args, **options):
+        if SKIP_INGESTION:
+            logger.debug("Skipping ingestion of Pulse Tasks")
+            return
         # Specifies the Pulse services from which Treeherder will ingest push
         # information.  Sources can include properties `hgmo`, `github`, or both, to
         # listen to events from those sources.  The value is a JSON array of the form

treeherder/etl/management/commands/pulse_listener_pushes.py

@@ -1,9 +1,11 @@
 import environ
+import logging
 from django.core.management.base import BaseCommand
-
+from treeherder.config.settings import SKIP_INGESTION
 from treeherder.services.pulse import PushConsumer, prepare_consumers
 
 env = environ.Env()
+logger = logging.getLogger(__name__)
 
 
 class Command(BaseCommand):
@@ -17,8 +19,8 @@ class Command(BaseCommand):
     help = "Read pushes from a set of pulse exchanges and queue for ingestion"
 
     def handle(self, *args, **options):
-        if env.bool('SKIP_INGESTION', default=False):
-            self.stdout.write("Skipping ingestion of Pulse Pushes")
+        if SKIP_INGESTION:
+            logger.debug("Skipping ingestion of Pulse Tasks")
             return
         # Specifies the Pulse services from which Treeherder will ingest push
         # information.  Sources can include properties `hgmo`, `github`, or both, to

treeherder/etl/management/commands/pulse_listener_tasks.py

@@ -1,9 +1,11 @@
 import environ
+import logging
 from django.core.management.base import BaseCommand
-
+from treeherder.config.settings import SKIP_INGESTION
 from treeherder.services.pulse import TaskConsumer, prepare_consumers
 
 env = environ.Env()
+logger = logging.getLogger(__name__)
 
 
 class Command(BaseCommand):
@@ -17,8 +19,8 @@ class Command(BaseCommand):
     help = "Read jobs from a set of pulse exchanges and queue for ingestion"
 
     def handle(self, *args, **options):
-        if env.bool('SKIP_INGESTION', default=False):
-            self.stdout.write("Skipping ingestion of Pulse Tasks")
+        if SKIP_INGESTION:
+            logger.debug("Skipping ingestion of Pulse Tasks")
             return
         # Specifies the Pulse services from which Treeherder will consume task
         # information.  This value is a JSON array of the form [{pulse_url: ..,