Bug 1281821 - Split PULSE_DATA_INGESTION_SOURCES into three separate settings

docs/pulseload.md

@@ -95,25 +95,63 @@ See [Running the unminified UI with Vagrant] for more info.
 Advanced Configuration
 ----------------------
 
-### Changing which data to ingest
+### Changing which Data to Ingest
 
-If you don't want all the sources provided by default in ``settings.py``, you
-can specify the exchange(s) to listen to for jobs by modifying
-``PULSE_DATA_INGESTION_SOURCES``.  For instance, you could specify the projects
-as only ``try`` and ``mozilla-central`` by setting:
-
-```bash
-export PULSE_DATA_INGESTION_SOURCES='[{"exchange": "exchange/taskcluster-treeherder/v1/jobs", "destinations": ["#"], "projects": ["try", "mozilla-central"]}]'
-```
-
-To change which exchanges you listen to for pushes, you would modify
-``PULSE_PUSH_SOURCES``.  For instance, to get only **Gitbub** pushes for Bugzilla,
-you would set:
+``treeherder.services.pulse.sources`` provides default sources for both Jobs and Pushes.
+However you can override these defaults using the standard env methods show below.
 
+#### Pushes
+``PULSE_PUSH_SOURCES`` defines a list of dictionaries with exchange and routing key strings.
 ```bash
 export PULSE_PUSH_SOURCES='[{"exchange": "exchange/taskcluster-github/v1/push","routing_keys": ["bugzilla#"]}]'
 ```
 
+#### Jobs
+``PULSE_JOB_EXCHANGES`` defines a list of exchanges to listen to.
+```bash
+export PULSE_JOB_EXCHANGES="exchange/taskcluster-treeherder/v1/jobs,exchange/fxtesteng/jobs"
+```
+
+To change which exchanges you listen to for pushes, you would modify
+``PULSE_PUSH_SOURCES``.  For instance, to get only Gitbub pushes for Bugzilla,
+you would set:
+
+``PULSE_JOB_PROJECTS`` defines a list of projects to listen to.
+```bash
+export PULSE_JOB_PROJECTS="try,mozilla-central"
+```
+
+``PULSE_JOB_DESTINATIONS`` defines a list of destinations to push to.
+```bash
+export PULSE_JOB_DESTINATIONS="#"
+```
+
+The source settings are combined such that all `projects` and `destinations` are applied to **each** `exchange`.
+The example settings above would produce the following settings:
+
+```python
+[{
+    "exchange": "exchange/taskcluster-treeherder/v1/jobs",
+    "projects": [
+        "try",
+        "mozilla-central",
+    ],
+    "destinations": [
+        "#",
+    ],
+}, {
+    "exchange": "exchange/fxtesteng/jobs",
+    "projects": [
+        "try",
+        "mozilla-central",
+    ],
+    "destinations": [
+        "#",
+    ],
+}]
+```
+
+
 ### Advanced Celery options
 
 If you only want to ingest the Pushes and Jobs, but don't care about log parsing
@@ -128,7 +166,7 @@ celery -A treeherder worker -B -Q pushlog,store_pulse_jobs,store_pulse_resultset
 * The ``store_pulse_resultsets`` queue will ingest all the pushes from the exchanges
   specified in ``PULSE_PUSH_SOURCES``.  This can be Mercurial and Github
 * The ``store_pulse_jobs`` queue will ingest all the jobs from the exchanges
-  specified in ``PULSE_DATA_INGESTION_SOURCES``.
+  specified in ``PULSE_JOB_EXCHANGES``.
 
 ```eval_rst
 .. note:: Any job that comes from **Pulse** that does not have an associated push will be skipped.

treeherder/services/pulse/connection.py

@@ -4,16 +4,14 @@ from kombu import Connection
 env = environ.Env()
 
 # Used to specify the PulseGuardian account that will be used to create
-# ingestion queues for the exchanges specified in ``PULSE_DATA_INGESTION_SOURCES``.
+# ingestion queues for the exchanges specified in ``PULSE_SOURCE_EXCHANGES``.
 # See https://pulse.mozilla.org/whats_pulse for more info.
 # Example: "amqp://myuserid:mypassword@pulse.mozilla.org:5672/?ssl=1"
 config = env.url("PULSE_URL")
 
 
 def build_connection(url):
-    """
-    Build a Kombu Broker connection with the given url
-    """
+    """Build a Kombu Broker connection to Mozilla Pulse with the given url."""
     return Connection(url)
 
 

treeherder/services/pulse/sources.py

@@ -3,6 +3,25 @@ import environ
 env = environ.Env()
 
 
+exchanges = env.list("PULSE_JOB_EXCHANGES", default=[
+    "exchange/taskcluster-treeherder/v1/jobs",
+    # "exchange/fxtesteng/jobs",
+    # ... other CI systems
+])
+projects = env.list("PULSE_JOB_PROJECTS", default=[
+    "#",
+    # some specific repos TC can ingest from
+    # "mozilla-central.#",
+    # "mozilla-inbound.#",
+])
+destinations = env.list("PULSE_JOB_DESTINATIONS", default=[
+    "#",
+    # "production",
+    # "staging",
+    # "tc-treeherder",
+])
+
+
 def get_job_sources():
     """
     Get Job ingestion source locations.
@@ -14,28 +33,12 @@ def get_job_sources():
     <destination>.<project> Wildcards such as ``#`` and ``*`` are supported for
     either field.
     """
-    sources = env.json(
-        "PULSE_DATA_INGESTION_SOURCES",
-        default=[
-            {
-                "exchange": "exchange/taskcluster-treeherder/v1/jobs",
-                "projects": [
-                    '#'
-                    # some specific repos TC can ingest from
-                    # 'mozilla-central.#',
-                    # 'mozilla-inbound.#'
-                ],
-                "destinations": [
-                    '#'
-                    # 'production',
-                    # 'staging'
-                ],
-            },
-            # ... other CI systems
-        ],
-    )
-
-    return sources
+    for exchange in exchanges:
+        yield {
+            "exchange": exchange,
+            "projects": projects,
+            "destinations": destinations,
+        }
 
 
 def get_push_sources():
@@ -62,5 +65,5 @@ def get_push_sources():
     return sources
 
 
-job_sources = get_job_sources()
+job_sources = list(get_job_sources())
 push_sources = get_push_sources()