Revert "Remove the pulse destinations configuration" (#3927)

This reverts commit 1728dd306c (#3884), due to:

```
Pulse warning: queue "queue/treeherder-staging/jobs" is overgrowing

Warning: your queue "queue/treeherder-staging/jobs" on exchange "could not be determined" is
overgrowing (4083 ready messages, 4083 total messages).
...
```

docs/pulseload.md

@@ -121,7 +121,12 @@ you would set:
 export PULSE_JOB_PROJECTS="try,mozilla-central"
 ```
 
-The source settings are combined such that all `projects` are applied to **each** `exchange`.
+``PULSE_JOB_DESTINATIONS`` defines a list of destinations to push to.
+```bash
+export PULSE_JOB_DESTINATIONS="#"
+```
+
+The source settings are combined such that all `projects` and `destinations` are applied to **each** `exchange`.
 The example settings above would produce the following settings:
 
 ```python
@@ -131,12 +136,18 @@ The example settings above would produce the following settings:
         "try",
         "mozilla-central",
     ],
+    "destinations": [
+        "#",
+    ],
 }, {
     "exchange": "exchange/fxtesteng/jobs",
     "projects": [
         "try",
         "mozilla-central",
     ],
+    "destinations": [
+        "#",
+    ],
 }]
 ```
 

docs/submitting_data.md

@@ -79,20 +79,27 @@ Submit a [Treeherder bug] with the following information:
 ```python
 {
     "exchange": "exchange/my-pulse-user/v1/jobs",
+    "destinations": [
+        'treeherder'
+    ],
     "projects": [
         'mozilla-inbound._'
     ],
 },
 ```
 
-Treeherder will bind to the exchange looking for all combinations of it and the
+Treeherder will bind to the exchange looking for all combinations of routing
-``projects``.  For example with the above config, we will only load jobs from
+keys from ``destinations`` and ``projects`` listed above.  For example with
-the ``mozilla-inbound._`` project.
+the above config, we will only load jobs with routing keys of
+``treeherder.mozilla-inbound._``
 
-If you want all jobs from your exchange to be loaded, you can use the ``#``
+If you want all jobs from your exchange to be loaded, you could simplify the
-wildcard like so:
+config by having values:
 
 ```python
+"destinations": [
+    '#'
+],
 "projects": [
     '#'
 ],
@@ -100,7 +107,7 @@ wildcard like so:
 
 If you want one config to go to Treeherder Staging and a different one to go
 to Production, please specify that in the bug.  You could use the same exchange
-with different project settings, or two separate exchanges.  The choice is
+with different routing key settings, or two separate exchanges.  The choice is
 yours.
 
 

treeherder/etl/management/commands/read_pulse_jobs.py

@@ -25,15 +25,19 @@ class Command(BaseCommand):
                 exchange = get_exchange(connection, source["exchange"])
 
                 for project in source["projects"]:
-                    consumer.bind_to(exchange=exchange, routing_key="#.{}".format(project))
+                    for destination in source['destinations']:
-                    new_binding_str = consumer.get_binding_str(exchange.name, project)
+                        routing_key = "{}.{}".format(destination, project)
-                    new_bindings.append(new_binding_str)
+                        consumer.bind_to(exchange, routing_key)
+                        new_binding_str = consumer.get_binding_str(
+                            exchange.name,
+                            routing_key)
+                        new_bindings.append(new_binding_str)
 
-                    self.stdout.write(
+                        self.stdout.write(
-                        "Pulse queue {} bound to: {}".format(
+                            "Pulse queue {} bound to: {}".format(
-                            consumer.queue_name,
+                                consumer.queue_name,
-                            new_binding_str
+                                new_binding_str
-                        ))
+                            ))
 
             consumer.prune_bindings(new_bindings)
 

treeherder/services/pulse/sources.py

@@ -14,16 +14,26 @@ projects = env.list("PULSE_JOB_PROJECTS", default=[
     # "mozilla-central.#",
     # "mozilla-inbound.#",
 ])
+destinations = env.list("PULSE_JOB_DESTINATIONS", default=[
+    "#",
+    # "production",
+    # "staging",
+    # "tc-treeherder",
+])
 
 
-# Specifies the Pulse exchanges Treeherder will ingest data from for Jobs. This
+# Get Job ingestion source locations.
-# list will be updated as new applications come online that Treeherder
+
-# supports.  Treeherder will subscribe with routing keys that are the project
+# Specifies the Pulse exchanges Treeherder will ingest data from for Job data.
-# names.  Wildcards such as ``#`` and ``*`` are supported for the project
+# This list will be updated as new applications come online that Treeherder
-# field.
+# supports. Treeherder will subscribe with routing keys that are all
+# combinations of ``project`` and ``destination`` in the form of:
+# <destination>.<project> Wildcards such as ``#`` and ``*`` are supported for
+# either field.
 job_sources = [{
     "exchange": exchange,
     "projects": projects,
+    "destinations": destinations,
 } for exchange in exchanges]