Pagerduty bug 1471730 (#149)

* pagerduty: remove unused test_user_has_escalation_policy

* requirements: remove pygerduty

* pagerduty: switch to remote monitoring extension checks

* make: ignore pagerduty/ for doctests

Makefile

@@ -35,7 +35,7 @@ clean-python:
 	find . -type d -name venv -prune -o -type d -name __pycache__ -print0 | xargs -0 rm -rf
 
 doctest: check_venv
-	pytest --doctest-modules -s --offline --debug-calls
+	pytest --doctest-modules -s --offline --debug-calls --ignore pagerduty/
 
 coverage: check_venv
 	pytest --cov-config .coveragerc --cov=. \

README.md

@@ -229,6 +229,10 @@ aws:
       ports:
         - 22
         - 2222
+pagerduty:
+  users_with_remote_access_monitoring: 'pd_users.json'
+  bastion_users: 'hierahash/*hierahash.json'
+  alternate_usernames: 'alternate_usernames.json'
 ```
 
 ### Test Exemptions
@@ -420,8 +424,8 @@ aws:
 
 ### GSuite Config
 
-pytest-services has a suite of GSuite tests. This section of the custom config includes configuration options specific
-to these tests.
+pytest-services has a suite of GSuite tests. This section of the
+custom config includes configuration options specific to these tests.
 
 **Make sure to [setup GSuite](#setting-up-gsuite-tests) before running GSuite tests**
 
@@ -438,6 +442,75 @@ gsuite:
       months: 0
 ```
 
+### Pagerduty Config
+
+pytest-services does not query the pagerduty API, but can run tests against output from it.
+
+The config looks like:
+```
+pagerduty:
+  users_with_remote_access_monitoring: 'pd_users.json'
+  bastion_users: 'hierahash/*hierahash.json'
+  alternate_usernames: 'alternate_usernames.json'
+```
+
+Where `users_with_remote_access_monitoring` and `bastion_users` are
+globs for multiple files relative to the current working directory and
+`alternate_usernames` is the path to a single file.
+
+The files have examples formats as follows:
+
+* `users_with_remote_access_monitoring`:
+
+```json
+[
+  {
+    "avatar_url": "https://secure.gravatar.com/avatar/...",
+    "billed": true,
+    "color": "sea-green",
+    "contact_methods": [],
+    "description": null,
+    "email": "example@example.com",
+    "html_url": "https://example.pagerduty.com/users/AAA0999",
+    "id": "AAA0999",
+    "invitation_sent": false,
+    "job_title": null,
+    "name": "Example Examplerton",
+    "notification_rules": [],
+    "role": "user",
+    "self_": "https://api.pagerduty.com/users/AAA0999",
+    "summary": "C. Hobbes",
+    "teams": [],
+    "time_zone": "America/New_York",
+    "type": "user"
+  },
+  ...
+]
+```
+
+* `bastion_users`:
+
+```json
+{
+  "chobbes": {
+    "groups": [""],
+    "root_ssh": true
+  },
+  "movedon": {
+    "ensure": "absent"
+  },
+  ...
+}
+```
+
+* `alternate_usernames`:
+
+```json
+{
+  "chobbes": ["calvin", "spacemanspiff"]
+}
+```
+
 ### Test Accuracy
 
 There are two important things to note about `pytest-services` tests that may be different from your expectations.

aws/iam/resources.py

@@ -288,3 +288,20 @@ def user_is_admin(user):
         if isinstance(policy, dict):
             if "admin" in policy.get("PolicyName", "").lower():
                 return True
+
+
+def get_all_users_that_can_access_aws_account():
+    """
+    Returns users with console or API access to an AWS account.
+    """
+    profile_usernames = [
+        profile["UserName"]
+        for profile in iam_user_login_profiles()
+        if profile is not None
+    ]
+    access_key_usernames = [
+        akey["UserName"]
+        for akey in iam_get_all_access_keys()
+        if akey["Status"] == "Active"
+    ]
+    return set(profile_usernames + access_key_usernames)

config.yaml.example

@@ -49,3 +49,7 @@ gsuite:
     no_activity_since:
       years: 1
       months: 0
+pagerduty:
+  users_with_remote_access_monitoring: 'pd_users.json'
+  bastion_users: 'hierahash/*hierahash.json'
+  alternate_usernames: 'alternate_usernames.json'

custom_config.py

@@ -16,6 +16,8 @@ class CustomConfig:
             parsed_config = yaml.load(config_fd)
         self.aws = AWSConfig(parsed_config.get("aws", {}))
         self.gsuite = GSuiteConfig(parsed_config.get("gsuite", {}))
+        self.pagerduty = PagerdutyConfig(parsed_config.get("pagerduty", {}))
+
         self.exemptions = exemptions.load(parsed_config.get("exemptions"))
         self.severities = severity.load(parsed_config.get("severities"))
         self.regressions = regressions.load(parsed_config.get("regressions"))
@@ -96,3 +98,13 @@ class GSuiteConfig(CustomConfigMixin):
     def __init__(self, config):
         self.domain = config.get("domain", "")
         super().__init__(config)
+
+
+class PagerdutyConfig(CustomConfigMixin):
+    def __init__(self, config):
+        self.users_with_remote_access_monitoring = config.get(
+            "users_with_remote_access_monitoring", ""
+        )
+        self.bastion_users = config.get("bastion_users", "")
+        self.alternate_usernames = config.get("alternate_usernames", "")
+        super().__init__(config)

pagerduty/helpers.py

@@ -0,0 +1,10 @@
+
+
+from pagerduty.resources import alternate_usernames
+
+
+def alternate_names_for_user(username):
+    """
+    Returns a list of the username and any alternates or nicknames for it
+    """
+    return [username] + alternate_usernames().get(username, [])

pagerduty/resources.py

@@ -0,0 +1,40 @@
+
+
+import functools
+import json
+from pathlib import Path
+
+import pytest
+
+
+@pytest.fixture
+def users_with_remote_access_escalation_policy_and_extension_configured(pytestconfig):
+    p = Path(".")
+    users = set()
+    for user_file in p.glob(
+        pytestconfig.custom_config.pagerduty.users_with_remote_access_monitoring
+    ):
+        with user_file.open("r") as fin:
+            for user in json.load(fin):
+                users.add(user["email"].split("@", 1)[0])
+
+    return users
+
+
+@functools.lru_cache()
+def bastion_users():
+    p = Path(".")
+    users = set()
+    for user_file in p.glob(pytest.config.custom_config.pagerduty.bastion_users):
+        with user_file.open("r") as fin:
+            for user, val in json.load(fin).items():
+                if val.get("ensure", None) != "absent":
+                    users.add(user)
+
+    return sorted(list(users))
+
+
+@functools.lru_cache()
+def alternate_usernames():
+    with open(pytest.config.custom_config.pagerduty.alternate_usernames, "r") as fin:
+        return json.load(fin)

pagerduty/test_bastion_users_have_pd_remote_monitoring_configured.py

@@ -0,0 +1,23 @@
+
+
+import pytest
+
+from pagerduty.helpers import alternate_names_for_user
+from pagerduty.resources import (
+    bastion_users,
+    users_with_remote_access_escalation_policy_and_extension_configured,
+)
+
+
+@pytest.mark.pagerduty
+@pytest.mark.parametrize("bastion_user", bastion_users())
+def test_bastion_users_have_pd_remote_monitoring_configured(
+    bastion_user, users_with_remote_access_escalation_policy_and_extension_configured
+):
+    """
+    Checks that users with SSH access have pagerduty remote monitoring properly configured.
+    """
+    assert any(
+        username in users_with_remote_access_escalation_policy_and_extension_configured
+        for username in alternate_names_for_user(bastion_user)
+    ), "bastion user {} is not configured in pagerduty".format(bastion_user)

pagerduty/test_iam_users_have_pd_remote_monitoring_configured.py

@@ -0,0 +1,22 @@
+
+
+import pytest
+from aws.iam.resources import get_all_users_that_can_access_aws_account
+from pagerduty.helpers import alternate_names_for_user
+from pagerduty.resources import (
+    users_with_remote_access_escalation_policy_and_extension_configured
+)
+
+
+@pytest.mark.pagerduty
+@pytest.mark.parametrize("iam_user", list(get_all_users_that_can_access_aws_account()))
+def test_iam_users_have_pd_remote_monitoring_configured(
+    iam_user, users_with_remote_access_escalation_policy_and_extension_configured
+):
+    """
+    Checks that users with access to an AWS account have pagerduty  remote monitoring properly configured.
+    """
+    assert any(
+        username in users_with_remote_access_escalation_policy_and_extension_configured
+        for username in alternate_names_for_user(iam_user)
+    ), "iam user {} is not configured in pagerduty".format(iam_user)

pagerduty/test_user_has_escalation_policy.py

@@ -1,50 +0,0 @@
-
-
-import os
-
-import pytest
-import pygerduty.v2
-
-
-PD_RO_KEY = os.environ.get("pagerdutyRoKey", None)
-
-
-def pd_client():
-    if not PD_RO_KEY:
-        return None
-    return pygerduty.v2.PagerDuty(PD_RO_KEY)
-
-
-def pd_users():
-    pager = pd_client()
-    if not pager:
-        return []
-
-    return pager.users.list()
-
-
-def pd_users_escalation_policies():
-    pager = pd_client()
-    if not pager:
-        return []
-
-    return [
-        pager.escalation_policies.list(user_ids=[user["id"]]) for user in pd_users()
-    ]
-
-
-@pytest.mark.pagerduty
-@pytest.mark.skipif(
-    lambda: not PD_RO_KEY,
-    reason='Env var "pagerdutyRoKey" of Pagerduty API key not found.',
-)
-@pytest.mark.parametrize(
-    ["pd_user", "pd_user_escalation_policy"],
-    zip(pd_users(), pd_users_escalation_policies()),
-)
-def test_user_has_ooh_escalation_policy(pd_user, pd_user_escalation_policy):
-    assert (
-        pd_user_escalation_policy
-    ), "{} does not have an out of hours escalation policy in Pagerduty".format(
-        pd_user
-    )

requirements.txt

@@ -3,7 +3,6 @@ coverage==4.5.1
 black==18.6b4
 google_api_python_client==1.6.1
 PyYAML==3.13
-pygerduty==0.37.0
 pytest-cov==2.5.1
 pytest-html==1.19.0
 pytest-json==0.4.0