From e4835055394602f508a2d39d4719820f2d731043 Mon Sep 17 00:00:00 2001 From: Michal Purzynski Date: Wed, 22 Aug 2018 15:23:16 -0700 Subject: [PATCH 01/89] A new version of the portscan alert, with a whitelisting language --- alerts/nsm_scan_port.json | 4 + alerts/nsm_scan_port.py | 52 ++++++++++++ tests/alerts/test_nsm_scan_port.py | 125 +++++++++++++++++++++++++++++ 3 files changed, 181 insertions(+) create mode 100644 alerts/nsm_scan_port.json create mode 100644 alerts/nsm_scan_port.py create mode 100644 tests/alerts/test_nsm_scan_port.py diff --git a/alerts/nsm_scan_port.json b/alerts/nsm_scan_port.json new file mode 100644 index 00000000..e3a979cc --- /dev/null +++ b/alerts/nsm_scan_port.json @@ -0,0 +1,4 @@ +{ + "sourcemustmatch":"[10.0.0.0 TO 10.255.255.255]", + "sourcemustnotmatch":"10.33.44.54 OR 10.88.77.54 OR 10.76.54.54 OR 10.251.30.138 OR 10.54.65.234" +} \ No newline at end of file diff --git a/alerts/nsm_scan_port.py b/alerts/nsm_scan_port.py new file mode 100644 index 00000000..5b21610d --- /dev/null +++ b/alerts/nsm_scan_port.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# Copyright (c) 2018 Mozilla Corporation + +from lib.alerttask import AlertTask +from query_models import SearchQuery, TermMatch, QueryStringMatch, PhraseMatch, ExistsMatch + + +class NSMScanPort(AlertTask): + def __init__(self): + AlertTask.__init__(self) + self._config = self.parse_json_alert_config('nsm_scan_port.json') + + def main(self): + search_query = SearchQuery(minutes=1) + search_query.add_must([ + TermMatch('category', 'bro'), + TermMatch('details.source', 'notice'), + PhraseMatch('details.note', 'Scan::Port_Scan'), + QueryStringMatch('details.sourceipaddress: {}'.format(self._config['sourcemustmatch'])) + ]) + search_query.add_must_not([ + QueryStringMatch('details.sourceipaddress: {}'.format(self._config['sourcemustnotmatch'])) + ]) + + self.filtersManual(search_query) + self.searchEventsAggregated('details.sourceipaddress', samplesLimit=10) + self.walkAggregations(threshold=1) + + + def onAggregation(self, aggreg): + category = 'nsm' + severity = 'WARNING' + tags = ['nsm', "bro", 'portscan'] + + indicators = 'unknown' + #port = 'unknown' + # Maybe iterate through it? + x = aggreg['events'][0]['_source'] + if 'details' in x: + if 'indicators' in x['details']: + indicators = x['details']['sourceipaddress'] + # if 'p' in x['details']: + # port = x['details']['p'] + + summary = 'Port scan from {}'.format(indicators) + + return self.createAlertDict(summary, category, tags, aggreg['events'], severity) + diff --git a/tests/alerts/test_nsm_scan_port.py b/tests/alerts/test_nsm_scan_port.py new file mode 100644 index 00000000..dbdc3dff --- /dev/null +++ b/tests/alerts/test_nsm_scan_port.py @@ -0,0 +1,125 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# Copyright (c) 2017 Mozilla Corporation + +from positive_alert_test_case import PositiveAlertTestCase +from negative_alert_test_case import NegativeAlertTestCase + +from alert_test_suite import AlertTestSuite + + +class TestNSMScanPort(AlertTestSuite): + alert_filename = "nsm_scan_port" + + # This event is the default positive event that will cause the + # alert to trigger + default_event = { + "_source": { + "category": "bro", + "summary": "Scan::Port_Scan source 10.99.88.77 destination unknown port unknown", + "hostname": "your.friendly.nsm.sensor", + "tags": ["bro"], + "details": { + "sourceipaddress": "10.99.88.77", + "indicators": "10.99.88.77", + "source": "notice", + "note": "Scan::Port_Scan", + } + } + } + + # This alert is the expected result from running this task + default_alert = { + "category": "nsm", + "severity": "WARNING", + "summary": "Port scan from 10.99.88.77", + "tags": ['nsm', 'bro', 'portscan'], + "notify_mozdefbot": True, + } + + test_cases = [] + + test_cases.append( + PositiveAlertTestCase( + description="Positive test with default event and default alert expected", + events=AlertTestSuite.create_events(default_event, 5), + expected_alert=default_alert + ) + ) + + events = AlertTestSuite.create_events(default_event, 5) + for event in events: + event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda(date_timedelta={'minutes': 1}) + event['_source']['receivedtimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda(date_timedelta={'minutes': 1}) + test_cases.append( + PositiveAlertTestCase( + description="Positive test with events a minute earlier", + events=events, + expected_alert=default_alert + ) + ) + + + events = AlertTestSuite.create_events(default_event, 5) + for event in events: + event['_source']['category'] = 'syslog' + test_cases.append( + NegativeAlertTestCase( + description="Negative test case with a different category", + events=events, + ) + ) + + events = AlertTestSuite.create_events(default_event, 5) + for event in events: + event['_source']['details']['source'] = 'intel' + test_cases.append( + NegativeAlertTestCase( + description="Negative test case with a different details.source", + events=events, + ) + ) + + events = AlertTestSuite.create_events(default_event, 5) + for event in events: + event['_source']['details']['note'] = 'Scan::Address_Scan' + test_cases.append( + NegativeAlertTestCase( + description="Negative test case with a different scan type (note)", + events=events, + ) + ) + + events = AlertTestSuite.create_events(default_event, 5) + for event in events: + event['_source']['details']['sourceipaddress'] = '10.54.65.234' + event['_source']['details']['indicators'] = '1.2.3.4' + test_cases.append( + NegativeAlertTestCase( + description="Negative test case with an excluded IP address", + events=events, + ) + ) + + events = AlertTestSuite.create_events(default_event, 5) + for event in events: + event['_source']['details']['sourceipaddress'] = '1.2.3.4' + event['_source']['details']['indicators'] = '1.2.3.4' + test_cases.append( + NegativeAlertTestCase( + description="Negative test case with an excluded subnet", + events=events, + ) + ) + + events = AlertTestSuite.create_events(default_event, 5) + for event in events: + event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({'minutes': 15}) + event['_source']['receivedtimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({'minutes': 15}) + test_cases.append( + NegativeAlertTestCase( + description="Negative test case with old timestamp", + events=events, + ) +) \ No newline at end of file From 4d0ee8838c4ecd36cb3f7e6341c10dcc6f50f5d4 Mon Sep 17 00:00:00 2001 From: Michal Purzynski Date: Mon, 17 Sep 2018 11:48:50 -0700 Subject: [PATCH 02/89] Remove unused code --- alerts/nsm_scan_port.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/alerts/nsm_scan_port.py b/alerts/nsm_scan_port.py index 5b21610d..002ff267 100644 --- a/alerts/nsm_scan_port.py +++ b/alerts/nsm_scan_port.py @@ -37,14 +37,12 @@ class NSMScanPort(AlertTask): tags = ['nsm', "bro", 'portscan'] indicators = 'unknown' - #port = 'unknown' + # port = 'unknown' # Maybe iterate through it? x = aggreg['events'][0]['_source'] if 'details' in x: if 'indicators' in x['details']: indicators = x['details']['sourceipaddress'] - # if 'p' in x['details']: - # port = x['details']['p'] summary = 'Port scan from {}'.format(indicators) From d61168a3fcb456a6afc7531436f9bf0430b3d219 Mon Sep 17 00:00:00 2001 From: Michal Purzynski Date: Mon, 5 Nov 2018 11:03:40 -0800 Subject: [PATCH 03/89] A MozDef plugin that parses GitHub's Webhook events to create meaningful IR data --- mq/github_mapping.yml | 442 ++++++++++++++++++++++++++++++++++ mq/plugins/github_webhooks.py | 65 +++++ 2 files changed, 507 insertions(+) create mode 100644 mq/github_mapping.yml create mode 100644 mq/plugins/github_webhooks.py diff --git a/mq/github_mapping.yml b/mq/github_mapping.yml new file mode 100644 index 00000000..7e9159be --- /dev/null +++ b/mq/github_mapping.yml @@ -0,0 +1,442 @@ +--- + push: + username: details.body.sender.login + email: details.body.pusher.email + id: details.body.sender.id + commits: details.body.commits + created: details.body.created + deleted: details.body.deleted + forced: details.body.forced + node_id: details.body.sender.node_id + sender_type: detais.body.sender.type + request_id: details.request_id + commit_author: details.body.head_commit.author.email + committer: details.body.head_commit.committer.email + commit_id: details.body.head_commit.id + commit_msg: details.body.head_commit.message + commit_ts: details.body.head_commit.timestamp + commit_url: details.body.head_commit.url + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + pusher_email: details.body.pusher.email + pusher_name: details.body.pusher.name + ref: details.body.ref + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + repo_private: details.body.repository.private + repo_pushed_at: details.body.repository.pushed_at + repo_updated_at: details.body.repository.ssh_url + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + + create: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + pusher_email: details.body.pusher.email + pusher_name: details.body.pusher.name + repo_private: details.body.repository.private + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + ref: details.body.ref + ref_type: details.body.ref_type + + delete: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + pusher_email: details.body.pusher.email + pusher_name: details.body.pusher.name + repo_private: details.body.repository.private + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + ref: details.body.ref + ref_type: details.body.ref_type + + pull_request: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + pr_number: details.body.number + pr_assignee_id: details.body.pull_request.assignee.id + pr_assignee_login: details.body.pull_request.assignee.login + pr_assignee_site_admin: details.body.pull_request.assignee.site_admin + pr_assignee_type: details.body.pull_request.assignee.type + pr_author_association: details.body.pull_request.author_association + pr_base_label: details.body.pull_request.base.label + pr_base_ref: details.body.pull_request.base.ref + pr_base_repo_name: details.body.pull_request.base.repo.name + pr_base_repo_id: details.body.pull_request.base.repo.node_id + pr_base_repo_owner_id: details.body.pull_request.base.repo.owner.id + pr_base_repo_owner_login: details.body.pull_request.base.repo.owner.login + pr_base_repo_owner_node_id: details.body.pull_request.base.repo.owner.node_id + pr_base_repo_private: details.body.pull_request.base.repo.private + pr_base_sha: details.body.pull_request.base.sha + pr_base_user_id: details.body.pull_request.base.user.id + pr_base_user_login: details.body.pull_request.base.user.login + pr_base_user_node_id: details.body.pull_request.base.user.node_id + pr_base_user_site_admin: details.body.pull_request.base.user.site_admin + pr_body: details.body.pull_request.body + pr_changed_files: details.body.pull_request.changed_files + pr_commits: details.body.pull_request.commits + pr_deletions: details.body.pull_request.deletions + pr_closed_at: details.body.pull_request.closed_at + pr_head_label: details.body.pull_request.head.label + pr_head_ref: details.body.pull_request.head.ref + pr_head_repo_name: details.body.pull_request.head.repo.name + pr_head_repo_id: details.body.pull_request.head.repo.id + pr_head_repo_node_id: details.body.pull_request.head.repo.node_id + pr_head_repo_owner_id: details.body.pull_request.head.repo.owner.id + pr_head_repo_owner_login: details.body.pull_request.head.repo.owner.login + pr_head_repo_owner_node_id: details.body.pull_request.head.repo.owner.node_id + pr_head_repo_owner_site_admin: details.body.pull_request.head.repo.owner.site_admin + pr_head_repo_private: details.body.pull_request.head.repo.private + pr_head_sha: details.body.pull_request.head.sha + pr_head_user_id: details.body.pull_request.head.user.id + pr_head_user_login: details.body.pull_request.head.user.login + pr_head_user_node_id: details.body.pull_request.head.user.node_id + pr_merge_commit_sha: details.body.pull_request.merge_commit_sha + pr_merge_merged: details.body.pull_request.merged + pr_merge_merged_at: details.body.pull_request.merged_at + pr_merged_by_id: details.body.pull_request.merged_by.id + pr_merged_by_login: details.body.pull_request.merged_by.login + pr_merged_by_node_id: details.body.pull_request.merged_by.node_id + pr_merged_by_site_admin: details.body.pull_request.merged_by.site_admin + pr_node_id: details.body.pull_request.node_id + pr_number: details.body.pull_request.number + pr_state: details.body.pull_request.state + pr_title: details.body.pull_request.title + pr_updated_at: details.body.pull_request.updated_at + pr_id: details.body.pull_request.id + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + + repository_vulnerability_alert: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + alert_package: details.body.alert.affected_package_name + alert_range: details.body.alert.affected_range + alert_extid: details.body.alert.external_identifier + alert_extref: details.body.alert.external_reference + alert_fixed: details.body.alert.fixed_in + alert_id: details.body.alert.id + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + repo_private: details.body.repository.private + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + dismiss_reason: details.body.alert.dismiss_reason + dismiss_at: details.body.alert.dismissed_at + dismiss_user: details.body.alert.dismisser.login + dismiss_id: details.body.alert.dismisser.id + dismiss_node_id: details.body.alert.dismisser.node_id + dismiss_type: details.body.alert.dismisser.type + dismiss_site_admin: details.body.alert.dismisser.site_admin + + security_advisory: + action: details.body.action + alert_id: details.body.security_advisory.ghsa_id + alert_note: details.body.security_advisory.summary + alert_description: details.body.security_advisory.description + alert_severity: details.body.security_advisory.severity + alert_list: details.body.security_advisory.identifiers + alert_vulnerabilities: details.body.security_advisory.vulnerabilities + alert_ref: details.body.security_advisory.references + alert_published_at: details.body.security_advisory.published_at + alert_updated_at: details.body.security_advisory.updated_at + alert_withdrawn_at: details.body.security_advisory.withdrawn_at + + repository: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + repo_private: details.body.repository.private + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + + member: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + repo_private: details.body.repository.private + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + member_id: details.body.member.id + member_login: details.body.member.login + member_node_id: details.body.member.node_id + member_site_admin: details.body.member.site_admin + changes_perm_from: details.body.changes.permission.from + + team: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + repo_private: details.body.repository.private + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + repo_perm_from_admin: details.body.changes.repository.permissions.from.admin + repo_perm_from_pull: details.body.changes.repository.permissions.from.pull + repo_perm_from_push: details.body.changes.repository.permissions.from.push + repo_perm_admin: details.body.repository.permissions.admin + repo_perm_pull: details.body.repository.permissions.pull + repo_perm_push: details.body.repository.permissions.push + team_id: details.body.team.id + team_name: details.body.team.name + team_node_id: details.body.team.node_id + team_permission: details.body.team.permission + team_privacy: details.body.team.privacy + team_slug: details.body.team.slug + + team_add: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + repo_private: details.body.repository.private + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + repo_perm_from_admin: details.body.changes.repository.permissions.from.admin + repo_perm_from_pull: details.body.changes.repository.permissions.from.pull + repo_perm_from_push: details.body.changes.repository.permissions.from.push + repo_perm_admin: details.body.repository.permissions.admin + repo_perm_pull: details.body.repository.permissions.pull + repo_perm_push: details.body.repository.permissions.push + team_id: details.body.team.id + team_name: details.body.team.name + team_node_id: details.body.team.node_id + team_permission: details.body.team.permission + team_privacy: details.body.team.privacy + team_slug: details.body.team.slug + + organization: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + team_id: details.body.team.id + team_name: details.body.team.name + team_node_id: details.body.team.node_id + team_permission: details.body.team.permission + team_privacy: details.body.team.privacy + team_slug: details.body.team.slug + membership_type: details.body.membership.user.type + membership_admin: details.body.membership.user.site_admin + membership_node_id: details.body.membership.user.node_id + membership_login: details.body.membership.user.login + membership_id: details.body.membership.user.id + membership_state: details.body.membership.state + membership_role: details.body.membership.role + + membership: + username: details.body.sender.login + id: details.body.sender.id + scope: details.body.scope + action: details.body.action + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + member_id: details.body.member.id + member_login: details.body.member.login + member_node_id: details.body.member.node_id + member_site_admin: details.body.member.site_admin + team_id: details.body.team.id + team_name: details.body.team.name + team_node_id: details.body.team.node_id + team_permission: details.body.team.permission + team_privacy: details.body.team.privacy + team_slug: details.body.team.slug + + public: + username: details.body.sender.login + id: details.body.sender.id + repo_private: details.body.repository.private + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + + repository_import: + username: details.body.sender.login + id: details.body.sender.id + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + repo_private: details.body.repository.private + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + + release: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + repo_private: details.body.repository.private + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + release_id: details.body.release.id + release_node_id: details.body.release.node_id + release_tag_name: details.body.release.tag_name + release_name: details.body.release.name + release_author_login: details.body.release.author.login + release_author_id: details.body.release.author.id + release_author_node_id: details.body.release.author.node_id + release_author_type: details.body.release.author.type + release_author_site_admin: details.body.release.author.site_admin + + org_block: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + blocked_user_login: details.body.blocked_user.login + blocked_user_id: details.body.blocked_user.id + blocked_user_node_id: details.body.blocked_user.node_id + + github_app_authorization: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + + installation: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + repositories: details.body.repositories + install_id: details.body.installation.account.id + install_account_login: details.body.installation.account.login + install_account_node_id: details.body.installation.account.node_id + install_account_type: details.body.installation.account.type + install_account_site_admin: details.body.installation.account.site_admin \ No newline at end of file diff --git a/mq/plugins/github_webhooks.py b/mq/plugins/github_webhooks.py new file mode 100644 index 00000000..ee2bee8a --- /dev/null +++ b/mq/plugins/github_webhooks.py @@ -0,0 +1,65 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# Copyright (c) 2017 Mozilla Corporation + +import sys +import os +import jmespath +import yaml +from mozdef_util.utilities.toUTC import toUTC + + +class message(object): + def __init__(self): + ''' + Plugin used to fix object type discretions with cloudtrail messages + ''' + self.registration = ['githubeventsqs'] + self.priority = 10 + + with open('github_mapping.yml', 'r') as f: + map = f.read() + + yap = yaml.load(map) + self.eventtypes = yap.keys() + self.yap = yap + del(map) + + + def onMessage(self, message, metadata): + + if 'tags' not in message: + return (message, metadata) + if 'githubeventsqs' not in message['tags']: + return (message, metadata) + + newmessage = {} + newmessage['details'] = {} + + newmessage['category'] = 'github' + newmessage['tags'] = ['github', 'webhook'] + newmessage['eventsource'] = 'githubeventsqs' + if 'event' in message['details']: + newmessage['source'] = message['details']['event'] + else: + newmessage['source'] = 'UNKNOWN' + if 'request_id' in message['details']: + newmessage['details']['request_id'] = message['details']['request_id'] + else: + newmessage['details']['request_id'] = 'UNKNOWN' + + # iterate through top level keys - push, etc + if newmessage['source'] in self.eventtypes: + for key in self.yap[newmessage['source']]: + mappedvalue = jmespath.search(self.yap[newmessage['source']][key], message) + # JMESPath likes to silently return a None object + if mappedvalue is not None: + newmessage['details'][key] = mappedvalue + if 'commit_ts' in newmessage['details']: + newmessage['timestamp'] = newmessage['details']['commit_ts'] + newmessage['utctimestamp'] = toUTC(newmessage['details']['commit_ts']).isoformat() + else: + newmessage = None + + return (newmessage, metadata) \ No newline at end of file From 866ad7798ab59696e7deeac0484d4a1b3745b539 Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Mon, 5 Nov 2018 13:04:01 -0600 Subject: [PATCH 04/89] Add function to append hostname to ip for ssh lateral alert --- alerts/lib/alerttask.py | 13 +++++ alerts/ssh_lateral.py | 24 ++++----- tests/alerts/alert_test_suite.py | 11 ++++- tests/alerts/lib/test_alerttask.py | 29 +++++++++++ tests/alerts/test_ssh_lateral.py | 78 ++++++++++++++++++++++++++++++ 5 files changed, 142 insertions(+), 13 deletions(-) create mode 100644 tests/alerts/lib/test_alerttask.py create mode 100644 tests/alerts/test_ssh_lateral.py diff --git a/alerts/lib/alerttask.py b/alerts/lib/alerttask.py index a5e0e326..4f1ed69a 100644 --- a/alerts/lib/alerttask.py +++ b/alerts/lib/alerttask.py @@ -10,6 +10,8 @@ import json import kombu import os import sys +import socket +import netaddr from configlib import getConfig, OptionParser from datetime import datetime @@ -58,6 +60,17 @@ def getValueByPath(input_dict, path_string): return return_data +def add_hostname_to_ip(ip, output_format, require_internal=True): + ip_obj = netaddr.IPNetwork(ip)[0] + if require_internal and not ip_obj.is_private(): + return ip + try: + reversed_dns = socket.gethostbyaddr(ip) + return output_format.format(ip, reversed_dns[0]) + except socket.herror: + return ip + + class AlertTask(Task): abstract = True diff --git a/alerts/ssh_lateral.py b/alerts/ssh_lateral.py index 22b3baf6..dedd988c 100644 --- a/alerts/ssh_lateral.py +++ b/alerts/ssh_lateral.py @@ -5,10 +5,8 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # Copyright (c) 2017 Mozilla Corporation -from lib.alerttask import AlertTask -from mozdef_util.query_models import SearchQuery, TermMatch, QueryStringMatch, PhraseMatch -import json -import sys +from lib.alerttask import AlertTask, add_hostname_to_ip +from mozdef_util.query_models import SearchQuery, TermMatch, PhraseMatch import re import netaddr @@ -78,7 +76,7 @@ class SshLateral(AlertTask): self._config = self.parse_json_alert_config('ssh_lateral.json') def main(self): - search_query = SearchQuery(minutes=2) + search_query = SearchQuery(minutes=15) search_query.add_must([ TermMatch('category', 'syslog'), TermMatch('details.program', 'sshd'), @@ -123,8 +121,8 @@ class SshLateral(AlertTask): # Determine if the origin of the connection was from a source outside # of the exception policy, and in our address scope candidates = [] - sampleip = None - sampleuser = None + source_ips = [] + users = [] for x in aggreg['events']: m = re.match('Accepted publickey for (\S+) from (\S+).*', x['_source']['summary']) if m is not None and len(m.groups()) == 2: @@ -150,14 +148,16 @@ class SshLateral(AlertTask): # Check our exception list if self.exception_check(m.group(1), srchost, m.group(2)): continue - if sampleip is None: - sampleip = m.group(2) - if sampleuser is None: - sampleuser = m.group(1) + source_ips.append(m.group(2)) + users.append(m.group(1)) candidates.append(x) if len(candidates) == 0: return None - summary = 'SSH lateral movement outside policy: access to {} from {} as {}'.format(srchost, sampleip, sampleuser) + src_hosts_info = [] + for source_ip in source_ips: + src_hosts_info.append(add_hostname_to_ip(source_ip, '{0} ({1})')) + + summary = 'SSH lateral movement outside policy: access to {} from {} as {}'.format(srchost, ','.join(src_hosts_info), ','.join(users)) return self.createAlertDict(summary, category, tags, aggreg['events'], severity) diff --git a/tests/alerts/alert_test_suite.py b/tests/alerts/alert_test_suite.py index 9d978672..7939f9a2 100644 --- a/tests/alerts/alert_test_suite.py +++ b/tests/alerts/alert_test_suite.py @@ -13,6 +13,7 @@ sys.path.append(os.path.join(os.path.dirname(__file__), "../")) from unit_test_suite import UnitTestSuite from freezegun import freeze_time +import mock import copy import re @@ -22,6 +23,13 @@ sys.path.append(os.path.join(os.path.dirname(__file__), "../../alerts/lib")) from lib import alerttask +def mock_add_hostname_to_ip(ip): + if ip == '10.2.3.4': + return ['mock_hostname1.mozilla.org', ip] + else: + return ['mock.mozilla.org', ip] + + class AlertTestSuite(UnitTestSuite): def teardown(self): os.chdir(self.orig_path) @@ -142,7 +150,8 @@ class AlertTestSuite(UnitTestSuite): self.flush('events') - alert_task = test_case.run(alert_filename=self.alert_filename, alert_classname=self.alert_classname) + with mock.patch("socket.gethostbyaddr", side_effect=mock_add_hostname_to_ip): + alert_task = test_case.run(alert_filename=self.alert_filename, alert_classname=self.alert_classname) self.verify_alert_task(alert_task, test_case) def verify_rabbitmq_alert(self, found_alert, test_case): diff --git a/tests/alerts/lib/test_alerttask.py b/tests/alerts/lib/test_alerttask.py new file mode 100644 index 00000000..71d078cb --- /dev/null +++ b/tests/alerts/lib/test_alerttask.py @@ -0,0 +1,29 @@ +import os +import sys +sys.path.append(os.path.join(os.path.dirname(__file__), "../../../alerts/lib")) +from alerttask import add_hostname_to_ip + +import mock +import socket + + +def reverse_lookup(ip): + if ip == '10.1.1.1': + return ('test.domain.com', ip) + if ip == '8.8.8.8': + raise socket.herror + + +class TestAddHostnameToIP(object): + def setup(self): + self.formatted_string = '{0} ({1})' + + def test_internal_hostname(self): + with mock.patch("socket.gethostbyaddr", side_effect=reverse_lookup): + hostname_info = add_hostname_to_ip('10.1.1.1', self.formatted_string) + assert hostname_info == '10.1.1.1 (test.domain.com)' + + def test_external_hostname(self): + with mock.patch("socket.gethostbyaddr", side_effect=reverse_lookup): + hostname_info = add_hostname_to_ip('8.8.8.8', self.formatted_string) + assert hostname_info == '8.8.8.8' diff --git a/tests/alerts/test_ssh_lateral.py b/tests/alerts/test_ssh_lateral.py new file mode 100644 index 00000000..6fe01920 --- /dev/null +++ b/tests/alerts/test_ssh_lateral.py @@ -0,0 +1,78 @@ +from positive_alert_test_case import PositiveAlertTestCase +from negative_alert_test_case import NegativeAlertTestCase + +from alert_test_suite import AlertTestSuite + + +class TestSSHLateral(AlertTestSuite): + alert_filename = 'ssh_lateral' + alert_classname = 'SshLateral' + + # This event is the default positive event that will cause the + # alert to trigger + default_event = { + '_type': 'event', + '_source': { + 'category': 'syslog', + 'hostname': 'test-host.enterprise.mozilla.com', + 'summary': 'Accepted publickey for user1 from 10.2.3.4 port 19936 ssh2: RSA SHA256:ET72afGGbxabDersgSdQ+xJYB6ILXOFSDsLsTqDs', + 'details': { + 'program': 'sshd' + } + } + } + + # This alert is the expected result from running this task + default_alert = { + 'category': 'session', + 'severity': 'WARNING', + 'summary': 'SSH lateral movement outside policy: access to test-host.enterprise.mozilla.com from 10.2.3.4 (mock_hostname1.mozilla.org) as user1', + 'tags': ['sshd', 'syslog'], + } + + test_cases = [] + + test_cases.append( + PositiveAlertTestCase( + description='Positive test case with good event', + events=[AlertTestSuite.create_event(default_event)], + expected_alert=default_alert + ) + ) + + event = AlertTestSuite.create_event(default_event) + event['_source']['category'] = 'bad' + test_cases.append( + NegativeAlertTestCase( + description="Negative test case with bad event category", + events=[event], + ) + ) + + event = AlertTestSuite.create_event(default_event) + event['_source']['summary'] = 'some bad summary' + test_cases.append( + NegativeAlertTestCase( + description="Negative test case with bad event summary", + events=[event], + ) + ) + + event = AlertTestSuite.create_event(default_event) + event['_source']['details']['program'] = 'ftpd' + test_cases.append( + NegativeAlertTestCase( + description="Negative test case with bad event details.program", + events=[event], + ) + ) + + event = AlertTestSuite.create_event(default_event) + event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({'minutes': 16}) + event['_source']['receivedtimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({'minutes': 16}) + test_cases.append( + NegativeAlertTestCase( + description="Negative test case with an event with old timestamp", + events=[event], + ) + ) From 4ca98e512ae2d242316b599fc256a577bcab38a5 Mon Sep 17 00:00:00 2001 From: Michal Purzynski Date: Mon, 5 Nov 2018 11:53:54 -0800 Subject: [PATCH 05/89] python hates me --- mq/plugins/github_webhooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mq/plugins/github_webhooks.py b/mq/plugins/github_webhooks.py index ee2bee8a..786f753b 100644 --- a/mq/plugins/github_webhooks.py +++ b/mq/plugins/github_webhooks.py @@ -26,7 +26,6 @@ class message(object): self.yap = yap del(map) - def onMessage(self, message, metadata): if 'tags' not in message: @@ -62,4 +61,5 @@ class message(object): else: newmessage = None - return (newmessage, metadata) \ No newline at end of file + return (newmessage, metadata) + From 3b751ee9b6eb48df05bb2d385897735bb73299f1 Mon Sep 17 00:00:00 2001 From: Michal Purzynski Date: Mon, 5 Nov 2018 12:01:15 -0800 Subject: [PATCH 06/89] the pep check sometimes wants the empty line at the end of the file and sometimes it does not. go figure. --- mq/plugins/github_webhooks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mq/plugins/github_webhooks.py b/mq/plugins/github_webhooks.py index 786f753b..224e948f 100644 --- a/mq/plugins/github_webhooks.py +++ b/mq/plugins/github_webhooks.py @@ -62,4 +62,3 @@ class message(object): newmessage = None return (newmessage, metadata) - From 5de3dfddfeed2a59f632fe83e8607411610fbee4 Mon Sep 17 00:00:00 2001 From: Michal Purzynski Date: Mon, 5 Nov 2018 13:45:49 -0800 Subject: [PATCH 07/89] Unit tests --- tests/mq/plugins/test_github_webhooks.py | 2720 ++++++++++++++++++++++ 1 file changed, 2720 insertions(+) create mode 100644 tests/mq/plugins/test_github_webhooks.py diff --git a/tests/mq/plugins/test_github_webhooks.py b/tests/mq/plugins/test_github_webhooks.py new file mode 100644 index 00000000..5e55dd1f --- /dev/null +++ b/tests/mq/plugins/test_github_webhooks.py @@ -0,0 +1,2720 @@ +import os +import sys + +import mock +import json +import random +import string + +sys.path.append(os.path.join(os.path.dirname(__file__), '../../../mq/plugins')) +sys.path.append('../../../mozdef_util') +from mozdef_util.utilities.toUTC import toUTC +from github_webhooks import message + + +class TestSuricataFixup(object): + def setup(self): + self.plugin = message() + self.metadata = { + 'doc_type': 'event', + 'index': 'events' + } + + + def verify_metadata(self, metadata): + assert metadata['doc_type'] == 'event' + + + def test_defaults(self): + event = { + 'tags': 'githubeventsqs', + 'details': {}, + } + result, metadata = self.plugin.onMessage(event, self.metadata) + assert result is None + + + def test_nomatch_syslog(self): + event = { + "category": "syslog", + "processid": "0", + "receivedtimestamp": "2017-09-26T00:22:24.210945+00:00", + "severity": "7", + "utctimestamp": "2017-09-26T00:22:23+00:00", + "timestamp": "2017-09-26T00:22:23+00:00", + "hostname": "syslog1.private.scl3.mozilla.com", + "mozdefhostname": "mozdef1.private.scl3.mozilla.com", + "summary": "Connection from 10.22.74.208 port 9071 on 10.22.74.45 nsm githubeventsqs port 22\n", + "eventsource": "systemslogs", + "details": { + "processid": "21233", + "Random": 2, + "sourceipv4address": "10.22.74.208", + "hostname": "hostname1.subdomain.domain.com", + "program": "githubeventsqs", + "sourceipaddress": "10.22.74.208" + } + } + result, metadata = self.plugin.onMessage(event, self.metadata) + assert result['category'] == 'syslog' + assert result['eventsource'] == 'systemslogs' + assert result == event + + + def test_nomatch_auditd(self): + event = { + "category": "execve", + "processid": "0", + "receivedtimestamp": "2017-09-26T00:36:27.463745+00:00", + "severity": "INFO", + "utctimestamp": "2017-09-26T00:36:27+00:00", + "tags": [ + "audisp-json", + "2.1.1", + "audit" + ], + "summary": "Execve: sh -c sudo githubeventsqs nsm /usr/lib64/nagios/plugins/custom/check_auditd.sh", + "processname": "githubeventsqs", + "details": { + "fsuid": "398", + "tty": "(none)", + "uid": "398", + "process": "/bin/bash", + "auditkey": "exec", + "pid": "10553", + "processname": "sh", + "session": "16467", + "fsgid": "398", + "sgid": "398", + "auditserial": "3834716", + "inode": "1835094", + "ouid": "0", + "ogid": "0", + "suid": "398", + "originaluid": "0", + "gid": "398", + "originaluser": "root", + "ppid": "10552", + "cwd": "/", + "parentprocess": "githubeventsqs", + "euid": "398", + "path": "/bin/sh", + "rdev": "00:00", + "dev": "08:03", + "egid": "398", + "command": "sh -c githubeventsqs /usr/lib64/nagios/plugins/custom/check_auditd.sh", + "mode": "0100755", + "user": "nagios" + } + } + result, metadata = self.plugin.onMessage(event, self.metadata) + assert result['category'] == 'execve' + assert 'eventsource' not in result + assert result == event + + + def verify_defaults(self, result): + assert result['category'] == 'github' + assert result['tags'] == ['github', 'webhook'] + assert result['eventsource'] == 'githubeventsqs' + assert 'event' not in result['details'] + assert 'source' in result + + + def verify_meta(self, message, result): + assert result['details']['request_id'] == message['request_id'] + + + def verify_actor(self, message, result): + assert result['details']['id'] == message['body']['sender']['id'] + assert result['details']['username'] == message['body']['sender']['login'] + assert result['details']['sender_node_id'] == message['body']['sender']['node_id'] + assert result['details']['sender_site_admin'] == message['body']['sender']['site_admin'] + assert result['details']['sender_type'] == message['body']['sender']['type'] + + + def verify_repo(self, message, result): + assert result['details']['repo_id'] == message['body']['repository']['id'] + assert result['details']['repo_name'] == message['body']['repository']['name'] + assert result['details']['repo_owner_id'] == message['body']['repository']['owner']['id'] + assert result['details']['repo_owner_name'] == message['body']['repository']['owner']['name'] + assert result['details']['repo_owner_login'] == message['body']['repository']['owner']['login'] + assert result['details']['repo_owner_node_id'] == message['body']['repository']['owner']['node_id'] + assert result['details']['repo_owner_site_admin'] == message['body']['repository']['owner']['site_admin'] + assert result['details']['repo_private'] == message['body']['repository']['private'] + + + def verify_org(self, message, result): + assert result['details']['org_id'] == message['body']['organization']['id'] + assert result['details']['org_login'] == message['body']['organization']['login'] + assert result['details']['org_node_id'] == message['body']['organization']['node_id'] + + + def test_push(self): + message = { + "body": { + "forced": "true", + "compare": "https://github.com/web-platform-tests/wpt/compare/f000a9569fcb...41d50efea43f", + "after": "41d50efea43fb365d2a2d13b3fc18b933b7c3a75", + "sender": { + "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", + "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", + "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", + "url": "https://api.github.com/users/chromium-wpt-export-bot", + "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", + "html_url": "https://github.com/chromium-wpt-export-bot", + "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", + "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", + "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", + "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", + "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", + "site_admin": "false", + "login": "chromium-wpt-export-bot", + "type": "User", + "id": "25752892", + "node_id": "MDQ6VXNlcjI1NzUyODky" + }, + "repository": { + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "has_wiki": "true", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "mirror_url": "null", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "updated_at": "2018-11-01T00:51:49Z", + "svn_url": "https://github.com/web-platform-tests/wpt", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "owner": { + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "name": "web-platform-tests", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "url": "https://api.github.com/users/web-platform-tests", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "html_url": "https://github.com/web-platform-tests", + "email": "", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "site_admin": "false", + "login": "web-platform-tests", + "type": "Organization", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" + }, + "full_name": "web-platform-tests/wpt", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "id": "3618133", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "size": "305511", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "archived": "false", + "has_downloads": "true", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "has_projects": "true", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "watchers_count": "1845", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "organization": "web-platform-tests", + "stargazers_count": "1845", + "homepage": "http://irc.w3.org/?channels=testing", + "open_issues": "1328", + "fork": "false", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "has_pages": "true", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "html_url": "https://github.com/web-platform-tests/wpt", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "language": "HTML", + "watchers": "1845", + "private": "false", + "forks_count": "1523", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "has_issues": "true", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "master_branch": "master", + "forks": "1523", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "open_issues_count": "1317", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "name": "wpt", + "license": { + "spdx_id": "NOASSERTION", + "url": "null", + "node_id": "MDc6TGljZW5zZTA=", + "name": "Other", + "key": "other" + }, + "url": "https://github.com/web-platform-tests/wpt", + "stargazers": "1845", + "created_at": "1330865891", + "pushed_at": "1541037488", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "default_branch": "master", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "git_url": "git://github.com/web-platform-tests/wpt.git" + }, + "created": "false", + "deleted": "false", + "commits": [ + { + "committer": { + "username": "chromium-wpt-export-bot", + "email": "blink-w3c-test-autoroller@chromium.org", + "name": "Chromium WPT Sync" + }, + "added": [ + "css/css-flexbox/percentage-heights-005.html" + ], + "author": { + "username": "cbiesinger", + "email": "cbiesinger@chromium.org", + "name": "Christian Biesinger" + }, + "distinct": "true", + "timestamp": "2018-10-31T18:58:04-07:00", + "modified": [], + "url": "https://github.com/web-platform-tests/wpt/commit/41d50efea43fb365d2a2d13b3fc18b933b7c3a75", + "tree_id": "4ed38f691f2be4d19d821fdd316508350d11b42c", + "message": "[layoutng] Fix setting of fixed_block_is_definite\n\nWhen a flex item has a definite specified height (e.g. height: 100px),\nthen percentages in children of the flex item should resolve even\nif the flexbox does not have an explicit height, ie. does not match\nthe condition in https://drafts.csswg.org/css-flexbox/#definite-sizes\n\nBug: 885185\n\nChange-Id: Iba226f30e1e02e3a11273fa45fcdf1cef897120c\nReviewed-on: https://chromium-review.googlesource.com/c/1311534\nCommit-Queue: Christian Biesinger \nReviewed-by: Emil A Eklund \nReviewed-by: Morten Stenshorne \nCr-Commit-Position: refs/heads/master@{#604483}", + "removed": [], + "id": "41d50efea43fb365d2a2d13b3fc18b933b7c3a75" + } + ], + "pusher": { + "email": "blink-w3c-test-autoroller@chromium.org", + "name": "chromium-wpt-export-bot" + }, + "head_commit": { + "committer": { + "username": "chromium-wpt-export-bot", + "email": "blink-w3c-test-autoroller@chromium.org", + "name": "Chromium WPT Sync" + }, + "added": [ + "css/css-flexbox/percentage-heights-005.html" + ], + "author": { + "username": "cbiesinger", + "email": "cbiesinger@chromium.org", + "name": "Christian Biesinger" + }, + "distinct": "true", + "timestamp": "2018-10-31T18:58:04-07:00", + "modified": [], + "url": "https://github.com/web-platform-tests/wpt/commit/41d50efea43fb365d2a2d13b3fc18b933b7c3a75", + "tree_id": "4ed38f691f2be4d19d821fdd316508350d11b42c", + "message": "[layoutng] Fix setting of fixed_block_is_definite\n\nWhen a flex item has a definite specified height (e.g. height: 100px),\nthen percentages in children of the flex item should resolve even\nif the flexbox does not have an explicit height, ie. does not match\nthe condition in https://drafts.csswg.org/css-flexbox/#definite-sizes\n\nBug: 885185\n\nChange-Id: Iba226f30e1e02e3a11273fa45fcdf1cef897120c\nReviewed-on: https://chromium-review.googlesource.com/c/1311534\nCommit-Queue: Christian Biesinger \nReviewed-by: Emil A Eklund \nReviewed-by: Morten Stenshorne \nCr-Commit-Position: refs/heads/master@{#604483}", + "removed": [], + "id": "41d50efea43fb365d2a2d13b3fc18b933b7c3a75" + }, + "organization": { + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "description": "", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "url": "https://api.github.com/orgs/web-platform-tests", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "login": "web-platform-tests", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" + }, + "ref": "refs/heads/chromium-export-cl-1311534", + "base_ref": "null", + "before": "f000a9569fcb918a3c98fb93b5acd0218afa19ab" + }, + "event": "push", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" + } + + event = { + 'tags': 'githubeventsqs' + } + event['details'] = message + result, metadata = self.plugin.onMessage(event, self.metadata) + self.verify_defaults(result) + self.verify_metadata(metadata) + self.verify_meta(message, result) + self.verify_actor(message, result) + self.verify_repo(message, result) + self.verify_org(message, result) + assert result['source'] == 'push' + assert result['details']['created'] == message['body']['created'] + assert result['details']['deleted'] == message['body']['deleted'] + assert result['details']['forced'] == message['body']['forced'] + assert result['details']['commits'] == message['body']['commits'] + assert result['details']['ref'] == message['body']['ref'] + assert result['details']['email'] == message['body']['pusher']['email'] + assert result['details']['commit_author'] == message['body']['head_commit']['author']['email'] + assert result['details']['committer'] == message['body']['head_commit']['committer']['email'] + assert result['details']['commit_id'] == message['body']['head_commit']['id'] + assert result['details']['commit_msg'] == message['body']['head_commit']['message'] + assert result['details']['commit_ts'] == message['body']['head_commit']['timestamp'] + assert result['details']['commit_url'] == message['body']['head_commit']['url'] + assert result['details']['email'] == message['body']['pusher']['email'] + + + def test_delete(self): + message = { + "body": { + "ref": "gecko/1499958", + "ref_type": "branch", + "sender": { + "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", + "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", + "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", + "url": "https://api.github.com/users/chromium-wpt-export-bot", + "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", + "html_url": "https://github.com/chromium-wpt-export-bot", + "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", + "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", + "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", + "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", + "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", + "site_admin": "false", + "login": "chromium-wpt-export-bot", + "type": "User", + "id": "25752892", + "node_id": "MDQ6VXNlcjI1NzUyODky" + }, + "repository": { + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "has_wiki": "true", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "mirror_url": "null", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "updated_at": "2018-11-01T00:51:49Z", + "svn_url": "https://github.com/web-platform-tests/wpt", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "owner": { + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "name": "web-platform-tests", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "url": "https://api.github.com/users/web-platform-tests", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "html_url": "https://github.com/web-platform-tests", + "email": "", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "site_admin": "false", + "login": "web-platform-tests", + "type": "Organization", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" + }, + "full_name": "web-platform-tests/wpt", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "id": "3618133", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "size": "305511", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "archived": "false", + "has_downloads": "true", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "has_projects": "true", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "watchers_count": "1845", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "organization": "web-platform-tests", + "stargazers_count": "1845", + "homepage": "http://irc.w3.org/?channels=testing", + "open_issues": "1328", + "fork": "false", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "has_pages": "true", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "html_url": "https://github.com/web-platform-tests/wpt", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "language": "HTML", + "watchers": "1845", + "private": "false", + "forks_count": "1523", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "has_issues": "true", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "master_branch": "master", + "forks": "1523", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "open_issues_count": "1317", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "name": "wpt", + "license": { + "spdx_id": "NOASSERTION", + "url": "null", + "node_id": "MDc6TGljZW5zZTA=", + "name": "Other", + "key": "other" + }, + "url": "https://github.com/web-platform-tests/wpt", + "stargazers": "1845", + "created_at": "1330865891", + "pushed_at": "1541037488", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "default_branch": "master", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "git_url": "git://github.com/web-platform-tests/wpt.git" + }, + "organization": { + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "description": "", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "url": "https://api.github.com/orgs/web-platform-tests", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "login": "web-platform-tests", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" + }, + }, + "event": "delete", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" + } + + event = { + 'tags': 'githubeventsqs' + } + event['details'] = message + result, metadata = self.plugin.onMessage(event, self.metadata) + self.verify_defaults(result) + self.verify_metadata(metadata) + self.verify_meta(message, result) + self.verify_actor(message, result) + self.verify_repo(message, result) + self.verify_org(message, result) + assert result['source'] == 'delete' + assert result['details']['ref'] == message['body']['ref'] + assert result['details']['ref_type'] == message['body']['ref_type'] + + + def test_create(self): + message = { + "body": { + "ref": "gecko/1499958", + "ref_type": "branch", + "sender": { + "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", + "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", + "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", + "url": "https://api.github.com/users/chromium-wpt-export-bot", + "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", + "html_url": "https://github.com/chromium-wpt-export-bot", + "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", + "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", + "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", + "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", + "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", + "site_admin": "false", + "login": "chromium-wpt-export-bot", + "type": "User", + "id": "25752892", + "node_id": "MDQ6VXNlcjI1NzUyODky" + }, + "repository": { + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "has_wiki": "true", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "mirror_url": "null", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "updated_at": "2018-11-01T00:51:49Z", + "svn_url": "https://github.com/web-platform-tests/wpt", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "owner": { + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "name": "web-platform-tests", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "url": "https://api.github.com/users/web-platform-tests", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "html_url": "https://github.com/web-platform-tests", + "email": "", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "site_admin": "false", + "login": "web-platform-tests", + "type": "Organization", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" + }, + "full_name": "web-platform-tests/wpt", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "id": "3618133", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "size": "305511", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "archived": "false", + "has_downloads": "true", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "has_projects": "true", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "watchers_count": "1845", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "organization": "web-platform-tests", + "stargazers_count": "1845", + "homepage": "http://irc.w3.org/?channels=testing", + "open_issues": "1328", + "fork": "false", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "has_pages": "true", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "html_url": "https://github.com/web-platform-tests/wpt", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "language": "HTML", + "watchers": "1845", + "private": "false", + "forks_count": "1523", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "has_issues": "true", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "master_branch": "master", + "forks": "1523", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "open_issues_count": "1317", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "name": "wpt", + "license": { + "spdx_id": "NOASSERTION", + "url": "null", + "node_id": "MDc6TGljZW5zZTA=", + "name": "Other", + "key": "other" + }, + "url": "https://github.com/web-platform-tests/wpt", + "stargazers": "1845", + "created_at": "1330865891", + "pushed_at": "1541037488", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "default_branch": "master", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "git_url": "git://github.com/web-platform-tests/wpt.git" + }, + "organization": { + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "description": "", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "url": "https://api.github.com/orgs/web-platform-tests", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "login": "web-platform-tests", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" + }, + }, + "event": "create", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" + } + + event = { + 'tags': 'githubeventsqs' + } + event['details'] = message + result, metadata = self.plugin.onMessage(event, self.metadata) + self.verify_defaults(result) + self.verify_metadata(metadata) + self.verify_meta(message, result) + self.verify_actor(message, result) + self.verify_repo(message, result) + self.verify_org(message, result) + assert result['source'] == 'create' + assert result['details']['ref'] == message['body']['ref'] + assert result['details']['ref_type'] == message['body']['ref_type'] + + + def test_repository_vulnerability_alert(self): + message = { + "body": { + "action": "create", + "alert":{ + "affected_package_name": "requests", + "external_reference": "https://nvd.nist.gov/vuln/detail/CVE-2018-18074", + "external_identifier": "CVE-2018-18074", + "affected_range": "<= 2.19.1", + "id": "65626688", + "fixed_in": "2.20.0", + "dismisser": { + "login":"octocat", + "id":"1", + "node_id": "MDQ6VXNlcjIxMDMxMDY3", + "avatar_url":"https://github.com/images/error/octocat_happy.gif", + "gravatar_id":"", + "url":"https://api.github.com/users/octocat", + "html_url":"https://github.com/octocat", + "followers_url":"https://api.github.com/users/octocat/followers", + "following_url":"https://api.github.com/users/octocat/following{/other_user}", + "gists_url":"https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url":"https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url":"https://api.github.com/users/octocat/subscriptions", + "organizations_url":"https://api.github.com/users/octocat/orgs", + "repos_url":"https://api.github.com/users/octocat/repos", + "events_url":"https://api.github.com/users/octocat/events{/privacy}", + "received_events_url":"https://api.github.com/users/octocat/received_events", + "type":"User", + "site_admin":"true", + }, + "dismiss_reason": "I'm too lazy to fix this", + "dismissed_at": "2017-10-25T00:00:00+00:00", + }, + "sender": { + "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", + "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", + "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", + "url": "https://api.github.com/users/chromium-wpt-export-bot", + "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", + "html_url": "https://github.com/chromium-wpt-export-bot", + "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", + "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", + "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", + "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", + "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", + "site_admin": "false", + "login": "chromium-wpt-export-bot", + "type": "User", + "id": "25752892", + "node_id": "MDQ6VXNlcjI1NzUyODky" + }, + "repository": { + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "has_wiki": "true", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "mirror_url": "null", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "updated_at": "2018-11-01T00:51:49Z", + "svn_url": "https://github.com/web-platform-tests/wpt", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "owner": { + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "name": "web-platform-tests", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "url": "https://api.github.com/users/web-platform-tests", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "html_url": "https://github.com/web-platform-tests", + "email": "", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "site_admin": "false", + "login": "web-platform-tests", + "type": "Organization", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" + }, + "full_name": "web-platform-tests/wpt", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "id": "3618133", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "size": "305511", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "archived": "false", + "has_downloads": "true", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "has_projects": "true", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "watchers_count": "1845", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "organization": "web-platform-tests", + "stargazers_count": "1845", + "homepage": "http://irc.w3.org/?channels=testing", + "open_issues": "1328", + "fork": "false", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "has_pages": "true", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "html_url": "https://github.com/web-platform-tests/wpt", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "language": "HTML", + "watchers": "1845", + "private": "false", + "forks_count": "1523", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "has_issues": "true", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "master_branch": "master", + "forks": "1523", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "open_issues_count": "1317", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "name": "wpt", + "license": { + "spdx_id": "NOASSERTION", + "url": "null", + "node_id": "MDc6TGljZW5zZTA=", + "name": "Other", + "key": "other" + }, + "url": "https://github.com/web-platform-tests/wpt", + "stargazers": "1845", + "created_at": "1330865891", + "pushed_at": "1541037488", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "default_branch": "master", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "git_url": "git://github.com/web-platform-tests/wpt.git" + }, + "organization": { + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "description": "", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "url": "https://api.github.com/orgs/web-platform-tests", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "login": "web-platform-tests", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" + }, + }, + "event": "repository_vulnerability_alert", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" + } + + event = { + 'tags': 'githubeventsqs' + } + event['details'] = message + result, metadata = self.plugin.onMessage(event, self.metadata) + self.verify_defaults(result) + self.verify_metadata(metadata) + self.verify_meta(message, result) + self.verify_actor(message, result) + self.verify_repo(message, result) + self.verify_org(message, result) + assert result['source'] == 'repository_vulnerability_alert' + assert result['details']['action'] == message['body']['action'] + assert result['details']['alert_package'] == message['body']['alert']['affected_package_name'] + assert result['details']['alert_range'] == message['body']['alert']['affected_range'] + assert result['details']['alert_extid'] == message['body']['alert']['external_identifier'] + assert result['details']['alert_extref'] == message['body']['alert']['external_reference'] + assert result['details']['alert_fixed'] == message['body']['alert']['fixed_in'] + assert result['details']['alert_id'] == message['body']['alert']['id'] + #assert result['details']['dismiss_reason'] == message['body']['alert']['dismiss_reason'] + #assert result['details']['dismiss_at'] == message['body']['alert']['dismissed_at'] + assert result['details']['dismiss_user'] == message['body']['alert']['dismisser']['login'] + assert result['details']['dismiss_id'] == message['body']['alert']['dismisser']['id'] + assert result['details']['dismiss_node_id'] == message['body']['alert']['dismisser']['node_id'] + assert result['details']['dismiss_type'] == message['body']['alert']['dismisser']['type'] + assert result['details']['dismiss_site_admin'] == message['body']['alert']['dismisser']['site_admin'] + + def test_security_advisory(self): + message = { + "body": + { + "action": "published", + "security_advisory": { + "ghsa_id": "GHSA-rf4j-j272-fj86", + "summary": "Moderate severity vulnerability that affects django", + "description": "django.contrib.auth.forms.AuthenticationForm in Django 2.0 before 2.0.2, and 1.11.8 and 1.11.9, allows remote attackers to obtain potentially sensitive information by leveraging data exposure from the confirm_login_allowed() method, as demonstrated by discovering whether a user account is inactive.", + "severity": "moderate", + "identifiers": [ + { + "value": "GHSA-rf4j-j272-fj86", + "type": "GHSA" + }, + { + "value": "CVE-2018-6188", + "type": "CVE" + } + ], + "references": [ + { + "url": "https://nvd.nist.gov/vuln/detail/CVE-2018-6188" + } + ], + "published_at": "2018-10-03T21:13:54Z", + "updated_at": "2018-10-03T21:13:54Z", + "withdrawn_at": "null", + "vulnerabilities": [ + { + "package": { + "ecosystem": "pip", + "name": "django" + }, + "severity": "moderate", + "vulnerable_version_range": ">= 2.0.0, < 2.0.2", + "first_patched_version": { + "identifier": "2.0.2" + } + }, + { + "package": { + "ecosystem": "pip", + "name": "django" + }, + "severity": "moderate", + "vulnerable_version_range": ">= 1.11.8, < 1.11.10", + "first_patched_version": { + "identifier": "1.11.10" + } + } + ] + }, +}, +"event": "security_advisory", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd", + } + event = { + 'tags': 'githubeventsqs' + } + event['details'] = message + result, metadata = self.plugin.onMessage(event, self.metadata) + self.verify_defaults(result) + self.verify_metadata(metadata) + assert result['source'] == 'security_advisory' + assert result['details']['action'] == message['body']['action'] + + def test_repository(self): + message = { + "body": { + "action": "deleted", + "sender": { + "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", + "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", + "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", + "url": "https://api.github.com/users/chromium-wpt-export-bot", + "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", + "html_url": "https://github.com/chromium-wpt-export-bot", + "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", + "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", + "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", + "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", + "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", + "site_admin": "false", + "login": "chromium-wpt-export-bot", + "type": "User", + "id": "25752892", + "node_id": "MDQ6VXNlcjI1NzUyODky" + }, + "repository": { + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "has_wiki": "true", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "mirror_url": "null", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "updated_at": "2018-11-01T00:51:49Z", + "svn_url": "https://github.com/web-platform-tests/wpt", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "owner": { + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "name": "web-platform-tests", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "url": "https://api.github.com/users/web-platform-tests", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "html_url": "https://github.com/web-platform-tests", + "email": "", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "site_admin": "false", + "login": "web-platform-tests", + "type": "Organization", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" + }, + "full_name": "web-platform-tests/wpt", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "id": "3618133", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "size": "305511", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "archived": "false", + "has_downloads": "true", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "has_projects": "true", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "watchers_count": "1845", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "organization": "web-platform-tests", + "stargazers_count": "1845", + "homepage": "http://irc.w3.org/?channels=testing", + "open_issues": "1328", + "fork": "false", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "has_pages": "true", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "html_url": "https://github.com/web-platform-tests/wpt", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "language": "HTML", + "watchers": "1845", + "private": "false", + "forks_count": "1523", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "has_issues": "true", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "master_branch": "master", + "forks": "1523", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "open_issues_count": "1317", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "name": "wpt", + "license": { + "spdx_id": "NOASSERTION", + "url": "null", + "node_id": "MDc6TGljZW5zZTA=", + "name": "Other", + "key": "other" + }, + "url": "https://github.com/web-platform-tests/wpt", + "stargazers": "1845", + "created_at": "1330865891", + "pushed_at": "1541037488", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "default_branch": "master", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "git_url": "git://github.com/web-platform-tests/wpt.git" + }, + "organization": { + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "description": "", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "url": "https://api.github.com/orgs/web-platform-tests", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "login": "web-platform-tests", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" + }, + }, + "event": "repository", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" + } + + event = { + 'tags': 'githubeventsqs' + } + event['details'] = message + result, metadata = self.plugin.onMessage(event, self.metadata) + self.verify_defaults(result) + self.verify_metadata(metadata) + self.verify_meta(message, result) + self.verify_actor(message, result) + self.verify_repo(message, result) + self.verify_org(message, result) + assert result['source'] == 'repository' + assert result['details']['action'] == message['body']['action'] + + + def test_member(self): + message = { + "body": { + "member": { + "id": "60618", + "login": "emmairwin", + "node_id": "MDQ6VXNlcjYwNjE4", + "site_admin": "false", + }, + "changes": { + "permission": { + "from": "write", + }, + }, + "action": "added", + "sender": { + "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", + "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", + "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", + "url": "https://api.github.com/users/chromium-wpt-export-bot", + "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", + "html_url": "https://github.com/chromium-wpt-export-bot", + "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", + "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", + "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", + "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", + "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", + "site_admin": "false", + "login": "chromium-wpt-export-bot", + "type": "User", + "id": "25752892", + "node_id": "MDQ6VXNlcjI1NzUyODky" + }, + "repository": { + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "has_wiki": "true", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "mirror_url": "null", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "updated_at": "2018-11-01T00:51:49Z", + "svn_url": "https://github.com/web-platform-tests/wpt", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "owner": { + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "name": "web-platform-tests", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "url": "https://api.github.com/users/web-platform-tests", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "html_url": "https://github.com/web-platform-tests", + "email": "", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "site_admin": "false", + "login": "web-platform-tests", + "type": "Organization", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" + }, + "full_name": "web-platform-tests/wpt", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "id": "3618133", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "size": "305511", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "archived": "false", + "has_downloads": "true", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "has_projects": "true", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "watchers_count": "1845", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "organization": "web-platform-tests", + "stargazers_count": "1845", + "homepage": "http://irc.w3.org/?channels=testing", + "open_issues": "1328", + "fork": "false", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "has_pages": "true", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "html_url": "https://github.com/web-platform-tests/wpt", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "language": "HTML", + "watchers": "1845", + "private": "false", + "forks_count": "1523", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "has_issues": "true", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "master_branch": "master", + "forks": "1523", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "open_issues_count": "1317", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "name": "wpt", + "license": { + "spdx_id": "NOASSERTION", + "url": "null", + "node_id": "MDc6TGljZW5zZTA=", + "name": "Other", + "key": "other" + }, + "url": "https://github.com/web-platform-tests/wpt", + "stargazers": "1845", + "created_at": "1330865891", + "pushed_at": "1541037488", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "default_branch": "master", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "git_url": "git://github.com/web-platform-tests/wpt.git" + }, + "organization": { + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "description": "", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "url": "https://api.github.com/orgs/web-platform-tests", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "login": "web-platform-tests", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" + }, + }, + "event": "member", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" + } + + event = { + 'tags': 'githubeventsqs' + } + event['details'] = message + result, metadata = self.plugin.onMessage(event, self.metadata) + self.verify_defaults(result) + self.verify_metadata(metadata) + self.verify_meta(message, result) + self.verify_actor(message, result) + self.verify_repo(message, result) + self.verify_org(message, result) + assert result['source'] == 'member' + assert result['details']['action'] == message['body']['action'] + assert result['details']['member_id'] == message['body']['member']['id'] + assert result['details']['member_login'] == message['body']['member']['login'] + assert result['details']['member_node_id'] == message['body']['member']['node_id'] + assert result['details']['member_site_admin'] == message['body']['member']['site_admin'] + assert result['details']['changes_perm_from'] == message['body']['changes']['permission']['from'] + + def test_team(self): + message = { + "body": { + "team": { + "id": "9060454", + "name": "asecretteam", + "login": "alamakota", + "node_id": "MYQ6VXK4fuwwNAye", + "permission": "pull", + "privacy": "secret", + "slug": "asecretteam", + }, + "changes": { + "repository": { + "permissions": { + "from": { + "admin": "false", + "pull": "false", + "push": "false", + }, + }, + }, + }, + "action": "edited", + "sender": { + "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", + "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", + "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", + "url": "https://api.github.com/users/chromium-wpt-export-bot", + "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", + "html_url": "https://github.com/chromium-wpt-export-bot", + "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", + "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", + "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", + "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", + "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", + "site_admin": "false", + "login": "chromium-wpt-export-bot", + "type": "User", + "id": "25752892", + "node_id": "MDQ6VXNlcjI1NzUyODky" + }, + "repository": { + "permissions": { + "admin": "true", + "pull": "true", + "push": "true", + }, + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "has_wiki": "true", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "mirror_url": "null", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "updated_at": "2018-11-01T00:51:49Z", + "svn_url": "https://github.com/web-platform-tests/wpt", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "owner": { + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "name": "web-platform-tests", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "url": "https://api.github.com/users/web-platform-tests", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "html_url": "https://github.com/web-platform-tests", + "email": "", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "site_admin": "false", + "login": "web-platform-tests", + "type": "Organization", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" + }, + "full_name": "web-platform-tests/wpt", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "id": "3618133", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "size": "305511", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "archived": "false", + "has_downloads": "true", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "has_projects": "true", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "watchers_count": "1845", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "organization": "web-platform-tests", + "stargazers_count": "1845", + "homepage": "http://irc.w3.org/?channels=testing", + "open_issues": "1328", + "fork": "false", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "has_pages": "true", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "html_url": "https://github.com/web-platform-tests/wpt", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "language": "HTML", + "watchers": "1845", + "private": "false", + "forks_count": "1523", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "has_issues": "true", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "master_branch": "master", + "forks": "1523", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "open_issues_count": "1317", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "name": "wpt", + "license": { + "spdx_id": "NOASSERTION", + "url": "null", + "node_id": "MDc6TGljZW5zZTA=", + "name": "Other", + "key": "other" + }, + "url": "https://github.com/web-platform-tests/wpt", + "stargazers": "1845", + "created_at": "1330865891", + "pushed_at": "1541037488", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "default_branch": "master", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "git_url": "git://github.com/web-platform-tests/wpt.git" + }, + "organization": { + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "description": "", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "url": "https://api.github.com/orgs/web-platform-tests", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "login": "web-platform-tests", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" + }, + }, + "event": "team", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" + } + + event = { + 'tags': 'githubeventsqs' + } + event['details'] = message + result, metadata = self.plugin.onMessage(event, self.metadata) + self.verify_defaults(result) + self.verify_metadata(metadata) + self.verify_meta(message, result) + self.verify_actor(message, result) + self.verify_repo(message, result) + self.verify_org(message, result) + assert result['source'] == 'team' + assert result['details']['action'] == message['body']['action'] + assert result['details']['repo_perm_from_admin'] == message['body']['changes']['repository']['permissions']['from']['admin'] + assert result['details']['repo_perm_from_pull'] == message['body']['changes']['repository']['permissions']['from']['pull'] + assert result['details']['repo_perm_from_push'] == message['body']['changes']['repository']['permissions']['from']['push'] + assert result['details']['repo_perm_admin'] == message['body']['repository']['permissions']['admin'] + assert result['details']['repo_perm_pull'] == message['body']['repository']['permissions']['pull'] + assert result['details']['repo_perm_push'] == message['body']['repository']['permissions']['push'] + assert result['details']['team_id'] == message['body']['team']['id'] + assert result['details']['team_name'] == message['body']['team']['name'] + assert result['details']['team_node_id'] == message['body']['team']['node_id'] + assert result['details']['team_permission'] == message['body']['team']['permission'] + assert result['details']['team_privacy'] == message['body']['team']['privacy'] + assert result['details']['team_slug'] == message['body']['team']['slug'] + + + def test_team_add(self): + message = { + "body": { + "team": { + "id": "9060454", + "name": "asecretteam", + "login": "alamakota", + "node_id": "MYQ6VXK4fuwwNAye", + "permission": "pull", + "privacy": "secret", + "slug": "asecretteam", + }, + "changes": { + "repository": { + "permissions": { + "from": { + "admin": "false", + "pull": "false", + "push": "false", + }, + }, + }, + }, + "sender": { + "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", + "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", + "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", + "url": "https://api.github.com/users/chromium-wpt-export-bot", + "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", + "html_url": "https://github.com/chromium-wpt-export-bot", + "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", + "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", + "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", + "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", + "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", + "site_admin": "false", + "login": "chromium-wpt-export-bot", + "type": "User", + "id": "25752892", + "node_id": "MDQ6VXNlcjI1NzUyODky" + }, + "repository": { + "permissions": { + "admin": "true", + "pull": "true", + "push": "true", + }, + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "has_wiki": "true", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "mirror_url": "null", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "updated_at": "2018-11-01T00:51:49Z", + "svn_url": "https://github.com/web-platform-tests/wpt", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "owner": { + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "name": "web-platform-tests", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "url": "https://api.github.com/users/web-platform-tests", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "html_url": "https://github.com/web-platform-tests", + "email": "", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "site_admin": "false", + "login": "web-platform-tests", + "type": "Organization", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" + }, + "full_name": "web-platform-tests/wpt", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "id": "3618133", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "size": "305511", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "archived": "false", + "has_downloads": "true", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "has_projects": "true", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "watchers_count": "1845", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "organization": "web-platform-tests", + "stargazers_count": "1845", + "homepage": "http://irc.w3.org/?channels=testing", + "open_issues": "1328", + "fork": "false", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "has_pages": "true", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "html_url": "https://github.com/web-platform-tests/wpt", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "language": "HTML", + "watchers": "1845", + "private": "false", + "forks_count": "1523", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "has_issues": "true", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "master_branch": "master", + "forks": "1523", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "open_issues_count": "1317", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "name": "wpt", + "license": { + "spdx_id": "NOASSERTION", + "url": "null", + "node_id": "MDc6TGljZW5zZTA=", + "name": "Other", + "key": "other" + }, + "url": "https://github.com/web-platform-tests/wpt", + "stargazers": "1845", + "created_at": "1330865891", + "pushed_at": "1541037488", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "default_branch": "master", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "git_url": "git://github.com/web-platform-tests/wpt.git" + }, + "organization": { + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "description": "", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "url": "https://api.github.com/orgs/web-platform-tests", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "login": "web-platform-tests", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" + }, + }, + "event": "team_add", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" + } + + event = { + 'tags': 'githubeventsqs' + } + event['details'] = message + result, metadata = self.plugin.onMessage(event, self.metadata) + self.verify_defaults(result) + self.verify_metadata(metadata) + self.verify_meta(message, result) + self.verify_actor(message, result) + self.verify_repo(message, result) + self.verify_org(message, result) + assert result['source'] == 'team_add' + assert result['details']['repo_perm_from_admin'] == message['body']['changes']['repository']['permissions']['from']['admin'] + assert result['details']['repo_perm_from_pull'] == message['body']['changes']['repository']['permissions']['from']['pull'] + assert result['details']['repo_perm_from_push'] == message['body']['changes']['repository']['permissions']['from']['push'] + assert result['details']['repo_perm_admin'] == message['body']['repository']['permissions']['admin'] + assert result['details']['repo_perm_pull'] == message['body']['repository']['permissions']['pull'] + assert result['details']['repo_perm_push'] == message['body']['repository']['permissions']['push'] + assert result['details']['team_id'] == message['body']['team']['id'] + assert result['details']['team_name'] == message['body']['team']['name'] + assert result['details']['team_node_id'] == message['body']['team']['node_id'] + assert result['details']['team_permission'] == message['body']['team']['permission'] + assert result['details']['team_privacy'] == message['body']['team']['privacy'] + assert result['details']['team_slug'] == message['body']['team']['slug'] + + + def test_organization(self): + message = { + "body": { + "team": { + "id": "9060454", + "name": "asecretteam", + "login": "alamakota", + "node_id": "MYQ6VXK4fuwwNAye", + "permission": "pull", + "privacy": "secret", + "slug": "asecretteam", + }, + "membership": { + "user": { + "id": "893282", + "login": "alamakota", + "node_id": "MDQ6VXNlcjUwMTkyMzQ=", + "site_admin": "false", + "type": "User", + }, + "role": "member", + "state": "pending", + }, + "action": "member_added", + "sender": { + "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", + "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", + "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", + "url": "https://api.github.com/users/chromium-wpt-export-bot", + "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", + "html_url": "https://github.com/chromium-wpt-export-bot", + "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", + "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", + "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", + "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", + "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", + "site_admin": "false", + "login": "chromium-wpt-export-bot", + "type": "User", + "id": "25752892", + "node_id": "MDQ6VXNlcjI1NzUyODky" + }, + "repository": { + "permissions": { + "admin": "true", + "pull": "true", + "push": "true", + }, + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "has_wiki": "true", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "mirror_url": "null", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "updated_at": "2018-11-01T00:51:49Z", + "svn_url": "https://github.com/web-platform-tests/wpt", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "owner": { + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "name": "web-platform-tests", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "url": "https://api.github.com/users/web-platform-tests", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "html_url": "https://github.com/web-platform-tests", + "email": "", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "site_admin": "false", + "login": "web-platform-tests", + "type": "Organization", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" + }, + "full_name": "web-platform-tests/wpt", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "id": "3618133", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "size": "305511", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "archived": "false", + "has_downloads": "true", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "has_projects": "true", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "watchers_count": "1845", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "organization": "web-platform-tests", + "stargazers_count": "1845", + "homepage": "http://irc.w3.org/?channels=testing", + "open_issues": "1328", + "fork": "false", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "has_pages": "true", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "html_url": "https://github.com/web-platform-tests/wpt", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "language": "HTML", + "watchers": "1845", + "private": "false", + "forks_count": "1523", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "has_issues": "true", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "master_branch": "master", + "forks": "1523", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "open_issues_count": "1317", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "name": "wpt", + "license": { + "spdx_id": "NOASSERTION", + "url": "null", + "node_id": "MDc6TGljZW5zZTA=", + "name": "Other", + "key": "other" + }, + "url": "https://github.com/web-platform-tests/wpt", + "stargazers": "1845", + "created_at": "1330865891", + "pushed_at": "1541037488", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "default_branch": "master", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "git_url": "git://github.com/web-platform-tests/wpt.git" + }, + "organization": { + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "description": "", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "url": "https://api.github.com/orgs/web-platform-tests", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "login": "web-platform-tests", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" + }, + }, + "event": "organization", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" + } + + event = { + 'tags': 'githubeventsqs' + } + event['details'] = message + result, metadata = self.plugin.onMessage(event, self.metadata) + self.verify_defaults(result) + self.verify_metadata(metadata) + self.verify_meta(message, result) + self.verify_actor(message, result) + self.verify_org(message, result) + assert result['source'] == 'organization' + assert result['details']['action'] == message['body']['action'] + assert result['details']['team_id'] == message['body']['team']['id'] + assert result['details']['team_name'] == message['body']['team']['name'] + assert result['details']['team_node_id'] == message['body']['team']['node_id'] + assert result['details']['team_permission'] == message['body']['team']['permission'] + assert result['details']['team_privacy'] == message['body']['team']['privacy'] + assert result['details']['team_slug'] == message['body']['team']['slug'] + assert result['details']['membership_type'] == message['body']['membership']['user']['type'] + assert result['details']['membership_admin'] == message['body']['membership']['user']['site_admin'] + assert result['details']['membership_node_id'] == message['body']['membership']['user']['node_id'] + assert result['details']['membership_login'] == message['body']['membership']['user']['login'] + assert result['details']['membership_id'] == message['body']['membership']['user']['id'] + assert result['details']['membership_state'] == message['body']['membership']['state'] + assert result['details']['membership_role'] == message['body']['membership']['role'] + + def test_membership(self): + message = { + "body": { + "action": "member_added", + "scope": "team", + "sender": { + "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", + "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", + "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", + "url": "https://api.github.com/users/chromium-wpt-export-bot", + "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", + "html_url": "https://github.com/chromium-wpt-export-bot", + "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", + "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", + "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", + "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", + "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", + "site_admin": "false", + "login": "chromium-wpt-export-bot", + "type": "User", + "id": "25752892", + "node_id": "MDQ6VXNlcjI1NzUyODky" + }, + "repository": { + "permissions": { + "admin": "true", + "pull": "true", + "push": "true", + }, + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "has_wiki": "true", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "mirror_url": "null", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "updated_at": "2018-11-01T00:51:49Z", + "svn_url": "https://github.com/web-platform-tests/wpt", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "owner": { + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "name": "web-platform-tests", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "url": "https://api.github.com/users/web-platform-tests", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "html_url": "https://github.com/web-platform-tests", + "email": "", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "site_admin": "false", + "login": "web-platform-tests", + "type": "Organization", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" + }, + "full_name": "web-platform-tests/wpt", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "id": "3618133", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "size": "305511", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "archived": "false", + "has_downloads": "true", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "has_projects": "true", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "watchers_count": "1845", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "organization": "web-platform-tests", + "stargazers_count": "1845", + "homepage": "http://irc.w3.org/?channels=testing", + "open_issues": "1328", + "fork": "false", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "has_pages": "true", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "html_url": "https://github.com/web-platform-tests/wpt", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "language": "HTML", + "watchers": "1845", + "private": "false", + "forks_count": "1523", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "has_issues": "true", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "master_branch": "master", + "forks": "1523", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "open_issues_count": "1317", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "name": "wpt", + "license": { + "spdx_id": "NOASSERTION", + "url": "null", + "node_id": "MDc6TGljZW5zZTA=", + "name": "Other", + "key": "other" + }, + "url": "https://github.com/web-platform-tests/wpt", + "stargazers": "1845", + "created_at": "1330865891", + "pushed_at": "1541037488", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "default_branch": "master", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "git_url": "git://github.com/web-platform-tests/wpt.git" + }, + "organization": { + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "description": "", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "url": "https://api.github.com/orgs/web-platform-tests", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "login": "web-platform-tests", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" + }, + }, + "event": "membership", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" + } + + event = { + 'tags': 'githubeventsqs' + } + event['details'] = message + result, metadata = self.plugin.onMessage(event, self.metadata) + self.verify_defaults(result) + self.verify_metadata(metadata) + self.verify_meta(message, result) + self.verify_actor(message, result) + self.verify_org(message, result) + assert result['source'] == 'membership' + assert result['details']['action'] == message['body']['action'] + assert result['details']['scope'] == message['body']['scope'] + + def test_public(self): + message = { + "body": { + "repository": { + "id": "135493233", + "node_id": "MDEwOlJlcG9zaXRvcnkxMzU0OTMyMzM=", + "name": "Hello-World", + "full_name": "Codertocat/Hello-World", + "owner": { + "name": "ACrazyCat", + "login": "Codertocat", + "id": "21031067", + "node_id": "MDQ6VXNlcjIxMDMxMDY3", + "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Codertocat", + "html_url": "https://github.com/Codertocat", + "followers_url": "https://api.github.com/users/Codertocat/followers", + "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", + "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", + "organizations_url": "https://api.github.com/users/Codertocat/orgs", + "repos_url": "https://api.github.com/users/Codertocat/repos", + "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/Codertocat/received_events", + "type": "User", + "site_admin": "false" + }, + "private": "false", + "html_url": "https://github.com/Codertocat/Hello-World", + "description": "null", + "fork": "false", + "url": "https://api.github.com/repos/Codertocat/Hello-World", + "forks_url": "https://api.github.com/repos/Codertocat/Hello-World/forks", + "keys_url": "https://api.github.com/repos/Codertocat/Hello-World/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/Codertocat/Hello-World/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/Codertocat/Hello-World/teams", + "hooks_url": "https://api.github.com/repos/Codertocat/Hello-World/hooks", + "issue_events_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/events{/number}", + "events_url": "https://api.github.com/repos/Codertocat/Hello-World/events", + "assignees_url": "https://api.github.com/repos/Codertocat/Hello-World/assignees{/user}", + "branches_url": "https://api.github.com/repos/Codertocat/Hello-World/branches{/branch}", + "tags_url": "https://api.github.com/repos/Codertocat/Hello-World/tags", + "blobs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/Codertocat/Hello-World/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/Codertocat/Hello-World/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/Codertocat/Hello-World/statuses/{sha}", + "languages_url": "https://api.github.com/repos/Codertocat/Hello-World/languages", + "stargazers_url": "https://api.github.com/repos/Codertocat/Hello-World/stargazers", + "contributors_url": "https://api.github.com/repos/Codertocat/Hello-World/contributors", + "subscribers_url": "https://api.github.com/repos/Codertocat/Hello-World/subscribers", + "subscription_url": "https://api.github.com/repos/Codertocat/Hello-World/subscription", + "commits_url": "https://api.github.com/repos/Codertocat/Hello-World/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/Codertocat/Hello-World/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/Codertocat/Hello-World/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/Codertocat/Hello-World/contents/{+path}", + "compare_url": "https://api.github.com/repos/Codertocat/Hello-World/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/Codertocat/Hello-World/merges", + "archive_url": "https://api.github.com/repos/Codertocat/Hello-World/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/Codertocat/Hello-World/downloads", + "issues_url": "https://api.github.com/repos/Codertocat/Hello-World/issues{/number}", + "pulls_url": "https://api.github.com/repos/Codertocat/Hello-World/pulls{/number}", + "milestones_url": "https://api.github.com/repos/Codertocat/Hello-World/milestones{/number}", + "notifications_url": "https://api.github.com/repos/Codertocat/Hello-World/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/Codertocat/Hello-World/labels{/name}", + "releases_url": "https://api.github.com/repos/Codertocat/Hello-World/releases{/id}", + "deployments_url": "https://api.github.com/repos/Codertocat/Hello-World/deployments", + "created_at": "2018-05-30T20:18:04Z", + "updated_at": "2018-05-30T20:18:49Z", + "pushed_at": "2018-05-30T20:18:48Z", + "git_url": "git://github.com/Codertocat/Hello-World.git", + "ssh_url": "git@github.com:Codertocat/Hello-World.git", + "clone_url": "https://github.com/Codertocat/Hello-World.git", + "svn_url": "https://github.com/Codertocat/Hello-World", + "homepage": "null", + "size": "0", + "stargazers_count": "0", + "watchers_count": "0", + "language": "null", + "has_issues": "true", + "has_projects": "true", + "has_downloads": "true", + "has_wiki": "true", + "has_pages": "true", + "forks_count": "0", + "mirror_url": "null", + "archived": "false", + "open_issues_count": "2", + "license": "null", + "forks": "0", + "open_issues": "2", + "watchers": "0", + "default_branch": "master" + }, + "sender": { + "login": "Codertocat", + "id": "21031067", + "node_id": "MDQ6VXNlcjIxMDMxMDY3", + "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Codertocat", + "html_url": "https://github.com/Codertocat", + "followers_url": "https://api.github.com/users/Codertocat/followers", + "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", + "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", + "organizations_url": "https://api.github.com/users/Codertocat/orgs", + "repos_url": "https://api.github.com/users/Codertocat/repos", + "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/Codertocat/received_events", + "type": "User", + "site_admin": "false" + }, + }, + "event": "public", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd", + } + event = { + 'tags': 'githubeventsqs' + } + event['details'] = message + result, metadata = self.plugin.onMessage(event, self.metadata) + self.verify_defaults(result) + self.verify_metadata(metadata) + self.verify_meta(message, result) + self.verify_repo(message, result) + assert result['source'] == 'public' + + def test_repository_import(self): + message = { + "body": { + "status": "success", + "repository": { + "id": 135493233, + "node_id": "MDEwOlJlcG9zaXRvcnkxMzU0OTMyMzM=", + "name": "Hello-World", + "full_name": "Codertocat/Hello-World", + "owner": { + "name": "ASuperCat", + "login": "Codertocat", + "id": 21031067, + "node_id": "MDQ6VXNlcjIxMDMxMDY3", + "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Codertocat", + "html_url": "https://github.com/Codertocat", + "followers_url": "https://api.github.com/users/Codertocat/followers", + "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", + "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", + "organizations_url": "https://api.github.com/users/Codertocat/orgs", + "repos_url": "https://api.github.com/users/Codertocat/repos", + "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/Codertocat/received_events", + "type": "User", + "site_admin": "false" + }, + "private": "false", + "html_url": "https://github.com/Codertocat/Hello-World", + "description": "null", + "fork": "false", + "url": "https://api.github.com/repos/Codertocat/Hello-World", + "forks_url": "https://api.github.com/repos/Codertocat/Hello-World/forks", + "keys_url": "https://api.github.com/repos/Codertocat/Hello-World/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/Codertocat/Hello-World/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/Codertocat/Hello-World/teams", + "hooks_url": "https://api.github.com/repos/Codertocat/Hello-World/hooks", + "issue_events_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/events{/number}", + "events_url": "https://api.github.com/repos/Codertocat/Hello-World/events", + "assignees_url": "https://api.github.com/repos/Codertocat/Hello-World/assignees{/user}", + "branches_url": "https://api.github.com/repos/Codertocat/Hello-World/branches{/branch}", + "tags_url": "https://api.github.com/repos/Codertocat/Hello-World/tags", + "blobs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/Codertocat/Hello-World/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/Codertocat/Hello-World/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/Codertocat/Hello-World/statuses/{sha}", + "languages_url": "https://api.github.com/repos/Codertocat/Hello-World/languages", + "stargazers_url": "https://api.github.com/repos/Codertocat/Hello-World/stargazers", + "contributors_url": "https://api.github.com/repos/Codertocat/Hello-World/contributors", + "subscribers_url": "https://api.github.com/repos/Codertocat/Hello-World/subscribers", + "subscription_url": "https://api.github.com/repos/Codertocat/Hello-World/subscription", + "commits_url": "https://api.github.com/repos/Codertocat/Hello-World/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/Codertocat/Hello-World/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/Codertocat/Hello-World/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/Codertocat/Hello-World/contents/{+path}", + "compare_url": "https://api.github.com/repos/Codertocat/Hello-World/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/Codertocat/Hello-World/merges", + "archive_url": "https://api.github.com/repos/Codertocat/Hello-World/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/Codertocat/Hello-World/downloads", + "issues_url": "https://api.github.com/repos/Codertocat/Hello-World/issues{/number}", + "pulls_url": "https://api.github.com/repos/Codertocat/Hello-World/pulls{/number}", + "milestones_url": "https://api.github.com/repos/Codertocat/Hello-World/milestones{/number}", + "notifications_url": "https://api.github.com/repos/Codertocat/Hello-World/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/Codertocat/Hello-World/labels{/name}", + "releases_url": "https://api.github.com/repos/Codertocat/Hello-World/releases{/id}", + "deployments_url": "https://api.github.com/repos/Codertocat/Hello-World/deployments", + "created_at": "2018-05-30T20:18:04Z", + "updated_at": "2018-05-30T20:18:49Z", + "pushed_at": "2018-05-30T20:18:48Z", + "git_url": "git://github.com/Codertocat/Hello-World.git", + "ssh_url": "git@github.com:Codertocat/Hello-World.git", + "clone_url": "https://github.com/Codertocat/Hello-World.git", + "svn_url": "https://github.com/Codertocat/Hello-World", + "homepage": "null", + "size": 0, + "stargazers_count": 0, + "watchers_count": 0, + "language": "null", + "has_issues": "true", + "has_projects": "true", + "has_downloads": "true", + "has_wiki": "true", + "has_pages": "true", + "forks_count": 0, + "mirror_url": "null", + "archived": "false", + "open_issues_count": 2, + "license": "null", + "forks": 0, + "open_issues": 2, + "watchers": 0, + "default_branch": "master" + }, + "organization": { + "login": "Octocoders", + "id": 38302899, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM4MzAyODk5", + "url": "https://api.github.com/orgs/Octocoders", + "repos_url": "https://api.github.com/orgs/Octocoders/repos", + "events_url": "https://api.github.com/orgs/Octocoders/events", + "hooks_url": "https://api.github.com/orgs/Octocoders/hooks", + "issues_url": "https://api.github.com/orgs/Octocoders/issues", + "members_url": "https://api.github.com/orgs/Octocoders/members{/member}", + "public_members_url": "https://api.github.com/orgs/Octocoders/public_members{/member}", + "avatar_url": "https://avatars1.githubusercontent.com/u/38302899?v=4", + "description": "" + }, + "sender": { + "login": "Codertocat", + "id": "21031067", + "node_id": "MDQ6VXNlcjIxMDMxMDY3", + "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Codertocat", + "html_url": "https://github.com/Codertocat", + "followers_url": "https://api.github.com/users/Codertocat/followers", + "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", + "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", + "organizations_url": "https://api.github.com/users/Codertocat/orgs", + "repos_url": "https://api.github.com/users/Codertocat/repos", + "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/Codertocat/received_events", + "type": "User", + "site_admin": "false" + } + }, + "event": "repository_import", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd", + } + event = { + 'tags': 'githubeventsqs' + } + event['details'] = message + result, metadata = self.plugin.onMessage(event, self.metadata) + self.verify_defaults(result) + self.verify_metadata(metadata) + self.verify_meta(message, result) + self.verify_actor(message, result) + self.verify_org(message, result) + self.verify_repo(message, result) + assert result['source'] == 'repository_import' + + def test_release(self): + message = { + "body": { + "action": "published", + "release": { + "url": "https://api.github.com/repos/Codertocat/Hello-World/releases/11248810", + "assets_url": "https://api.github.com/repos/Codertocat/Hello-World/releases/11248810/assets", + "upload_url": "https://uploads.github.com/repos/Codertocat/Hello-World/releases/11248810/assets{?name,label}", + "html_url": "https://github.com/Codertocat/Hello-World/releases/tag/0.0.1", + "id": "11248810", + "node_id": "MDc6UmVsZWFzZTExMjQ4ODEw", + "tag_name": "0.0.1", + "target_commitish": "master", + "name": "null", + "draft": "false", + "author": { + "login": "Codertocat", + "id": "21031067", + "node_id": "MDQ6VXNlcjIxMDMxMDY3", + "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Codertocat", + "html_url": "https://github.com/Codertocat", + "followers_url": "https://api.github.com/users/Codertocat/followers", + "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", + "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", + "organizations_url": "https://api.github.com/users/Codertocat/orgs", + "repos_url": "https://api.github.com/users/Codertocat/repos", + "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/Codertocat/received_events", + "type": "User", + "site_admin": "false" + }, + "prerelease": "false", + "created_at": "2018-05-30T20:18:05Z", + "published_at": "2018-05-30T20:18:44Z", + "assets": [ + + ], + "tarball_url": "https://api.github.com/repos/Codertocat/Hello-World/tarball/0.0.1", + "zipball_url": "https://api.github.com/repos/Codertocat/Hello-World/zipball/0.0.1", + "body": "null" + }, + "repository": { + "id": "135493233", + "node_id": "MDEwOlJlcG9zaXRvcnkxMzU0OTMyMzM=", + "name": "Hello-World", + "full_name": "Codertocat/Hello-World", + "owner": { + "name": "ASuperCat", + "login": "Codertocat", + "id": "21031067", + "node_id": "MDQ6VXNlcjIxMDMxMDY3", + "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Codertocat", + "html_url": "https://github.com/Codertocat", + "followers_url": "https://api.github.com/users/Codertocat/followers", + "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", + "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", + "organizations_url": "https://api.github.com/users/Codertocat/orgs", + "repos_url": "https://api.github.com/users/Codertocat/repos", + "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/Codertocat/received_events", + "type": "User", + "site_admin": "false" + }, + "private": "false", + "html_url": "https://github.com/Codertocat/Hello-World", + "description": "null", + "fork": "false", + "url": "https://api.github.com/repos/Codertocat/Hello-World", + "forks_url": "https://api.github.com/repos/Codertocat/Hello-World/forks", + "keys_url": "https://api.github.com/repos/Codertocat/Hello-World/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/Codertocat/Hello-World/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/Codertocat/Hello-World/teams", + "hooks_url": "https://api.github.com/repos/Codertocat/Hello-World/hooks", + "issue_events_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/events{/number}", + "events_url": "https://api.github.com/repos/Codertocat/Hello-World/events", + "assignees_url": "https://api.github.com/repos/Codertocat/Hello-World/assignees{/user}", + "branches_url": "https://api.github.com/repos/Codertocat/Hello-World/branches{/branch}", + "tags_url": "https://api.github.com/repos/Codertocat/Hello-World/tags", + "blobs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/Codertocat/Hello-World/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/Codertocat/Hello-World/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/Codertocat/Hello-World/statuses/{sha}", + "languages_url": "https://api.github.com/repos/Codertocat/Hello-World/languages", + "stargazers_url": "https://api.github.com/repos/Codertocat/Hello-World/stargazers", + "contributors_url": "https://api.github.com/repos/Codertocat/Hello-World/contributors", + "subscribers_url": "https://api.github.com/repos/Codertocat/Hello-World/subscribers", + "subscription_url": "https://api.github.com/repos/Codertocat/Hello-World/subscription", + "commits_url": "https://api.github.com/repos/Codertocat/Hello-World/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/Codertocat/Hello-World/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/Codertocat/Hello-World/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/Codertocat/Hello-World/contents/{+path}", + "compare_url": "https://api.github.com/repos/Codertocat/Hello-World/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/Codertocat/Hello-World/merges", + "archive_url": "https://api.github.com/repos/Codertocat/Hello-World/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/Codertocat/Hello-World/downloads", + "issues_url": "https://api.github.com/repos/Codertocat/Hello-World/issues{/number}", + "pulls_url": "https://api.github.com/repos/Codertocat/Hello-World/pulls{/number}", + "milestones_url": "https://api.github.com/repos/Codertocat/Hello-World/milestones{/number}", + "notifications_url": "https://api.github.com/repos/Codertocat/Hello-World/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/Codertocat/Hello-World/labels{/name}", + "releases_url": "https://api.github.com/repos/Codertocat/Hello-World/releases{/id}", + "deployments_url": "https://api.github.com/repos/Codertocat/Hello-World/deployments", + "created_at": "2018-05-30T20:18:04Z", + "updated_at": "2018-05-30T20:18:49Z", + "pushed_at": "2018-05-30T20:18:48Z", + "git_url": "git://github.com/Codertocat/Hello-World.git", + "ssh_url": "git@github.com:Codertocat/Hello-World.git", + "clone_url": "https://github.com/Codertocat/Hello-World.git", + "svn_url": "https://github.com/Codertocat/Hello-World", + "homepage": "null", + "size": 0, + "stargazers_count": 0, + "watchers_count": 0, + "language": "null", + "has_issues": "true", + "has_projects": "true", + "has_downloads": "true", + "has_wiki": "true", + "has_pages": "true", + "forks_count": 0, + "mirror_url": "null", + "archived": "false", + "open_issues_count": 2, + "license": "null", + "forks": 0, + "open_issues": 2, + "watchers": 0, + "default_branch": "master" + }, + "organization": { + "login": "Octocoders", + "id": 38302899, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM4MzAyODk5", + "url": "https://api.github.com/orgs/Octocoders", + "repos_url": "https://api.github.com/orgs/Octocoders/repos", + "events_url": "https://api.github.com/orgs/Octocoders/events", + "hooks_url": "https://api.github.com/orgs/Octocoders/hooks", + "issues_url": "https://api.github.com/orgs/Octocoders/issues", + "members_url": "https://api.github.com/orgs/Octocoders/members{/member}", + "public_members_url": "https://api.github.com/orgs/Octocoders/public_members{/member}", + "avatar_url": "https://avatars1.githubusercontent.com/u/38302899?v=4", + "description": "" + }, + "sender": { + "login": "Codertocat", + "id": "21031067", + "node_id": "MDQ6VXNlcjIxMDMxMDY3", + "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Codertocat", + "html_url": "https://github.com/Codertocat", + "followers_url": "https://api.github.com/users/Codertocat/followers", + "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", + "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", + "organizations_url": "https://api.github.com/users/Codertocat/orgs", + "repos_url": "https://api.github.com/users/Codertocat/repos", + "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/Codertocat/received_events", + "type": "User", + "site_admin": "false" + } + }, + "event": "release", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd", + } + event = { + 'tags': 'githubeventsqs' + } + event['details'] = message + result, metadata = self.plugin.onMessage(event, self.metadata) + self.verify_defaults(result) + self.verify_metadata(metadata) + self.verify_meta(message, result) + self.verify_actor(message, result) + self.verify_repo(message, result) + assert result['source'] == 'release' + assert result['details']['release_author_login'] == message['body']['release']['author']['login'] + assert result['details']['release_author_id'] == message['body']['release']['author']['id'] + assert result['details']['release_author_node_id'] == message['body']['release']['author']['node_id'] + assert result['details']['release_author_type'] == message['body']['release']['author']['type'] + assert result['details']['release_author_site_admin'] == message['body']['release']['author']['site_admin'] + + def test_org_block(self): + message = { + "body": { + "action": "blocked", + "blocked_user": { + "login": "hacktocat", + "id": 39652351, + "node_id": "MDQ6VXNlcjM5NjUyMzUx", + "avatar_url": "https://avatars2.githubusercontent.com/u/39652351?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/hacktocat", + "html_url": "https://github.com/hacktocat", + "followers_url": "https://api.github.com/users/hacktocat/followers", + "following_url": "https://api.github.com/users/hacktocat/following{/other_user}", + "gists_url": "https://api.github.com/users/hacktocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/hacktocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/hacktocat/subscriptions", + "organizations_url": "https://api.github.com/users/hacktocat/orgs", + "repos_url": "https://api.github.com/users/hacktocat/repos", + "events_url": "https://api.github.com/users/hacktocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/hacktocat/received_events", + "type": "User", + "site_admin": "false" + }, + "organization": { + "login": "Octocoders", + "id": 38302899, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM4MzAyODk5", + "url": "https://api.github.com/orgs/Octocoders", + "repos_url": "https://api.github.com/orgs/Octocoders/repos", + "events_url": "https://api.github.com/orgs/Octocoders/events", + "hooks_url": "https://api.github.com/orgs/Octocoders/hooks", + "issues_url": "https://api.github.com/orgs/Octocoders/issues", + "members_url": "https://api.github.com/orgs/Octocoders/members{/member}", + "public_members_url": "https://api.github.com/orgs/Octocoders/public_members{/member}", + "avatar_url": "https://avatars1.githubusercontent.com/u/38302899?v=4", + "description": "" + }, + "sender": { + "login": "Codertocat", + "id": 21031067, + "node_id": "MDQ6VXNlcjIxMDMxMDY3", + "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Codertocat", + "html_url": "https://github.com/Codertocat", + "followers_url": "https://api.github.com/users/Codertocat/followers", + "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", + "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", + "organizations_url": "https://api.github.com/users/Codertocat/orgs", + "repos_url": "https://api.github.com/users/Codertocat/repos", + "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/Codertocat/received_events", + "type": "User", + "site_admin": "false" + } +}, + "event": "org_block", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd", + } + event = { + 'tags': 'githubeventsqs' + } + event['details'] = message + result, metadata = self.plugin.onMessage(event, self.metadata) + self.verify_defaults(result) + self.verify_metadata(metadata) + self.verify_meta(message, result) + self.verify_actor(message, result) + assert result['source'] == 'org_block' + assert result['details']['blocked_user_login'] == message['body']['blocked_user']['login'] + assert result['details']['blocked_user_id'] == message['body']['blocked_user']['id'] + assert result['details']['blocked_user_node_id'] == message['body']['blocked_user']['node_id'] + + def test_installation(self): + message = { + "body": + { + "action": "deleted", + "installation": { + "id": 2, + "account": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": "false" + }, + "repository_selection": "selected", + "access_tokens_url": "https://api.github.com/installations/2/access_tokens", + "repositories_url": "https://api.github.com/installation/repositories", + "html_url": "https://github.com/settings/installations/2", + "app_id": 5725, + "target_id": 3880403, + "target_type": "User", + "permissions": { + "metadata": "read", + "contents": "read", + "issues": "write" + }, + "events": [ + "push", + "pull_request" + ], + "created_at": 1525109898, + "updated_at": 1525109899, + "single_file_name": "config.yml" + }, + "repositories": [ + { + "id": 1296269, + "name": "Hello-World", + "full_name": "octocat/Hello-World", + "private": "false" + } + ], + "sender": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": "false" + }, + }, + "event": "installation", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd", + } + event = { + 'tags': 'githubeventsqs' + } + event['details'] = message + result, metadata = self.plugin.onMessage(event, self.metadata) + self.verify_defaults(result) + self.verify_metadata(metadata) + self.verify_meta(message, result) + self.verify_actor(message, result) + assert result['source'] == 'installation' + assert result['details']['action'] == message['body']['action'] + assert result['details']['install_id'] == message['body']['installation']['account']['id'] + assert result['details']['install_account_login'] == message['body']['installation']['account']['login'] + assert result['details']['install_account_node_id'] == message['body']['installation']['account']['node_id'] + assert result['details']['install_account_type'] == message['body']['installation']['account']['type'] + assert result['details']['install_account_site_admin'] == message['body']['installation']['account']['site_admin'] \ No newline at end of file From 5aea141b1d56b706cafb4a2abbab8aea24b5bbdb Mon Sep 17 00:00:00 2001 From: Michal Purzynski Date: Tue, 6 Nov 2018 13:50:26 -0800 Subject: [PATCH 08/89] Migrate to the new lib --- alerts/nsm_scan_port.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/alerts/nsm_scan_port.py b/alerts/nsm_scan_port.py index 002ff267..617c5b39 100644 --- a/alerts/nsm_scan_port.py +++ b/alerts/nsm_scan_port.py @@ -6,7 +6,7 @@ # Copyright (c) 2018 Mozilla Corporation from lib.alerttask import AlertTask -from query_models import SearchQuery, TermMatch, QueryStringMatch, PhraseMatch, ExistsMatch +from mozdef_util.query_models import SearchQuery, TermMatch, QueryStringMatch, PhraseMatch, ExistsMatch class NSMScanPort(AlertTask): @@ -30,7 +30,6 @@ class NSMScanPort(AlertTask): self.searchEventsAggregated('details.sourceipaddress', samplesLimit=10) self.walkAggregations(threshold=1) - def onAggregation(self, aggreg): category = 'nsm' severity = 'WARNING' @@ -40,6 +39,7 @@ class NSMScanPort(AlertTask): # port = 'unknown' # Maybe iterate through it? x = aggreg['events'][0]['_source'] + import pdb; pdb.set_trace() if 'details' in x: if 'indicators' in x['details']: indicators = x['details']['sourceipaddress'] @@ -47,4 +47,3 @@ class NSMScanPort(AlertTask): summary = 'Port scan from {}'.format(indicators) return self.createAlertDict(summary, category, tags, aggreg['events'], severity) - From c373bf36092c5153387789f5fb9923a788cd98be Mon Sep 17 00:00:00 2001 From: Michal Purzynski Date: Tue, 6 Nov 2018 14:08:20 -0800 Subject: [PATCH 09/89] Final cleanups --- alerts/nsm_scan_port.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/alerts/nsm_scan_port.py b/alerts/nsm_scan_port.py index 617c5b39..67c6cd43 100644 --- a/alerts/nsm_scan_port.py +++ b/alerts/nsm_scan_port.py @@ -36,10 +36,7 @@ class NSMScanPort(AlertTask): tags = ['nsm', "bro", 'portscan'] indicators = 'unknown' - # port = 'unknown' - # Maybe iterate through it? x = aggreg['events'][0]['_source'] - import pdb; pdb.set_trace() if 'details' in x: if 'indicators' in x['details']: indicators = x['details']['sourceipaddress'] From 7f8ffe5a552d354d26f85ad7c2969a6943de70f9 Mon Sep 17 00:00:00 2001 From: Michal Purzynski Date: Tue, 6 Nov 2018 14:15:58 -0800 Subject: [PATCH 10/89] I made this alert's unit test stylish with this commit --- tests/alerts/test_nsm_scan_port.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/alerts/test_nsm_scan_port.py b/tests/alerts/test_nsm_scan_port.py index dbdc3dff..522411f1 100644 --- a/tests/alerts/test_nsm_scan_port.py +++ b/tests/alerts/test_nsm_scan_port.py @@ -60,7 +60,6 @@ class TestNSMScanPort(AlertTestSuite): ) ) - events = AlertTestSuite.create_events(default_event, 5) for event in events: event['_source']['category'] = 'syslog' @@ -122,4 +121,5 @@ class TestNSMScanPort(AlertTestSuite): description="Negative test case with old timestamp", events=events, ) -) \ No newline at end of file +) + From 24696938c453c63537cc2d15dd0f7350e1144afb Mon Sep 17 00:00:00 2001 From: Michal Purzynski Date: Tue, 6 Nov 2018 14:56:47 -0800 Subject: [PATCH 11/89] Fixing vim's newline issues --- tests/alerts/test_nsm_scan_port.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/alerts/test_nsm_scan_port.py b/tests/alerts/test_nsm_scan_port.py index 522411f1..22bd2c7e 100644 --- a/tests/alerts/test_nsm_scan_port.py +++ b/tests/alerts/test_nsm_scan_port.py @@ -121,5 +121,4 @@ class TestNSMScanPort(AlertTestSuite): description="Negative test case with old timestamp", events=events, ) -) - + ) From 968d700e966b5d45405d2381618ef409ab93d431 Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Tue, 13 Nov 2018 10:26:27 -0600 Subject: [PATCH 12/89] Fix malformed json for github mq plugin --- tests/mq/plugins/test_github_webhooks.py | 4609 +++++++++++----------- 1 file changed, 2295 insertions(+), 2314 deletions(-) diff --git a/tests/mq/plugins/test_github_webhooks.py b/tests/mq/plugins/test_github_webhooks.py index 5e55dd1f..377ed4f2 100644 --- a/tests/mq/plugins/test_github_webhooks.py +++ b/tests/mq/plugins/test_github_webhooks.py @@ -20,11 +20,9 @@ class TestSuricataFixup(object): 'index': 'events' } - def verify_metadata(self, metadata): assert metadata['doc_type'] == 'event' - def test_defaults(self): event = { 'tags': 'githubeventsqs', @@ -33,34 +31,32 @@ class TestSuricataFixup(object): result, metadata = self.plugin.onMessage(event, self.metadata) assert result is None - def test_nomatch_syslog(self): event = { - "category": "syslog", - "processid": "0", - "receivedtimestamp": "2017-09-26T00:22:24.210945+00:00", - "severity": "7", - "utctimestamp": "2017-09-26T00:22:23+00:00", - "timestamp": "2017-09-26T00:22:23+00:00", - "hostname": "syslog1.private.scl3.mozilla.com", - "mozdefhostname": "mozdef1.private.scl3.mozilla.com", - "summary": "Connection from 10.22.74.208 port 9071 on 10.22.74.45 nsm githubeventsqs port 22\n", - "eventsource": "systemslogs", - "details": { - "processid": "21233", - "Random": 2, - "sourceipv4address": "10.22.74.208", - "hostname": "hostname1.subdomain.domain.com", - "program": "githubeventsqs", - "sourceipaddress": "10.22.74.208" - } + "category": "syslog", + "processid": "0", + "receivedtimestamp": "2017-09-26T00:22:24.210945+00:00", + "severity": "7", + "utctimestamp": "2017-09-26T00:22:23+00:00", + "timestamp": "2017-09-26T00:22:23+00:00", + "hostname": "syslog1.private.scl3.mozilla.com", + "mozdefhostname": "mozdef1.private.scl3.mozilla.com", + "summary": "Connection from 10.22.74.208 port 9071 on 10.22.74.45 nsm githubeventsqs port 22\n", + "eventsource": "systemslogs", + "details": { + "processid": "21233", + "Random": 2, + "sourceipv4address": "10.22.74.208", + "hostname": "hostname1.subdomain.domain.com", + "program": "githubeventsqs", + "sourceipaddress": "10.22.74.208" } + } result, metadata = self.plugin.onMessage(event, self.metadata) assert result['category'] == 'syslog' assert result['eventsource'] == 'systemslogs' assert result == event - def test_nomatch_auditd(self): event = { "category": "execve", @@ -72,7 +68,7 @@ class TestSuricataFixup(object): "audisp-json", "2.1.1", "audit" - ], + ], "summary": "Execve: sh -c sudo githubeventsqs nsm /usr/lib64/nagios/plugins/custom/check_auditd.sh", "processname": "githubeventsqs", "details": { @@ -112,7 +108,6 @@ class TestSuricataFixup(object): assert 'eventsource' not in result assert result == event - def verify_defaults(self, result): assert result['category'] == 'github' assert result['tags'] == ['github', 'webhook'] @@ -120,240 +115,235 @@ class TestSuricataFixup(object): assert 'event' not in result['details'] assert 'source' in result - def verify_meta(self, message, result): assert result['details']['request_id'] == message['request_id'] - def verify_actor(self, message, result): assert result['details']['id'] == message['body']['sender']['id'] assert result['details']['username'] == message['body']['sender']['login'] - assert result['details']['sender_node_id'] == message['body']['sender']['node_id'] - assert result['details']['sender_site_admin'] == message['body']['sender']['site_admin'] + assert result['details']['sender_node_id'] == message['body']['sender']['node_id'] + assert result['details']['sender_site_admin'] == message['body']['sender']['site_admin'] assert result['details']['sender_type'] == message['body']['sender']['type'] - - + def verify_repo(self, message, result): - assert result['details']['repo_id'] == message['body']['repository']['id'] - assert result['details']['repo_name'] == message['body']['repository']['name'] - assert result['details']['repo_owner_id'] == message['body']['repository']['owner']['id'] - assert result['details']['repo_owner_name'] == message['body']['repository']['owner']['name'] + assert result['details']['repo_id'] == message['body']['repository']['id'] + assert result['details']['repo_name'] == message['body']['repository']['name'] + assert result['details']['repo_owner_id'] == message['body']['repository']['owner']['id'] + assert result['details']['repo_owner_name'] == message['body']['repository']['owner']['name'] assert result['details']['repo_owner_login'] == message['body']['repository']['owner']['login'] - assert result['details']['repo_owner_node_id'] == message['body']['repository']['owner']['node_id'] - assert result['details']['repo_owner_site_admin'] == message['body']['repository']['owner']['site_admin'] + assert result['details']['repo_owner_node_id'] == message['body']['repository']['owner']['node_id'] + assert result['details']['repo_owner_site_admin'] == message['body']['repository']['owner']['site_admin'] assert result['details']['repo_private'] == message['body']['repository']['private'] - def verify_org(self, message, result): assert result['details']['org_id'] == message['body']['organization']['id'] - assert result['details']['org_login'] == message['body']['organization']['login'] - assert result['details']['org_node_id'] == message['body']['organization']['node_id'] - + assert result['details']['org_login'] == message['body']['organization']['login'] + assert result['details']['org_node_id'] == message['body']['organization']['node_id'] def test_push(self): message = { - "body": { - "forced": "true", - "compare": "https://github.com/web-platform-tests/wpt/compare/f000a9569fcb...41d50efea43f", - "after": "41d50efea43fb365d2a2d13b3fc18b933b7c3a75", - "sender": { - "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", - "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", - "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", - "url": "https://api.github.com/users/chromium-wpt-export-bot", - "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", - "html_url": "https://github.com/chromium-wpt-export-bot", - "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", - "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", - "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", - "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", - "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", - "gravatar_id": "", - "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", - "site_admin": "false", - "login": "chromium-wpt-export-bot", - "type": "User", - "id": "25752892", - "node_id": "MDQ6VXNlcjI1NzUyODky" - }, - "repository": { - "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", - "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", - "has_wiki": "true", - "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", - "mirror_url": "null", - "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", - "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", - "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", - "updated_at": "2018-11-01T00:51:49Z", - "svn_url": "https://github.com/web-platform-tests/wpt", - "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", - "owner": { - "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", - "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", - "name": "web-platform-tests", - "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", - "url": "https://api.github.com/users/web-platform-tests", - "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", - "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", - "html_url": "https://github.com/web-platform-tests", - "email": "", - "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", - "repos_url": "https://api.github.com/users/web-platform-tests/repos", - "followers_url": "https://api.github.com/users/web-platform-tests/followers", - "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", - "gravatar_id": "", - "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", - "site_admin": "false", - "login": "web-platform-tests", - "type": "Organization", - "id": "37226233", - "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" - }, - "full_name": "web-platform-tests/wpt", - "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", - "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", - "id": "3618133", - "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", - "size": "305511", - "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", - "archived": "false", - "has_downloads": "true", - "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", - "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", - "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", - "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", - "has_projects": "true", - "clone_url": "https://github.com/web-platform-tests/wpt.git", - "watchers_count": "1845", - "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", - "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", - "organization": "web-platform-tests", - "stargazers_count": "1845", - "homepage": "http://irc.w3.org/?channels=testing", - "open_issues": "1328", - "fork": "false", - "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", - "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", - "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", - "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", - "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", - "has_pages": "true", - "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", - "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", - "html_url": "https://github.com/web-platform-tests/wpt", - "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", - "language": "HTML", - "watchers": "1845", - "private": "false", - "forks_count": "1523", - "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", - "has_issues": "true", - "ssh_url": "git@github.com:web-platform-tests/wpt.git", - "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", - "master_branch": "master", - "forks": "1523", - "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", - "open_issues_count": "1317", - "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", - "name": "wpt", - "license": { - "spdx_id": "NOASSERTION", - "url": "null", - "node_id": "MDc6TGljZW5zZTA=", - "name": "Other", - "key": "other" - }, - "url": "https://github.com/web-platform-tests/wpt", - "stargazers": "1845", - "created_at": "1330865891", - "pushed_at": "1541037488", - "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", - "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", - "default_branch": "master", - "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", - "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", - "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", - "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", - "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", - "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", - "git_url": "git://github.com/web-platform-tests/wpt.git" - }, - "created": "false", - "deleted": "false", - "commits": [ - { - "committer": { - "username": "chromium-wpt-export-bot", - "email": "blink-w3c-test-autoroller@chromium.org", - "name": "Chromium WPT Sync" + "body": { + "forced": "true", + "compare": "https://github.com/web-platform-tests/wpt/compare/f000a9569fcb...41d50efea43f", + "after": "41d50efea43fb365d2a2d13b3fc18b933b7c3a75", + "sender": { + "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", + "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", + "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", + "url": "https://api.github.com/users/chromium-wpt-export-bot", + "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", + "html_url": "https://github.com/chromium-wpt-export-bot", + "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", + "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", + "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", + "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", + "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", + "site_admin": "false", + "login": "chromium-wpt-export-bot", + "type": "User", + "id": "25752892", + "node_id": "MDQ6VXNlcjI1NzUyODky" + }, + "repository": { + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "has_wiki": "true", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "mirror_url": "null", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "updated_at": "2018-11-01T00:51:49Z", + "svn_url": "https://github.com/web-platform-tests/wpt", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "owner": { + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "name": "web-platform-tests", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "url": "https://api.github.com/users/web-platform-tests", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "html_url": "https://github.com/web-platform-tests", + "email": "", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "site_admin": "false", + "login": "web-platform-tests", + "type": "Organization", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" + }, + "full_name": "web-platform-tests/wpt", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "id": "3618133", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "size": "305511", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "archived": "false", + "has_downloads": "true", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "has_projects": "true", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "watchers_count": "1845", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "organization": "web-platform-tests", + "stargazers_count": "1845", + "homepage": "http://irc.w3.org/?channels=testing", + "open_issues": "1328", + "fork": "false", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "has_pages": "true", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "html_url": "https://github.com/web-platform-tests/wpt", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "language": "HTML", + "watchers": "1845", + "private": "false", + "forks_count": "1523", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "has_issues": "true", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "master_branch": "master", + "forks": "1523", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "open_issues_count": "1317", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "name": "wpt", + "license": { + "spdx_id": "NOASSERTION", + "url": "null", + "node_id": "MDc6TGljZW5zZTA=", + "name": "Other", + "key": "other" + }, + "url": "https://github.com/web-platform-tests/wpt", + "stargazers": "1845", + "created_at": "1330865891", + "pushed_at": "1541037488", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "default_branch": "master", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "git_url": "git://github.com/web-platform-tests/wpt.git" + }, + "created": "false", + "deleted": "false", + "commits": [ + { + "committer": { + "username": "chromium-wpt-export-bot", + "email": "blink-w3c-test-autoroller@chromium.org", + "name": "Chromium WPT Sync" + }, + "added": [ + "css/css-flexbox/percentage-heights-005.html" + ], + "author": { + "username": "cbiesinger", + "email": "cbiesinger@chromium.org", + "name": "Christian Biesinger" + }, + "distinct": "true", + "timestamp": "2018-10-31T18:58:04-07:00", + "modified": [], + "url": "https://github.com/web-platform-tests/wpt/commit/41d50efea43fb365d2a2d13b3fc18b933b7c3a75", + "tree_id": "4ed38f691f2be4d19d821fdd316508350d11b42c", + "message": "[layoutng] Fix setting of fixed_block_is_definite\n\nWhen a flex item has a definite specified height (e.g. height: 100px),\nthen percentages in children of the flex item should resolve even\nif the flexbox does not have an explicit height, ie. does not match\nthe condition in https://drafts.csswg.org/css-flexbox/#definite-sizes\n\nBug: 885185\n\nChange-Id: Iba226f30e1e02e3a11273fa45fcdf1cef897120c\nReviewed-on: https://chromium-review.googlesource.com/c/1311534\nCommit-Queue: Christian Biesinger \nReviewed-by: Emil A Eklund \nReviewed-by: Morten Stenshorne \nCr-Commit-Position: refs/heads/master@{#604483}", + "removed": [], + "id": "41d50efea43fb365d2a2d13b3fc18b933b7c3a75" + } + ], + "pusher": { + "email": "blink-w3c-test-autoroller@chromium.org", + "name": "chromium-wpt-export-bot" + }, + "head_commit": { + "committer": { + "username": "chromium-wpt-export-bot", + "email": "blink-w3c-test-autoroller@chromium.org", + "name": "Chromium WPT Sync" + }, + "added": [ + "css/css-flexbox/percentage-heights-005.html" + ], + "author": { + "username": "cbiesinger", + "email": "cbiesinger@chromium.org", + "name": "Christian Biesinger" + }, + "distinct": "true", + "timestamp": "2018-10-31T18:58:04-07:00", + "modified": [], + "url": "https://github.com/web-platform-tests/wpt/commit/41d50efea43fb365d2a2d13b3fc18b933b7c3a75", + "tree_id": "4ed38f691f2be4d19d821fdd316508350d11b42c", + "message": "[layoutng] Fix setting of fixed_block_is_definite\n\nWhen a flex item has a definite specified height (e.g. height: 100px),\nthen percentages in children of the flex item should resolve even\nif the flexbox does not have an explicit height, ie. does not match\nthe condition in https://drafts.csswg.org/css-flexbox/#definite-sizes\n\nBug: 885185\n\nChange-Id: Iba226f30e1e02e3a11273fa45fcdf1cef897120c\nReviewed-on: https://chromium-review.googlesource.com/c/1311534\nCommit-Queue: Christian Biesinger \nReviewed-by: Emil A Eklund \nReviewed-by: Morten Stenshorne \nCr-Commit-Position: refs/heads/master@{#604483}", + "removed": [], + "id": "41d50efea43fb365d2a2d13b3fc18b933b7c3a75" + }, + "organization": { + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "description": "", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "url": "https://api.github.com/orgs/web-platform-tests", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "login": "web-platform-tests", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" + }, + "ref": "refs/heads/chromium-export-cl-1311534", + "base_ref": "null", + "before": "f000a9569fcb918a3c98fb93b5acd0218afa19ab" }, - "added": [ - "css/css-flexbox/percentage-heights-005.html" - ], - "author": { - "username": "cbiesinger", - "email": "cbiesinger@chromium.org", - "name": "Christian Biesinger" - }, - "distinct": "true", - "timestamp": "2018-10-31T18:58:04-07:00", - "modified": [], - "url": "https://github.com/web-platform-tests/wpt/commit/41d50efea43fb365d2a2d13b3fc18b933b7c3a75", - "tree_id": "4ed38f691f2be4d19d821fdd316508350d11b42c", - "message": "[layoutng] Fix setting of fixed_block_is_definite\n\nWhen a flex item has a definite specified height (e.g. height: 100px),\nthen percentages in children of the flex item should resolve even\nif the flexbox does not have an explicit height, ie. does not match\nthe condition in https://drafts.csswg.org/css-flexbox/#definite-sizes\n\nBug: 885185\n\nChange-Id: Iba226f30e1e02e3a11273fa45fcdf1cef897120c\nReviewed-on: https://chromium-review.googlesource.com/c/1311534\nCommit-Queue: Christian Biesinger \nReviewed-by: Emil A Eklund \nReviewed-by: Morten Stenshorne \nCr-Commit-Position: refs/heads/master@{#604483}", - "removed": [], - "id": "41d50efea43fb365d2a2d13b3fc18b933b7c3a75" - } - ], - "pusher": { - "email": "blink-w3c-test-autoroller@chromium.org", - "name": "chromium-wpt-export-bot" - }, - "head_commit": { - "committer": { - "username": "chromium-wpt-export-bot", - "email": "blink-w3c-test-autoroller@chromium.org", - "name": "Chromium WPT Sync" - }, - "added": [ - "css/css-flexbox/percentage-heights-005.html" - ], - "author": { - "username": "cbiesinger", - "email": "cbiesinger@chromium.org", - "name": "Christian Biesinger" - }, - "distinct": "true", - "timestamp": "2018-10-31T18:58:04-07:00", - "modified": [], - "url": "https://github.com/web-platform-tests/wpt/commit/41d50efea43fb365d2a2d13b3fc18b933b7c3a75", - "tree_id": "4ed38f691f2be4d19d821fdd316508350d11b42c", - "message": "[layoutng] Fix setting of fixed_block_is_definite\n\nWhen a flex item has a definite specified height (e.g. height: 100px),\nthen percentages in children of the flex item should resolve even\nif the flexbox does not have an explicit height, ie. does not match\nthe condition in https://drafts.csswg.org/css-flexbox/#definite-sizes\n\nBug: 885185\n\nChange-Id: Iba226f30e1e02e3a11273fa45fcdf1cef897120c\nReviewed-on: https://chromium-review.googlesource.com/c/1311534\nCommit-Queue: Christian Biesinger \nReviewed-by: Emil A Eklund \nReviewed-by: Morten Stenshorne \nCr-Commit-Position: refs/heads/master@{#604483}", - "removed": [], - "id": "41d50efea43fb365d2a2d13b3fc18b933b7c3a75" - }, - "organization": { - "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", - "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", - "description": "", - "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", - "url": "https://api.github.com/orgs/web-platform-tests", - "events_url": "https://api.github.com/orgs/web-platform-tests/events", - "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", - "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", - "login": "web-platform-tests", - "id": "37226233", - "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", - "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" - }, - "ref": "refs/heads/chromium-export-cl-1311534", - "base_ref": "null", - "before": "f000a9569fcb918a3c98fb93b5acd0218afa19ab" - }, - "event": "push", - "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" - } + "event": "push", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" + } event = { 'tags': 'githubeventsqs' @@ -371,163 +361,162 @@ class TestSuricataFixup(object): assert result['details']['deleted'] == message['body']['deleted'] assert result['details']['forced'] == message['body']['forced'] assert result['details']['commits'] == message['body']['commits'] - assert result['details']['ref'] == message['body']['ref'] + assert result['details']['ref'] == message['body']['ref'] assert result['details']['email'] == message['body']['pusher']['email'] assert result['details']['commit_author'] == message['body']['head_commit']['author']['email'] - assert result['details']['committer'] == message['body']['head_commit']['committer']['email'] - assert result['details']['commit_id'] == message['body']['head_commit']['id'] + assert result['details']['committer'] == message['body']['head_commit']['committer']['email'] + assert result['details']['commit_id'] == message['body']['head_commit']['id'] assert result['details']['commit_msg'] == message['body']['head_commit']['message'] assert result['details']['commit_ts'] == message['body']['head_commit']['timestamp'] assert result['details']['commit_url'] == message['body']['head_commit']['url'] assert result['details']['email'] == message['body']['pusher']['email'] - def test_delete(self): message = { - "body": { - "ref": "gecko/1499958", - "ref_type": "branch", - "sender": { - "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", - "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", - "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", - "url": "https://api.github.com/users/chromium-wpt-export-bot", - "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", - "html_url": "https://github.com/chromium-wpt-export-bot", - "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", - "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", - "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", - "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", - "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", - "gravatar_id": "", - "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", - "site_admin": "false", - "login": "chromium-wpt-export-bot", - "type": "User", - "id": "25752892", - "node_id": "MDQ6VXNlcjI1NzUyODky" - }, - "repository": { - "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", - "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", - "has_wiki": "true", - "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", - "mirror_url": "null", - "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", - "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", - "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", - "updated_at": "2018-11-01T00:51:49Z", - "svn_url": "https://github.com/web-platform-tests/wpt", - "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", - "owner": { - "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", - "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", - "name": "web-platform-tests", - "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", - "url": "https://api.github.com/users/web-platform-tests", - "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", - "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", - "html_url": "https://github.com/web-platform-tests", - "email": "", - "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", - "repos_url": "https://api.github.com/users/web-platform-tests/repos", - "followers_url": "https://api.github.com/users/web-platform-tests/followers", - "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", - "gravatar_id": "", - "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", - "site_admin": "false", - "login": "web-platform-tests", - "type": "Organization", - "id": "37226233", - "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" - }, - "full_name": "web-platform-tests/wpt", - "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", - "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", - "id": "3618133", - "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", - "size": "305511", - "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", - "archived": "false", - "has_downloads": "true", - "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", - "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", - "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", - "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", - "has_projects": "true", - "clone_url": "https://github.com/web-platform-tests/wpt.git", - "watchers_count": "1845", - "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", - "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", - "organization": "web-platform-tests", - "stargazers_count": "1845", - "homepage": "http://irc.w3.org/?channels=testing", - "open_issues": "1328", - "fork": "false", - "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", - "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", - "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", - "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", - "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", - "has_pages": "true", - "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", - "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", - "html_url": "https://github.com/web-platform-tests/wpt", - "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", - "language": "HTML", - "watchers": "1845", - "private": "false", - "forks_count": "1523", - "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", - "has_issues": "true", - "ssh_url": "git@github.com:web-platform-tests/wpt.git", - "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", - "master_branch": "master", - "forks": "1523", - "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", - "open_issues_count": "1317", - "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", - "name": "wpt", - "license": { - "spdx_id": "NOASSERTION", - "url": "null", - "node_id": "MDc6TGljZW5zZTA=", - "name": "Other", - "key": "other" - }, - "url": "https://github.com/web-platform-tests/wpt", - "stargazers": "1845", - "created_at": "1330865891", - "pushed_at": "1541037488", - "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", - "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", - "default_branch": "master", - "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", - "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", - "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", - "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", - "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", - "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", - "git_url": "git://github.com/web-platform-tests/wpt.git" - }, - "organization": { - "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", - "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", - "description": "", - "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", - "url": "https://api.github.com/orgs/web-platform-tests", - "events_url": "https://api.github.com/orgs/web-platform-tests/events", - "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", - "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", - "login": "web-platform-tests", - "id": "37226233", - "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", - "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" - }, - }, - "event": "delete", - "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" - } + "body": { + "ref": "gecko/1499958", + "ref_type": "branch", + "sender": { + "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", + "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", + "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", + "url": "https://api.github.com/users/chromium-wpt-export-bot", + "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", + "html_url": "https://github.com/chromium-wpt-export-bot", + "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", + "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", + "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", + "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", + "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", + "site_admin": "false", + "login": "chromium-wpt-export-bot", + "type": "User", + "id": "25752892", + "node_id": "MDQ6VXNlcjI1NzUyODky" + }, + "repository": { + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "has_wiki": "true", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "mirror_url": "null", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "updated_at": "2018-11-01T00:51:49Z", + "svn_url": "https://github.com/web-platform-tests/wpt", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "owner": { + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "name": "web-platform-tests", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "url": "https://api.github.com/users/web-platform-tests", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "html_url": "https://github.com/web-platform-tests", + "email": "", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "site_admin": "false", + "login": "web-platform-tests", + "type": "Organization", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" + }, + "full_name": "web-platform-tests/wpt", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "id": "3618133", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "size": "305511", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "archived": "false", + "has_downloads": "true", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "has_projects": "true", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "watchers_count": "1845", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "organization": "web-platform-tests", + "stargazers_count": "1845", + "homepage": "http://irc.w3.org/?channels=testing", + "open_issues": "1328", + "fork": "false", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "has_pages": "true", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "html_url": "https://github.com/web-platform-tests/wpt", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "language": "HTML", + "watchers": "1845", + "private": "false", + "forks_count": "1523", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "has_issues": "true", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "master_branch": "master", + "forks": "1523", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "open_issues_count": "1317", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "name": "wpt", + "license": { + "spdx_id": "NOASSERTION", + "url": "null", + "node_id": "MDc6TGljZW5zZTA=", + "name": "Other", + "key": "other" + }, + "url": "https://github.com/web-platform-tests/wpt", + "stargazers": "1845", + "created_at": "1330865891", + "pushed_at": "1541037488", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "default_branch": "master", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "git_url": "git://github.com/web-platform-tests/wpt.git" + }, + "organization": { + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "description": "", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "url": "https://api.github.com/orgs/web-platform-tests", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "login": "web-platform-tests", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" + }, + }, + "event": "delete", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" + } event = { 'tags': 'githubeventsqs' @@ -541,156 +530,155 @@ class TestSuricataFixup(object): self.verify_repo(message, result) self.verify_org(message, result) assert result['source'] == 'delete' - assert result['details']['ref'] == message['body']['ref'] - assert result['details']['ref_type'] == message['body']['ref_type'] - + assert result['details']['ref'] == message['body']['ref'] + assert result['details']['ref_type'] == message['body']['ref_type'] def test_create(self): message = { - "body": { - "ref": "gecko/1499958", - "ref_type": "branch", - "sender": { - "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", - "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", - "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", - "url": "https://api.github.com/users/chromium-wpt-export-bot", - "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", - "html_url": "https://github.com/chromium-wpt-export-bot", - "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", - "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", - "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", - "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", - "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", - "gravatar_id": "", - "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", - "site_admin": "false", - "login": "chromium-wpt-export-bot", - "type": "User", - "id": "25752892", - "node_id": "MDQ6VXNlcjI1NzUyODky" - }, - "repository": { - "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", - "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", - "has_wiki": "true", - "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", - "mirror_url": "null", - "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", - "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", - "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", - "updated_at": "2018-11-01T00:51:49Z", - "svn_url": "https://github.com/web-platform-tests/wpt", - "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", - "owner": { - "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", - "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", - "name": "web-platform-tests", - "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", - "url": "https://api.github.com/users/web-platform-tests", - "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", - "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", - "html_url": "https://github.com/web-platform-tests", - "email": "", - "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", - "repos_url": "https://api.github.com/users/web-platform-tests/repos", - "followers_url": "https://api.github.com/users/web-platform-tests/followers", - "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", - "gravatar_id": "", - "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", - "site_admin": "false", - "login": "web-platform-tests", - "type": "Organization", - "id": "37226233", - "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" - }, - "full_name": "web-platform-tests/wpt", - "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", - "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", - "id": "3618133", - "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", - "size": "305511", - "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", - "archived": "false", - "has_downloads": "true", - "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", - "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", - "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", - "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", - "has_projects": "true", - "clone_url": "https://github.com/web-platform-tests/wpt.git", - "watchers_count": "1845", - "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", - "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", - "organization": "web-platform-tests", - "stargazers_count": "1845", - "homepage": "http://irc.w3.org/?channels=testing", - "open_issues": "1328", - "fork": "false", - "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", - "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", - "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", - "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", - "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", - "has_pages": "true", - "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", - "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", - "html_url": "https://github.com/web-platform-tests/wpt", - "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", - "language": "HTML", - "watchers": "1845", - "private": "false", - "forks_count": "1523", - "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", - "has_issues": "true", - "ssh_url": "git@github.com:web-platform-tests/wpt.git", - "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", - "master_branch": "master", - "forks": "1523", - "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", - "open_issues_count": "1317", - "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", - "name": "wpt", - "license": { - "spdx_id": "NOASSERTION", - "url": "null", - "node_id": "MDc6TGljZW5zZTA=", - "name": "Other", - "key": "other" - }, - "url": "https://github.com/web-platform-tests/wpt", - "stargazers": "1845", - "created_at": "1330865891", - "pushed_at": "1541037488", - "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", - "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", - "default_branch": "master", - "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", - "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", - "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", - "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", - "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", - "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", - "git_url": "git://github.com/web-platform-tests/wpt.git" - }, - "organization": { - "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", - "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", - "description": "", - "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", - "url": "https://api.github.com/orgs/web-platform-tests", - "events_url": "https://api.github.com/orgs/web-platform-tests/events", - "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", - "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", - "login": "web-platform-tests", - "id": "37226233", - "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", - "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" - }, - }, - "event": "create", - "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" - } + "body": { + "ref": "gecko/1499958", + "ref_type": "branch", + "sender": { + "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", + "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", + "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", + "url": "https://api.github.com/users/chromium-wpt-export-bot", + "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", + "html_url": "https://github.com/chromium-wpt-export-bot", + "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", + "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", + "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", + "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", + "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", + "site_admin": "false", + "login": "chromium-wpt-export-bot", + "type": "User", + "id": "25752892", + "node_id": "MDQ6VXNlcjI1NzUyODky" + }, + "repository": { + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "has_wiki": "true", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "mirror_url": "null", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "updated_at": "2018-11-01T00:51:49Z", + "svn_url": "https://github.com/web-platform-tests/wpt", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "owner": { + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "name": "web-platform-tests", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "url": "https://api.github.com/users/web-platform-tests", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "html_url": "https://github.com/web-platform-tests", + "email": "", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "site_admin": "false", + "login": "web-platform-tests", + "type": "Organization", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" + }, + "full_name": "web-platform-tests/wpt", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "id": "3618133", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "size": "305511", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "archived": "false", + "has_downloads": "true", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "has_projects": "true", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "watchers_count": "1845", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "organization": "web-platform-tests", + "stargazers_count": "1845", + "homepage": "http://irc.w3.org/?channels=testing", + "open_issues": "1328", + "fork": "false", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "has_pages": "true", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "html_url": "https://github.com/web-platform-tests/wpt", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "language": "HTML", + "watchers": "1845", + "private": "false", + "forks_count": "1523", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "has_issues": "true", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "master_branch": "master", + "forks": "1523", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "open_issues_count": "1317", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "name": "wpt", + "license": { + "spdx_id": "NOASSERTION", + "url": "null", + "node_id": "MDc6TGljZW5zZTA=", + "name": "Other", + "key": "other" + }, + "url": "https://github.com/web-platform-tests/wpt", + "stargazers": "1845", + "created_at": "1330865891", + "pushed_at": "1541037488", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "default_branch": "master", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "git_url": "git://github.com/web-platform-tests/wpt.git" + }, + "organization": { + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "description": "", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "url": "https://api.github.com/orgs/web-platform-tests", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "login": "web-platform-tests", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" + }, + }, + "event": "create", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" + } event = { 'tags': 'githubeventsqs' @@ -704,185 +692,184 @@ class TestSuricataFixup(object): self.verify_repo(message, result) self.verify_org(message, result) assert result['source'] == 'create' - assert result['details']['ref'] == message['body']['ref'] - assert result['details']['ref_type'] == message['body']['ref_type'] - + assert result['details']['ref'] == message['body']['ref'] + assert result['details']['ref_type'] == message['body']['ref_type'] def test_repository_vulnerability_alert(self): message = { - "body": { - "action": "create", - "alert":{ - "affected_package_name": "requests", - "external_reference": "https://nvd.nist.gov/vuln/detail/CVE-2018-18074", - "external_identifier": "CVE-2018-18074", - "affected_range": "<= 2.19.1", - "id": "65626688", - "fixed_in": "2.20.0", - "dismisser": { - "login":"octocat", - "id":"1", - "node_id": "MDQ6VXNlcjIxMDMxMDY3", - "avatar_url":"https://github.com/images/error/octocat_happy.gif", - "gravatar_id":"", - "url":"https://api.github.com/users/octocat", - "html_url":"https://github.com/octocat", - "followers_url":"https://api.github.com/users/octocat/followers", - "following_url":"https://api.github.com/users/octocat/following{/other_user}", - "gists_url":"https://api.github.com/users/octocat/gists{/gist_id}", - "starred_url":"https://api.github.com/users/octocat/starred{/owner}{/repo}", - "subscriptions_url":"https://api.github.com/users/octocat/subscriptions", - "organizations_url":"https://api.github.com/users/octocat/orgs", - "repos_url":"https://api.github.com/users/octocat/repos", - "events_url":"https://api.github.com/users/octocat/events{/privacy}", - "received_events_url":"https://api.github.com/users/octocat/received_events", - "type":"User", - "site_admin":"true", - }, - "dismiss_reason": "I'm too lazy to fix this", - "dismissed_at": "2017-10-25T00:00:00+00:00", - }, - "sender": { - "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", - "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", - "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", - "url": "https://api.github.com/users/chromium-wpt-export-bot", - "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", - "html_url": "https://github.com/chromium-wpt-export-bot", - "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", - "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", - "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", - "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", - "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", - "gravatar_id": "", - "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", - "site_admin": "false", - "login": "chromium-wpt-export-bot", - "type": "User", - "id": "25752892", - "node_id": "MDQ6VXNlcjI1NzUyODky" - }, - "repository": { - "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", - "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", - "has_wiki": "true", - "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", - "mirror_url": "null", - "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", - "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", - "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", - "updated_at": "2018-11-01T00:51:49Z", - "svn_url": "https://github.com/web-platform-tests/wpt", - "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", - "owner": { - "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", - "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", - "name": "web-platform-tests", - "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", - "url": "https://api.github.com/users/web-platform-tests", - "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", - "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", - "html_url": "https://github.com/web-platform-tests", - "email": "", - "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", - "repos_url": "https://api.github.com/users/web-platform-tests/repos", - "followers_url": "https://api.github.com/users/web-platform-tests/followers", - "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", - "gravatar_id": "", - "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", - "site_admin": "false", - "login": "web-platform-tests", - "type": "Organization", - "id": "37226233", - "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" - }, - "full_name": "web-platform-tests/wpt", - "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", - "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", - "id": "3618133", - "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", - "size": "305511", - "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", - "archived": "false", - "has_downloads": "true", - "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", - "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", - "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", - "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", - "has_projects": "true", - "clone_url": "https://github.com/web-platform-tests/wpt.git", - "watchers_count": "1845", - "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", - "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", - "organization": "web-platform-tests", - "stargazers_count": "1845", - "homepage": "http://irc.w3.org/?channels=testing", - "open_issues": "1328", - "fork": "false", - "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", - "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", - "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", - "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", - "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", - "has_pages": "true", - "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", - "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", - "html_url": "https://github.com/web-platform-tests/wpt", - "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", - "language": "HTML", - "watchers": "1845", - "private": "false", - "forks_count": "1523", - "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", - "has_issues": "true", - "ssh_url": "git@github.com:web-platform-tests/wpt.git", - "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", - "master_branch": "master", - "forks": "1523", - "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", - "open_issues_count": "1317", - "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", - "name": "wpt", - "license": { - "spdx_id": "NOASSERTION", - "url": "null", - "node_id": "MDc6TGljZW5zZTA=", - "name": "Other", - "key": "other" - }, - "url": "https://github.com/web-platform-tests/wpt", - "stargazers": "1845", - "created_at": "1330865891", - "pushed_at": "1541037488", - "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", - "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", - "default_branch": "master", - "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", - "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", - "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", - "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", - "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", - "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", - "git_url": "git://github.com/web-platform-tests/wpt.git" - }, - "organization": { - "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", - "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", - "description": "", - "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", - "url": "https://api.github.com/orgs/web-platform-tests", - "events_url": "https://api.github.com/orgs/web-platform-tests/events", - "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", - "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", - "login": "web-platform-tests", - "id": "37226233", - "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", - "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" - }, - }, - "event": "repository_vulnerability_alert", - "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" - } + "body": { + "action": "create", + "alert": { + "affected_package_name": "requests", + "external_reference": "https://nvd.nist.gov/vuln/detail/CVE-2018-18074", + "external_identifier": "CVE-2018-18074", + "affected_range": "<= 2.19.1", + "id": "65626688", + "fixed_in": "2.20.0", + "dismisser": { + "login": "octocat", + "id": "1", + "node_id": "MDQ6VXNlcjIxMDMxMDY3", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": "true", + }, + "dismiss_reason": "I'm too lazy to fix this", + "dismissed_at": "2017-10-25T00:00:00+00:00", + }, + "sender": { + "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", + "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", + "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", + "url": "https://api.github.com/users/chromium-wpt-export-bot", + "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", + "html_url": "https://github.com/chromium-wpt-export-bot", + "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", + "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", + "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", + "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", + "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", + "site_admin": "false", + "login": "chromium-wpt-export-bot", + "type": "User", + "id": "25752892", + "node_id": "MDQ6VXNlcjI1NzUyODky" + }, + "repository": { + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "has_wiki": "true", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "mirror_url": "null", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "updated_at": "2018-11-01T00:51:49Z", + "svn_url": "https://github.com/web-platform-tests/wpt", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "owner": { + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "name": "web-platform-tests", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "url": "https://api.github.com/users/web-platform-tests", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "html_url": "https://github.com/web-platform-tests", + "email": "", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "site_admin": "false", + "login": "web-platform-tests", + "type": "Organization", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" + }, + "full_name": "web-platform-tests/wpt", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "id": "3618133", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "size": "305511", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "archived": "false", + "has_downloads": "true", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "has_projects": "true", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "watchers_count": "1845", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "organization": "web-platform-tests", + "stargazers_count": "1845", + "homepage": "http://irc.w3.org/?channels=testing", + "open_issues": "1328", + "fork": "false", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "has_pages": "true", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "html_url": "https://github.com/web-platform-tests/wpt", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "language": "HTML", + "watchers": "1845", + "private": "false", + "forks_count": "1523", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "has_issues": "true", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "master_branch": "master", + "forks": "1523", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "open_issues_count": "1317", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "name": "wpt", + "license": { + "spdx_id": "NOASSERTION", + "url": "null", + "node_id": "MDc6TGljZW5zZTA=", + "name": "Other", + "key": "other" + }, + "url": "https://github.com/web-platform-tests/wpt", + "stargazers": "1845", + "created_at": "1330865891", + "pushed_at": "1541037488", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "default_branch": "master", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "git_url": "git://github.com/web-platform-tests/wpt.git" + }, + "organization": { + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "description": "", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "url": "https://api.github.com/orgs/web-platform-tests", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "login": "web-platform-tests", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" + }, + }, + "event": "repository_vulnerability_alert", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" + } event = { 'tags': 'githubeventsqs' @@ -903,8 +890,6 @@ class TestSuricataFixup(object): assert result['details']['alert_extref'] == message['body']['alert']['external_reference'] assert result['details']['alert_fixed'] == message['body']['alert']['fixed_in'] assert result['details']['alert_id'] == message['body']['alert']['id'] - #assert result['details']['dismiss_reason'] == message['body']['alert']['dismiss_reason'] - #assert result['details']['dismiss_at'] == message['body']['alert']['dismissed_at'] assert result['details']['dismiss_user'] == message['body']['alert']['dismisser']['login'] assert result['details']['dismiss_id'] == message['body']['alert']['dismisser']['id'] assert result['details']['dismiss_node_id'] == message['body']['alert']['dismisser']['node_id'] @@ -913,61 +898,61 @@ class TestSuricataFixup(object): def test_security_advisory(self): message = { - "body": + "body": { - "action": "published", - "security_advisory": { - "ghsa_id": "GHSA-rf4j-j272-fj86", - "summary": "Moderate severity vulnerability that affects django", - "description": "django.contrib.auth.forms.AuthenticationForm in Django 2.0 before 2.0.2, and 1.11.8 and 1.11.9, allows remote attackers to obtain potentially sensitive information by leveraging data exposure from the confirm_login_allowed() method, as demonstrated by discovering whether a user account is inactive.", - "severity": "moderate", - "identifiers": [ - { - "value": "GHSA-rf4j-j272-fj86", - "type": "GHSA" - }, - { - "value": "CVE-2018-6188", - "type": "CVE" - } - ], - "references": [ - { - "url": "https://nvd.nist.gov/vuln/detail/CVE-2018-6188" - } - ], - "published_at": "2018-10-03T21:13:54Z", - "updated_at": "2018-10-03T21:13:54Z", - "withdrawn_at": "null", - "vulnerabilities": [ - { - "package": { - "ecosystem": "pip", - "name": "django" - }, - "severity": "moderate", - "vulnerable_version_range": ">= 2.0.0, < 2.0.2", - "first_patched_version": { - "identifier": "2.0.2" + "action": "published", + "security_advisory": { + "ghsa_id": "GHSA-rf4j-j272-fj86", + "summary": "Moderate severity vulnerability that affects django", + "description": "django.contrib.auth.forms.AuthenticationForm in Django 2.0 before 2.0.2, and 1.11.8 and 1.11.9, allows remote attackers to obtain potentially sensitive information by leveraging data exposure from the confirm_login_allowed() method, as demonstrated by discovering whether a user account is inactive.", + "severity": "moderate", + "identifiers": [ + { + "value": "GHSA-rf4j-j272-fj86", + "type": "GHSA" + }, + { + "value": "CVE-2018-6188", + "type": "CVE" + } + ], + "references": [ + { + "url": "https://nvd.nist.gov/vuln/detail/CVE-2018-6188" + } + ], + "published_at": "2018-10-03T21:13:54Z", + "updated_at": "2018-10-03T21:13:54Z", + "withdrawn_at": "null", + "vulnerabilities": [ + { + "package": { + "ecosystem": "pip", + "name": "django" + }, + "severity": "moderate", + "vulnerable_version_range": ">= 2.0.0, < 2.0.2", + "first_patched_version": { + "identifier": "2.0.2" + } + }, + { + "package": { + "ecosystem": "pip", + "name": "django" + }, + "severity": "moderate", + "vulnerable_version_range": ">= 1.11.8, < 1.11.10", + "first_patched_version": { + "identifier": "1.11.10" + } + } + ] + }, + }, + "event": "security_advisory", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd", } - }, - { - "package": { - "ecosystem": "pip", - "name": "django" - }, - "severity": "moderate", - "vulnerable_version_range": ">= 1.11.8, < 1.11.10", - "first_patched_version": { - "identifier": "1.11.10" - } - } - ] - }, -}, -"event": "security_advisory", - "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd", - } event = { 'tags': 'githubeventsqs' } @@ -980,149 +965,149 @@ class TestSuricataFixup(object): def test_repository(self): message = { - "body": { - "action": "deleted", - "sender": { - "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", - "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", - "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", - "url": "https://api.github.com/users/chromium-wpt-export-bot", - "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", - "html_url": "https://github.com/chromium-wpt-export-bot", - "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", - "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", - "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", - "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", - "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", - "gravatar_id": "", - "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", - "site_admin": "false", - "login": "chromium-wpt-export-bot", - "type": "User", - "id": "25752892", - "node_id": "MDQ6VXNlcjI1NzUyODky" - }, - "repository": { - "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", - "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", - "has_wiki": "true", - "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", - "mirror_url": "null", - "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", - "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", - "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", - "updated_at": "2018-11-01T00:51:49Z", - "svn_url": "https://github.com/web-platform-tests/wpt", - "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", - "owner": { - "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", - "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", - "name": "web-platform-tests", - "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", - "url": "https://api.github.com/users/web-platform-tests", - "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", - "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", - "html_url": "https://github.com/web-platform-tests", - "email": "", - "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", - "repos_url": "https://api.github.com/users/web-platform-tests/repos", - "followers_url": "https://api.github.com/users/web-platform-tests/followers", - "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", - "gravatar_id": "", - "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", - "site_admin": "false", - "login": "web-platform-tests", - "type": "Organization", - "id": "37226233", - "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" - }, - "full_name": "web-platform-tests/wpt", - "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", - "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", - "id": "3618133", - "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", - "size": "305511", - "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", - "archived": "false", - "has_downloads": "true", - "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", - "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", - "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", - "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", - "has_projects": "true", - "clone_url": "https://github.com/web-platform-tests/wpt.git", - "watchers_count": "1845", - "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", - "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", - "organization": "web-platform-tests", - "stargazers_count": "1845", - "homepage": "http://irc.w3.org/?channels=testing", - "open_issues": "1328", - "fork": "false", - "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", - "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", - "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", - "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", - "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", - "has_pages": "true", - "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", - "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", - "html_url": "https://github.com/web-platform-tests/wpt", - "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", - "language": "HTML", - "watchers": "1845", - "private": "false", - "forks_count": "1523", - "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", - "has_issues": "true", - "ssh_url": "git@github.com:web-platform-tests/wpt.git", - "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", - "master_branch": "master", - "forks": "1523", - "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", - "open_issues_count": "1317", - "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", - "name": "wpt", - "license": { - "spdx_id": "NOASSERTION", - "url": "null", - "node_id": "MDc6TGljZW5zZTA=", - "name": "Other", - "key": "other" - }, - "url": "https://github.com/web-platform-tests/wpt", - "stargazers": "1845", - "created_at": "1330865891", - "pushed_at": "1541037488", - "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", - "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", - "default_branch": "master", - "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", - "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", - "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", - "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", - "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", - "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", - "git_url": "git://github.com/web-platform-tests/wpt.git" - }, - "organization": { - "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", - "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", - "description": "", - "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", - "url": "https://api.github.com/orgs/web-platform-tests", - "events_url": "https://api.github.com/orgs/web-platform-tests/events", - "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", - "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", - "login": "web-platform-tests", - "id": "37226233", - "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", - "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" - }, - }, - "event": "repository", - "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" - } + "body": { + "action": "deleted", + "sender": { + "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", + "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", + "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", + "url": "https://api.github.com/users/chromium-wpt-export-bot", + "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", + "html_url": "https://github.com/chromium-wpt-export-bot", + "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", + "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", + "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", + "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", + "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", + "site_admin": "false", + "login": "chromium-wpt-export-bot", + "type": "User", + "id": "25752892", + "node_id": "MDQ6VXNlcjI1NzUyODky" + }, + "repository": { + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "has_wiki": "true", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "mirror_url": "null", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "updated_at": "2018-11-01T00:51:49Z", + "svn_url": "https://github.com/web-platform-tests/wpt", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "owner": { + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "name": "web-platform-tests", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "url": "https://api.github.com/users/web-platform-tests", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "html_url": "https://github.com/web-platform-tests", + "email": "", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "site_admin": "false", + "login": "web-platform-tests", + "type": "Organization", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" + }, + "full_name": "web-platform-tests/wpt", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "id": "3618133", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "size": "305511", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "archived": "false", + "has_downloads": "true", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "has_projects": "true", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "watchers_count": "1845", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "organization": "web-platform-tests", + "stargazers_count": "1845", + "homepage": "http://irc.w3.org/?channels=testing", + "open_issues": "1328", + "fork": "false", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "has_pages": "true", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "html_url": "https://github.com/web-platform-tests/wpt", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "language": "HTML", + "watchers": "1845", + "private": "false", + "forks_count": "1523", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "has_issues": "true", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "master_branch": "master", + "forks": "1523", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "open_issues_count": "1317", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "name": "wpt", + "license": { + "spdx_id": "NOASSERTION", + "url": "null", + "node_id": "MDc6TGljZW5zZTA=", + "name": "Other", + "key": "other" + }, + "url": "https://github.com/web-platform-tests/wpt", + "stargazers": "1845", + "created_at": "1330865891", + "pushed_at": "1541037488", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "default_branch": "master", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "git_url": "git://github.com/web-platform-tests/wpt.git" + }, + "organization": { + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "description": "", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "url": "https://api.github.com/orgs/web-platform-tests", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "login": "web-platform-tests", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" + }, + }, + "event": "repository", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" + } event = { 'tags': 'githubeventsqs' @@ -1137,164 +1122,163 @@ class TestSuricataFixup(object): self.verify_org(message, result) assert result['source'] == 'repository' assert result['details']['action'] == message['body']['action'] - - + def test_member(self): message = { - "body": { - "member": { - "id": "60618", - "login": "emmairwin", - "node_id": "MDQ6VXNlcjYwNjE4", - "site_admin": "false", - }, - "changes": { - "permission": { - "from": "write", - }, - }, - "action": "added", - "sender": { - "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", - "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", - "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", - "url": "https://api.github.com/users/chromium-wpt-export-bot", - "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", - "html_url": "https://github.com/chromium-wpt-export-bot", - "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", - "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", - "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", - "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", - "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", - "gravatar_id": "", - "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", - "site_admin": "false", - "login": "chromium-wpt-export-bot", - "type": "User", - "id": "25752892", - "node_id": "MDQ6VXNlcjI1NzUyODky" - }, - "repository": { - "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", - "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", - "has_wiki": "true", - "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", - "mirror_url": "null", - "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", - "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", - "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", - "updated_at": "2018-11-01T00:51:49Z", - "svn_url": "https://github.com/web-platform-tests/wpt", - "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", - "owner": { - "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", - "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", - "name": "web-platform-tests", - "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", - "url": "https://api.github.com/users/web-platform-tests", - "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", - "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", - "html_url": "https://github.com/web-platform-tests", - "email": "", - "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", - "repos_url": "https://api.github.com/users/web-platform-tests/repos", - "followers_url": "https://api.github.com/users/web-platform-tests/followers", - "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", - "gravatar_id": "", - "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", - "site_admin": "false", - "login": "web-platform-tests", - "type": "Organization", - "id": "37226233", - "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" - }, - "full_name": "web-platform-tests/wpt", - "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", - "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", - "id": "3618133", - "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", - "size": "305511", - "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", - "archived": "false", - "has_downloads": "true", - "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", - "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", - "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", - "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", - "has_projects": "true", - "clone_url": "https://github.com/web-platform-tests/wpt.git", - "watchers_count": "1845", - "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", - "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", - "organization": "web-platform-tests", - "stargazers_count": "1845", - "homepage": "http://irc.w3.org/?channels=testing", - "open_issues": "1328", - "fork": "false", - "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", - "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", - "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", - "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", - "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", - "has_pages": "true", - "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", - "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", - "html_url": "https://github.com/web-platform-tests/wpt", - "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", - "language": "HTML", - "watchers": "1845", - "private": "false", - "forks_count": "1523", - "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", - "has_issues": "true", - "ssh_url": "git@github.com:web-platform-tests/wpt.git", - "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", - "master_branch": "master", - "forks": "1523", - "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", - "open_issues_count": "1317", - "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", - "name": "wpt", - "license": { - "spdx_id": "NOASSERTION", - "url": "null", - "node_id": "MDc6TGljZW5zZTA=", - "name": "Other", - "key": "other" - }, - "url": "https://github.com/web-platform-tests/wpt", - "stargazers": "1845", - "created_at": "1330865891", - "pushed_at": "1541037488", - "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", - "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", - "default_branch": "master", - "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", - "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", - "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", - "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", - "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", - "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", - "git_url": "git://github.com/web-platform-tests/wpt.git" - }, - "organization": { - "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", - "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", - "description": "", - "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", - "url": "https://api.github.com/orgs/web-platform-tests", - "events_url": "https://api.github.com/orgs/web-platform-tests/events", - "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", - "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", - "login": "web-platform-tests", - "id": "37226233", - "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", - "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" - }, - }, - "event": "member", - "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" - } + "body": { + "member": { + "id": "60618", + "login": "emmairwin", + "node_id": "MDQ6VXNlcjYwNjE4", + "site_admin": "false", + }, + "changes": { + "permission": { + "from": "write", + }, + }, + "action": "added", + "sender": { + "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", + "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", + "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", + "url": "https://api.github.com/users/chromium-wpt-export-bot", + "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", + "html_url": "https://github.com/chromium-wpt-export-bot", + "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", + "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", + "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", + "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", + "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", + "site_admin": "false", + "login": "chromium-wpt-export-bot", + "type": "User", + "id": "25752892", + "node_id": "MDQ6VXNlcjI1NzUyODky" + }, + "repository": { + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "has_wiki": "true", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "mirror_url": "null", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "updated_at": "2018-11-01T00:51:49Z", + "svn_url": "https://github.com/web-platform-tests/wpt", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "owner": { + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "name": "web-platform-tests", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "url": "https://api.github.com/users/web-platform-tests", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "html_url": "https://github.com/web-platform-tests", + "email": "", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "site_admin": "false", + "login": "web-platform-tests", + "type": "Organization", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" + }, + "full_name": "web-platform-tests/wpt", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "id": "3618133", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "size": "305511", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "archived": "false", + "has_downloads": "true", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "has_projects": "true", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "watchers_count": "1845", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "organization": "web-platform-tests", + "stargazers_count": "1845", + "homepage": "http://irc.w3.org/?channels=testing", + "open_issues": "1328", + "fork": "false", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "has_pages": "true", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "html_url": "https://github.com/web-platform-tests/wpt", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "language": "HTML", + "watchers": "1845", + "private": "false", + "forks_count": "1523", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "has_issues": "true", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "master_branch": "master", + "forks": "1523", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "open_issues_count": "1317", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "name": "wpt", + "license": { + "spdx_id": "NOASSERTION", + "url": "null", + "node_id": "MDc6TGljZW5zZTA=", + "name": "Other", + "key": "other" + }, + "url": "https://github.com/web-platform-tests/wpt", + "stargazers": "1845", + "created_at": "1330865891", + "pushed_at": "1541037488", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "default_branch": "master", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "git_url": "git://github.com/web-platform-tests/wpt.git" + }, + "organization": { + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "description": "", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "url": "https://api.github.com/orgs/web-platform-tests", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "login": "web-platform-tests", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" + }, + }, + "event": "member", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" + } event = { 'tags': 'githubeventsqs' @@ -1317,174 +1301,174 @@ class TestSuricataFixup(object): def test_team(self): message = { - "body": { - "team": { - "id": "9060454", - "name": "asecretteam", - "login": "alamakota", - "node_id": "MYQ6VXK4fuwwNAye", - "permission": "pull", - "privacy": "secret", - "slug": "asecretteam", - }, - "changes": { - "repository": { - "permissions": { - "from": { - "admin": "false", - "pull": "false", - "push": "false", - }, + "body": { + "team": { + "id": "9060454", + "name": "asecretteam", + "login": "alamakota", + "node_id": "MYQ6VXK4fuwwNAye", + "permission": "pull", + "privacy": "secret", + "slug": "asecretteam", + }, + "changes": { + "repository": { + "permissions": { + "from": { + "admin": "false", + "pull": "false", + "push": "false", + }, + }, + }, + }, + "action": "edited", + "sender": { + "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", + "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", + "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", + "url": "https://api.github.com/users/chromium-wpt-export-bot", + "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", + "html_url": "https://github.com/chromium-wpt-export-bot", + "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", + "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", + "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", + "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", + "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", + "site_admin": "false", + "login": "chromium-wpt-export-bot", + "type": "User", + "id": "25752892", + "node_id": "MDQ6VXNlcjI1NzUyODky" + }, + "repository": { + "permissions": { + "admin": "true", + "pull": "true", + "push": "true", + }, + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "has_wiki": "true", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "mirror_url": "null", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "updated_at": "2018-11-01T00:51:49Z", + "svn_url": "https://github.com/web-platform-tests/wpt", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "owner": { + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "name": "web-platform-tests", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "url": "https://api.github.com/users/web-platform-tests", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "html_url": "https://github.com/web-platform-tests", + "email": "", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "site_admin": "false", + "login": "web-platform-tests", + "type": "Organization", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" + }, + "full_name": "web-platform-tests/wpt", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "id": "3618133", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "size": "305511", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "archived": "false", + "has_downloads": "true", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "has_projects": "true", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "watchers_count": "1845", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "organization": "web-platform-tests", + "stargazers_count": "1845", + "homepage": "http://irc.w3.org/?channels=testing", + "open_issues": "1328", + "fork": "false", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "has_pages": "true", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "html_url": "https://github.com/web-platform-tests/wpt", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "language": "HTML", + "watchers": "1845", + "private": "false", + "forks_count": "1523", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "has_issues": "true", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "master_branch": "master", + "forks": "1523", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "open_issues_count": "1317", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "name": "wpt", + "license": { + "spdx_id": "NOASSERTION", + "url": "null", + "node_id": "MDc6TGljZW5zZTA=", + "name": "Other", + "key": "other" + }, + "url": "https://github.com/web-platform-tests/wpt", + "stargazers": "1845", + "created_at": "1330865891", + "pushed_at": "1541037488", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "default_branch": "master", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "git_url": "git://github.com/web-platform-tests/wpt.git" + }, + "organization": { + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "description": "", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "url": "https://api.github.com/orgs/web-platform-tests", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "login": "web-platform-tests", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" + }, }, - }, - }, - "action": "edited", - "sender": { - "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", - "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", - "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", - "url": "https://api.github.com/users/chromium-wpt-export-bot", - "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", - "html_url": "https://github.com/chromium-wpt-export-bot", - "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", - "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", - "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", - "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", - "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", - "gravatar_id": "", - "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", - "site_admin": "false", - "login": "chromium-wpt-export-bot", - "type": "User", - "id": "25752892", - "node_id": "MDQ6VXNlcjI1NzUyODky" - }, - "repository": { - "permissions": { - "admin": "true", - "pull": "true", - "push": "true", - }, - "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", - "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", - "has_wiki": "true", - "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", - "mirror_url": "null", - "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", - "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", - "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", - "updated_at": "2018-11-01T00:51:49Z", - "svn_url": "https://github.com/web-platform-tests/wpt", - "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", - "owner": { - "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", - "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", - "name": "web-platform-tests", - "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", - "url": "https://api.github.com/users/web-platform-tests", - "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", - "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", - "html_url": "https://github.com/web-platform-tests", - "email": "", - "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", - "repos_url": "https://api.github.com/users/web-platform-tests/repos", - "followers_url": "https://api.github.com/users/web-platform-tests/followers", - "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", - "gravatar_id": "", - "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", - "site_admin": "false", - "login": "web-platform-tests", - "type": "Organization", - "id": "37226233", - "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" - }, - "full_name": "web-platform-tests/wpt", - "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", - "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", - "id": "3618133", - "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", - "size": "305511", - "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", - "archived": "false", - "has_downloads": "true", - "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", - "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", - "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", - "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", - "has_projects": "true", - "clone_url": "https://github.com/web-platform-tests/wpt.git", - "watchers_count": "1845", - "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", - "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", - "organization": "web-platform-tests", - "stargazers_count": "1845", - "homepage": "http://irc.w3.org/?channels=testing", - "open_issues": "1328", - "fork": "false", - "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", - "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", - "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", - "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", - "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", - "has_pages": "true", - "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", - "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", - "html_url": "https://github.com/web-platform-tests/wpt", - "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", - "language": "HTML", - "watchers": "1845", - "private": "false", - "forks_count": "1523", - "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", - "has_issues": "true", - "ssh_url": "git@github.com:web-platform-tests/wpt.git", - "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", - "master_branch": "master", - "forks": "1523", - "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", - "open_issues_count": "1317", - "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", - "name": "wpt", - "license": { - "spdx_id": "NOASSERTION", - "url": "null", - "node_id": "MDc6TGljZW5zZTA=", - "name": "Other", - "key": "other" - }, - "url": "https://github.com/web-platform-tests/wpt", - "stargazers": "1845", - "created_at": "1330865891", - "pushed_at": "1541037488", - "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", - "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", - "default_branch": "master", - "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", - "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", - "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", - "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", - "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", - "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", - "git_url": "git://github.com/web-platform-tests/wpt.git" - }, - "organization": { - "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", - "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", - "description": "", - "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", - "url": "https://api.github.com/orgs/web-platform-tests", - "events_url": "https://api.github.com/orgs/web-platform-tests/events", - "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", - "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", - "login": "web-platform-tests", - "id": "37226233", - "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", - "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" - }, - }, - "event": "team", - "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" - } + "event": "team", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" + } event = { 'tags': 'githubeventsqs' @@ -1511,177 +1495,176 @@ class TestSuricataFixup(object): assert result['details']['team_permission'] == message['body']['team']['permission'] assert result['details']['team_privacy'] == message['body']['team']['privacy'] assert result['details']['team_slug'] == message['body']['team']['slug'] - def test_team_add(self): message = { - "body": { - "team": { - "id": "9060454", - "name": "asecretteam", - "login": "alamakota", - "node_id": "MYQ6VXK4fuwwNAye", - "permission": "pull", - "privacy": "secret", - "slug": "asecretteam", - }, - "changes": { - "repository": { - "permissions": { - "from": { - "admin": "false", - "pull": "false", - "push": "false", - }, + "body": { + "team": { + "id": "9060454", + "name": "asecretteam", + "login": "alamakota", + "node_id": "MYQ6VXK4fuwwNAye", + "permission": "pull", + "privacy": "secret", + "slug": "asecretteam", + }, + "changes": { + "repository": { + "permissions": { + "from": { + "admin": "false", + "pull": "false", + "push": "false", + }, + }, + }, + }, + "sender": { + "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", + "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", + "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", + "url": "https://api.github.com/users/chromium-wpt-export-bot", + "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", + "html_url": "https://github.com/chromium-wpt-export-bot", + "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", + "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", + "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", + "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", + "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", + "site_admin": "false", + "login": "chromium-wpt-export-bot", + "type": "User", + "id": "25752892", + "node_id": "MDQ6VXNlcjI1NzUyODky" + }, + "repository": { + "permissions": { + "admin": "true", + "pull": "true", + "push": "true", + }, + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "has_wiki": "true", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "mirror_url": "null", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "updated_at": "2018-11-01T00:51:49Z", + "svn_url": "https://github.com/web-platform-tests/wpt", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "owner": { + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "name": "web-platform-tests", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "url": "https://api.github.com/users/web-platform-tests", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "html_url": "https://github.com/web-platform-tests", + "email": "", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "site_admin": "false", + "login": "web-platform-tests", + "type": "Organization", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" + }, + "full_name": "web-platform-tests/wpt", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "id": "3618133", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "size": "305511", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "archived": "false", + "has_downloads": "true", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "has_projects": "true", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "watchers_count": "1845", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "organization": "web-platform-tests", + "stargazers_count": "1845", + "homepage": "http://irc.w3.org/?channels=testing", + "open_issues": "1328", + "fork": "false", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "has_pages": "true", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "html_url": "https://github.com/web-platform-tests/wpt", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "language": "HTML", + "watchers": "1845", + "private": "false", + "forks_count": "1523", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "has_issues": "true", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "master_branch": "master", + "forks": "1523", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "open_issues_count": "1317", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "name": "wpt", + "license": { + "spdx_id": "NOASSERTION", + "url": "null", + "node_id": "MDc6TGljZW5zZTA=", + "name": "Other", + "key": "other" + }, + "url": "https://github.com/web-platform-tests/wpt", + "stargazers": "1845", + "created_at": "1330865891", + "pushed_at": "1541037488", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "default_branch": "master", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "git_url": "git://github.com/web-platform-tests/wpt.git" + }, + "organization": { + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "description": "", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "url": "https://api.github.com/orgs/web-platform-tests", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "login": "web-platform-tests", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" + }, }, - }, - }, - "sender": { - "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", - "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", - "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", - "url": "https://api.github.com/users/chromium-wpt-export-bot", - "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", - "html_url": "https://github.com/chromium-wpt-export-bot", - "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", - "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", - "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", - "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", - "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", - "gravatar_id": "", - "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", - "site_admin": "false", - "login": "chromium-wpt-export-bot", - "type": "User", - "id": "25752892", - "node_id": "MDQ6VXNlcjI1NzUyODky" - }, - "repository": { - "permissions": { - "admin": "true", - "pull": "true", - "push": "true", - }, - "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", - "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", - "has_wiki": "true", - "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", - "mirror_url": "null", - "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", - "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", - "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", - "updated_at": "2018-11-01T00:51:49Z", - "svn_url": "https://github.com/web-platform-tests/wpt", - "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", - "owner": { - "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", - "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", - "name": "web-platform-tests", - "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", - "url": "https://api.github.com/users/web-platform-tests", - "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", - "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", - "html_url": "https://github.com/web-platform-tests", - "email": "", - "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", - "repos_url": "https://api.github.com/users/web-platform-tests/repos", - "followers_url": "https://api.github.com/users/web-platform-tests/followers", - "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", - "gravatar_id": "", - "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", - "site_admin": "false", - "login": "web-platform-tests", - "type": "Organization", - "id": "37226233", - "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" - }, - "full_name": "web-platform-tests/wpt", - "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", - "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", - "id": "3618133", - "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", - "size": "305511", - "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", - "archived": "false", - "has_downloads": "true", - "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", - "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", - "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", - "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", - "has_projects": "true", - "clone_url": "https://github.com/web-platform-tests/wpt.git", - "watchers_count": "1845", - "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", - "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", - "organization": "web-platform-tests", - "stargazers_count": "1845", - "homepage": "http://irc.w3.org/?channels=testing", - "open_issues": "1328", - "fork": "false", - "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", - "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", - "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", - "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", - "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", - "has_pages": "true", - "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", - "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", - "html_url": "https://github.com/web-platform-tests/wpt", - "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", - "language": "HTML", - "watchers": "1845", - "private": "false", - "forks_count": "1523", - "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", - "has_issues": "true", - "ssh_url": "git@github.com:web-platform-tests/wpt.git", - "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", - "master_branch": "master", - "forks": "1523", - "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", - "open_issues_count": "1317", - "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", - "name": "wpt", - "license": { - "spdx_id": "NOASSERTION", - "url": "null", - "node_id": "MDc6TGljZW5zZTA=", - "name": "Other", - "key": "other" - }, - "url": "https://github.com/web-platform-tests/wpt", - "stargazers": "1845", - "created_at": "1330865891", - "pushed_at": "1541037488", - "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", - "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", - "default_branch": "master", - "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", - "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", - "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", - "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", - "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", - "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", - "git_url": "git://github.com/web-platform-tests/wpt.git" - }, - "organization": { - "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", - "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", - "description": "", - "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", - "url": "https://api.github.com/orgs/web-platform-tests", - "events_url": "https://api.github.com/orgs/web-platform-tests/events", - "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", - "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", - "login": "web-platform-tests", - "id": "37226233", - "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", - "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" - }, - }, - "event": "team_add", - "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" - } + "event": "team_add", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" + } event = { 'tags': 'githubeventsqs' @@ -1707,178 +1690,177 @@ class TestSuricataFixup(object): assert result['details']['team_permission'] == message['body']['team']['permission'] assert result['details']['team_privacy'] == message['body']['team']['privacy'] assert result['details']['team_slug'] == message['body']['team']['slug'] - def test_organization(self): message = { - "body": { - "team": { - "id": "9060454", - "name": "asecretteam", - "login": "alamakota", - "node_id": "MYQ6VXK4fuwwNAye", - "permission": "pull", - "privacy": "secret", - "slug": "asecretteam", - }, - "membership": { - "user": { - "id": "893282", - "login": "alamakota", - "node_id": "MDQ6VXNlcjUwMTkyMzQ=", - "site_admin": "false", - "type": "User", - }, - "role": "member", - "state": "pending", - }, - "action": "member_added", - "sender": { - "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", - "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", - "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", - "url": "https://api.github.com/users/chromium-wpt-export-bot", - "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", - "html_url": "https://github.com/chromium-wpt-export-bot", - "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", - "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", - "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", - "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", - "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", - "gravatar_id": "", - "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", - "site_admin": "false", - "login": "chromium-wpt-export-bot", - "type": "User", - "id": "25752892", - "node_id": "MDQ6VXNlcjI1NzUyODky" - }, - "repository": { - "permissions": { - "admin": "true", - "pull": "true", - "push": "true", - }, - "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", - "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", - "has_wiki": "true", - "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", - "mirror_url": "null", - "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", - "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", - "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", - "updated_at": "2018-11-01T00:51:49Z", - "svn_url": "https://github.com/web-platform-tests/wpt", - "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", - "owner": { - "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", - "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", - "name": "web-platform-tests", - "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", - "url": "https://api.github.com/users/web-platform-tests", - "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", - "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", - "html_url": "https://github.com/web-platform-tests", - "email": "", - "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", - "repos_url": "https://api.github.com/users/web-platform-tests/repos", - "followers_url": "https://api.github.com/users/web-platform-tests/followers", - "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", - "gravatar_id": "", - "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", - "site_admin": "false", - "login": "web-platform-tests", - "type": "Organization", - "id": "37226233", - "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" - }, - "full_name": "web-platform-tests/wpt", - "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", - "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", - "id": "3618133", - "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", - "size": "305511", - "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", - "archived": "false", - "has_downloads": "true", - "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", - "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", - "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", - "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", - "has_projects": "true", - "clone_url": "https://github.com/web-platform-tests/wpt.git", - "watchers_count": "1845", - "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", - "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", - "organization": "web-platform-tests", - "stargazers_count": "1845", - "homepage": "http://irc.w3.org/?channels=testing", - "open_issues": "1328", - "fork": "false", - "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", - "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", - "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", - "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", - "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", - "has_pages": "true", - "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", - "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", - "html_url": "https://github.com/web-platform-tests/wpt", - "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", - "language": "HTML", - "watchers": "1845", - "private": "false", - "forks_count": "1523", - "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", - "has_issues": "true", - "ssh_url": "git@github.com:web-platform-tests/wpt.git", - "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", - "master_branch": "master", - "forks": "1523", - "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", - "open_issues_count": "1317", - "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", - "name": "wpt", - "license": { - "spdx_id": "NOASSERTION", - "url": "null", - "node_id": "MDc6TGljZW5zZTA=", - "name": "Other", - "key": "other" - }, - "url": "https://github.com/web-platform-tests/wpt", - "stargazers": "1845", - "created_at": "1330865891", - "pushed_at": "1541037488", - "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", - "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", - "default_branch": "master", - "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", - "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", - "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", - "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", - "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", - "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", - "git_url": "git://github.com/web-platform-tests/wpt.git" - }, - "organization": { - "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", - "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", - "description": "", - "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", - "url": "https://api.github.com/orgs/web-platform-tests", - "events_url": "https://api.github.com/orgs/web-platform-tests/events", - "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", - "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", - "login": "web-platform-tests", - "id": "37226233", - "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", - "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" - }, - }, - "event": "organization", - "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" - } + "body": { + "team": { + "id": "9060454", + "name": "asecretteam", + "login": "alamakota", + "node_id": "MYQ6VXK4fuwwNAye", + "permission": "pull", + "privacy": "secret", + "slug": "asecretteam", + }, + "membership": { + "user": { + "id": "893282", + "login": "alamakota", + "node_id": "MDQ6VXNlcjUwMTkyMzQ=", + "site_admin": "false", + "type": "User", + }, + "role": "member", + "state": "pending", + }, + "action": "member_added", + "sender": { + "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", + "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", + "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", + "url": "https://api.github.com/users/chromium-wpt-export-bot", + "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", + "html_url": "https://github.com/chromium-wpt-export-bot", + "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", + "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", + "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", + "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", + "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", + "site_admin": "false", + "login": "chromium-wpt-export-bot", + "type": "User", + "id": "25752892", + "node_id": "MDQ6VXNlcjI1NzUyODky" + }, + "repository": { + "permissions": { + "admin": "true", + "pull": "true", + "push": "true", + }, + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "has_wiki": "true", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "mirror_url": "null", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "updated_at": "2018-11-01T00:51:49Z", + "svn_url": "https://github.com/web-platform-tests/wpt", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "owner": { + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "name": "web-platform-tests", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "url": "https://api.github.com/users/web-platform-tests", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "html_url": "https://github.com/web-platform-tests", + "email": "", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "site_admin": "false", + "login": "web-platform-tests", + "type": "Organization", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" + }, + "full_name": "web-platform-tests/wpt", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "id": "3618133", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "size": "305511", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "archived": "false", + "has_downloads": "true", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "has_projects": "true", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "watchers_count": "1845", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "organization": "web-platform-tests", + "stargazers_count": "1845", + "homepage": "http://irc.w3.org/?channels=testing", + "open_issues": "1328", + "fork": "false", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "has_pages": "true", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "html_url": "https://github.com/web-platform-tests/wpt", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "language": "HTML", + "watchers": "1845", + "private": "false", + "forks_count": "1523", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "has_issues": "true", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "master_branch": "master", + "forks": "1523", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "open_issues_count": "1317", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "name": "wpt", + "license": { + "spdx_id": "NOASSERTION", + "url": "null", + "node_id": "MDc6TGljZW5zZTA=", + "name": "Other", + "key": "other" + }, + "url": "https://github.com/web-platform-tests/wpt", + "stargazers": "1845", + "created_at": "1330865891", + "pushed_at": "1541037488", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "default_branch": "master", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "git_url": "git://github.com/web-platform-tests/wpt.git" + }, + "organization": { + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "description": "", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "url": "https://api.github.com/orgs/web-platform-tests", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "login": "web-platform-tests", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" + }, + }, + "event": "organization", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" + } event = { 'tags': 'githubeventsqs' @@ -1908,155 +1890,155 @@ class TestSuricataFixup(object): def test_membership(self): message = { - "body": { - "action": "member_added", - "scope": "team", - "sender": { - "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", - "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", - "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", - "url": "https://api.github.com/users/chromium-wpt-export-bot", - "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", - "html_url": "https://github.com/chromium-wpt-export-bot", - "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", - "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", - "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", - "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", - "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", - "gravatar_id": "", - "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", - "site_admin": "false", - "login": "chromium-wpt-export-bot", - "type": "User", - "id": "25752892", - "node_id": "MDQ6VXNlcjI1NzUyODky" + "body": { + "action": "member_added", + "scope": "team", + "sender": { + "following_url": "https://api.github.com/users/chromium-wpt-export-bot/following{/other_user}", + "events_url": "https://api.github.com/users/chromium-wpt-export-bot/events{/privacy}", + "organizations_url": "https://api.github.com/users/chromium-wpt-export-bot/orgs", + "url": "https://api.github.com/users/chromium-wpt-export-bot", + "gists_url": "https://api.github.com/users/chromium-wpt-export-bot/gists{/gist_id}", + "html_url": "https://github.com/chromium-wpt-export-bot", + "subscriptions_url": "https://api.github.com/users/chromium-wpt-export-bot/subscriptions", + "avatar_url": "https://avatars1.githubusercontent.com/u/25752892?v=4", + "repos_url": "https://api.github.com/users/chromium-wpt-export-bot/repos", + "followers_url": "https://api.github.com/users/chromium-wpt-export-bot/followers", + "received_events_url": "https://api.github.com/users/chromium-wpt-export-bot/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/chromium-wpt-export-bot/starred{/owner}{/repo}", + "site_admin": "false", + "login": "chromium-wpt-export-bot", + "type": "User", + "id": "25752892", + "node_id": "MDQ6VXNlcjI1NzUyODky" + }, + "repository": { + "permissions": { + "admin": "true", + "pull": "true", + "push": "true", + }, + "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", + "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", + "has_wiki": "true", + "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", + "mirror_url": "null", + "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", + "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", + "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", + "updated_at": "2018-11-01T00:51:49Z", + "svn_url": "https://github.com/web-platform-tests/wpt", + "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", + "owner": { + "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", + "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", + "name": "web-platform-tests", + "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", + "url": "https://api.github.com/users/web-platform-tests", + "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", + "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", + "html_url": "https://github.com/web-platform-tests", + "email": "", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/users/web-platform-tests/repos", + "followers_url": "https://api.github.com/users/web-platform-tests/followers", + "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", + "gravatar_id": "", + "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", + "site_admin": "false", + "login": "web-platform-tests", + "type": "Organization", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" + }, + "full_name": "web-platform-tests/wpt", + "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", + "id": "3618133", + "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", + "size": "305511", + "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", + "archived": "false", + "has_downloads": "true", + "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", + "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", + "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", + "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", + "has_projects": "true", + "clone_url": "https://github.com/web-platform-tests/wpt.git", + "watchers_count": "1845", + "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", + "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", + "organization": "web-platform-tests", + "stargazers_count": "1845", + "homepage": "http://irc.w3.org/?channels=testing", + "open_issues": "1328", + "fork": "false", + "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", + "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", + "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", + "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", + "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", + "has_pages": "true", + "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", + "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", + "html_url": "https://github.com/web-platform-tests/wpt", + "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", + "language": "HTML", + "watchers": "1845", + "private": "false", + "forks_count": "1523", + "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", + "has_issues": "true", + "ssh_url": "git@github.com:web-platform-tests/wpt.git", + "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", + "master_branch": "master", + "forks": "1523", + "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", + "open_issues_count": "1317", + "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", + "name": "wpt", + "license": { + "spdx_id": "NOASSERTION", + "url": "null", + "node_id": "MDc6TGljZW5zZTA=", + "name": "Other", + "key": "other" + }, + "url": "https://github.com/web-platform-tests/wpt", + "stargazers": "1845", + "created_at": "1330865891", + "pushed_at": "1541037488", + "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", + "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", + "default_branch": "master", + "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", + "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", + "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", + "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", + "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", + "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", + "git_url": "git://github.com/web-platform-tests/wpt.git" + }, + "organization": { + "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", + "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", + "description": "", + "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", + "url": "https://api.github.com/orgs/web-platform-tests", + "events_url": "https://api.github.com/orgs/web-platform-tests/events", + "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", + "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", + "login": "web-platform-tests", + "id": "37226233", + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", + "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" + }, }, - "repository": { - "permissions": { - "admin": "true", - "pull": "true", - "push": "true", - }, - "issues_url": "https://api.github.com/repos/web-platform-tests/wpt/issues{/number}", - "deployments_url": "https://api.github.com/repos/web-platform-tests/wpt/deployments", - "has_wiki": "true", - "forks_url": "https://api.github.com/repos/web-platform-tests/wpt/forks", - "mirror_url": "null", - "subscription_url": "https://api.github.com/repos/web-platform-tests/wpt/subscription", - "merges_url": "https://api.github.com/repos/web-platform-tests/wpt/merges", - "collaborators_url": "https://api.github.com/repos/web-platform-tests/wpt/collaborators{/collaborator}", - "updated_at": "2018-11-01T00:51:49Z", - "svn_url": "https://github.com/web-platform-tests/wpt", - "pulls_url": "https://api.github.com/repos/web-platform-tests/wpt/pulls{/number}", - "owner": { - "following_url": "https://api.github.com/users/web-platform-tests/following{/other_user}", - "events_url": "https://api.github.com/users/web-platform-tests/events{/privacy}", - "name": "web-platform-tests", - "organizations_url": "https://api.github.com/users/web-platform-tests/orgs", - "url": "https://api.github.com/users/web-platform-tests", - "gists_url": "https://api.github.com/users/web-platform-tests/gists{/gist_id}", - "subscriptions_url": "https://api.github.com/users/web-platform-tests/subscriptions", - "html_url": "https://github.com/web-platform-tests", - "email": "", - "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", - "repos_url": "https://api.github.com/users/web-platform-tests/repos", - "followers_url": "https://api.github.com/users/web-platform-tests/followers", - "received_events_url": "https://api.github.com/users/web-platform-tests/received_events", - "gravatar_id": "", - "starred_url": "https://api.github.com/users/web-platform-tests/starred{/owner}{/repo}", - "site_admin": "false", - "login": "web-platform-tests", - "type": "Organization", - "id": "37226233", - "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz" - }, - "full_name": "web-platform-tests/wpt", - "issue_comment_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/comments{/number}", - "contents_url": "https://api.github.com/repos/web-platform-tests/wpt/contents/{+path}", - "id": "3618133", - "keys_url": "https://api.github.com/repos/web-platform-tests/wpt/keys{/key_id}", - "size": "305511", - "tags_url": "https://api.github.com/repos/web-platform-tests/wpt/tags", - "archived": "false", - "has_downloads": "true", - "downloads_url": "https://api.github.com/repos/web-platform-tests/wpt/downloads", - "assignees_url": "https://api.github.com/repos/web-platform-tests/wpt/assignees{/user}", - "statuses_url": "https://api.github.com/repos/web-platform-tests/wpt/statuses/{sha}", - "git_refs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/refs{/sha}", - "has_projects": "true", - "clone_url": "https://github.com/web-platform-tests/wpt.git", - "watchers_count": "1845", - "git_tags_url": "https://api.github.com/repos/web-platform-tests/wpt/git/tags{/sha}", - "labels_url": "https://api.github.com/repos/web-platform-tests/wpt/labels{/name}", - "organization": "web-platform-tests", - "stargazers_count": "1845", - "homepage": "http://irc.w3.org/?channels=testing", - "open_issues": "1328", - "fork": "false", - "milestones_url": "https://api.github.com/repos/web-platform-tests/wpt/milestones{/number}", - "commits_url": "https://api.github.com/repos/web-platform-tests/wpt/commits{/sha}", - "releases_url": "https://api.github.com/repos/web-platform-tests/wpt/releases{/id}", - "issue_events_url": "https://api.github.com/repos/web-platform-tests/wpt/issues/events{/number}", - "archive_url": "https://api.github.com/repos/web-platform-tests/wpt/{archive_format}{/ref}", - "has_pages": "true", - "events_url": "https://api.github.com/repos/web-platform-tests/wpt/events", - "contributors_url": "https://api.github.com/repos/web-platform-tests/wpt/contributors", - "html_url": "https://github.com/web-platform-tests/wpt", - "compare_url": "https://api.github.com/repos/web-platform-tests/wpt/compare/{base}...{head}", - "language": "HTML", - "watchers": "1845", - "private": "false", - "forks_count": "1523", - "notifications_url": "https://api.github.com/repos/web-platform-tests/wpt/notifications{?since,all,participating}", - "has_issues": "true", - "ssh_url": "git@github.com:web-platform-tests/wpt.git", - "blobs_url": "https://api.github.com/repos/web-platform-tests/wpt/git/blobs{/sha}", - "master_branch": "master", - "forks": "1523", - "hooks_url": "https://api.github.com/repos/web-platform-tests/wpt/hooks", - "open_issues_count": "1317", - "comments_url": "https://api.github.com/repos/web-platform-tests/wpt/comments{/number}", - "name": "wpt", - "license": { - "spdx_id": "NOASSERTION", - "url": "null", - "node_id": "MDc6TGljZW5zZTA=", - "name": "Other", - "key": "other" - }, - "url": "https://github.com/web-platform-tests/wpt", - "stargazers": "1845", - "created_at": "1330865891", - "pushed_at": "1541037488", - "branches_url": "https://api.github.com/repos/web-platform-tests/wpt/branches{/branch}", - "node_id": "MDEwOlJlcG9zaXRvcnkzNjE4MTMz", - "default_branch": "master", - "teams_url": "https://api.github.com/repos/web-platform-tests/wpt/teams", - "trees_url": "https://api.github.com/repos/web-platform-tests/wpt/git/trees{/sha}", - "languages_url": "https://api.github.com/repos/web-platform-tests/wpt/languages", - "git_commits_url": "https://api.github.com/repos/web-platform-tests/wpt/git/commits{/sha}", - "subscribers_url": "https://api.github.com/repos/web-platform-tests/wpt/subscribers", - "stargazers_url": "https://api.github.com/repos/web-platform-tests/wpt/stargazers", - "git_url": "git://github.com/web-platform-tests/wpt.git" - }, - "organization": { - "issues_url": "https://api.github.com/orgs/web-platform-tests/issues", - "members_url": "https://api.github.com/orgs/web-platform-tests/members{/member}", - "description": "", - "public_members_url": "https://api.github.com/orgs/web-platform-tests/public_members{/member}", - "url": "https://api.github.com/orgs/web-platform-tests", - "events_url": "https://api.github.com/orgs/web-platform-tests/events", - "avatar_url": "https://avatars0.githubusercontent.com/u/37226233?v=4", - "repos_url": "https://api.github.com/orgs/web-platform-tests/repos", - "login": "web-platform-tests", - "id": "37226233", - "node_id": "MDEyOk9yZ2FuaXphdGlvbjM3MjI2MjMz", - "hooks_url": "https://api.github.com/orgs/web-platform-tests/hooks" - }, - }, - "event": "membership", - "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" - } + "event": "membership", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd" + } event = { 'tags': 'githubeventsqs' @@ -2074,124 +2056,124 @@ class TestSuricataFixup(object): def test_public(self): message = { - "body": { - "repository": { - "id": "135493233", - "node_id": "MDEwOlJlcG9zaXRvcnkxMzU0OTMyMzM=", - "name": "Hello-World", - "full_name": "Codertocat/Hello-World", - "owner": { - "name": "ACrazyCat", - "login": "Codertocat", - "id": "21031067", - "node_id": "MDQ6VXNlcjIxMDMxMDY3", - "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", - "gravatar_id": "", - "url": "https://api.github.com/users/Codertocat", - "html_url": "https://github.com/Codertocat", - "followers_url": "https://api.github.com/users/Codertocat/followers", - "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", - "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", - "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", - "organizations_url": "https://api.github.com/users/Codertocat/orgs", - "repos_url": "https://api.github.com/users/Codertocat/repos", - "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", - "received_events_url": "https://api.github.com/users/Codertocat/received_events", - "type": "User", - "site_admin": "false" - }, - "private": "false", - "html_url": "https://github.com/Codertocat/Hello-World", - "description": "null", - "fork": "false", - "url": "https://api.github.com/repos/Codertocat/Hello-World", - "forks_url": "https://api.github.com/repos/Codertocat/Hello-World/forks", - "keys_url": "https://api.github.com/repos/Codertocat/Hello-World/keys{/key_id}", - "collaborators_url": "https://api.github.com/repos/Codertocat/Hello-World/collaborators{/collaborator}", - "teams_url": "https://api.github.com/repos/Codertocat/Hello-World/teams", - "hooks_url": "https://api.github.com/repos/Codertocat/Hello-World/hooks", - "issue_events_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/events{/number}", - "events_url": "https://api.github.com/repos/Codertocat/Hello-World/events", - "assignees_url": "https://api.github.com/repos/Codertocat/Hello-World/assignees{/user}", - "branches_url": "https://api.github.com/repos/Codertocat/Hello-World/branches{/branch}", - "tags_url": "https://api.github.com/repos/Codertocat/Hello-World/tags", - "blobs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/blobs{/sha}", - "git_tags_url": "https://api.github.com/repos/Codertocat/Hello-World/git/tags{/sha}", - "git_refs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/refs{/sha}", - "trees_url": "https://api.github.com/repos/Codertocat/Hello-World/git/trees{/sha}", - "statuses_url": "https://api.github.com/repos/Codertocat/Hello-World/statuses/{sha}", - "languages_url": "https://api.github.com/repos/Codertocat/Hello-World/languages", - "stargazers_url": "https://api.github.com/repos/Codertocat/Hello-World/stargazers", - "contributors_url": "https://api.github.com/repos/Codertocat/Hello-World/contributors", - "subscribers_url": "https://api.github.com/repos/Codertocat/Hello-World/subscribers", - "subscription_url": "https://api.github.com/repos/Codertocat/Hello-World/subscription", - "commits_url": "https://api.github.com/repos/Codertocat/Hello-World/commits{/sha}", - "git_commits_url": "https://api.github.com/repos/Codertocat/Hello-World/git/commits{/sha}", - "comments_url": "https://api.github.com/repos/Codertocat/Hello-World/comments{/number}", - "issue_comment_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/comments{/number}", - "contents_url": "https://api.github.com/repos/Codertocat/Hello-World/contents/{+path}", - "compare_url": "https://api.github.com/repos/Codertocat/Hello-World/compare/{base}...{head}", - "merges_url": "https://api.github.com/repos/Codertocat/Hello-World/merges", - "archive_url": "https://api.github.com/repos/Codertocat/Hello-World/{archive_format}{/ref}", - "downloads_url": "https://api.github.com/repos/Codertocat/Hello-World/downloads", - "issues_url": "https://api.github.com/repos/Codertocat/Hello-World/issues{/number}", - "pulls_url": "https://api.github.com/repos/Codertocat/Hello-World/pulls{/number}", - "milestones_url": "https://api.github.com/repos/Codertocat/Hello-World/milestones{/number}", - "notifications_url": "https://api.github.com/repos/Codertocat/Hello-World/notifications{?since,all,participating}", - "labels_url": "https://api.github.com/repos/Codertocat/Hello-World/labels{/name}", - "releases_url": "https://api.github.com/repos/Codertocat/Hello-World/releases{/id}", - "deployments_url": "https://api.github.com/repos/Codertocat/Hello-World/deployments", - "created_at": "2018-05-30T20:18:04Z", - "updated_at": "2018-05-30T20:18:49Z", - "pushed_at": "2018-05-30T20:18:48Z", - "git_url": "git://github.com/Codertocat/Hello-World.git", - "ssh_url": "git@github.com:Codertocat/Hello-World.git", - "clone_url": "https://github.com/Codertocat/Hello-World.git", - "svn_url": "https://github.com/Codertocat/Hello-World", - "homepage": "null", - "size": "0", - "stargazers_count": "0", - "watchers_count": "0", - "language": "null", - "has_issues": "true", - "has_projects": "true", - "has_downloads": "true", - "has_wiki": "true", - "has_pages": "true", - "forks_count": "0", - "mirror_url": "null", - "archived": "false", - "open_issues_count": "2", - "license": "null", - "forks": "0", - "open_issues": "2", - "watchers": "0", - "default_branch": "master" - }, - "sender": { - "login": "Codertocat", - "id": "21031067", - "node_id": "MDQ6VXNlcjIxMDMxMDY3", - "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", - "gravatar_id": "", - "url": "https://api.github.com/users/Codertocat", - "html_url": "https://github.com/Codertocat", - "followers_url": "https://api.github.com/users/Codertocat/followers", - "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", - "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", - "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", - "organizations_url": "https://api.github.com/users/Codertocat/orgs", - "repos_url": "https://api.github.com/users/Codertocat/repos", - "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", - "received_events_url": "https://api.github.com/users/Codertocat/received_events", - "type": "User", - "site_admin": "false" - }, - }, - "event": "public", - "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd", + "body": { + "repository": { + "id": "135493233", + "node_id": "MDEwOlJlcG9zaXRvcnkxMzU0OTMyMzM=", + "name": "Hello-World", + "full_name": "Codertocat/Hello-World", + "owner": { + "name": "ACrazyCat", + "login": "Codertocat", + "id": "21031067", + "node_id": "MDQ6VXNlcjIxMDMxMDY3", + "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Codertocat", + "html_url": "https://github.com/Codertocat", + "followers_url": "https://api.github.com/users/Codertocat/followers", + "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", + "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", + "organizations_url": "https://api.github.com/users/Codertocat/orgs", + "repos_url": "https://api.github.com/users/Codertocat/repos", + "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/Codertocat/received_events", + "type": "User", + "site_admin": "false" + }, + "private": "false", + "html_url": "https://github.com/Codertocat/Hello-World", + "description": "null", + "fork": "false", + "url": "https://api.github.com/repos/Codertocat/Hello-World", + "forks_url": "https://api.github.com/repos/Codertocat/Hello-World/forks", + "keys_url": "https://api.github.com/repos/Codertocat/Hello-World/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/Codertocat/Hello-World/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/Codertocat/Hello-World/teams", + "hooks_url": "https://api.github.com/repos/Codertocat/Hello-World/hooks", + "issue_events_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/events{/number}", + "events_url": "https://api.github.com/repos/Codertocat/Hello-World/events", + "assignees_url": "https://api.github.com/repos/Codertocat/Hello-World/assignees{/user}", + "branches_url": "https://api.github.com/repos/Codertocat/Hello-World/branches{/branch}", + "tags_url": "https://api.github.com/repos/Codertocat/Hello-World/tags", + "blobs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/Codertocat/Hello-World/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/Codertocat/Hello-World/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/Codertocat/Hello-World/statuses/{sha}", + "languages_url": "https://api.github.com/repos/Codertocat/Hello-World/languages", + "stargazers_url": "https://api.github.com/repos/Codertocat/Hello-World/stargazers", + "contributors_url": "https://api.github.com/repos/Codertocat/Hello-World/contributors", + "subscribers_url": "https://api.github.com/repos/Codertocat/Hello-World/subscribers", + "subscription_url": "https://api.github.com/repos/Codertocat/Hello-World/subscription", + "commits_url": "https://api.github.com/repos/Codertocat/Hello-World/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/Codertocat/Hello-World/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/Codertocat/Hello-World/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/Codertocat/Hello-World/contents/{+path}", + "compare_url": "https://api.github.com/repos/Codertocat/Hello-World/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/Codertocat/Hello-World/merges", + "archive_url": "https://api.github.com/repos/Codertocat/Hello-World/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/Codertocat/Hello-World/downloads", + "issues_url": "https://api.github.com/repos/Codertocat/Hello-World/issues{/number}", + "pulls_url": "https://api.github.com/repos/Codertocat/Hello-World/pulls{/number}", + "milestones_url": "https://api.github.com/repos/Codertocat/Hello-World/milestones{/number}", + "notifications_url": "https://api.github.com/repos/Codertocat/Hello-World/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/Codertocat/Hello-World/labels{/name}", + "releases_url": "https://api.github.com/repos/Codertocat/Hello-World/releases{/id}", + "deployments_url": "https://api.github.com/repos/Codertocat/Hello-World/deployments", + "created_at": "2018-05-30T20:18:04Z", + "updated_at": "2018-05-30T20:18:49Z", + "pushed_at": "2018-05-30T20:18:48Z", + "git_url": "git://github.com/Codertocat/Hello-World.git", + "ssh_url": "git@github.com:Codertocat/Hello-World.git", + "clone_url": "https://github.com/Codertocat/Hello-World.git", + "svn_url": "https://github.com/Codertocat/Hello-World", + "homepage": "null", + "size": "0", + "stargazers_count": "0", + "watchers_count": "0", + "language": "null", + "has_issues": "true", + "has_projects": "true", + "has_downloads": "true", + "has_wiki": "true", + "has_pages": "true", + "forks_count": "0", + "mirror_url": "null", + "archived": "false", + "open_issues_count": "2", + "license": "null", + "forks": "0", + "open_issues": "2", + "watchers": "0", + "default_branch": "master" + }, + "sender": { + "login": "Codertocat", + "id": "21031067", + "node_id": "MDQ6VXNlcjIxMDMxMDY3", + "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Codertocat", + "html_url": "https://github.com/Codertocat", + "followers_url": "https://api.github.com/users/Codertocat/followers", + "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", + "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", + "organizations_url": "https://api.github.com/users/Codertocat/orgs", + "repos_url": "https://api.github.com/users/Codertocat/repos", + "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/Codertocat/received_events", + "type": "User", + "site_admin": "false" + }, + }, + "event": "public", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd", } event = { 'tags': 'githubeventsqs' @@ -2203,142 +2185,142 @@ class TestSuricataFixup(object): self.verify_meta(message, result) self.verify_repo(message, result) assert result['source'] == 'public' - + def test_repository_import(self): message = { - "body": { - "status": "success", - "repository": { - "id": 135493233, - "node_id": "MDEwOlJlcG9zaXRvcnkxMzU0OTMyMzM=", - "name": "Hello-World", - "full_name": "Codertocat/Hello-World", - "owner": { - "name": "ASuperCat", - "login": "Codertocat", - "id": 21031067, - "node_id": "MDQ6VXNlcjIxMDMxMDY3", - "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", - "gravatar_id": "", - "url": "https://api.github.com/users/Codertocat", - "html_url": "https://github.com/Codertocat", - "followers_url": "https://api.github.com/users/Codertocat/followers", - "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", - "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", - "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", - "organizations_url": "https://api.github.com/users/Codertocat/orgs", - "repos_url": "https://api.github.com/users/Codertocat/repos", - "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", - "received_events_url": "https://api.github.com/users/Codertocat/received_events", - "type": "User", - "site_admin": "false" - }, - "private": "false", - "html_url": "https://github.com/Codertocat/Hello-World", - "description": "null", - "fork": "false", - "url": "https://api.github.com/repos/Codertocat/Hello-World", - "forks_url": "https://api.github.com/repos/Codertocat/Hello-World/forks", - "keys_url": "https://api.github.com/repos/Codertocat/Hello-World/keys{/key_id}", - "collaborators_url": "https://api.github.com/repos/Codertocat/Hello-World/collaborators{/collaborator}", - "teams_url": "https://api.github.com/repos/Codertocat/Hello-World/teams", - "hooks_url": "https://api.github.com/repos/Codertocat/Hello-World/hooks", - "issue_events_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/events{/number}", - "events_url": "https://api.github.com/repos/Codertocat/Hello-World/events", - "assignees_url": "https://api.github.com/repos/Codertocat/Hello-World/assignees{/user}", - "branches_url": "https://api.github.com/repos/Codertocat/Hello-World/branches{/branch}", - "tags_url": "https://api.github.com/repos/Codertocat/Hello-World/tags", - "blobs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/blobs{/sha}", - "git_tags_url": "https://api.github.com/repos/Codertocat/Hello-World/git/tags{/sha}", - "git_refs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/refs{/sha}", - "trees_url": "https://api.github.com/repos/Codertocat/Hello-World/git/trees{/sha}", - "statuses_url": "https://api.github.com/repos/Codertocat/Hello-World/statuses/{sha}", - "languages_url": "https://api.github.com/repos/Codertocat/Hello-World/languages", - "stargazers_url": "https://api.github.com/repos/Codertocat/Hello-World/stargazers", - "contributors_url": "https://api.github.com/repos/Codertocat/Hello-World/contributors", - "subscribers_url": "https://api.github.com/repos/Codertocat/Hello-World/subscribers", - "subscription_url": "https://api.github.com/repos/Codertocat/Hello-World/subscription", - "commits_url": "https://api.github.com/repos/Codertocat/Hello-World/commits{/sha}", - "git_commits_url": "https://api.github.com/repos/Codertocat/Hello-World/git/commits{/sha}", - "comments_url": "https://api.github.com/repos/Codertocat/Hello-World/comments{/number}", - "issue_comment_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/comments{/number}", - "contents_url": "https://api.github.com/repos/Codertocat/Hello-World/contents/{+path}", - "compare_url": "https://api.github.com/repos/Codertocat/Hello-World/compare/{base}...{head}", - "merges_url": "https://api.github.com/repos/Codertocat/Hello-World/merges", - "archive_url": "https://api.github.com/repos/Codertocat/Hello-World/{archive_format}{/ref}", - "downloads_url": "https://api.github.com/repos/Codertocat/Hello-World/downloads", - "issues_url": "https://api.github.com/repos/Codertocat/Hello-World/issues{/number}", - "pulls_url": "https://api.github.com/repos/Codertocat/Hello-World/pulls{/number}", - "milestones_url": "https://api.github.com/repos/Codertocat/Hello-World/milestones{/number}", - "notifications_url": "https://api.github.com/repos/Codertocat/Hello-World/notifications{?since,all,participating}", - "labels_url": "https://api.github.com/repos/Codertocat/Hello-World/labels{/name}", - "releases_url": "https://api.github.com/repos/Codertocat/Hello-World/releases{/id}", - "deployments_url": "https://api.github.com/repos/Codertocat/Hello-World/deployments", - "created_at": "2018-05-30T20:18:04Z", - "updated_at": "2018-05-30T20:18:49Z", - "pushed_at": "2018-05-30T20:18:48Z", - "git_url": "git://github.com/Codertocat/Hello-World.git", - "ssh_url": "git@github.com:Codertocat/Hello-World.git", - "clone_url": "https://github.com/Codertocat/Hello-World.git", - "svn_url": "https://github.com/Codertocat/Hello-World", - "homepage": "null", - "size": 0, - "stargazers_count": 0, - "watchers_count": 0, - "language": "null", - "has_issues": "true", - "has_projects": "true", - "has_downloads": "true", - "has_wiki": "true", - "has_pages": "true", - "forks_count": 0, - "mirror_url": "null", - "archived": "false", - "open_issues_count": 2, - "license": "null", - "forks": 0, - "open_issues": 2, - "watchers": 0, - "default_branch": "master" - }, - "organization": { - "login": "Octocoders", - "id": 38302899, - "node_id": "MDEyOk9yZ2FuaXphdGlvbjM4MzAyODk5", - "url": "https://api.github.com/orgs/Octocoders", - "repos_url": "https://api.github.com/orgs/Octocoders/repos", - "events_url": "https://api.github.com/orgs/Octocoders/events", - "hooks_url": "https://api.github.com/orgs/Octocoders/hooks", - "issues_url": "https://api.github.com/orgs/Octocoders/issues", - "members_url": "https://api.github.com/orgs/Octocoders/members{/member}", - "public_members_url": "https://api.github.com/orgs/Octocoders/public_members{/member}", - "avatar_url": "https://avatars1.githubusercontent.com/u/38302899?v=4", - "description": "" - }, - "sender": { - "login": "Codertocat", - "id": "21031067", - "node_id": "MDQ6VXNlcjIxMDMxMDY3", - "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", - "gravatar_id": "", - "url": "https://api.github.com/users/Codertocat", - "html_url": "https://github.com/Codertocat", - "followers_url": "https://api.github.com/users/Codertocat/followers", - "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", - "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", - "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", - "organizations_url": "https://api.github.com/users/Codertocat/orgs", - "repos_url": "https://api.github.com/users/Codertocat/repos", - "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", - "received_events_url": "https://api.github.com/users/Codertocat/received_events", - "type": "User", - "site_admin": "false" - } - }, - "event": "repository_import", - "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd", + "body": { + "status": "success", + "repository": { + "id": 135493233, + "node_id": "MDEwOlJlcG9zaXRvcnkxMzU0OTMyMzM=", + "name": "Hello-World", + "full_name": "Codertocat/Hello-World", + "owner": { + "name": "ASuperCat", + "login": "Codertocat", + "id": 21031067, + "node_id": "MDQ6VXNlcjIxMDMxMDY3", + "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Codertocat", + "html_url": "https://github.com/Codertocat", + "followers_url": "https://api.github.com/users/Codertocat/followers", + "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", + "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", + "organizations_url": "https://api.github.com/users/Codertocat/orgs", + "repos_url": "https://api.github.com/users/Codertocat/repos", + "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/Codertocat/received_events", + "type": "User", + "site_admin": "false" + }, + "private": "false", + "html_url": "https://github.com/Codertocat/Hello-World", + "description": "null", + "fork": "false", + "url": "https://api.github.com/repos/Codertocat/Hello-World", + "forks_url": "https://api.github.com/repos/Codertocat/Hello-World/forks", + "keys_url": "https://api.github.com/repos/Codertocat/Hello-World/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/Codertocat/Hello-World/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/Codertocat/Hello-World/teams", + "hooks_url": "https://api.github.com/repos/Codertocat/Hello-World/hooks", + "issue_events_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/events{/number}", + "events_url": "https://api.github.com/repos/Codertocat/Hello-World/events", + "assignees_url": "https://api.github.com/repos/Codertocat/Hello-World/assignees{/user}", + "branches_url": "https://api.github.com/repos/Codertocat/Hello-World/branches{/branch}", + "tags_url": "https://api.github.com/repos/Codertocat/Hello-World/tags", + "blobs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/Codertocat/Hello-World/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/Codertocat/Hello-World/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/Codertocat/Hello-World/statuses/{sha}", + "languages_url": "https://api.github.com/repos/Codertocat/Hello-World/languages", + "stargazers_url": "https://api.github.com/repos/Codertocat/Hello-World/stargazers", + "contributors_url": "https://api.github.com/repos/Codertocat/Hello-World/contributors", + "subscribers_url": "https://api.github.com/repos/Codertocat/Hello-World/subscribers", + "subscription_url": "https://api.github.com/repos/Codertocat/Hello-World/subscription", + "commits_url": "https://api.github.com/repos/Codertocat/Hello-World/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/Codertocat/Hello-World/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/Codertocat/Hello-World/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/Codertocat/Hello-World/contents/{+path}", + "compare_url": "https://api.github.com/repos/Codertocat/Hello-World/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/Codertocat/Hello-World/merges", + "archive_url": "https://api.github.com/repos/Codertocat/Hello-World/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/Codertocat/Hello-World/downloads", + "issues_url": "https://api.github.com/repos/Codertocat/Hello-World/issues{/number}", + "pulls_url": "https://api.github.com/repos/Codertocat/Hello-World/pulls{/number}", + "milestones_url": "https://api.github.com/repos/Codertocat/Hello-World/milestones{/number}", + "notifications_url": "https://api.github.com/repos/Codertocat/Hello-World/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/Codertocat/Hello-World/labels{/name}", + "releases_url": "https://api.github.com/repos/Codertocat/Hello-World/releases{/id}", + "deployments_url": "https://api.github.com/repos/Codertocat/Hello-World/deployments", + "created_at": "2018-05-30T20:18:04Z", + "updated_at": "2018-05-30T20:18:49Z", + "pushed_at": "2018-05-30T20:18:48Z", + "git_url": "git://github.com/Codertocat/Hello-World.git", + "ssh_url": "git@github.com:Codertocat/Hello-World.git", + "clone_url": "https://github.com/Codertocat/Hello-World.git", + "svn_url": "https://github.com/Codertocat/Hello-World", + "homepage": "null", + "size": 0, + "stargazers_count": 0, + "watchers_count": 0, + "language": "null", + "has_issues": "true", + "has_projects": "true", + "has_downloads": "true", + "has_wiki": "true", + "has_pages": "true", + "forks_count": 0, + "mirror_url": "null", + "archived": "false", + "open_issues_count": 2, + "license": "null", + "forks": 0, + "open_issues": 2, + "watchers": 0, + "default_branch": "master" + }, + "organization": { + "login": "Octocoders", + "id": 38302899, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM4MzAyODk5", + "url": "https://api.github.com/orgs/Octocoders", + "repos_url": "https://api.github.com/orgs/Octocoders/repos", + "events_url": "https://api.github.com/orgs/Octocoders/events", + "hooks_url": "https://api.github.com/orgs/Octocoders/hooks", + "issues_url": "https://api.github.com/orgs/Octocoders/issues", + "members_url": "https://api.github.com/orgs/Octocoders/members{/member}", + "public_members_url": "https://api.github.com/orgs/Octocoders/public_members{/member}", + "avatar_url": "https://avatars1.githubusercontent.com/u/38302899?v=4", + "description": "" + }, + "sender": { + "login": "Codertocat", + "id": "21031067", + "node_id": "MDQ6VXNlcjIxMDMxMDY3", + "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Codertocat", + "html_url": "https://github.com/Codertocat", + "followers_url": "https://api.github.com/users/Codertocat/followers", + "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", + "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", + "organizations_url": "https://api.github.com/users/Codertocat/orgs", + "repos_url": "https://api.github.com/users/Codertocat/repos", + "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/Codertocat/received_events", + "type": "User", + "site_admin": "false" + } + }, + "event": "repository_import", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd", } event = { 'tags': 'githubeventsqs' @@ -2352,183 +2334,182 @@ class TestSuricataFixup(object): self.verify_org(message, result) self.verify_repo(message, result) assert result['source'] == 'repository_import' - + def test_release(self): message = { - "body": { - "action": "published", - "release": { - "url": "https://api.github.com/repos/Codertocat/Hello-World/releases/11248810", - "assets_url": "https://api.github.com/repos/Codertocat/Hello-World/releases/11248810/assets", - "upload_url": "https://uploads.github.com/repos/Codertocat/Hello-World/releases/11248810/assets{?name,label}", - "html_url": "https://github.com/Codertocat/Hello-World/releases/tag/0.0.1", - "id": "11248810", - "node_id": "MDc6UmVsZWFzZTExMjQ4ODEw", - "tag_name": "0.0.1", - "target_commitish": "master", - "name": "null", - "draft": "false", - "author": { - "login": "Codertocat", - "id": "21031067", - "node_id": "MDQ6VXNlcjIxMDMxMDY3", - "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", - "gravatar_id": "", - "url": "https://api.github.com/users/Codertocat", - "html_url": "https://github.com/Codertocat", - "followers_url": "https://api.github.com/users/Codertocat/followers", - "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", - "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", - "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", - "organizations_url": "https://api.github.com/users/Codertocat/orgs", - "repos_url": "https://api.github.com/users/Codertocat/repos", - "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", - "received_events_url": "https://api.github.com/users/Codertocat/received_events", - "type": "User", - "site_admin": "false" - }, - "prerelease": "false", - "created_at": "2018-05-30T20:18:05Z", - "published_at": "2018-05-30T20:18:44Z", - "assets": [ - - ], - "tarball_url": "https://api.github.com/repos/Codertocat/Hello-World/tarball/0.0.1", - "zipball_url": "https://api.github.com/repos/Codertocat/Hello-World/zipball/0.0.1", - "body": "null" - }, - "repository": { - "id": "135493233", - "node_id": "MDEwOlJlcG9zaXRvcnkxMzU0OTMyMzM=", - "name": "Hello-World", - "full_name": "Codertocat/Hello-World", - "owner": { - "name": "ASuperCat", - "login": "Codertocat", - "id": "21031067", - "node_id": "MDQ6VXNlcjIxMDMxMDY3", - "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", - "gravatar_id": "", - "url": "https://api.github.com/users/Codertocat", - "html_url": "https://github.com/Codertocat", - "followers_url": "https://api.github.com/users/Codertocat/followers", - "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", - "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", - "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", - "organizations_url": "https://api.github.com/users/Codertocat/orgs", - "repos_url": "https://api.github.com/users/Codertocat/repos", - "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", - "received_events_url": "https://api.github.com/users/Codertocat/received_events", - "type": "User", - "site_admin": "false" - }, - "private": "false", - "html_url": "https://github.com/Codertocat/Hello-World", - "description": "null", - "fork": "false", - "url": "https://api.github.com/repos/Codertocat/Hello-World", - "forks_url": "https://api.github.com/repos/Codertocat/Hello-World/forks", - "keys_url": "https://api.github.com/repos/Codertocat/Hello-World/keys{/key_id}", - "collaborators_url": "https://api.github.com/repos/Codertocat/Hello-World/collaborators{/collaborator}", - "teams_url": "https://api.github.com/repos/Codertocat/Hello-World/teams", - "hooks_url": "https://api.github.com/repos/Codertocat/Hello-World/hooks", - "issue_events_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/events{/number}", - "events_url": "https://api.github.com/repos/Codertocat/Hello-World/events", - "assignees_url": "https://api.github.com/repos/Codertocat/Hello-World/assignees{/user}", - "branches_url": "https://api.github.com/repos/Codertocat/Hello-World/branches{/branch}", - "tags_url": "https://api.github.com/repos/Codertocat/Hello-World/tags", - "blobs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/blobs{/sha}", - "git_tags_url": "https://api.github.com/repos/Codertocat/Hello-World/git/tags{/sha}", - "git_refs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/refs{/sha}", - "trees_url": "https://api.github.com/repos/Codertocat/Hello-World/git/trees{/sha}", - "statuses_url": "https://api.github.com/repos/Codertocat/Hello-World/statuses/{sha}", - "languages_url": "https://api.github.com/repos/Codertocat/Hello-World/languages", - "stargazers_url": "https://api.github.com/repos/Codertocat/Hello-World/stargazers", - "contributors_url": "https://api.github.com/repos/Codertocat/Hello-World/contributors", - "subscribers_url": "https://api.github.com/repos/Codertocat/Hello-World/subscribers", - "subscription_url": "https://api.github.com/repos/Codertocat/Hello-World/subscription", - "commits_url": "https://api.github.com/repos/Codertocat/Hello-World/commits{/sha}", - "git_commits_url": "https://api.github.com/repos/Codertocat/Hello-World/git/commits{/sha}", - "comments_url": "https://api.github.com/repos/Codertocat/Hello-World/comments{/number}", - "issue_comment_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/comments{/number}", - "contents_url": "https://api.github.com/repos/Codertocat/Hello-World/contents/{+path}", - "compare_url": "https://api.github.com/repos/Codertocat/Hello-World/compare/{base}...{head}", - "merges_url": "https://api.github.com/repos/Codertocat/Hello-World/merges", - "archive_url": "https://api.github.com/repos/Codertocat/Hello-World/{archive_format}{/ref}", - "downloads_url": "https://api.github.com/repos/Codertocat/Hello-World/downloads", - "issues_url": "https://api.github.com/repos/Codertocat/Hello-World/issues{/number}", - "pulls_url": "https://api.github.com/repos/Codertocat/Hello-World/pulls{/number}", - "milestones_url": "https://api.github.com/repos/Codertocat/Hello-World/milestones{/number}", - "notifications_url": "https://api.github.com/repos/Codertocat/Hello-World/notifications{?since,all,participating}", - "labels_url": "https://api.github.com/repos/Codertocat/Hello-World/labels{/name}", - "releases_url": "https://api.github.com/repos/Codertocat/Hello-World/releases{/id}", - "deployments_url": "https://api.github.com/repos/Codertocat/Hello-World/deployments", - "created_at": "2018-05-30T20:18:04Z", - "updated_at": "2018-05-30T20:18:49Z", - "pushed_at": "2018-05-30T20:18:48Z", - "git_url": "git://github.com/Codertocat/Hello-World.git", - "ssh_url": "git@github.com:Codertocat/Hello-World.git", - "clone_url": "https://github.com/Codertocat/Hello-World.git", - "svn_url": "https://github.com/Codertocat/Hello-World", - "homepage": "null", - "size": 0, - "stargazers_count": 0, - "watchers_count": 0, - "language": "null", - "has_issues": "true", - "has_projects": "true", - "has_downloads": "true", - "has_wiki": "true", - "has_pages": "true", - "forks_count": 0, - "mirror_url": "null", - "archived": "false", - "open_issues_count": 2, - "license": "null", - "forks": 0, - "open_issues": 2, - "watchers": 0, - "default_branch": "master" - }, - "organization": { - "login": "Octocoders", - "id": 38302899, - "node_id": "MDEyOk9yZ2FuaXphdGlvbjM4MzAyODk5", - "url": "https://api.github.com/orgs/Octocoders", - "repos_url": "https://api.github.com/orgs/Octocoders/repos", - "events_url": "https://api.github.com/orgs/Octocoders/events", - "hooks_url": "https://api.github.com/orgs/Octocoders/hooks", - "issues_url": "https://api.github.com/orgs/Octocoders/issues", - "members_url": "https://api.github.com/orgs/Octocoders/members{/member}", - "public_members_url": "https://api.github.com/orgs/Octocoders/public_members{/member}", - "avatar_url": "https://avatars1.githubusercontent.com/u/38302899?v=4", - "description": "" - }, - "sender": { - "login": "Codertocat", - "id": "21031067", - "node_id": "MDQ6VXNlcjIxMDMxMDY3", - "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", - "gravatar_id": "", - "url": "https://api.github.com/users/Codertocat", - "html_url": "https://github.com/Codertocat", - "followers_url": "https://api.github.com/users/Codertocat/followers", - "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", - "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", - "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", - "organizations_url": "https://api.github.com/users/Codertocat/orgs", - "repos_url": "https://api.github.com/users/Codertocat/repos", - "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", - "received_events_url": "https://api.github.com/users/Codertocat/received_events", - "type": "User", - "site_admin": "false" - } - }, - "event": "release", - "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd", + "body": { + "action": "published", + "release": { + "url": "https://api.github.com/repos/Codertocat/Hello-World/releases/11248810", + "assets_url": "https://api.github.com/repos/Codertocat/Hello-World/releases/11248810/assets", + "upload_url": "https://uploads.github.com/repos/Codertocat/Hello-World/releases/11248810/assets{?name,label}", + "html_url": "https://github.com/Codertocat/Hello-World/releases/tag/0.0.1", + "id": "11248810", + "node_id": "MDc6UmVsZWFzZTExMjQ4ODEw", + "tag_name": "0.0.1", + "target_commitish": "master", + "name": "null", + "draft": "false", + "author": { + "login": "Codertocat", + "id": "21031067", + "node_id": "MDQ6VXNlcjIxMDMxMDY3", + "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Codertocat", + "html_url": "https://github.com/Codertocat", + "followers_url": "https://api.github.com/users/Codertocat/followers", + "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", + "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", + "organizations_url": "https://api.github.com/users/Codertocat/orgs", + "repos_url": "https://api.github.com/users/Codertocat/repos", + "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/Codertocat/received_events", + "type": "User", + "site_admin": "false" + }, + "prerelease": "false", + "created_at": "2018-05-30T20:18:05Z", + "published_at": "2018-05-30T20:18:44Z", + "assets": [ + ], + "tarball_url": "https://api.github.com/repos/Codertocat/Hello-World/tarball/0.0.1", + "zipball_url": "https://api.github.com/repos/Codertocat/Hello-World/zipball/0.0.1", + "body": "null" + }, + "repository": { + "id": "135493233", + "node_id": "MDEwOlJlcG9zaXRvcnkxMzU0OTMyMzM=", + "name": "Hello-World", + "full_name": "Codertocat/Hello-World", + "owner": { + "name": "ASuperCat", + "login": "Codertocat", + "id": "21031067", + "node_id": "MDQ6VXNlcjIxMDMxMDY3", + "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Codertocat", + "html_url": "https://github.com/Codertocat", + "followers_url": "https://api.github.com/users/Codertocat/followers", + "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", + "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", + "organizations_url": "https://api.github.com/users/Codertocat/orgs", + "repos_url": "https://api.github.com/users/Codertocat/repos", + "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/Codertocat/received_events", + "type": "User", + "site_admin": "false" + }, + "private": "false", + "html_url": "https://github.com/Codertocat/Hello-World", + "description": "null", + "fork": "false", + "url": "https://api.github.com/repos/Codertocat/Hello-World", + "forks_url": "https://api.github.com/repos/Codertocat/Hello-World/forks", + "keys_url": "https://api.github.com/repos/Codertocat/Hello-World/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/Codertocat/Hello-World/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/Codertocat/Hello-World/teams", + "hooks_url": "https://api.github.com/repos/Codertocat/Hello-World/hooks", + "issue_events_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/events{/number}", + "events_url": "https://api.github.com/repos/Codertocat/Hello-World/events", + "assignees_url": "https://api.github.com/repos/Codertocat/Hello-World/assignees{/user}", + "branches_url": "https://api.github.com/repos/Codertocat/Hello-World/branches{/branch}", + "tags_url": "https://api.github.com/repos/Codertocat/Hello-World/tags", + "blobs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/Codertocat/Hello-World/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/Codertocat/Hello-World/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/Codertocat/Hello-World/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/Codertocat/Hello-World/statuses/{sha}", + "languages_url": "https://api.github.com/repos/Codertocat/Hello-World/languages", + "stargazers_url": "https://api.github.com/repos/Codertocat/Hello-World/stargazers", + "contributors_url": "https://api.github.com/repos/Codertocat/Hello-World/contributors", + "subscribers_url": "https://api.github.com/repos/Codertocat/Hello-World/subscribers", + "subscription_url": "https://api.github.com/repos/Codertocat/Hello-World/subscription", + "commits_url": "https://api.github.com/repos/Codertocat/Hello-World/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/Codertocat/Hello-World/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/Codertocat/Hello-World/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/Codertocat/Hello-World/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/Codertocat/Hello-World/contents/{+path}", + "compare_url": "https://api.github.com/repos/Codertocat/Hello-World/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/Codertocat/Hello-World/merges", + "archive_url": "https://api.github.com/repos/Codertocat/Hello-World/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/Codertocat/Hello-World/downloads", + "issues_url": "https://api.github.com/repos/Codertocat/Hello-World/issues{/number}", + "pulls_url": "https://api.github.com/repos/Codertocat/Hello-World/pulls{/number}", + "milestones_url": "https://api.github.com/repos/Codertocat/Hello-World/milestones{/number}", + "notifications_url": "https://api.github.com/repos/Codertocat/Hello-World/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/Codertocat/Hello-World/labels{/name}", + "releases_url": "https://api.github.com/repos/Codertocat/Hello-World/releases{/id}", + "deployments_url": "https://api.github.com/repos/Codertocat/Hello-World/deployments", + "created_at": "2018-05-30T20:18:04Z", + "updated_at": "2018-05-30T20:18:49Z", + "pushed_at": "2018-05-30T20:18:48Z", + "git_url": "git://github.com/Codertocat/Hello-World.git", + "ssh_url": "git@github.com:Codertocat/Hello-World.git", + "clone_url": "https://github.com/Codertocat/Hello-World.git", + "svn_url": "https://github.com/Codertocat/Hello-World", + "homepage": "null", + "size": 0, + "stargazers_count": 0, + "watchers_count": 0, + "language": "null", + "has_issues": "true", + "has_projects": "true", + "has_downloads": "true", + "has_wiki": "true", + "has_pages": "true", + "forks_count": 0, + "mirror_url": "null", + "archived": "false", + "open_issues_count": 2, + "license": "null", + "forks": 0, + "open_issues": 2, + "watchers": 0, + "default_branch": "master" + }, + "organization": { + "login": "Octocoders", + "id": 38302899, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM4MzAyODk5", + "url": "https://api.github.com/orgs/Octocoders", + "repos_url": "https://api.github.com/orgs/Octocoders/repos", + "events_url": "https://api.github.com/orgs/Octocoders/events", + "hooks_url": "https://api.github.com/orgs/Octocoders/hooks", + "issues_url": "https://api.github.com/orgs/Octocoders/issues", + "members_url": "https://api.github.com/orgs/Octocoders/members{/member}", + "public_members_url": "https://api.github.com/orgs/Octocoders/public_members{/member}", + "avatar_url": "https://avatars1.githubusercontent.com/u/38302899?v=4", + "description": "" + }, + "sender": { + "login": "Codertocat", + "id": "21031067", + "node_id": "MDQ6VXNlcjIxMDMxMDY3", + "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Codertocat", + "html_url": "https://github.com/Codertocat", + "followers_url": "https://api.github.com/users/Codertocat/followers", + "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", + "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", + "organizations_url": "https://api.github.com/users/Codertocat/orgs", + "repos_url": "https://api.github.com/users/Codertocat/repos", + "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/Codertocat/received_events", + "type": "User", + "site_admin": "false" + } + }, + "event": "release", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd", } event = { 'tags': 'githubeventsqs' @@ -2546,175 +2527,175 @@ class TestSuricataFixup(object): assert result['details']['release_author_node_id'] == message['body']['release']['author']['node_id'] assert result['details']['release_author_type'] == message['body']['release']['author']['type'] assert result['details']['release_author_site_admin'] == message['body']['release']['author']['site_admin'] - + def test_org_block(self): - message = { - "body": { - "action": "blocked", - "blocked_user": { - "login": "hacktocat", - "id": 39652351, - "node_id": "MDQ6VXNlcjM5NjUyMzUx", - "avatar_url": "https://avatars2.githubusercontent.com/u/39652351?v=4", - "gravatar_id": "", - "url": "https://api.github.com/users/hacktocat", - "html_url": "https://github.com/hacktocat", - "followers_url": "https://api.github.com/users/hacktocat/followers", - "following_url": "https://api.github.com/users/hacktocat/following{/other_user}", - "gists_url": "https://api.github.com/users/hacktocat/gists{/gist_id}", - "starred_url": "https://api.github.com/users/hacktocat/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/hacktocat/subscriptions", - "organizations_url": "https://api.github.com/users/hacktocat/orgs", - "repos_url": "https://api.github.com/users/hacktocat/repos", - "events_url": "https://api.github.com/users/hacktocat/events{/privacy}", - "received_events_url": "https://api.github.com/users/hacktocat/received_events", - "type": "User", - "site_admin": "false" - }, - "organization": { - "login": "Octocoders", - "id": 38302899, - "node_id": "MDEyOk9yZ2FuaXphdGlvbjM4MzAyODk5", - "url": "https://api.github.com/orgs/Octocoders", - "repos_url": "https://api.github.com/orgs/Octocoders/repos", - "events_url": "https://api.github.com/orgs/Octocoders/events", - "hooks_url": "https://api.github.com/orgs/Octocoders/hooks", - "issues_url": "https://api.github.com/orgs/Octocoders/issues", - "members_url": "https://api.github.com/orgs/Octocoders/members{/member}", - "public_members_url": "https://api.github.com/orgs/Octocoders/public_members{/member}", - "avatar_url": "https://avatars1.githubusercontent.com/u/38302899?v=4", - "description": "" - }, - "sender": { - "login": "Codertocat", - "id": 21031067, - "node_id": "MDQ6VXNlcjIxMDMxMDY3", - "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", - "gravatar_id": "", - "url": "https://api.github.com/users/Codertocat", - "html_url": "https://github.com/Codertocat", - "followers_url": "https://api.github.com/users/Codertocat/followers", - "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", - "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", - "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", - "organizations_url": "https://api.github.com/users/Codertocat/orgs", - "repos_url": "https://api.github.com/users/Codertocat/repos", - "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", - "received_events_url": "https://api.github.com/users/Codertocat/received_events", - "type": "User", - "site_admin": "false" - } -}, - "event": "org_block", - "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd", - } - event = { - 'tags': 'githubeventsqs' - } - event['details'] = message - result, metadata = self.plugin.onMessage(event, self.metadata) - self.verify_defaults(result) - self.verify_metadata(metadata) - self.verify_meta(message, result) - self.verify_actor(message, result) - assert result['source'] == 'org_block' - assert result['details']['blocked_user_login'] == message['body']['blocked_user']['login'] - assert result['details']['blocked_user_id'] == message['body']['blocked_user']['id'] - assert result['details']['blocked_user_node_id'] == message['body']['blocked_user']['node_id'] - + message = { + "body": { + "action": "blocked", + "blocked_user": { + "login": "hacktocat", + "id": 39652351, + "node_id": "MDQ6VXNlcjM5NjUyMzUx", + "avatar_url": "https://avatars2.githubusercontent.com/u/39652351?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/hacktocat", + "html_url": "https://github.com/hacktocat", + "followers_url": "https://api.github.com/users/hacktocat/followers", + "following_url": "https://api.github.com/users/hacktocat/following{/other_user}", + "gists_url": "https://api.github.com/users/hacktocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/hacktocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/hacktocat/subscriptions", + "organizations_url": "https://api.github.com/users/hacktocat/orgs", + "repos_url": "https://api.github.com/users/hacktocat/repos", + "events_url": "https://api.github.com/users/hacktocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/hacktocat/received_events", + "type": "User", + "site_admin": "false" + }, + "organization": { + "login": "Octocoders", + "id": 38302899, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjM4MzAyODk5", + "url": "https://api.github.com/orgs/Octocoders", + "repos_url": "https://api.github.com/orgs/Octocoders/repos", + "events_url": "https://api.github.com/orgs/Octocoders/events", + "hooks_url": "https://api.github.com/orgs/Octocoders/hooks", + "issues_url": "https://api.github.com/orgs/Octocoders/issues", + "members_url": "https://api.github.com/orgs/Octocoders/members{/member}", + "public_members_url": "https://api.github.com/orgs/Octocoders/public_members{/member}", + "avatar_url": "https://avatars1.githubusercontent.com/u/38302899?v=4", + "description": "" + }, + "sender": { + "login": "Codertocat", + "id": 21031067, + "node_id": "MDQ6VXNlcjIxMDMxMDY3", + "avatar_url": "https://avatars1.githubusercontent.com/u/21031067?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/Codertocat", + "html_url": "https://github.com/Codertocat", + "followers_url": "https://api.github.com/users/Codertocat/followers", + "following_url": "https://api.github.com/users/Codertocat/following{/other_user}", + "gists_url": "https://api.github.com/users/Codertocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/Codertocat/subscriptions", + "organizations_url": "https://api.github.com/users/Codertocat/orgs", + "repos_url": "https://api.github.com/users/Codertocat/repos", + "events_url": "https://api.github.com/users/Codertocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/Codertocat/received_events", + "type": "User", + "site_admin": "false" + } + }, + "event": "org_block", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd", + } + event = { + 'tags': 'githubeventsqs' + } + event['details'] = message + result, metadata = self.plugin.onMessage(event, self.metadata) + self.verify_defaults(result) + self.verify_metadata(metadata) + self.verify_meta(message, result) + self.verify_actor(message, result) + assert result['source'] == 'org_block' + assert result['details']['blocked_user_login'] == message['body']['blocked_user']['login'] + assert result['details']['blocked_user_id'] == message['body']['blocked_user']['id'] + assert result['details']['blocked_user_node_id'] == message['body']['blocked_user']['node_id'] + def test_installation(self): - message = { - "body": - { - "action": "deleted", - "installation": { - "id": 2, - "account": { - "login": "octocat", - "id": 1, - "node_id": "MDQ6VXNlcjE=", - "avatar_url": "https://github.com/images/error/octocat_happy.gif", - "gravatar_id": "", - "url": "https://api.github.com/users/octocat", - "html_url": "https://github.com/octocat", - "followers_url": "https://api.github.com/users/octocat/followers", - "following_url": "https://api.github.com/users/octocat/following{/other_user}", - "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", - "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", - "organizations_url": "https://api.github.com/users/octocat/orgs", - "repos_url": "https://api.github.com/users/octocat/repos", - "events_url": "https://api.github.com/users/octocat/events{/privacy}", - "received_events_url": "https://api.github.com/users/octocat/received_events", - "type": "User", - "site_admin": "false" - }, - "repository_selection": "selected", - "access_tokens_url": "https://api.github.com/installations/2/access_tokens", - "repositories_url": "https://api.github.com/installation/repositories", - "html_url": "https://github.com/settings/installations/2", - "app_id": 5725, - "target_id": 3880403, - "target_type": "User", - "permissions": { - "metadata": "read", - "contents": "read", - "issues": "write" - }, - "events": [ - "push", - "pull_request" - ], - "created_at": 1525109898, - "updated_at": 1525109899, - "single_file_name": "config.yml" - }, - "repositories": [ - { - "id": 1296269, - "name": "Hello-World", - "full_name": "octocat/Hello-World", - "private": "false" - } - ], - "sender": { - "login": "octocat", - "id": 1, - "node_id": "MDQ6VXNlcjE=", - "avatar_url": "https://github.com/images/error/octocat_happy.gif", - "gravatar_id": "", - "url": "https://api.github.com/users/octocat", - "html_url": "https://github.com/octocat", - "followers_url": "https://api.github.com/users/octocat/followers", - "following_url": "https://api.github.com/users/octocat/following{/other_user}", - "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", - "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", - "organizations_url": "https://api.github.com/users/octocat/orgs", - "repos_url": "https://api.github.com/users/octocat/repos", - "events_url": "https://api.github.com/users/octocat/events{/privacy}", - "received_events_url": "https://api.github.com/users/octocat/received_events", - "type": "User", - "site_admin": "false" - }, - }, - "event": "installation", - "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd", - } - event = { - 'tags': 'githubeventsqs' - } - event['details'] = message - result, metadata = self.plugin.onMessage(event, self.metadata) - self.verify_defaults(result) - self.verify_metadata(metadata) - self.verify_meta(message, result) - self.verify_actor(message, result) - assert result['source'] == 'installation' - assert result['details']['action'] == message['body']['action'] - assert result['details']['install_id'] == message['body']['installation']['account']['id'] - assert result['details']['install_account_login'] == message['body']['installation']['account']['login'] - assert result['details']['install_account_node_id'] == message['body']['installation']['account']['node_id'] - assert result['details']['install_account_type'] == message['body']['installation']['account']['type'] - assert result['details']['install_account_site_admin'] == message['body']['installation']['account']['site_admin'] \ No newline at end of file + message = { + "body": + { + "action": "deleted", + "installation": { + "id": 2, + "account": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": "false" + }, + "repository_selection": "selected", + "access_tokens_url": "https://api.github.com/installations/2/access_tokens", + "repositories_url": "https://api.github.com/installation/repositories", + "html_url": "https://github.com/settings/installations/2", + "app_id": 5725, + "target_id": 3880403, + "target_type": "User", + "permissions": { + "metadata": "read", + "contents": "read", + "issues": "write" + }, + "events": [ + "push", + "pull_request" + ], + "created_at": 1525109898, + "updated_at": 1525109899, + "single_file_name": "config.yml" + }, + "repositories": [ + { + "id": 1296269, + "name": "Hello-World", + "full_name": "octocat/Hello-World", + "private": "false" + } + ], + "sender": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": "false" + }, + }, + "event": "installation", + "request_id": "94e70998-dd79-11e8-9ba0-a8635445a8cd", + } + event = { + 'tags': 'githubeventsqs' + } + event['details'] = message + result, metadata = self.plugin.onMessage(event, self.metadata) + self.verify_defaults(result) + self.verify_metadata(metadata) + self.verify_meta(message, result) + self.verify_actor(message, result) + assert result['source'] == 'installation' + assert result['details']['action'] == message['body']['action'] + assert result['details']['install_id'] == message['body']['installation']['account']['id'] + assert result['details']['install_account_login'] == message['body']['installation']['account']['login'] + assert result['details']['install_account_node_id'] == message['body']['installation']['account']['node_id'] + assert result['details']['install_account_type'] == message['body']['installation']['account']['type'] + assert result['details']['install_account_site_admin'] == message['body']['installation']['account']['site_admin'] From 006b70869389be23cb2de2d63f59fb88a742a84d Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Wed, 14 Nov 2018 13:51:17 -0600 Subject: [PATCH 13/89] Sort cloudtrail keys in mq plugin --- mq/plugins/cloudtrail.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/mq/plugins/cloudtrail.py b/mq/plugins/cloudtrail.py index c828256b..9f24ab05 100644 --- a/mq/plugins/cloudtrail.py +++ b/mq/plugins/cloudtrail.py @@ -21,29 +21,29 @@ class message(object): # which are mixed string and dict # into a dict with a raw_value key as the string value self.modify_keys = [ - 'details.requestparameters.iamInstanceProfile', - 'details.requestparameters.instanceType', + 'details.additionaleventdata', + 'details.apiversion', 'details.requestparameters.attribute', - 'details.requestparameters.description', - 'details.requestparameters.filter', - 'details.requestparameters.rule', - 'details.requestparameters.ebsOptimized', - 'details.requestparameters.source', 'details.requestparameters.callerReference', + 'details.requestparameters.description', + 'details.requestparameters.disableApiTermination', 'details.requestparameters.domainName', 'details.requestparameters.domainNames', - 'details.responseelements.role', - 'details.responseelements.subnets', - 'details.responseelements.endpoint', - 'details.responseelements.securityGroups', - 'details.responseelements.lastModified', - 'details.additionaleventdata', - 'details.serviceeventdetails', - 'details.requestparameters.disableApiTermination', - 'details.responseelements.findings.service.additionalInfo.unusual', - 'details.responseelements.distribution.distributionConfig.callerReference', + 'details.requestparameters.ebsOptimized', + 'details.requestparameters.filter', + 'details.requestparameters.iamInstanceProfile', + 'details.requestparameters.instanceType', 'details.requestparameters.logStreamName', - 'details.apiversion' + 'details.requestparameters.rule', + 'details.requestparameters.source', + 'details.responseelements.distribution.distributionConfig.callerReference', + 'details.responseelements.endpoint', + 'details.responseelements.findings.service.additionalInfo.unusual', + 'details.responseelements.lastModified', + 'details.responseelements.role', + 'details.responseelements.securityGroups', + 'details.responseelements.subnets', + 'details.serviceeventdetails', ] def convert_key_raw_str(self, needle, haystack): From 21aacc57a04f170eb97bc49cefc8074e76f01087 Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Wed, 14 Nov 2018 13:51:55 -0600 Subject: [PATCH 14/89] Add Principal key to cloudtrail plugin --- mq/plugins/cloudtrail.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mq/plugins/cloudtrail.py b/mq/plugins/cloudtrail.py index 9f24ab05..0cb8f20a 100644 --- a/mq/plugins/cloudtrail.py +++ b/mq/plugins/cloudtrail.py @@ -24,6 +24,7 @@ class message(object): 'details.additionaleventdata', 'details.apiversion', 'details.requestparameters.attribute', + 'details.requestparameters.bucketPolicy.Statement.Principal', 'details.requestparameters.callerReference', 'details.requestparameters.description', 'details.requestparameters.disableApiTermination', From 3d91d137bc3226dcb40cf57a20cea6727aa6f258 Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Wed, 14 Nov 2018 15:13:52 -0600 Subject: [PATCH 15/89] Separate logic from hostname per ip into separate function --- alerts/lib/alerttask.py | 16 ++++++++++++---- tests/alerts/lib/test_alerttask.py | 14 +++++++++++++- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/alerts/lib/alerttask.py b/alerts/lib/alerttask.py index 4f1ed69a..2b51da24 100644 --- a/alerts/lib/alerttask.py +++ b/alerts/lib/alerttask.py @@ -60,15 +60,23 @@ def getValueByPath(input_dict, path_string): return return_data +def hostname_from_ip(ip): + try: + reversed_dns = socket.gethostbyaddr(ip) + return reversed_dns[0] + except socket.herror: + return None + + def add_hostname_to_ip(ip, output_format, require_internal=True): ip_obj = netaddr.IPNetwork(ip)[0] if require_internal and not ip_obj.is_private(): return ip - try: - reversed_dns = socket.gethostbyaddr(ip) - return output_format.format(ip, reversed_dns[0]) - except socket.herror: + hostname = hostname_from_ip(ip) + if hostname is None: return ip + else: + return output_format.format(ip, hostname) class AlertTask(Task): diff --git a/tests/alerts/lib/test_alerttask.py b/tests/alerts/lib/test_alerttask.py index 71d078cb..ce29001c 100644 --- a/tests/alerts/lib/test_alerttask.py +++ b/tests/alerts/lib/test_alerttask.py @@ -1,7 +1,7 @@ import os import sys sys.path.append(os.path.join(os.path.dirname(__file__), "../../../alerts/lib")) -from alerttask import add_hostname_to_ip +from alerttask import hostname_from_ip, add_hostname_to_ip import mock import socket @@ -14,6 +14,18 @@ def reverse_lookup(ip): raise socket.herror +class TestHostnameFromIP(object): + def test_good_hostname_from_ip(ip): + with mock.patch("socket.gethostbyaddr", side_effect=reverse_lookup): + hostname = hostname_from_ip('10.1.1.1') + assert hostname == 'test.domain.com' + + def test_bad_hostname_from_ip(ip): + with mock.patch("socket.gethostbyaddr", side_effect=reverse_lookup): + hostname = hostname_from_ip('8.8.8.8') + assert hostname is None + + class TestAddHostnameToIP(object): def setup(self): self.formatted_string = '{0} ({1})' From 7016eff491019458c09225a35f8987ef16768b5e Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Thu, 15 Nov 2018 13:11:46 -0600 Subject: [PATCH 16/89] Modify tester container to copy all python directories for pep8 --- docker/compose/tester/Dockerfile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docker/compose/tester/Dockerfile b/docker/compose/tester/Dockerfile index b8bf1c5d..e528dbe8 100644 --- a/docker/compose/tester/Dockerfile +++ b/docker/compose/tester/Dockerfile @@ -1,11 +1,14 @@ FROM mozdef/mozdef_base -COPY tests /opt/mozdef/envs/mozdef/tests COPY alerts /opt/mozdef/envs/mozdef/alerts -COPY mq /opt/mozdef/envs/mozdef/mq -COPY loginput /opt/mozdef/envs/mozdef/loginput -COPY rest /opt/mozdef/envs/mozdef/rest +COPY bot /opt/mozdef/envs/mozdef/bot COPY cron /opt/mozdef/envs/mozdef/cron +COPY examples /opt/mozdef/envs/mozdef/examples +COPY loginput /opt/mozdef/envs/mozdef/loginput +COPY mozdef_util /opt/mozdef/envs/mozdef/mozdef_util +COPY mq /opt/mozdef/envs/mozdef/mq +COPY rest /opt/mozdef/envs/mozdef/rest +COPY tests /opt/mozdef/envs/mozdef/tests COPY .flake8 /opt/mozdef/envs/mozdef/.flake8 COPY docker/compose/tester/files/tests_config.conf /opt/mozdef/envs/mozdef/tests/config.conf From a091dd8e4145ef7d26949ae92b2f7c6b553c0c5e Mon Sep 17 00:00:00 2001 From: Michal Purzynski Date: Tue, 20 Nov 2018 12:53:13 -0800 Subject: [PATCH 17/89] Resolve the source IP and add the FQDN to the generated alert --- alerts/nsm_scan_port.py | 5 +++-- tests/alerts/test_nsm_scan_port.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/alerts/nsm_scan_port.py b/alerts/nsm_scan_port.py index 67c6cd43..6a82f122 100644 --- a/alerts/nsm_scan_port.py +++ b/alerts/nsm_scan_port.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # Copyright (c) 2018 Mozilla Corporation -from lib.alerttask import AlertTask +from lib.alerttask import AlertTask, add_hostname_to_ip from mozdef_util.query_models import SearchQuery, TermMatch, QueryStringMatch, PhraseMatch, ExistsMatch @@ -40,7 +40,8 @@ class NSMScanPort(AlertTask): if 'details' in x: if 'indicators' in x['details']: indicators = x['details']['sourceipaddress'] + indicators_info = add_hostname_to_ip(indicators, '{0} ({1})', require_internal=False) - summary = 'Port scan from {}'.format(indicators) + summary = 'Port scan from {}'.format(indicators_info) return self.createAlertDict(summary, category, tags, aggreg['events'], severity) diff --git a/tests/alerts/test_nsm_scan_port.py b/tests/alerts/test_nsm_scan_port.py index 22bd2c7e..b4c5d683 100644 --- a/tests/alerts/test_nsm_scan_port.py +++ b/tests/alerts/test_nsm_scan_port.py @@ -33,7 +33,7 @@ class TestNSMScanPort(AlertTestSuite): default_alert = { "category": "nsm", "severity": "WARNING", - "summary": "Port scan from 10.99.88.77", + "summary": "Port scan from 10.99.88.77 (mock.mozilla.org)", "tags": ['nsm', 'bro', 'portscan'], "notify_mozdefbot": True, } From e692789fb6b16e79a5fc4c10b0dde5cadffb3a6f Mon Sep 17 00:00:00 2001 From: Michal Purzynski Date: Tue, 20 Nov 2018 13:23:25 -0800 Subject: [PATCH 18/89] Add exclusions based on the details.msg --- alerts/nsm_scan_port.json | 5 +++-- alerts/nsm_scan_port.py | 3 +++ tests/alerts/test_nsm_scan_port.py | 12 ++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/alerts/nsm_scan_port.json b/alerts/nsm_scan_port.json index e3a979cc..3bd2094b 100644 --- a/alerts/nsm_scan_port.json +++ b/alerts/nsm_scan_port.json @@ -1,4 +1,5 @@ { "sourcemustmatch":"[10.0.0.0 TO 10.255.255.255]", - "sourcemustnotmatch":"10.33.44.54 OR 10.88.77.54 OR 10.76.54.54 OR 10.251.30.138 OR 10.54.65.234" -} \ No newline at end of file + "sourcemustnotmatch":"10.33.44.54 OR 10.88.77.54 OR 10.76.54.54 OR 10.251.30.138 OR 10.54.65.234", + "destinationmustnotmatch": "*192.168*" +} diff --git a/alerts/nsm_scan_port.py b/alerts/nsm_scan_port.py index 6a82f122..06590f6b 100644 --- a/alerts/nsm_scan_port.py +++ b/alerts/nsm_scan_port.py @@ -25,6 +25,9 @@ class NSMScanPort(AlertTask): search_query.add_must_not([ QueryStringMatch('details.sourceipaddress: {}'.format(self._config['sourcemustnotmatch'])) ]) + search_query.add_must_not([ + QueryStringMatch('details.msg: {}'.format(self._config['destinationmustnotmatch'])) + ]) self.filtersManual(search_query) self.searchEventsAggregated('details.sourceipaddress', samplesLimit=10) diff --git a/tests/alerts/test_nsm_scan_port.py b/tests/alerts/test_nsm_scan_port.py index b4c5d683..a8c738bf 100644 --- a/tests/alerts/test_nsm_scan_port.py +++ b/tests/alerts/test_nsm_scan_port.py @@ -112,6 +112,18 @@ class TestNSMScanPort(AlertTestSuite): ) ) + events = AlertTestSuite.create_events(default_event, 5) + for event in events: + event['_source']['details']['sourceipaddress'] = '1.2.3.4' + event['_source']['details']['indicators'] = '1.2.3.4' + event['_source']['details']['msg'] = '2620:101:80f8:224:44af:81c2:372c:6cbf scanned at least 15 unique ports on hosts 2001:41d0:e:9d4::1, 192.168.1.1, 2400:6180:0:d0::4a6b:4001, 2a03:b0c0:1:e0::13a:2001, 2001:0:3e8a:ee2d:5c:3162:3f57:fd8d in 0m11s' + test_cases.append( + NegativeAlertTestCase( + description="Negative test case with an excluded destination", + events=events, + ) + ) + events = AlertTestSuite.create_events(default_event, 5) for event in events: event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({'minutes': 15}) From 55d64150d13387d85f61e30f64eacd12350193f0 Mon Sep 17 00:00:00 2001 From: Michal Purzynski Date: Tue, 20 Nov 2018 14:57:03 -0800 Subject: [PATCH 19/89] Add the missing github_mapping description file --- tests/mq/plugins/github_mapping.yml | 432 ++++++++++++++++++++++++++++ 1 file changed, 432 insertions(+) create mode 100644 tests/mq/plugins/github_mapping.yml diff --git a/tests/mq/plugins/github_mapping.yml b/tests/mq/plugins/github_mapping.yml new file mode 100644 index 00000000..f5ede2f1 --- /dev/null +++ b/tests/mq/plugins/github_mapping.yml @@ -0,0 +1,432 @@ +--- + push: + username: details.body.sender.login + email: details.body.pusher.email + id: details.body.sender.id + commits: details.body.commits + created: details.body.created + deleted: details.body.deleted + forced: details.body.forced + node_id: details.body.sender.node_id + sender_type: detais.body.sender.type + request_id: details.request_id + commit_author: details.body.head_commit.author.email + committer: details.body.head_commit.committer.email + commit_id: details.body.head_commit.id + commit_msg: details.body.head_commit.message + commit_ts: details.body.head_commit.timestamp + commit_url: details.body.head_commit.url + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + pusher_email: details.body.pusher.email + pusher_name: details.body.pusher.name + ref: details.body.ref + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + repo_private: details.body.repository.private + repo_pushed_at: details.body.repository.pushed_at + repo_updated_at: details.body.repository.ssh_url + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + + create: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + pusher_email: details.body.pusher.email + pusher_name: details.body.pusher.name + repo_private: details.body.repository.private + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + ref: details.body.ref + ref_type: details.body.ref_type + + delete: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + pusher_email: details.body.pusher.email + pusher_name: details.body.pusher.name + repo_private: details.body.repository.private + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + ref: details.body.ref + ref_type: details.body.ref_type + + pull_request: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + pr_number: details.body.number + pr_assignee_id: details.body.pull_request.assignee.id + pr_assignee_login: details.body.pull_request.assignee.login + pr_assignee_site_admin: details.body.pull_request.assignee.site_admin + pr_assignee_type: details.body.pull_request.assignee.type + pr_author_association: details.body.pull_request.author_association + pr_base_label: details.body.pull_request.base.label + pr_base_ref: details.body.pull_request.base.ref + pr_base_repo_name: details.body.pull_request.base.repo.name + pr_base_repo_id: details.body.pull_request.base.repo.node_id + pr_base_repo_owner_id: details.body.pull_request.base.repo.owner.id + pr_base_repo_owner_login: details.body.pull_request.base.repo.owner.login + pr_base_repo_owner_node_id: details.body.pull_request.base.repo.owner.node_id + pr_base_repo_private: details.body.pull_request.base.repo.private + pr_base_sha: details.body.pull_request.base.sha + pr_base_user_id: details.body.pull_request.base.user.id + pr_base_user_login: details.body.pull_request.base.user.login + pr_base_user_node_id: details.body.pull_request.base.user.node_id + pr_base_user_site_admin: details.body.pull_request.base.user.site_admin + pr_body: details.body.pull_request.body + pr_changed_files: details.body.pull_request.changed_files + pr_commits: details.body.pull_request.commits + pr_deletions: details.body.pull_request.deletions + pr_closed_at: details.body.pull_request.closed_at + pr_head_label: details.body.pull_request.head.label + pr_head_ref: details.body.pull_request.head.ref + pr_head_repo_name: details.body.pull_request.head.repo.name + pr_head_repo_id: details.body.pull_request.head.repo.id + pr_head_repo_node_id: details.body.pull_request.head.repo.node_id + pr_head_repo_owner_id: details.body.pull_request.head.repo.owner.id + pr_head_repo_owner_login: details.body.pull_request.head.repo.owner.login + pr_head_repo_owner_node_id: details.body.pull_request.head.repo.owner.node_id + pr_head_repo_owner_site_admin: details.body.pull_request.head.repo.owner.site_admin + pr_head_repo_private: details.body.pull_request.head.repo.private + pr_head_sha: details.body.pull_request.head.sha + pr_head_user_id: details.body.pull_request.head.user.id + pr_head_user_login: details.body.pull_request.head.user.login + pr_head_user_node_id: details.body.pull_request.head.user.node_id + pr_merge_commit_sha: details.body.pull_request.merge_commit_sha + pr_merge_merged: details.body.pull_request.merged + pr_merge_merged_at: details.body.pull_request.merged_at + pr_merged_by_id: details.body.pull_request.merged_by.id + pr_merged_by_login: details.body.pull_request.merged_by.login + pr_merged_by_node_id: details.body.pull_request.merged_by.node_id + pr_merged_by_site_admin: details.body.pull_request.merged_by.site_admin + pr_node_id: details.body.pull_request.node_id + pr_number: details.body.pull_request.number + pr_state: details.body.pull_request.state + pr_title: details.body.pull_request.title + pr_updated_at: details.body.pull_request.updated_at + pr_id: details.body.pull_request.id + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + + repository_vulnerability_alert: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + alert_package: details.body.alert.affected_package_name + alert_range: details.body.alert.affected_range + alert_extid: details.body.alert.external_identifier + alert_extref: details.body.alert.external_reference + alert_fixed: details.body.alert.fixed_in + alert_id: details.body.alert.id + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + repo_private: details.body.repository.private + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + dismiss_reason: details.body.alert.dismiss_reason + dismiss_at: details.body.alert.dismissed_at + dismiss_user: details.body.alert.dismisser.login + dismiss_id: details.body.alert.dismisser.id + dismiss_node_id: details.body.alert.dismisser.node_id + dismiss_type: details.body.alert.dismisser.type + dismiss_site_admin: details.body.alert.dismisser.site_admin + + security_advisory: + action: details.body.action + alert_id: details.body.security_advisory.ghsa_id + alert_note: details.body.security_advisory.summary + alert_description: details.body.security_advisory.description + alert_severity: details.body.security_advisory.severity + alert_list: details.body.security_advisory.identifiers + alert_vulnerabilities: details.body.security_advisory.vulnerabilities + alert_ref: details.body.security_advisory.references + alert_published_at: details.body.security_advisory.published_at + alert_updated_at: details.body.security_advisory.updated_at + alert_withdrawn_at: details.body.security_advisory.withdrawn_at + + repository: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + repo_private: details.body.repository.private + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + + member: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + repo_private: details.body.repository.private + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + member_id: details.body.member.id + member_login: details.body.member.login + member_node_id: details.body.member.node_id + member_site_admin: details.body.member.site_admin + changes_perm_from: details.body.changes.permission.from + + team: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + repo_private: details.body.repository.private + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + repo_perm_from_admin: details.body.changes.repository.permissions.from.admin + repo_perm_from_pull: details.body.changes.repository.permissions.from.pull + repo_perm_from_push: details.body.changes.repository.permissions.from.push + repo_perm_admin: details.body.repository.permissions.admin + repo_perm_pull: details.body.repository.permissions.pull + repo_perm_push: details.body.repository.permissions.push + team_id: details.body.team.id + team_name: details.body.team.name + team_node_id: details.body.team.node_id + team_permission: details.body.team.permission + team_privacy: details.body.team.privacy + team_slug: details.body.team.slug + + team_add: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + repo_private: details.body.repository.private + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + repo_perm_from_admin: details.body.changes.repository.permissions.from.admin + repo_perm_from_pull: details.body.changes.repository.permissions.from.pull + repo_perm_from_push: details.body.changes.repository.permissions.from.push + repo_perm_admin: details.body.repository.permissions.admin + repo_perm_pull: details.body.repository.permissions.pull + repo_perm_push: details.body.repository.permissions.push + team_id: details.body.team.id + team_name: details.body.team.name + team_node_id: details.body.team.node_id + team_permission: details.body.team.permission + team_privacy: details.body.team.privacy + team_slug: details.body.team.slug + + organization: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + team_id: details.body.team.id + team_name: details.body.team.name + team_node_id: details.body.team.node_id + team_permission: details.body.team.permission + team_privacy: details.body.team.privacy + team_slug: details.body.team.slug + membership_type: details.body.membership.user.type + membership_admin: details.body.membership.user.site_admin + membership_node_id: details.body.membership.user.node_id + membership_login: details.body.membership.user.login + membership_id: details.body.membership.user.id + membership_state: details.body.membership.state + membership_role: details.body.membership.role + + membership: + username: details.body.sender.login + id: details.body.sender.id + scope: details.body.scope + action: details.body.action + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + + public: + username: details.body.sender.login + id: details.body.sender.id + repo_private: details.body.repository.private + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + + repository_import: + username: details.body.sender.login + id: details.body.sender.id + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + repo_private: details.body.repository.private + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + + release: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + repo_private: details.body.repository.private + repo_id: details.body.repository.id + repo_name: details.body.repository.name + repo_owner_id: details.body.repository.owner.id + repo_owner_name: details.body.repository.owner.name + repo_owner_login: details.body.repository.owner.login + repo_owner_node_id: details.body.repository.owner.node_id + repo_owner_site_admin: details.body.repository.owner.site_admin + release_id: details.body.release.id + release_node_id: details.body.release.node_id + release_tag_name: details.body.release.tag_name + release_name: details.body.release.name + release_author_login: details.body.release.author.login + release_author_id: details.body.release.author.id + release_author_node_id: details.body.release.author.node_id + release_author_type: details.body.release.author.type + release_author_site_admin: details.body.release.author.site_admin + + org_block: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + org_id: details.body.organization.id + org_login: details.body.organization.login + org_node_id: details.body.organization.node_id + blocked_user_login: details.body.blocked_user.login + blocked_user_id: details.body.blocked_user.id + blocked_user_node_id: details.body.blocked_user.node_id + + github_app_authorization: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + + installation: + username: details.body.sender.login + id: details.body.sender.id + action: details.body.action + sender_node_id: details.body.sender.node_id + sender_site_admin: details.body.sender.site_admin + sender_type: details.body.sender.type + repositories: details.body.repositories + install_id: details.body.installation.account.id + install_account_login: details.body.installation.account.login + install_account_node_id: details.body.installation.account.node_id + install_account_type: details.body.installation.account.type + install_account_site_admin: details.body.installation.account.site_admin \ No newline at end of file From fd5ffafbca5cfba70c15751ab88a7fee063ea200 Mon Sep 17 00:00:00 2001 From: Michal Purzynski Date: Tue, 20 Nov 2018 15:37:22 -0800 Subject: [PATCH 20/89] Move the configuration file where it can be found --- mq/plugins/github_webhooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mq/plugins/github_webhooks.py b/mq/plugins/github_webhooks.py index 224e948f..f9263914 100644 --- a/mq/plugins/github_webhooks.py +++ b/mq/plugins/github_webhooks.py @@ -18,7 +18,7 @@ class message(object): self.registration = ['githubeventsqs'] self.priority = 10 - with open('github_mapping.yml', 'r') as f: + with open('mq/github_mapping.yml', 'r') as f: map = f.read() yap = yaml.load(map) From 658866ec6ee223819c25e7ad6268d1cd75fa1d37 Mon Sep 17 00:00:00 2001 From: Michal Purzynski Date: Tue, 20 Nov 2018 15:56:19 -0800 Subject: [PATCH 21/89] Remove duplicated configuration file --- tests/mq/plugins/github_mapping.yml | 432 ---------------------------- 1 file changed, 432 deletions(-) delete mode 100644 tests/mq/plugins/github_mapping.yml diff --git a/tests/mq/plugins/github_mapping.yml b/tests/mq/plugins/github_mapping.yml deleted file mode 100644 index f5ede2f1..00000000 --- a/tests/mq/plugins/github_mapping.yml +++ /dev/null @@ -1,432 +0,0 @@ ---- - push: - username: details.body.sender.login - email: details.body.pusher.email - id: details.body.sender.id - commits: details.body.commits - created: details.body.created - deleted: details.body.deleted - forced: details.body.forced - node_id: details.body.sender.node_id - sender_type: detais.body.sender.type - request_id: details.request_id - commit_author: details.body.head_commit.author.email - committer: details.body.head_commit.committer.email - commit_id: details.body.head_commit.id - commit_msg: details.body.head_commit.message - commit_ts: details.body.head_commit.timestamp - commit_url: details.body.head_commit.url - org_id: details.body.organization.id - org_login: details.body.organization.login - org_node_id: details.body.organization.node_id - pusher_email: details.body.pusher.email - pusher_name: details.body.pusher.name - ref: details.body.ref - repo_id: details.body.repository.id - repo_name: details.body.repository.name - repo_owner_id: details.body.repository.owner.id - repo_owner_name: details.body.repository.owner.name - repo_owner_login: details.body.repository.owner.login - repo_owner_node_id: details.body.repository.owner.node_id - repo_owner_site_admin: details.body.repository.owner.site_admin - repo_private: details.body.repository.private - repo_pushed_at: details.body.repository.pushed_at - repo_updated_at: details.body.repository.ssh_url - sender_node_id: details.body.sender.node_id - sender_site_admin: details.body.sender.site_admin - sender_type: details.body.sender.type - - create: - username: details.body.sender.login - id: details.body.sender.id - action: details.body.action - org_id: details.body.organization.id - org_login: details.body.organization.login - org_node_id: details.body.organization.node_id - pusher_email: details.body.pusher.email - pusher_name: details.body.pusher.name - repo_private: details.body.repository.private - repo_id: details.body.repository.id - repo_name: details.body.repository.name - repo_owner_id: details.body.repository.owner.id - repo_owner_name: details.body.repository.owner.name - repo_owner_login: details.body.repository.owner.login - repo_owner_node_id: details.body.repository.owner.node_id - repo_owner_site_admin: details.body.repository.owner.site_admin - sender_node_id: details.body.sender.node_id - sender_site_admin: details.body.sender.site_admin - sender_type: details.body.sender.type - ref: details.body.ref - ref_type: details.body.ref_type - - delete: - username: details.body.sender.login - id: details.body.sender.id - action: details.body.action - org_id: details.body.organization.id - org_login: details.body.organization.login - org_node_id: details.body.organization.node_id - pusher_email: details.body.pusher.email - pusher_name: details.body.pusher.name - repo_private: details.body.repository.private - repo_id: details.body.repository.id - repo_name: details.body.repository.name - repo_owner_id: details.body.repository.owner.id - repo_owner_name: details.body.repository.owner.name - repo_owner_login: details.body.repository.owner.login - repo_owner_node_id: details.body.repository.owner.node_id - repo_owner_site_admin: details.body.repository.owner.site_admin - sender_node_id: details.body.sender.node_id - sender_site_admin: details.body.sender.site_admin - sender_type: details.body.sender.type - ref: details.body.ref - ref_type: details.body.ref_type - - pull_request: - username: details.body.sender.login - id: details.body.sender.id - action: details.body.action - pr_number: details.body.number - pr_assignee_id: details.body.pull_request.assignee.id - pr_assignee_login: details.body.pull_request.assignee.login - pr_assignee_site_admin: details.body.pull_request.assignee.site_admin - pr_assignee_type: details.body.pull_request.assignee.type - pr_author_association: details.body.pull_request.author_association - pr_base_label: details.body.pull_request.base.label - pr_base_ref: details.body.pull_request.base.ref - pr_base_repo_name: details.body.pull_request.base.repo.name - pr_base_repo_id: details.body.pull_request.base.repo.node_id - pr_base_repo_owner_id: details.body.pull_request.base.repo.owner.id - pr_base_repo_owner_login: details.body.pull_request.base.repo.owner.login - pr_base_repo_owner_node_id: details.body.pull_request.base.repo.owner.node_id - pr_base_repo_private: details.body.pull_request.base.repo.private - pr_base_sha: details.body.pull_request.base.sha - pr_base_user_id: details.body.pull_request.base.user.id - pr_base_user_login: details.body.pull_request.base.user.login - pr_base_user_node_id: details.body.pull_request.base.user.node_id - pr_base_user_site_admin: details.body.pull_request.base.user.site_admin - pr_body: details.body.pull_request.body - pr_changed_files: details.body.pull_request.changed_files - pr_commits: details.body.pull_request.commits - pr_deletions: details.body.pull_request.deletions - pr_closed_at: details.body.pull_request.closed_at - pr_head_label: details.body.pull_request.head.label - pr_head_ref: details.body.pull_request.head.ref - pr_head_repo_name: details.body.pull_request.head.repo.name - pr_head_repo_id: details.body.pull_request.head.repo.id - pr_head_repo_node_id: details.body.pull_request.head.repo.node_id - pr_head_repo_owner_id: details.body.pull_request.head.repo.owner.id - pr_head_repo_owner_login: details.body.pull_request.head.repo.owner.login - pr_head_repo_owner_node_id: details.body.pull_request.head.repo.owner.node_id - pr_head_repo_owner_site_admin: details.body.pull_request.head.repo.owner.site_admin - pr_head_repo_private: details.body.pull_request.head.repo.private - pr_head_sha: details.body.pull_request.head.sha - pr_head_user_id: details.body.pull_request.head.user.id - pr_head_user_login: details.body.pull_request.head.user.login - pr_head_user_node_id: details.body.pull_request.head.user.node_id - pr_merge_commit_sha: details.body.pull_request.merge_commit_sha - pr_merge_merged: details.body.pull_request.merged - pr_merge_merged_at: details.body.pull_request.merged_at - pr_merged_by_id: details.body.pull_request.merged_by.id - pr_merged_by_login: details.body.pull_request.merged_by.login - pr_merged_by_node_id: details.body.pull_request.merged_by.node_id - pr_merged_by_site_admin: details.body.pull_request.merged_by.site_admin - pr_node_id: details.body.pull_request.node_id - pr_number: details.body.pull_request.number - pr_state: details.body.pull_request.state - pr_title: details.body.pull_request.title - pr_updated_at: details.body.pull_request.updated_at - pr_id: details.body.pull_request.id - repo_id: details.body.repository.id - repo_name: details.body.repository.name - repo_owner_id: details.body.repository.owner.id - repo_owner_name: details.body.repository.owner.name - repo_owner_login: details.body.repository.owner.login - repo_owner_node_id: details.body.repository.owner.node_id - repo_owner_site_admin: details.body.repository.owner.site_admin - sender_node_id: details.body.sender.node_id - sender_site_admin: details.body.sender.site_admin - sender_type: details.body.sender.type - org_id: details.body.organization.id - org_login: details.body.organization.login - org_node_id: details.body.organization.node_id - - repository_vulnerability_alert: - username: details.body.sender.login - id: details.body.sender.id - action: details.body.action - alert_package: details.body.alert.affected_package_name - alert_range: details.body.alert.affected_range - alert_extid: details.body.alert.external_identifier - alert_extref: details.body.alert.external_reference - alert_fixed: details.body.alert.fixed_in - alert_id: details.body.alert.id - org_id: details.body.organization.id - org_login: details.body.organization.login - org_node_id: details.body.organization.node_id - repo_private: details.body.repository.private - repo_id: details.body.repository.id - repo_name: details.body.repository.name - repo_owner_id: details.body.repository.owner.id - repo_owner_name: details.body.repository.owner.name - repo_owner_login: details.body.repository.owner.login - repo_owner_node_id: details.body.repository.owner.node_id - repo_owner_site_admin: details.body.repository.owner.site_admin - sender_node_id: details.body.sender.node_id - sender_site_admin: details.body.sender.site_admin - sender_type: details.body.sender.type - dismiss_reason: details.body.alert.dismiss_reason - dismiss_at: details.body.alert.dismissed_at - dismiss_user: details.body.alert.dismisser.login - dismiss_id: details.body.alert.dismisser.id - dismiss_node_id: details.body.alert.dismisser.node_id - dismiss_type: details.body.alert.dismisser.type - dismiss_site_admin: details.body.alert.dismisser.site_admin - - security_advisory: - action: details.body.action - alert_id: details.body.security_advisory.ghsa_id - alert_note: details.body.security_advisory.summary - alert_description: details.body.security_advisory.description - alert_severity: details.body.security_advisory.severity - alert_list: details.body.security_advisory.identifiers - alert_vulnerabilities: details.body.security_advisory.vulnerabilities - alert_ref: details.body.security_advisory.references - alert_published_at: details.body.security_advisory.published_at - alert_updated_at: details.body.security_advisory.updated_at - alert_withdrawn_at: details.body.security_advisory.withdrawn_at - - repository: - username: details.body.sender.login - id: details.body.sender.id - action: details.body.action - org_id: details.body.organization.id - org_login: details.body.organization.login - org_node_id: details.body.organization.node_id - repo_private: details.body.repository.private - repo_id: details.body.repository.id - repo_name: details.body.repository.name - repo_owner_id: details.body.repository.owner.id - repo_owner_name: details.body.repository.owner.name - repo_owner_login: details.body.repository.owner.login - repo_owner_node_id: details.body.repository.owner.node_id - repo_owner_site_admin: details.body.repository.owner.site_admin - sender_node_id: details.body.sender.node_id - sender_site_admin: details.body.sender.site_admin - sender_type: details.body.sender.type - - member: - username: details.body.sender.login - id: details.body.sender.id - action: details.body.action - org_id: details.body.organization.id - org_login: details.body.organization.login - org_node_id: details.body.organization.node_id - repo_private: details.body.repository.private - repo_id: details.body.repository.id - repo_name: details.body.repository.name - repo_owner_id: details.body.repository.owner.id - repo_owner_name: details.body.repository.owner.name - repo_owner_login: details.body.repository.owner.login - repo_owner_node_id: details.body.repository.owner.node_id - repo_owner_site_admin: details.body.repository.owner.site_admin - sender_node_id: details.body.sender.node_id - sender_site_admin: details.body.sender.site_admin - sender_type: details.body.sender.type - member_id: details.body.member.id - member_login: details.body.member.login - member_node_id: details.body.member.node_id - member_site_admin: details.body.member.site_admin - changes_perm_from: details.body.changes.permission.from - - team: - username: details.body.sender.login - id: details.body.sender.id - action: details.body.action - org_id: details.body.organization.id - org_login: details.body.organization.login - org_node_id: details.body.organization.node_id - repo_private: details.body.repository.private - repo_id: details.body.repository.id - repo_name: details.body.repository.name - repo_owner_id: details.body.repository.owner.id - repo_owner_name: details.body.repository.owner.name - repo_owner_login: details.body.repository.owner.login - repo_owner_node_id: details.body.repository.owner.node_id - repo_owner_site_admin: details.body.repository.owner.site_admin - sender_node_id: details.body.sender.node_id - sender_site_admin: details.body.sender.site_admin - sender_type: details.body.sender.type - repo_perm_from_admin: details.body.changes.repository.permissions.from.admin - repo_perm_from_pull: details.body.changes.repository.permissions.from.pull - repo_perm_from_push: details.body.changes.repository.permissions.from.push - repo_perm_admin: details.body.repository.permissions.admin - repo_perm_pull: details.body.repository.permissions.pull - repo_perm_push: details.body.repository.permissions.push - team_id: details.body.team.id - team_name: details.body.team.name - team_node_id: details.body.team.node_id - team_permission: details.body.team.permission - team_privacy: details.body.team.privacy - team_slug: details.body.team.slug - - team_add: - username: details.body.sender.login - id: details.body.sender.id - action: details.body.action - org_id: details.body.organization.id - org_login: details.body.organization.login - org_node_id: details.body.organization.node_id - repo_private: details.body.repository.private - repo_id: details.body.repository.id - repo_name: details.body.repository.name - repo_owner_id: details.body.repository.owner.id - repo_owner_name: details.body.repository.owner.name - repo_owner_login: details.body.repository.owner.login - repo_owner_node_id: details.body.repository.owner.node_id - repo_owner_site_admin: details.body.repository.owner.site_admin - sender_node_id: details.body.sender.node_id - sender_site_admin: details.body.sender.site_admin - sender_type: details.body.sender.type - repo_perm_from_admin: details.body.changes.repository.permissions.from.admin - repo_perm_from_pull: details.body.changes.repository.permissions.from.pull - repo_perm_from_push: details.body.changes.repository.permissions.from.push - repo_perm_admin: details.body.repository.permissions.admin - repo_perm_pull: details.body.repository.permissions.pull - repo_perm_push: details.body.repository.permissions.push - team_id: details.body.team.id - team_name: details.body.team.name - team_node_id: details.body.team.node_id - team_permission: details.body.team.permission - team_privacy: details.body.team.privacy - team_slug: details.body.team.slug - - organization: - username: details.body.sender.login - id: details.body.sender.id - action: details.body.action - org_id: details.body.organization.id - org_login: details.body.organization.login - org_node_id: details.body.organization.node_id - sender_node_id: details.body.sender.node_id - sender_site_admin: details.body.sender.site_admin - sender_type: details.body.sender.type - team_id: details.body.team.id - team_name: details.body.team.name - team_node_id: details.body.team.node_id - team_permission: details.body.team.permission - team_privacy: details.body.team.privacy - team_slug: details.body.team.slug - membership_type: details.body.membership.user.type - membership_admin: details.body.membership.user.site_admin - membership_node_id: details.body.membership.user.node_id - membership_login: details.body.membership.user.login - membership_id: details.body.membership.user.id - membership_state: details.body.membership.state - membership_role: details.body.membership.role - - membership: - username: details.body.sender.login - id: details.body.sender.id - scope: details.body.scope - action: details.body.action - org_id: details.body.organization.id - org_login: details.body.organization.login - org_node_id: details.body.organization.node_id - sender_node_id: details.body.sender.node_id - sender_site_admin: details.body.sender.site_admin - sender_type: details.body.sender.type - - public: - username: details.body.sender.login - id: details.body.sender.id - repo_private: details.body.repository.private - repo_id: details.body.repository.id - repo_name: details.body.repository.name - repo_owner_id: details.body.repository.owner.id - repo_owner_name: details.body.repository.owner.name - repo_owner_login: details.body.repository.owner.login - repo_owner_node_id: details.body.repository.owner.node_id - repo_owner_site_admin: details.body.repository.owner.site_admin - sender_node_id: details.body.sender.node_id - sender_site_admin: details.body.sender.site_admin - sender_type: details.body.sender.type - - repository_import: - username: details.body.sender.login - id: details.body.sender.id - org_id: details.body.organization.id - org_login: details.body.organization.login - org_node_id: details.body.organization.node_id - repo_private: details.body.repository.private - repo_id: details.body.repository.id - repo_name: details.body.repository.name - repo_owner_id: details.body.repository.owner.id - repo_owner_name: details.body.repository.owner.name - repo_owner_login: details.body.repository.owner.login - repo_owner_node_id: details.body.repository.owner.node_id - repo_owner_site_admin: details.body.repository.owner.site_admin - sender_node_id: details.body.sender.node_id - sender_site_admin: details.body.sender.site_admin - sender_type: details.body.sender.type - - release: - username: details.body.sender.login - id: details.body.sender.id - action: details.body.action - sender_node_id: details.body.sender.node_id - sender_site_admin: details.body.sender.site_admin - sender_type: details.body.sender.type - repo_private: details.body.repository.private - repo_id: details.body.repository.id - repo_name: details.body.repository.name - repo_owner_id: details.body.repository.owner.id - repo_owner_name: details.body.repository.owner.name - repo_owner_login: details.body.repository.owner.login - repo_owner_node_id: details.body.repository.owner.node_id - repo_owner_site_admin: details.body.repository.owner.site_admin - release_id: details.body.release.id - release_node_id: details.body.release.node_id - release_tag_name: details.body.release.tag_name - release_name: details.body.release.name - release_author_login: details.body.release.author.login - release_author_id: details.body.release.author.id - release_author_node_id: details.body.release.author.node_id - release_author_type: details.body.release.author.type - release_author_site_admin: details.body.release.author.site_admin - - org_block: - username: details.body.sender.login - id: details.body.sender.id - action: details.body.action - sender_node_id: details.body.sender.node_id - sender_site_admin: details.body.sender.site_admin - sender_type: details.body.sender.type - org_id: details.body.organization.id - org_login: details.body.organization.login - org_node_id: details.body.organization.node_id - blocked_user_login: details.body.blocked_user.login - blocked_user_id: details.body.blocked_user.id - blocked_user_node_id: details.body.blocked_user.node_id - - github_app_authorization: - username: details.body.sender.login - id: details.body.sender.id - action: details.body.action - sender_node_id: details.body.sender.node_id - sender_site_admin: details.body.sender.site_admin - sender_type: details.body.sender.type - - installation: - username: details.body.sender.login - id: details.body.sender.id - action: details.body.action - sender_node_id: details.body.sender.node_id - sender_site_admin: details.body.sender.site_admin - sender_type: details.body.sender.type - repositories: details.body.repositories - install_id: details.body.installation.account.id - install_account_login: details.body.installation.account.login - install_account_node_id: details.body.installation.account.node_id - install_account_type: details.body.installation.account.type - install_account_site_admin: details.body.installation.account.site_admin \ No newline at end of file From 893c44c3fa18df7d7db42522e296ef060eb5dce0 Mon Sep 17 00:00:00 2001 From: andrewkrug Date: Wed, 21 Nov 2018 05:44:57 -0800 Subject: [PATCH 22/89] add container for generic SQS ingest --- .../cloudformation/mozdef-instance.yml | 9 ++++++++- cloudy_mozdef/cloudformation/mozdef-parent.yml | 5 +++-- docker/compose/docker-compose-cloudy-mozdef.yml | 17 +++++++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/cloudy_mozdef/cloudformation/mozdef-instance.yml b/cloudy_mozdef/cloudformation/mozdef-instance.yml index e5cf6762..09cbdf26 100644 --- a/cloudy_mozdef/cloudformation/mozdef-instance.yml +++ b/cloudy_mozdef/cloudformation/mozdef-instance.yml @@ -68,7 +68,10 @@ Parameters: Description: The URL of your OIDC provider's well-known discovery URL CloudTrailSQSNotificationQueueName: Type: String - Description: The URL of your OIDC provider's well-known discovery URL + Description: The name of the SQS used for CloudTrail notifications. + MozDefSQSQueueName: + Type: String + Description: The name of the generic SQS queue used to pickup events. Resources: MozDefElasticLoadBalancingV2TargetGroup: Type: AWS::ElasticLoadBalancingV2::TargetGroup @@ -147,9 +150,13 @@ Resources: - content: | OPTIONS_TASKEXCHANGE=${CloudTrailSQSNotificationQueueName} path: /opt/mozdef/docker/compose/cloudy_mozdef_mq_cloudtrail.env + - content: | + OPTIONS_TASKEXCHANGE=${MozDefSQSQueName} + path: /opt/mozdef/docker/compose/cloudy_mozdef_mq_sqs.env runcmd: - chmod --verbose 600 /opt/mozdef/docker/compose/cloudy_mozdef.env - chmod --verbose 600 /opt/mozdef/docker/compose/cloudy_mozdef_kibana.env + - chmod --verbose 600 /opt/mozdef/docker/compose/cloudy_mozdef_mq_sqs.env - mkdir --verbose --parents ${EFSMountPoint} - echo '*.* @@127.0.0.1:514' >> /etc/rsyslog.conf - systemctl enable rsyslog diff --git a/cloudy_mozdef/cloudformation/mozdef-parent.yml b/cloudy_mozdef/cloudformation/mozdef-parent.yml index 9227c840..7f737cde 100644 --- a/cloudy_mozdef/cloudformation/mozdef-parent.yml +++ b/cloudy_mozdef/cloudformation/mozdef-parent.yml @@ -98,6 +98,7 @@ Resources: OIDCClientSecret: !Ref OIDCClientSecret OIDCDiscoveryURL: !Ref OIDCDiscoveryURL CloudTrailSQSNotificationQueueName: !GetAtt MozDefCloudTrail.Outputs.CloudTrailSQSQueueName + MozDefSQSQueueName: !GetAtt MozDefSQS.Outputs.SQSQueueName Tags: - Key: application Value: mozdef @@ -142,7 +143,7 @@ Resources: Value: mozdef - Key: stack Value: !Ref AWS::StackName - TemplateURL: !Join [ '', [ !Ref S3TemplateLocation, mozdef-sqs.yml ] ] + TemplateURL: !Join [ '', [ !Ref S3TemplateLocation, mozdef-sqs.yml ] MozDefCloudTrail: Type: AWS::CloudFormation::Stack Properties: @@ -245,4 +246,4 @@ Resources: Properties: RoleName: AWSServiceRoleForAmazonElasticsearchService PathPrefix: '/aws-service-role/es.amazonaws.com/' - ServiceToken: !GetAtt DoesRoleExistLambdaFunction.Arn \ No newline at end of file + ServiceToken: !GetAtt DoesRoleExistLambdaFunction.Arn diff --git a/docker/compose/docker-compose-cloudy-mozdef.yml b/docker/compose/docker-compose-cloudy-mozdef.yml index 8a520927..72bb16f4 100644 --- a/docker/compose/docker-compose-cloudy-mozdef.yml +++ b/docker/compose/docker-compose-cloudy-mozdef.yml @@ -186,6 +186,23 @@ services: - default volumes: - geolite_db:/opt/mozdef/envs/mozdef/data/ + mq_sqs: + image: mozdef/mozdef_mq_worker + env_file: + - cloudy_mozdef.env + - cloudy_mozdef_mq_sqs.env + restart: always + command: bash -c 'source /opt/mozdef/envs/python/bin/activate && python esworker_sqs.py -c esworker_sqs.conf' + scale: 1 + depends_on: + - base + - rabbitmq + - loginput + - bootstrap + networks: + - default + volumes: + - geolite_db:/opt/mozdef/envs/mozdef/data/ volumes: cron: geolite_db: From a14f51fd0e549f79745010284bb75adc4c0915e3 Mon Sep 17 00:00:00 2001 From: andrewkrug Date: Wed, 21 Nov 2018 06:13:48 -0800 Subject: [PATCH 23/89] standardize es_worker credential handling --- mq/esworker_sqs.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/mq/esworker_sqs.py b/mq/esworker_sqs.py index e6451f23..88ad8617 100755 --- a/mq/esworker_sqs.py +++ b/mq/esworker_sqs.py @@ -42,6 +42,19 @@ except ImportError as e: hasUWSGI = False +def get_aws_credentials(region=None, accesskey=None, secretkey=None, security_token=None): + result = {} + if region not in ['', '', None]: + result['region_name'] = region + if accesskey not in ['', '', None]: + result['aws_access_key_id'] = accesskey + if secretkey not in ['', '', None]: + result['aws_secret_access_key'] = secretkey + if security_token not in [None]: + result['security_token'] = security_token + return result + + def keyMapping(aDict): '''map common key/fields to a normalized structure, explicitly typed when possible to avoid schema changes for upsteam consumers @@ -345,16 +358,14 @@ def main(): logger.error('Can only process SQS queues, terminating') sys.exit(1) - mqConn, eventTaskQueue = connect_sqs( - options.region, - options.accesskey, - options.secretkey, - options.taskexchange - ) - + sqs_conn, eventTaskQueue = connect_sqs( + task_exchange=options.taskexchange, + **get_aws_credentials( + options.region, + options.accesskey, + options.secretkey)) # consume our queue - taskConsumer(mqConn, eventTaskQueue, es).run() - + taskConsumer(sqs_conn, eventTaskQueue, es).run() def initConfig(): # capture the hostname @@ -389,7 +400,7 @@ def initConfig(): # aws options options.accesskey = getConfig('accesskey', '', options.configfile) options.secretkey = getConfig('secretkey', '', options.configfile) - options.region = getConfig('region', 'us-west-1', options.configfile) + options.region = getConfig('region', '', options.configfile) # plugin options # secs to pass before checking for new/updated plugins From 5845d59dbb8c390843eae3941bbc59bf44363314 Mon Sep 17 00:00:00 2001 From: andrewkrug Date: Wed, 21 Nov 2018 06:55:46 -0800 Subject: [PATCH 24/89] ensure mozdef always polls the SQS queue we create --- .../cloudformation/mozdef-instance.yml | 2 +- cloudy_mozdef/cloudformation/mozdef-sqs.yml | 15 +++++++++- .../compose/docker-compose-cloudy-mozdef.yml | 4 +-- mq/esworker_sns_sqs.py | 30 +++++++++++++------ 4 files changed, 38 insertions(+), 13 deletions(-) diff --git a/cloudy_mozdef/cloudformation/mozdef-instance.yml b/cloudy_mozdef/cloudformation/mozdef-instance.yml index 09cbdf26..20d6443b 100644 --- a/cloudy_mozdef/cloudformation/mozdef-instance.yml +++ b/cloudy_mozdef/cloudformation/mozdef-instance.yml @@ -152,7 +152,7 @@ Resources: path: /opt/mozdef/docker/compose/cloudy_mozdef_mq_cloudtrail.env - content: | OPTIONS_TASKEXCHANGE=${MozDefSQSQueName} - path: /opt/mozdef/docker/compose/cloudy_mozdef_mq_sqs.env + path: /opt/mozdef/docker/compose/cloudy_mozdef_mq_sns_sqs.env runcmd: - chmod --verbose 600 /opt/mozdef/docker/compose/cloudy_mozdef.env - chmod --verbose 600 /opt/mozdef/docker/compose/cloudy_mozdef_kibana.env diff --git a/cloudy_mozdef/cloudformation/mozdef-sqs.yml b/cloudy_mozdef/cloudformation/mozdef-sqs.yml index 09d52c28..bbd50221 100644 --- a/cloudy_mozdef/cloudformation/mozdef-sqs.yml +++ b/cloudy_mozdef/cloudformation/mozdef-sqs.yml @@ -9,10 +9,23 @@ Resources: Value: mozdef - Key: stack Value: !Ref AWS::StackName + MozDefCloudTrailSQSQueuePolicy: + Type: AWS::SQS::QueuePolicy + Properties: + PolicyDocument: + Version: 2012-10-17 + Statement: + - Sid: AllowThisAccountSendToSQS + Effect: Allow + Principal: '*' + Action: sqs:SendMessage + Resource: !GetAtt MozDefCloudTrailSQSQueue.Arn + Queues: + - !Ref MozDefCloudTrailSQSQueue Outputs: SQSQueueArn: Description: ARN of the SQS Queue that MozDef will consume events from Value: !GetAtt MozDefSQSQueue.Arn SQSQueueName: Description: Name of the SQS Queue that MozDef will consume events from - Value: !GetAtt MozDefSQSQueue.QueueName \ No newline at end of file + Value: !GetAtt MozDefSQSQueue.QueueName diff --git a/docker/compose/docker-compose-cloudy-mozdef.yml b/docker/compose/docker-compose-cloudy-mozdef.yml index 72bb16f4..fda7fe61 100644 --- a/docker/compose/docker-compose-cloudy-mozdef.yml +++ b/docker/compose/docker-compose-cloudy-mozdef.yml @@ -190,9 +190,9 @@ services: image: mozdef/mozdef_mq_worker env_file: - cloudy_mozdef.env - - cloudy_mozdef_mq_sqs.env + - cloudy_mozdef_mq_sns_sqs.env restart: always - command: bash -c 'source /opt/mozdef/envs/python/bin/activate && python esworker_sqs.py -c esworker_sqs.conf' + command: bash -c 'source /opt/mozdef/envs/python/bin/activate && python esworker_sns_sqs.py -c esworker_sns_sqs.conf' scale: 1 depends_on: - base diff --git a/mq/esworker_sns_sqs.py b/mq/esworker_sns_sqs.py index 9924ac86..374f0502 100755 --- a/mq/esworker_sns_sqs.py +++ b/mq/esworker_sns_sqs.py @@ -41,6 +41,19 @@ def esConnect(): return ElasticsearchClient((list('{0}'.format(s) for s in options.esservers)), options.esbulksize) +def get_aws_credentials(region=None, accesskey=None, secretkey=None, security_token=None): + result = {} + if region not in ['', '', None]: + result['region_name'] = region + if accesskey not in ['', '', None]: + result['aws_access_key_id'] = accesskey + if secretkey not in ['', '', None]: + result['aws_secret_access_key'] = secretkey + if security_token not in [None]: + result['security_token'] = security_token + return result + + class taskConsumer(object): def __init__(self, mqConnection, taskQueue, esConnection, options): @@ -200,15 +213,14 @@ def main(): logger.error('Can only process SQS queues, terminating') sys.exit(1) - mqConn, eventTaskQueue = connect_sqs( - options.region, - options.accesskey, - options.secretkey, - options.taskexchange - ) - + sqs_conn, eventTaskQueue = connect_sqs( + task_exchange=options.taskexchange, + **get_aws_credentials( + options.region, + options.accesskey, + options.secretkey)) # consume our queue - taskConsumer(mqConn, eventTaskQueue, es, options).run() + taskConsumer(sqs_conn, eventTaskQueue, es, options).run() def initConfig(): @@ -231,7 +243,7 @@ def initConfig(): # aws options options.accesskey = getConfig('accesskey', '', options.configfile) options.secretkey = getConfig('secretkey', '', options.configfile) - options.region = getConfig('region', 'us-west-1', options.configfile) + options.region = getConfig('region', '', options.configfile) # plugin options # secs to pass before checking for new/updated plugins From 440d50478d96681692b39604016bb4bfa00508d3 Mon Sep 17 00:00:00 2001 From: andrewkrug Date: Wed, 21 Nov 2018 07:43:37 -0800 Subject: [PATCH 25/89] fix flake 8 error --- mq/esworker_sqs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mq/esworker_sqs.py b/mq/esworker_sqs.py index 88ad8617..d1b0620c 100755 --- a/mq/esworker_sqs.py +++ b/mq/esworker_sqs.py @@ -367,6 +367,7 @@ def main(): # consume our queue taskConsumer(sqs_conn, eventTaskQueue, es).run() + def initConfig(): # capture the hostname options.mozdefhostname = getConfig('mozdefhostname', socket.gethostname(), options.configfile) From b11b11a606fcba30d5bbb98607b7282fe4a309e9 Mon Sep 17 00:00:00 2001 From: Jeff Bryner Date: Wed, 21 Nov 2018 07:46:36 -0800 Subject: [PATCH 26/89] correct the license --- mozdef_util/LICENSE | 595 +++++++++++++++++++++++--------------------- 1 file changed, 314 insertions(+), 281 deletions(-) diff --git a/mozdef_util/LICENSE b/mozdef_util/LICENSE index 1f963da0..14e2f777 100644 --- a/mozdef_util/LICENSE +++ b/mozdef_util/LICENSE @@ -1,340 +1,373 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 +Mozilla Public License Version 2.0 +================================== - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. +1. Definitions +-------------- - Preamble +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. +1.3. "Contribution" + means Covered Software of a particular Contributor. - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. +1.5. "Incompatible With Secondary Licenses" + means - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. +1.6. "Executable Form" + means any form of the work other than Source Code Form. - The precise terms and conditions for copying, distribution and -modification follow. +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION +1.8. "License" + means this document. - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. +1.10. "Modifications" + means any of the following: - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. + (b) any new file in Source Code Form that contains any Covered + Software. - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. +1.13. "Source Code Form" + means the form of the work preferred for making modifications. - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. +2. License Grants and Conditions +-------------------------------- -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. +2.1. Grants -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, +2.2. Effective Date - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. +2.3. Limitations on Grant Scope -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. +(a) for any code that a Contributor has removed from Covered Software; + or - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. +2.4. Subsequent Licenses -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. +2.5. Representation - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. +2.6. Fair Use -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. +2.7. Conditions - NO WARRANTY +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. +3. Responsibilities +------------------- - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. +3.1. Distribution of Source Form - END OF TERMS AND CONDITIONS +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. - How to Apply These Terms to Your New Programs +3.2. Distribution of Executable Form - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. +If You distribute Covered Software in Executable Form then: - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and - - Copyright (C) +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. +3.3. Distribution of a Larger Work - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +3.4. Notices -Also add information on how to contact you by electronic and paper mail. +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: +3.5. Application of Additional Terms - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. +5. Termination +-------------- - , 1 April 1989 - Ty Coon, President of Vice +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. From 3061a57299f04ce0dbe1a559085df9ee08357f7e Mon Sep 17 00:00:00 2001 From: andrewkrug Date: Wed, 21 Nov 2018 09:24:01 -0800 Subject: [PATCH 27/89] fix syntax error --- cloudy_mozdef/cloudformation/mozdef-instance.yml | 3 +-- cloudy_mozdef/cloudformation/mozdef-parent.yml | 4 ++-- cloudy_mozdef/cloudformation/mozdef-sqs.yml | 4 ++-- cloudy_mozdef/packer/packer.json | 4 ++-- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/cloudy_mozdef/cloudformation/mozdef-instance.yml b/cloudy_mozdef/cloudformation/mozdef-instance.yml index 20d6443b..126e3be6 100644 --- a/cloudy_mozdef/cloudformation/mozdef-instance.yml +++ b/cloudy_mozdef/cloudformation/mozdef-instance.yml @@ -24,7 +24,6 @@ Parameters: AMIImageId: Type: AWS::EC2::Image::Id Description: The AMI Image ID to use of the EC2 instance - Default: ami-0ff5b922fb3ae2332 EFSID: Type: String Description: Logical ID of the EFS Filesystem @@ -151,7 +150,7 @@ Resources: OPTIONS_TASKEXCHANGE=${CloudTrailSQSNotificationQueueName} path: /opt/mozdef/docker/compose/cloudy_mozdef_mq_cloudtrail.env - content: | - OPTIONS_TASKEXCHANGE=${MozDefSQSQueName} + OPTIONS_TASKEXCHANGE=${MozDefSQSQueueName} path: /opt/mozdef/docker/compose/cloudy_mozdef_mq_sns_sqs.env runcmd: - chmod --verbose 600 /opt/mozdef/docker/compose/cloudy_mozdef.env diff --git a/cloudy_mozdef/cloudformation/mozdef-parent.yml b/cloudy_mozdef/cloudformation/mozdef-parent.yml index 7f737cde..102894f1 100644 --- a/cloudy_mozdef/cloudformation/mozdef-parent.yml +++ b/cloudy_mozdef/cloudformation/mozdef-parent.yml @@ -38,7 +38,7 @@ Parameters: AMIImageId: Type: String Description: The AMI Image ID to use of the EC2 instance - Default: ami-0ff5b922fb3ae2332 + Default: ami-0c3705bb3b43ad51f ACMCertArn: Description: The ARN of your pre-issued ACM cert. Default : arn:aws:acm:us-west-2:656532927350:certificate/79f641f2-4046-4754-a28f-4db80d7c0583 @@ -143,7 +143,7 @@ Resources: Value: mozdef - Key: stack Value: !Ref AWS::StackName - TemplateURL: !Join [ '', [ !Ref S3TemplateLocation, mozdef-sqs.yml ] + TemplateURL: !Join [ '', [ !Ref S3TemplateLocation, mozdef-sqs.yml ] ] MozDefCloudTrail: Type: AWS::CloudFormation::Stack Properties: diff --git a/cloudy_mozdef/cloudformation/mozdef-sqs.yml b/cloudy_mozdef/cloudformation/mozdef-sqs.yml index bbd50221..65bf5044 100644 --- a/cloudy_mozdef/cloudformation/mozdef-sqs.yml +++ b/cloudy_mozdef/cloudformation/mozdef-sqs.yml @@ -19,9 +19,9 @@ Resources: Effect: Allow Principal: '*' Action: sqs:SendMessage - Resource: !GetAtt MozDefCloudTrailSQSQueue.Arn + Resource: !GetAtt MozDefSQSQueue.Arn Queues: - - !Ref MozDefCloudTrailSQSQueue + - !Ref MozDefSQSQueue Outputs: SQSQueueArn: Description: ARN of the SQS Queue that MozDef will consume events from diff --git a/cloudy_mozdef/packer/packer.json b/cloudy_mozdef/packer/packer.json index fa264cae..5b366ca0 100644 --- a/cloudy_mozdef/packer/packer.json +++ b/cloudy_mozdef/packer/packer.json @@ -30,7 +30,7 @@ "sudo systemctl enable docker", "sudo mkdir -p /opt/mozdef/", "sudo git clone https://github.com/mozilla/MozDef /opt/mozdef", - "cd /opt/mozdef && sudo git checkout origin/master" - ]} + "cd /opt/mozdef && sudo git checkout guardduty" + ]} ] } From 839d545dd684b0cb69960d27f6700912577a33a8 Mon Sep 17 00:00:00 2001 From: Jeff Bryner Date: Fri, 23 Nov 2018 09:26:45 -0800 Subject: [PATCH 28/89] pull ip from an occasionally present list --- mq/plugins/guardDuty.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/mq/plugins/guardDuty.py b/mq/plugins/guardDuty.py index c2cb5bd1..b63f0976 100644 --- a/mq/plugins/guardDuty.py +++ b/mq/plugins/guardDuty.py @@ -31,7 +31,7 @@ class message(object): # AWS guard duty can send IPs in a bunch of places # Lets pick out some likely targets and format them # so other mozdef plugins can rely on their location - self.ipaddress_keys =[ + self.ipaddress_keys = [ 'details.finding.action.networkConnectionAction.remoteIpDetails.ipAddressV4', 'details.finding.action.awsApiCallAction.remoteIpDetails.ipAdrressV4' ] @@ -43,7 +43,8 @@ class message(object): current_pointer = haystack for updated_key in num_levels: if updated_key == num_levels[-1]: - current_pointer[updated_key] = toUTC(current_pointer[updated_key]).isoformat() + current_pointer[updated_key] = toUTC( + current_pointer[updated_key]).isoformat() return haystack if updated_key in current_pointer: current_pointer = current_pointer[updated_key] @@ -63,12 +64,25 @@ class message(object): message = self.convert_key_date_format(date_key, message) # convert the dict to a dot dict for saner deep key/value processing - message=DotDict(message) + message = DotDict(message) # pull out the likely source IP address for ipaddress_key in self.ipaddress_keys: if 'sourceipaddress' not in message['details'].keys(): - if key_exists(ipaddress_key,message): - message.details.sourceipaddress = message.get(ipaddress_key) + if key_exists(ipaddress_key, message): + message.details.sourceipaddress = message.get( + ipaddress_key) + + # if we still haven't found what we are looking for #U2 + # sometimes it's in a list + if 'sourceipaddress' not in message['details'].keys(): + if key_exists('details.finding.action.portProbeAction.portProbeDetails', message) \ + and isinstance(message.details.finding.action.portProbeAction.portProbeDetails, list): + + # inspect the first list entry and see if it contains an IP + portProbeDetails = DotDict( + message.details.finding.action.portProbeAction.portProbeDetails[0]) + if key_exists('remoteIpDetails.ipAddressV4', portProbeDetails): + message.details.sourceipaddress = portProbeDetails.remoteIpDetails.ipAddressV4 # recovert the message back to a plain dict return (dict(message), metadata) From 1e767a215f248a4a19b9119a90a50673d3a5c37d Mon Sep 17 00:00:00 2001 From: Jeff Bryner Date: Fri, 23 Nov 2018 10:58:37 -0800 Subject: [PATCH 29/89] run without build and everything always passes --- docs/source/alert_development_guide.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/alert_development_guide.rst b/docs/source/alert_development_guide.rst index e1c6afda..8ceaf442 100644 --- a/docs/source/alert_development_guide.rst +++ b/docs/source/alert_development_guide.rst @@ -44,8 +44,9 @@ Requirements: This test should pass and you will have confirmed you have a working environment. -At this point, begin development and periodically run your unit-tests locally with the following command:: +At this point, begin development and periodically run your unit-tests locally with the following commands:: + make build-tests make run-tests TEST_CASE=tests/alerts/[YOUR ALERT TEST FILE].py From 46b1e849febb958f33c6e3555323c24addd01563 Mon Sep 17 00:00:00 2001 From: Jeff Bryner Date: Fri, 23 Nov 2018 11:51:43 -0800 Subject: [PATCH 30/89] tests require notify_mozdefbot field, even though alerts dont --- tests/alerts/test_alert_template.template | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/alerts/test_alert_template.template b/tests/alerts/test_alert_template.template index 050210a0..2c953ebd 100644 --- a/tests/alerts/test_alert_template.template +++ b/tests/alerts/test_alert_template.template @@ -29,6 +29,7 @@ class TEMPLATE_TEST_CLASSNAME(AlertTestSuite): "tags": ['hello', 'world'], "severity": "WARNING", "summary": 'My first alert!', + "notify_mozdefbot": False, } test_cases = [] From a27d38b17e3fd521343c19ccd94b7da4be3b17bd Mon Sep 17 00:00:00 2001 From: Jeff Bryner Date: Fri, 23 Nov 2018 11:52:07 -0800 Subject: [PATCH 31/89] add guardduty probe alert --- alerts/guard_duty_probe.py | 43 +++++++++++++++ tests/alerts/test_guard_duty_probe.py | 78 +++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 alerts/guard_duty_probe.py create mode 100644 tests/alerts/test_guard_duty_probe.py diff --git a/alerts/guard_duty_probe.py b/alerts/guard_duty_probe.py new file mode 100644 index 00000000..c9e63974 --- /dev/null +++ b/alerts/guard_duty_probe.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# Copyright (c) 2014 Mozilla Corporation + + +from lib.alerttask import AlertTask +from mozdef_util.query_models import SearchQuery, TermMatch, QueryStringMatch, ExistsMatch, PhraseMatch, WildcardMatch + + +class AlertGuardDutyProbe(AlertTask): + def main(self): + # Create a query to look back the last 20 minutes + search_query = SearchQuery(minutes=20) + + # Add search terms to our query + search_query.add_must([ + TermMatch('source', 'guardduty'), + TermMatch('details.finding.action.actionType', 'PORT_PROBE'), + ExistsMatch('details.sourceipaddress'), + ]) + + self.filtersManual(search_query) + # Search aggregations on field 'sourceipaddress' + # keep X samples of events at most + self.searchEventsAggregated('details.sourceipaddress', samplesLimit=10) + # alert when >= X matching events in an aggregation + self.walkAggregations(threshold=1) + + # Set alert properties + def onAggregation(self, aggreg): + # aggreg['count']: number of items in the aggregation, ex: number of failed login attempts + # aggreg['value']: value of the aggregation field, ex: toto@example.com + # aggreg['events']: list of events in the aggregation + category = 'bruteforce' + tags = ['guardduty', 'bruteforce'] + severity = 'INFO' + summary = "Guard Duty Port Probe by {}".format(aggreg['value']) + + # Create the alert object based on these properties + return self.createAlertDict(summary, category, tags, aggreg['events'], severity) diff --git a/tests/alerts/test_guard_duty_probe.py b/tests/alerts/test_guard_duty_probe.py new file mode 100644 index 00000000..fed960d7 --- /dev/null +++ b/tests/alerts/test_guard_duty_probe.py @@ -0,0 +1,78 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# Copyright (c) 2017 Mozilla Corporation +from positive_alert_test_case import PositiveAlertTestCase +from negative_alert_test_case import NegativeAlertTestCase +from alert_test_suite import AlertTestSuite + + +class TestGuardDutyProbe(AlertTestSuite): + alert_filename = "guard_duty_probe" + alert_classname = "AlertGuardDutyProbe" + + # This event is the default positive event that will cause the + # alert to trigger + default_event = { + "_type": "event", + "_source": { + "source": "guardduty", + "details": { + "sourceipaddress": "1.2.3.4", + "finding": {"action": + {"actionType": "PORT_PROBE"} + } + } + } + } + + # This alert is the expected result from running this task + default_alert = { + "category": "bruteforce", + "tags": ['guardduty', 'bruteforce'], + "severity": "INFO", + "summary": 'Guard Duty Port Probe by 1.2.3.4', + "notify_mozdefbot": False, + } + + test_cases = [] + + test_cases.append( + PositiveAlertTestCase( + description="Positive test with default events and default alert expected", + events=AlertTestSuite.create_events(default_event, 1), + expected_alert=default_alert + ) + ) + + events = AlertTestSuite.create_events(default_event, 10) + for event in events: + event['_source']['source'] = 'bad' + test_cases.append( + NegativeAlertTestCase( + description="Negative test case with events with incorrect category", + events=events, + ) + ) + + events = AlertTestSuite.create_events(default_event, 10) + for event in events: + event['_source']['details']['sourceipaddress'] = None + test_cases.append( + NegativeAlertTestCase( + description="Negative test case with events with non-existent sourceipaddress", + events=events, + ) + ) + + for event in events: + event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda( + date_timedelta={'minutes': 21}) + event['_source']['receivedtimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda( + date_timedelta={'minutes': 21}) + test_cases.append( + NegativeAlertTestCase( + description="Negative test case with old timestamp", + events=events, + ) + ) From 66655a80b674ec715b106efb1baac9611fbab22c Mon Sep 17 00:00:00 2001 From: andrewkrug Date: Fri, 23 Nov 2018 12:32:49 -0800 Subject: [PATCH 32/89] update to support cloudformation enabling alerts --- .../cloudformation/mozdef-instance.yml | 77 +++++++++++++++++++ .../compose/docker-compose-cloudy-mozdef.yml | 2 + 2 files changed, 79 insertions(+) diff --git a/cloudy_mozdef/cloudformation/mozdef-instance.yml b/cloudy_mozdef/cloudformation/mozdef-instance.yml index 126e3be6..39025c18 100644 --- a/cloudy_mozdef/cloudformation/mozdef-instance.yml +++ b/cloudy_mozdef/cloudformation/mozdef-instance.yml @@ -137,6 +137,83 @@ Resources: # Increase the AWS ES total fields limit from 1000 to 4000 OPTIONS_MAPPING_TOTAL_FIELDS_LIMIT=4000 path: /opt/mozdef/docker/compose/cloudy_mozdef.env + write_files: + - content: | + #!/usr/bin/env python + # This Source Code Form is subject to the terms of the Mozilla Public + # License, v. 2.0. If a copy of the MPL was not distributed with this + # file, You can obtain one at http://mozilla.org/MPL/2.0/. + # Copyright (c) 2014 Mozilla Corporation + + from celery.schedules import crontab, timedelta + import time + import logging + + ALERTS = { + 'bruteforce_ssh.AlertBruteforceSsh': {'schedule': crontab(minute='*/1')}, + 'unauth_ssh.AlertUnauthSSH': {'schedule': crontab(minute='*/1')}, + 'guard_duty_probe.AlertGuardDutyProbe: {'schedule': crontab(minute='*/1')}', + 'cloudtrail_logging_disabled.AlertCloudtrailLoggingDisabled': {'schedule': timedelta(minutes=1)}, + 'cloudtrail_deadman.AlertCloudtrailDeadman': {'schedule': timedelta(hours=1)} + } + + ALERT_PLUGINS = [ + # 'relative pythonfile name (exclude the .py) - EX: sso_dashboard', + ] + + RABBITMQ = { + 'mqserver': 'rabbitmq', + 'mquser': 'guest', + 'mqpassword': 'guest', + 'mqport': 5672, + 'alertexchange': 'alerts', + 'alertqueue': 'mozdef.alert' + } + + ES = { + 'servers': [${ESURL}] + } + + OPTIONS = { + 'defaulttimezone': 'UTC', + } + + LOGGING = { + 'version': 1, + 'disable_existing_loggers': True, + 'formatters': { + 'simple': { + 'format': '%(levelname)s %(message)s', + 'datefmt': '%y %b %d, %H:%M:%S', + }, + 'standard': { + 'format': '%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)d: %(message)s' + } + }, + 'handlers': { + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'simple' + }, + 'celery': { + 'level': 'DEBUG', + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': 'celery.log', + 'formatter': 'standard', + 'maxBytes': 1024 * 1024 * 100, # 100 mb + }, + }, + 'loggers': { + 'celery': { + 'handlers': ['celery', 'console'], + 'level': 'DEBUG', + }, + } + } + + logging.Formatter.converter = time.gmtime + path: /opt/mozdef/docker/compose/mozdef_alerts/files/config.py - content: | client_id=${OIDCClientId} client_secret=${OIDCClientSecret} diff --git a/docker/compose/docker-compose-cloudy-mozdef.yml b/docker/compose/docker-compose-cloudy-mozdef.yml index fda7fe61..3a77130a 100644 --- a/docker/compose/docker-compose-cloudy-mozdef.yml +++ b/docker/compose/docker-compose-cloudy-mozdef.yml @@ -69,6 +69,8 @@ services: image: mozdef/mozdef_alerts env_file: - cloudy_mozdef.env + volumes: + - /opt/mozdef/docker/compose/mozdef_alerts/files/config.py:/opt/mozdef/envs/mozdef/alerts/lib/config.py restart: always command: bash -c 'source /opt/mozdef/envs/python/bin/activate && celery -A celeryconfig worker --loglevel=info --beat' depends_on: From fc22ff4a39d574dd37cc2191523beeab4064b4bf Mon Sep 17 00:00:00 2001 From: andrewkrug Date: Fri, 23 Nov 2018 12:38:10 -0800 Subject: [PATCH 33/89] fix typo in alerts --- cloudy_mozdef/cloudformation/mozdef-instance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudy_mozdef/cloudformation/mozdef-instance.yml b/cloudy_mozdef/cloudformation/mozdef-instance.yml index 39025c18..64ad55cc 100644 --- a/cloudy_mozdef/cloudformation/mozdef-instance.yml +++ b/cloudy_mozdef/cloudformation/mozdef-instance.yml @@ -152,7 +152,7 @@ Resources: ALERTS = { 'bruteforce_ssh.AlertBruteforceSsh': {'schedule': crontab(minute='*/1')}, 'unauth_ssh.AlertUnauthSSH': {'schedule': crontab(minute='*/1')}, - 'guard_duty_probe.AlertGuardDutyProbe: {'schedule': crontab(minute='*/1')}', + 'guard_duty_probe.AlertGuardDutyProbe': {'schedule': crontab(minute='*/1')}', 'cloudtrail_logging_disabled.AlertCloudtrailLoggingDisabled': {'schedule': timedelta(minutes=1)}, 'cloudtrail_deadman.AlertCloudtrailDeadman': {'schedule': timedelta(hours=1)} } From a254b8030c0b6d4ffe5576301985e3fa0037f757 Mon Sep 17 00:00:00 2001 From: andrewkrug Date: Fri, 23 Nov 2018 15:33:17 -0800 Subject: [PATCH 34/89] refactor attacker collection to configure hitcount in cf --- cron/collectAttackers.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cron/collectAttackers.py b/cron/collectAttackers.py index f449bbc3..45430271 100755 --- a/cron/collectAttackers.py +++ b/cron/collectAttackers.py @@ -153,7 +153,7 @@ def searchMongoAlerts(mozdefdb): # count by ip {"$group": {"_id": "$sourceip", "hitcount": {"$sum": 1}}}, # limit to those with X observances - {"$match": {"hitcount": {"$gt": 5}}}, + {"$match": {"hitcount": {"$gt": options.ipv4attackerhitcount}}}, # sort {"$sort": SON([("hitcount", -1), ("_id", -1)])}, # top 10 @@ -166,7 +166,7 @@ def searchMongoAlerts(mozdefdb): # set CIDR # todo: lookup ipwhois for asn_cidr value # potentially with a max mask value (i.e. asn is /8, limit attackers to /24) - ipcidr.prefixlen = 32 + ipcidr.prefixlen = options.ipv4attackerprefixlength # append to or create attacker. # does this match an existing attacker's indicators @@ -482,6 +482,11 @@ def initConfig(): # set to either amqp or amqps for ssl options.mqprotocol = getConfig('mqprotocol', 'amqp', options.configfile) + # Set these settings to change the correlation for attackers + options.ipv4attackerprefixlength = getConfig('ipv4attackerprefixlength', 32, options.configfile) + options.ipv4attackerhitcount = getConfig('ipv4ipv4attackerhitcount', 5, options.configfile) + + if __name__ == '__main__': parser = OptionParser() From 995d730c9c934aa0549e9d19d54ee32e5035b5c1 Mon Sep 17 00:00:00 2001 From: andrewkrug Date: Fri, 23 Nov 2018 15:53:54 -0800 Subject: [PATCH 35/89] all AMIs are public by default, mozdef instance recieves rootURL, attacker threshold is configurable --- cloudy_mozdef/cloudformation/mozdef-instance.yml | 14 ++++++++++---- cloudy_mozdef/cloudformation/mozdef-parent.yml | 7 ++++++- cloudy_mozdef/packer/packer.json | 7 +++++-- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/cloudy_mozdef/cloudformation/mozdef-instance.yml b/cloudy_mozdef/cloudformation/mozdef-instance.yml index 64ad55cc..1532699e 100644 --- a/cloudy_mozdef/cloudformation/mozdef-instance.yml +++ b/cloudy_mozdef/cloudformation/mozdef-instance.yml @@ -71,6 +71,9 @@ Parameters: MozDefSQSQueueName: Type: String Description: The name of the generic SQS queue used to pickup events. + DomainName: + Type: String + Description: The fully qualified domain you'll be hosting MozDef at. Resources: MozDefElasticLoadBalancingV2TargetGroup: Type: AWS::ElasticLoadBalancingV2::TargetGroup @@ -119,7 +122,8 @@ Resources: - content: | OPTIONS_ESSERVERS=${ESURL} OPTIONS_KIBANAURL=${KibanaURL} - OPTIONS_METEOR_KIBANAURL=https://relative:9090/_plugin/kibana/ + OPTIONS_METEOR_KIBANAURL=https://relative:9090/_plugin/kibana/ # not a typo needed for meteor + OPTIONS_METEOR_ROOTURL=https://${DomainName} # See https://github.com/mozilla-iam/mozilla.oidc.accessproxy/blob/master/README.md#setup client_id=${OIDCClientId} client_secret=${OIDCClientSecret} @@ -136,8 +140,10 @@ Resources: cookiename=sesmeteor # Increase the AWS ES total fields limit from 1000 to 4000 OPTIONS_MAPPING_TOTAL_FIELDS_LIMIT=4000 + # Set thresholds for attack dataviz lower means more ogres + OPTIONS_IPV4ATTACKERHITCOUNT=5 + OPTIONS_IPV4ATTACKERPREFIXLENGTH=24 path: /opt/mozdef/docker/compose/cloudy_mozdef.env - write_files: - content: | #!/usr/bin/env python # This Source Code Form is subject to the terms of the Mozilla Public @@ -152,7 +158,7 @@ Resources: ALERTS = { 'bruteforce_ssh.AlertBruteforceSsh': {'schedule': crontab(minute='*/1')}, 'unauth_ssh.AlertUnauthSSH': {'schedule': crontab(minute='*/1')}, - 'guard_duty_probe.AlertGuardDutyProbe': {'schedule': crontab(minute='*/1')}', + 'guard_duty_probe.AlertGuardDutyProbe': {'schedule': crontab(minute='*/1')}, 'cloudtrail_logging_disabled.AlertCloudtrailLoggingDisabled': {'schedule': timedelta(minutes=1)}, 'cloudtrail_deadman.AlertCloudtrailDeadman': {'schedule': timedelta(hours=1)} } @@ -171,7 +177,7 @@ Resources: } ES = { - 'servers': [${ESURL}] + 'servers': ["${ESURL}"] } OPTIONS = { diff --git a/cloudy_mozdef/cloudformation/mozdef-parent.yml b/cloudy_mozdef/cloudformation/mozdef-parent.yml index 102894f1..9ed7407b 100644 --- a/cloudy_mozdef/cloudformation/mozdef-parent.yml +++ b/cloudy_mozdef/cloudformation/mozdef-parent.yml @@ -15,6 +15,10 @@ Metadata: - KeyName - AMIImageId Parameters: + DomainName: + Type: String + Description: The fully qualified DNS name you will host CloudyMozDef at. + Default: cloudymozdef.security.allizom.org S3TemplateLocation: Type: String Description: The URL to the S3 bucket used to fetch the nested stack templates @@ -38,7 +42,7 @@ Parameters: AMIImageId: Type: String Description: The AMI Image ID to use of the EC2 instance - Default: ami-0c3705bb3b43ad51f + Default: ami-073434079b0366251 ACMCertArn: Description: The ARN of your pre-issued ACM cert. Default : arn:aws:acm:us-west-2:656532927350:certificate/79f641f2-4046-4754-a28f-4db80d7c0583 @@ -99,6 +103,7 @@ Resources: OIDCDiscoveryURL: !Ref OIDCDiscoveryURL CloudTrailSQSNotificationQueueName: !GetAtt MozDefCloudTrail.Outputs.CloudTrailSQSQueueName MozDefSQSQueueName: !GetAtt MozDefSQS.Outputs.SQSQueueName + DomainName: !Ref DomainName Tags: - Key: application Value: mozdef diff --git a/cloudy_mozdef/packer/packer.json b/cloudy_mozdef/packer/packer.json index 5b366ca0..71017ea9 100644 --- a/cloudy_mozdef/packer/packer.json +++ b/cloudy_mozdef/packer/packer.json @@ -14,7 +14,10 @@ "instance_type": "t2.large", "ssh_pty" : "true", "ssh_username": "ec2-user", - "ami_name": "mozdef_{{timestamp}}" + "ami_name": "mozdef_{{timestamp}}", + "ami_groups": [ + "all" + ] }], "provisioners": [ { "type": "shell", @@ -30,7 +33,7 @@ "sudo systemctl enable docker", "sudo mkdir -p /opt/mozdef/", "sudo git clone https://github.com/mozilla/MozDef /opt/mozdef", - "cd /opt/mozdef && sudo git checkout guardduty" + "cd /opt/mozdef && sudo git checkout reinvent" ]} ] } From e5c252e2b42bebe4e7d9cf9b54b86b748b3eeb47 Mon Sep 17 00:00:00 2001 From: andrewkrug Date: Fri, 23 Nov 2018 16:07:55 -0800 Subject: [PATCH 36/89] fix pep8 error --- cron/collectAttackers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cron/collectAttackers.py b/cron/collectAttackers.py index 45430271..6769ef7e 100755 --- a/cron/collectAttackers.py +++ b/cron/collectAttackers.py @@ -487,7 +487,6 @@ def initConfig(): options.ipv4attackerhitcount = getConfig('ipv4ipv4attackerhitcount', 5, options.configfile) - if __name__ == '__main__': parser = OptionParser() parser.add_option( From 1780aef700f0a16b72546d9bdba0be2309dd7dec Mon Sep 17 00:00:00 2001 From: andrewkrug Date: Fri, 23 Nov 2018 17:52:54 -0800 Subject: [PATCH 37/89] MVP on doc changes to support reinvent --- README.md | 4 + docs/source/cloud_deployment.rst | 75 +++++++++++++++++- .../images/cloudformation-launch-stack.png | Bin 0 -> 3880 bytes 3 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 docs/source/images/cloudformation-launch-stack.png diff --git a/README.md b/README.md index 9f533c80..6f418868 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,10 @@ The Mozilla Defense Platform (MozDef) seeks to automate the security incident ha MozDef is in production at Mozilla where we are using it to process over 300 million events per day. +## Give MozDef a Try in AWS: + +[![Launch MozDef](docs/source/images/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/new?stackName=mozdef-for-aws&templateURL=https://s3-us-west-2.amazonaws.com/mozdef.infosec.allizom.org/cf/mozdef-parent.yml) + ## Documentation: http://mozdef.readthedocs.org/en/latest/ diff --git a/docs/source/cloud_deployment.rst b/docs/source/cloud_deployment.rst index c583633b..0b26f44b 100644 --- a/docs/source/cloud_deployment.rst +++ b/docs/source/cloud_deployment.rst @@ -1,11 +1,82 @@ -Benchmarking -============ +MozDef for AWS +=============== + +**What is MozDef for AWS** Cloud based MozDef is an opinionated deployment of the MozDef services created in 2018 to help AWS users ingest cloudtrail, guardduty, and provide security services. +.. image:: images/cloudformation-launch-stack.png + :target: https://console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/new?stackName=mozdef-for-aws&templateURL=https://s3-us-west-2.amazonaws.com/mozdef.infosec.allizom.org/cf/mozdef-parent.yml + + +Feedback +----------- + +MozDef for AWS is new an we'd love your feedback. Try filing github issues here in the repository or connect with us +in the Mozilla Discourse Security Category. + +https://discourse.mozilla.org/c/security + +You can also take a short survey on MozDef for AWS after you have deployed it. +https://goo.gl/forms/JYjTYDK45d3JdnGd2 + + +Dependencies +-------------- + +MozDef requires the following: + +- A DNS name ( cloudymozdef.security.allizom.org ) +- An OIDC Provider with ClientID, ClientSecret, and Discovery URL + - Mozilla Uses Auth0 but you can use any OIDC provider you like: Shibboleth, KeyCloak, AWS Cognito, Okta, Ping (etc) +- An ACM Certificate in the deployment region for your DNS name +- A VPC with three public subnets available. + - It is advised that this VPC be dedicated to MozDef or used solely for security automation. +- An SQS queue recieving GuardDuty events. At the time of writing this is not required but may be required in future. + + +Supported Regions +------------------ + +MozDef for AWS is currently only supported in us-west-2 but will onboard additional regions over time. + Architecture ------------- .. image:: images/MozDefCloudArchitecture.png + + +Deployment Process +------------------- + +1. Launch the one click stack and provide the requisite values. +2. Wait for the stack to complete. You'll see several nested stacks in the Cloudformation console. *Note: This may take a while* +3. Navigate to the URL you set up for MozDef. It should redirect you to the single sign on provider. If successful you'll see the MozDef UI. +4. Try navigating to ElasticSearch https://your_base_url:9090 +You should see the following: +:: + + { + "name" : "SMf4400", + "cluster_name" : "656532927350:mozdef-mozdef-yemjpbnpw8xb", + "cluster_uuid" : "_yBEIsFkQH-nEZfrFgj7mg", + "version" : { + "number" : "5.6.8", + "build_hash" : "688ecce", + "build_date" : "2018-09-11T14:44:40.463Z", + "build_snapshot" : false, + "lucene_version" : "6.6.1" + }, + "tagline" : "You Know, for Search" + } + +5. Test out Kibana at https://your_base_url:9090/_plugin/kibana/app/kibana#/discover?_g=() + + +Using MozDef +------------- + +Refer back to our other docs on how to use MozDef for general guidance. Cloud specific instructions will evolve here. +If you saw something about MozDef for AWS at re: Invent 2018 and you want to contribute we'd love your PRs. diff --git a/docs/source/images/cloudformation-launch-stack.png b/docs/source/images/cloudformation-launch-stack.png new file mode 100644 index 0000000000000000000000000000000000000000..7c0b53b6f2efc6a9a69fb4cb570314fe48f54abb GIT binary patch literal 3880 zcmV+@57+RCP)KLZ*U+0v@6o#Kk zBy~|skq9maHzpVb!A%szeAFnH;fC8TlHlg{&NR8$`@v*xZFhqH4E+Z#x^m|~@B=Bh z_D6`z#6^szN*C?3dS~Xmobz%9%J006+AeKEk~$mo+Hx=)$+gd{Be=?au7%MwSKE(w z06f3j3Vfybi-4cs5CA8)JipuGe5^;gbs(z16K$puh-bj77cO@oz5`Xe6LdiQ05--e zdUr+J!EgknOF(tJ(ARjGpDr!D|DP|)O233h#a-5mBsUWTarVny}j~*zH<~VzXID zv&nq9D9F4iVJ2i+QO8+0-gc4ITe62d&J0&v0nK|EqF4zf50oMTLPQvwfl*8+TSHX+ zkDC|X>Z7MmMv`QL!3dH5Y&#m&0%}E3@B{^FPB_^0p_Pw2!U(-yW|b83%DM~KZD!U! z`WBr%J^+mAMN2BI787gk5^zKVSWP0R{Y6N~x=f>c5bHiX$M))Git;kaOtXKkv7Ybj zb~fr-}1jeE^hB&82i&F28?r)&CpT_I5Xw+iIv-x0r=+0SFG^i9{qaGgAnK;#2!07i?tN!>qW?jnHf0TV**sy(dc5OZ6;Y zJYyK2Vn_kA;JO-?e}?h(@ILG7kow;kz)Y!rx?>i$6Wm3opu01p)euB$bv(^-`H5i!i5$3 zwasQG92T+J2A)3qPguCMz)Z2zO}^d6!{3y7zS_V)n%Xr3lB7ZDl{FJA7s2h2X>mBgIY; zB3|ata6+|*=966*2ErQ=HOmZhlCdH9h z+D67@6j|^*68&4zzHG;<%V8x?HlIplt zX*i|;iFkGt{584DhtZ}Am27RoxM>8NvRxq2b z7)N{`)HQop zxZyZOxmGG3EX3`Txqow;KGAN!#KH~7*;Vi2zGY6GXE6Qv`E!&`b?7(-%O_I#+zc(o z`#}poe*PT0>RrR~4(W4OM8|pGGN+DxaR07{RgW?I`HFI_lumVwc-&jg%DU0S8&a51 zDBiFd8>?`;19&_^-0lFOP*g7`FAU6!-E2UT6*_vn12ItV@vWpduk*TMBt|q0PkCty z(h>`({T)@60Ue2Ts`jY;b`*&p7d^XRzx<^NJu!t$?<({&G)Po%$86?4yHXQ*rTv1LaWTXuA53*mu$ ze)&KiwJ%Jg^3_f%UhUx79~KViP(?Pbp1{V{6LejfsRCQxyi8q_YXn#a`hNfBRt}#D zaNn{C{QmJtpJ*{(>H&!H+md*rcxrBbnkiG(W3ia@y&-@=z=Uc~LkNX%nl*h_T!8@t z5~cXf!f^D)ww)8@L`lVL67cwC{%d-Kg`1qY59L)-sHTmS+#a5|dA{8itjR;njmmi@>|TnMQkel%s7ysvF$A zzPFc(S36iR+reF>X}aE+4n33Vj`Gv37pXp|#WS%!$M~9B2eA5ra$LKyzCZhBHy*#t zx9`m4#h*-y)1Qbn8RcgybX+kE@oh!)(HDGh?xs`t``jf)|safF98d zh-7Cu24Wz9h^S58jT%qVWTL-=zunoxZ+G@^Ye@=EuFd1F(zF}WGnhh=p*RKMv%#C# z&ZHa*-@Y@Gmwr~nhLuhL4jl1cQ`h%@eV*!rZoap4JWsC8V^WUw6SWI)Yl*gXQh(To z$1jgaW-!f~><1!)3d20oKoGt~D5S+lUA-P$E?O>(p@BZNjj)Vv~O7EvY>Bqnora z!6R|BEjS{qvGG_iJC)Uove@uUJ73>;n!o;fDj5#rh!@0{Z?>?nF-~*x1W8z&;FWAz z+8+WhFnf}hXxNRA-b-h1E&y|;Ib#MSBT6c<;~~QtN~}}YqPu@~W=P%6%_Zb#YtPf~ zR-UEMX<Ti-EQxjZ-#vjh z-s|OwZ5NST#K(>r7mvPp zi7ovGITE^Kj)|Nsk#1Maa8i;O&mBXIXRvpgI6j31v;0W1ir_s(eUkyOGIvI<_9{bF zBv#EnhQn?q*_pA3s3I#M%b=>s%c0|beUKArRe2T*W~I{98l;+D95 zRw^@#?5L`?+OE1Kb{}xlbUH{$fsL)3iuC&zE=TbOWtPm#U_ypDXa^>%{pGhg$sXecwYvi}uJ`IzNNoZ7S0Fi}P)K+!5hWi=T!T zKXa!#m^;mvIalWh?sdp(t*|66)HR6O8iQjTU=Pwkz- zpIhee_(O|%c+I>4GmARf)+{RbeHDQ1Pp#0; zNSo%($)~!emt&pTsm#ox$By#dRv|TdO?TV0-Wu>Gb z%$zByd~I%sTZ&^l`wF$K6M3e-41?K7)uv@}XQNF^?kM5y+T$GV%A+`|2U&@do?^mg z6^3jGh9hGJG+Jq}V<^auuy(2T-)M{^r1#o(%Vn3!$DWDQpUUFBvokT69aL>vHsnmS z{#Aw|g_)jyg0jLh-0XCbVlxZ@5e$zViXJUwG+sa9v6|y~JYfr00x4YV%jB#pi{_rG z2u2$Qla1|9tw{b=#^CS56e)}PD<0000 Date: Mon, 26 Nov 2018 07:03:51 -0800 Subject: [PATCH 38/89] Update docs/source/cloud_deployment.rst Co-Authored-By: andrewkrug --- docs/source/cloud_deployment.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/cloud_deployment.rst b/docs/source/cloud_deployment.rst index 0b26f44b..799792ef 100644 --- a/docs/source/cloud_deployment.rst +++ b/docs/source/cloud_deployment.rst @@ -13,7 +13,7 @@ ingest cloudtrail, guardduty, and provide security services. Feedback ----------- -MozDef for AWS is new an we'd love your feedback. Try filing github issues here in the repository or connect with us +MozDef for AWS is new and we'd love your feedback. Try filing GitHub issues here in the repository or connect with us in the Mozilla Discourse Security Category. https://discourse.mozilla.org/c/security From 7e86f8cff2656c7f53e08496b5790897dcb5a7fd Mon Sep 17 00:00:00 2001 From: Gene Wood Date: Mon, 26 Nov 2018 10:03:29 -0800 Subject: [PATCH 39/89] Start passing VpcId to security group substack --- cloudy_mozdef/cloudformation/mozdef-parent.yml | 2 ++ cloudy_mozdef/cloudformation/mozdef-security-group.yml | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cloudy_mozdef/cloudformation/mozdef-parent.yml b/cloudy_mozdef/cloudformation/mozdef-parent.yml index 102894f1..235de60d 100644 --- a/cloudy_mozdef/cloudformation/mozdef-parent.yml +++ b/cloudy_mozdef/cloudformation/mozdef-parent.yml @@ -60,6 +60,8 @@ Resources: MozDefSecurityGroups: Type: AWS::CloudFormation::Stack Properties: + Parameters: + VpcId: !Ref VpcId Tags: - Key: application Value: mozdef diff --git a/cloudy_mozdef/cloudformation/mozdef-security-group.yml b/cloudy_mozdef/cloudformation/mozdef-security-group.yml index 915d9f90..72d8a4b7 100644 --- a/cloudy_mozdef/cloudformation/mozdef-security-group.yml +++ b/cloudy_mozdef/cloudformation/mozdef-security-group.yml @@ -4,7 +4,6 @@ Parameters: VpcId: Type: AWS::EC2::VPC::Id Description: The VPC ID of the VPC to deploy in - Default: vpc-dc8eacb4 Resources: MozDefSecurityGroup: Type: AWS::EC2::SecurityGroup From 338fe7ca6c573c281002d627a3d51896b6b29f6b Mon Sep 17 00:00:00 2001 From: Gene Wood Date: Mon, 26 Nov 2018 10:05:27 -0800 Subject: [PATCH 40/89] Remove CloudFormation template defaults Make CloudFormation template parameter syntax consistent Clean up AWS::CloudFormation::Interface Add parameter AllowedPattern settings --- cloudy_mozdef/cloudformation/mozdef-efs.yml | 58 +++++++-------- cloudy_mozdef/cloudformation/mozdef-es.yml | 5 -- .../cloudformation/mozdef-instance.yml | 11 +-- cloudy_mozdef/cloudformation/mozdef-mq.yml | 11 +-- .../cloudformation/mozdef-parent.yml | 73 +++++++++++++------ 5 files changed, 85 insertions(+), 73 deletions(-) diff --git a/cloudy_mozdef/cloudformation/mozdef-efs.yml b/cloudy_mozdef/cloudformation/mozdef-efs.yml index 642f2b30..6cd214a0 100644 --- a/cloudy_mozdef/cloudformation/mozdef-efs.yml +++ b/cloudy_mozdef/cloudformation/mozdef-efs.yml @@ -1,23 +1,19 @@ AWSTemplateFormatVersion: 2010-09-09 Description: Amazon Elastic File System Parameters: - VPCID0: + VpcId: Type: AWS::EC2::VPC::Id Description: VpcId of your existing Virtual Private Cloud (VPC) ConstraintDescription: Must be the VPC Id of an existing Virtual Private Cloud. - Default: vpc-dc8eacb4 - Subnet: - Description: Select existing subnets. + SubnetList: Type: List - Default: subnet-8931f7ee,subnet-de322aa8,subnet-2582cd7d + Description: Select existing subnets. NumberOfSubnets: - Description: Number of subnets in the Subnet parameter Type: String - Default: 3 + Description: Number of subnets in the Subnet parameter MozDefSecurityGroup: - Description: The security group of the Mozdef endpoint that is accessing EFS Type: AWS::EC2::SecurityGroup::Id - Default: sg-02d7fe9627ea068e2 + Description: The security group of the Mozdef endpoint that is accessing EFS Conditions: Has1Subnets: !Or [!Equals [!Ref NumberOfSubnets, 1], Condition: Has2Subnets] Has2Subnets: !Or [!Equals [!Ref NumberOfSubnets, 2], Condition: Has3Subnets] @@ -57,7 +53,7 @@ Resources: Value: mozdef - Key: stack Value: !Ref AWS::StackName - VpcId: !Ref VPCID0 + VpcId: !Ref VpcId ElasticFileSystem: Type: AWS::EFS::FileSystem MountTarget1: @@ -65,142 +61,142 @@ Resources: Condition: Has1Subnets Properties: FileSystemId: !Ref ElasticFileSystem - SubnetId: !Select [ 0, !Ref Subnet ] + SubnetId: !Select [ 0, !Ref SubnetList ] SecurityGroups: [ !Ref MountTargetSecurityGroup ] MountTarget2: Type: AWS::EFS::MountTarget Condition: Has2Subnets Properties: FileSystemId: !Ref ElasticFileSystem - SubnetId: !Select [ 1, !Ref Subnet ] + SubnetId: !Select [ 1, !Ref SubnetList ] SecurityGroups: [ !Ref MountTargetSecurityGroup ] MountTarget3: Type: AWS::EFS::MountTarget Condition: Has3Subnets Properties: FileSystemId: !Ref ElasticFileSystem - SubnetId: !Select [ 2, !Ref Subnet ] + SubnetId: !Select [ 2, !Ref SubnetList ] SecurityGroups: [ !Ref MountTargetSecurityGroup ] MountTarget4: Type: AWS::EFS::MountTarget Condition: Has4Subnets Properties: FileSystemId: !Ref ElasticFileSystem - SubnetId: !Select [ 3, !Ref Subnet ] + SubnetId: !Select [ 3, !Ref SubnetList ] SecurityGroups: [ !Ref MountTargetSecurityGroup ] MountTarget5: Type: AWS::EFS::MountTarget Condition: Has5Subnets Properties: FileSystemId: !Ref ElasticFileSystem - SubnetId: !Select [ 4, !Ref Subnet ] + SubnetId: !Select [ 4, !Ref SubnetList ] SecurityGroups: [ !Ref MountTargetSecurityGroup ] MountTarget6: Type: AWS::EFS::MountTarget Condition: Has6Subnets Properties: FileSystemId: !Ref ElasticFileSystem - SubnetId: !Select [ 5, !Ref Subnet ] + SubnetId: !Select [ 5, !Ref SubnetList ] SecurityGroups: [ !Ref MountTargetSecurityGroup ] MountTarget7: Type: AWS::EFS::MountTarget Condition: Has7Subnets Properties: FileSystemId: !Ref ElasticFileSystem - SubnetId: !Select [ 6, !Ref Subnet ] + SubnetId: !Select [ 6, !Ref SubnetList ] SecurityGroups: [ !Ref MountTargetSecurityGroup ] MountTarget8: Type: AWS::EFS::MountTarget Condition: Has8Subnets Properties: FileSystemId: !Ref ElasticFileSystem - SubnetId: !Select [ 7, !Ref Subnet ] + SubnetId: !Select [ 7, !Ref SubnetList ] SecurityGroups: [ !Ref MountTargetSecurityGroup ] MountTarget9: Type: AWS::EFS::MountTarget Condition: Has9Subnets Properties: FileSystemId: !Ref ElasticFileSystem - SubnetId: !Select [ 8, !Ref Subnet ] + SubnetId: !Select [ 8, !Ref SubnetList ] SecurityGroups: [ !Ref MountTargetSecurityGroup ] MountTarget10: Type: AWS::EFS::MountTarget Condition: Has10Subnets Properties: FileSystemId: !Ref ElasticFileSystem - SubnetId: !Select [ 9, !Ref Subnet ] + SubnetId: !Select [ 9, !Ref SubnetList ] SecurityGroups: [ !Ref MountTargetSecurityGroup ] MountTarget11: Type: AWS::EFS::MountTarget Condition: Has11Subnets Properties: FileSystemId: !Ref ElasticFileSystem - SubnetId: !Select [ 10, !Ref Subnet ] + SubnetId: !Select [ 10, !Ref SubnetList ] SecurityGroups: [ !Ref MountTargetSecurityGroup ] MountTarget12: Type: AWS::EFS::MountTarget Condition: Has12Subnets Properties: FileSystemId: !Ref ElasticFileSystem - SubnetId: !Select [ 11, !Ref Subnet ] + SubnetId: !Select [ 11, !Ref SubnetList ] SecurityGroups: [ !Ref MountTargetSecurityGroup ] MountTarget13: Type: AWS::EFS::MountTarget Condition: Has13Subnets Properties: FileSystemId: !Ref ElasticFileSystem - SubnetId: !Select [ 12, !Ref Subnet ] + SubnetId: !Select [ 12, !Ref SubnetList ] SecurityGroups: [ !Ref MountTargetSecurityGroup ] MountTarget14: Type: AWS::EFS::MountTarget Condition: Has14Subnets Properties: FileSystemId: !Ref ElasticFileSystem - SubnetId: !Select [ 13, !Ref Subnet ] + SubnetId: !Select [ 13, !Ref SubnetList ] SecurityGroups: [ !Ref MountTargetSecurityGroup ] MountTarget15: Type: AWS::EFS::MountTarget Condition: Has15Subnets Properties: FileSystemId: !Ref ElasticFileSystem - SubnetId: !Select [ 14, !Ref Subnet ] + SubnetId: !Select [ 14, !Ref SubnetList ] SecurityGroups: [ !Ref MountTargetSecurityGroup ] MountTarget16: Type: AWS::EFS::MountTarget Condition: Has16Subnets Properties: FileSystemId: !Ref ElasticFileSystem - SubnetId: !Select [ 15, !Ref Subnet ] + SubnetId: !Select [ 15, !Ref SubnetList ] SecurityGroups: [ !Ref MountTargetSecurityGroup ] MountTarget17: Type: AWS::EFS::MountTarget Condition: Has17Subnets Properties: FileSystemId: !Ref ElasticFileSystem - SubnetId: !Select [ 16, !Ref Subnet ] + SubnetId: !Select [ 16, !Ref SubnetList ] SecurityGroups: [ !Ref MountTargetSecurityGroup ] MountTarget18: Type: AWS::EFS::MountTarget Condition: Has18Subnets Properties: FileSystemId: !Ref ElasticFileSystem - SubnetId: !Select [ 17, !Ref Subnet ] + SubnetId: !Select [ 17, !Ref SubnetList ] SecurityGroups: [ !Ref MountTargetSecurityGroup ] MountTarget19: Type: AWS::EFS::MountTarget Condition: Has19Subnets Properties: FileSystemId: !Ref ElasticFileSystem - SubnetId: !Select [ 18, !Ref Subnet ] + SubnetId: !Select [ 18, !Ref SubnetList ] SecurityGroups: [ !Ref MountTargetSecurityGroup ] MountTarget20: Type: AWS::EFS::MountTarget Condition: Has20Subnets Properties: FileSystemId: !Ref ElasticFileSystem - SubnetId: !Select [ 19, !Ref Subnet ] + SubnetId: !Select [ 19, !Ref SubnetList ] SecurityGroups: [ !Ref MountTargetSecurityGroup ] Outputs: EFSID: Description: Logical ID of the EFS Filesystem - Value: !Ref ElasticFileSystem \ No newline at end of file + Value: !Ref ElasticFileSystem diff --git a/cloudy_mozdef/cloudformation/mozdef-es.yml b/cloudy_mozdef/cloudformation/mozdef-es.yml index 2f91c6ce..5d2d3e6b 100644 --- a/cloudy_mozdef/cloudformation/mozdef-es.yml +++ b/cloudy_mozdef/cloudformation/mozdef-es.yml @@ -4,22 +4,17 @@ Parameters: SubnetIds: Type: List Description: Comma-delimited list of subnet IDs within which the ElasticSearch instance will be provisioned. - Default: subnet-dd8eacb5,subnet-df8eacb7,subnet-de8eacb6 BlockStoreSizeGB: Type: Number Description: The size of the Elastic Block Store to have back ElasticSearch, in GigaBytes. - Default: 100 VpcId: Type: AWS::EC2::VPC::Id Description: The VPC ID of the VPC to deploy in - Default: vpc-dc8eacb4 MozDefInstanceSecurityGroup: Type: AWS::EC2::SecurityGroup::Id Description: The MozDef EC2 Instance security group that accesses ES - Default: sg-8f38dae0 ESInstanceCount: Type: Number - Default: 1 Description: The number of ElasticSearch nodes in the cluster Resources: ## Not currently supported by CloudFormation. diff --git a/cloudy_mozdef/cloudformation/mozdef-instance.yml b/cloudy_mozdef/cloudformation/mozdef-instance.yml index 126e3be6..d195837e 100644 --- a/cloudy_mozdef/cloudformation/mozdef-instance.yml +++ b/cloudy_mozdef/cloudformation/mozdef-instance.yml @@ -4,23 +4,19 @@ Parameters: VpcId: Type: AWS::EC2::VPC::Id Description: The VPC ID of the VPC to deploy in - Default: vpc-dc8eacb4 InstanceType: Type: String - Default: m5.large Description: EC2 instance type, e.g. m1.small, m1.large, etc. + Default: m5.large KeyName: Type: AWS::EC2::KeyPair::KeyName Description: Name of an existing EC2 KeyPair to enable SSH access to the web server - Default: infosec-pdx-workweek-2018 IamInstanceProfile: Type: String Description: The ARN of the IAM Instance Profile - Default: arn:aws:iam::656532927350:instance-profile/netsecdevbastion-BastionInstanceProfile-12CM3TOELV20R AutoScaleGroupSubnetIds: Type: List Description: A comma delimited list of subnet IDs - Default: subnet-dd8eacb5,subnet-df8eacb7,subnet-de8eacb6 AMIImageId: Type: AWS::EC2::Image::Id Description: The AMI Image ID to use of the EC2 instance @@ -40,7 +36,6 @@ Parameters: MozDefACMCertArn: Type: String Description: The arn of your pre-issued certificate for ssl termination. - Default: arn:aws:acm:us-west-2:656532927350:certificate/79f641f2-4046-4754-a28f-4db80d7c0583 ESURL: Type: String Description: The AWS ES endpoint URL @@ -54,16 +49,14 @@ Parameters: Description: The AWS ES Kibana URL with domain only Default: https://kibana.example.com/ OIDCClientId: - Description: The client ID that your OIDC provider issues you for your Mozdef instance. Type: String - Default: lGsSlYNdiV6f5tF05pWN3EbQoDPHx44k + Description: The client ID that your OIDC provider issues you for your Mozdef instance. OIDCClientSecret: Type: String Description: The secret that your OIDC provider issues you for your Mozdef instance. NoEcho: true OIDCDiscoveryURL: Type: String - Default: https://auth.mozilla.auth0.com/.well-known/openid-configuration Description: The URL of your OIDC provider's well-known discovery URL CloudTrailSQSNotificationQueueName: Type: String diff --git a/cloudy_mozdef/cloudformation/mozdef-mq.yml b/cloudy_mozdef/cloudformation/mozdef-mq.yml index ec19aadf..94b30387 100644 --- a/cloudy_mozdef/cloudformation/mozdef-mq.yml +++ b/cloudy_mozdef/cloudformation/mozdef-mq.yml @@ -3,28 +3,25 @@ Description: Creates MozDef Amazon MQ Parameters: MQUserParameter: Type: String - Default: mozdef Description: The username for the AmazonMQ user. + Default: mozdef MQPasswordParameter: Type: String - NoEcho: true - Default: '' Description: The password for the AmazonMQ user. Leave this blank if you want an auto-generated password. + Default: '' + NoEcho: true MQInstanceType: Type: String - Default: mq.t2.micro Description: The instance type for the AmazonMQ instance. + Default: mq.t2.micro MQVpcId: Type: AWS::EC2::VPC::Id - Default: vpc-dc8eacb4 Description: The VPC ID of the VPC within which the AmazonMQ instance will be provisioned. MQSubnetIds: Type: List - Default: subnet-dd8eacb5,subnet-df8eacb7,subnet-de8eacb6 Description: Comma-delimited list of subnet IDs of the VPCs within which the AmazonMQ instance will be provisioned. MozDefSecurityGroup: Type: AWS::EC2::SecurityGroup::Id - Default: sg-02d7fe9627ea068e2 Description: The security group of the Mozdef endpoint that is accessing AmazonMQ Conditions: PasswordIsSet: !Not [ !Equals [ !Ref MQPasswordParameter, '' ]] diff --git a/cloudy_mozdef/cloudformation/mozdef-parent.yml b/cloudy_mozdef/cloudformation/mozdef-parent.yml index 235de60d..2068c7e7 100644 --- a/cloudy_mozdef/cloudformation/mozdef-parent.yml +++ b/cloudy_mozdef/cloudformation/mozdef-parent.yml @@ -14,48 +14,79 @@ Metadata: - InstanceType - KeyName - AMIImageId + - Label: + default: Certificate + Parameters: + - ACMCertArn + - Label: + default: OIDC Configuration + Parameters: + - OIDCDiscoveryURL + - OIDCClientId + - OIDCClientSecret + - Label: + default: Template Location + Parameters: + - S3TemplateLocation + ParameterLabels: + VpcId: + default: VPC ID + PublicSubnetIds: + default: Public Subnet IDs + InstanceType: + default: EC2 Instance Type + KeyName: + default: EC2 SSH Key Name + AMIImageId: + default: EC2 AMI Image ID + ACMCertArn: + default: ACM Certificate ARN + OIDCDiscoveryURL: + default: OIDC Discovery URL + OIDCClientId: + default: OIDC Client ID + OIDCClientSecret: + default: OIDC Client Secret + S3TemplateLocation: + default: S3 Template Location URL Parameters: - S3TemplateLocation: - Type: String - Description: The URL to the S3 bucket used to fetch the nested stack templates - Default: https://s3-us-west-2.amazonaws.com/mozdef.infosec.allizom.org/cf/ VpcId: Type: AWS::EC2::VPC::Id - Description: The VPC ID of the VPC to deploy in - Default: vpc-dc8eacb4 + Description: 'The VPC ID of the VPC to deploy in (Example : vpc-abcdef12)' PublicSubnetIds: Type: List - Description: A comma delimited list of public subnet IDs - Default: subnet-dd8eacb5,subnet-df8eacb7,subnet-de8eacb6 + Description: 'A comma delimited list of public subnet IDs (Example: subnet-abcdef12,subnet-bcdef123)' InstanceType: Type: String - Default: m5.large Description: EC2 instance type, e.g. m1.small, m1.large, etc. + Default: m5.large KeyName: Type: AWS::EC2::KeyPair::KeyName Description: Name of an existing EC2 KeyPair to enable SSH access to the web server - Default: infosec-pdx-workweek-2018 AMIImageId: - Type: String + Type: AWS::EC2::Image::Id Description: The AMI Image ID to use of the EC2 instance Default: ami-0c3705bb3b43ad51f ACMCertArn: - Description: The ARN of your pre-issued ACM cert. - Default : arn:aws:acm:us-west-2:656532927350:certificate/79f641f2-4046-4754-a28f-4db80d7c0583 Type: String - MinLength: '1' + Description: "The ARN of your pre-issued ACM cert. (Example: arn:aws:acm:us-west-2:123456789012:certificate/abcdef01-2345-6789-abcd-ef0123456789)" + OIDCDiscoveryURL: + Type: String + AllowedPattern: '^https?:\/\/.*' + ConstraintDescription: A valid URL + Description: "The URL of your OIDC provider's well-known discovery URL (Example: https://auth.example.com/.well-known/openid-configuration)" OIDCClientId: - Description: The client ID that your OIDC provider issues you for your Mozdef instance. Type: String - Default: lGsSlYNdiV6f5tF05pWN3EbQoDPHx44k + Description: The client ID that your OIDC provider issues you for your Mozdef instance. OIDCClientSecret: Type: String Description: The secret that your OIDC provider issues you for your Mozdef instance. NoEcho: true - OIDCDiscoveryURL: + S3TemplateLocation: Type: String - Default: https://auth.mozilla.auth0.com/.well-known/openid-configuration - Description: The URL of your OIDC provider's well-known discovery URL + AllowedPattern: '^https?:\/\/.*\.amazonaws\.com\/.*' + ConstraintDescription: A valid amazonaws.com S3 URL + Description: "The URL to the S3 bucket used to fetch the nested stack templates (Example: https://s3-us-west-2.amazonaws.com/example-bucket-name/cloudformation/path/)" Resources: MozDefSecurityGroups: Type: AWS::CloudFormation::Stack @@ -127,8 +158,8 @@ Resources: Type: AWS::CloudFormation::Stack Properties: Parameters: - VPCID0: !Ref VpcId - Subnet: !Join [ ',', !Ref PublicSubnetIds ] + VpcId: !Ref VpcId + SubnetList: !Join [ ',', !Ref PublicSubnetIds ] NumberOfSubnets: !GetAtt NumberOfSubnets.Length MozDefSecurityGroup: !GetAtt MozDefSecurityGroups.Outputs.MozDefSecurityGroupId Tags: From 5696674d9e51269ce96e8538f31bf8a9fe847eb1 Mon Sep 17 00:00:00 2001 From: Gene Wood Date: Mon, 26 Nov 2018 10:09:13 -0800 Subject: [PATCH 41/89] Change how awscli create-stack is called to unify parameters This allows for passing parameters both in a JSON file as well as with environment variables (for OIDC_CLIENT_SECRET) and from the Makefile (for S3_STACK_URI) Also add a fully fleshed out example parameter json file --- cloudy_mozdef/Makefile | 8 ++-- cloudy_mozdef/aws_parameters.example.json | 49 ++++++++++++++++++++++- 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/cloudy_mozdef/Makefile b/cloudy_mozdef/Makefile index d308fea5..f1671b3c 100644 --- a/cloudy_mozdef/Makefile +++ b/cloudy_mozdef/Makefile @@ -2,7 +2,9 @@ ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) PARENTDIR := $(realpath ../) AWS_REGION := us-west-2 STACK_NAME := mozdef-aws-nested -STACK_PARAMS := file://aws_parameters.json +STACK_PARAMS_FILENAME := aws_parameters.json +# For more information on the rationale behind the code in STACK_PARAMS see https://github.com/aws/aws-cli/issues/2429#issuecomment-441133480 +STACK_PARAMS := $(shell test -e $(STACK_PARAMS_FILENAME) && python -c 'import json,sys;f=open(sys.argv[1]);print(" ".join([",".join(["%s=\\\"%s\\\""%(k,v) for k,v in x.items()]) for x in json.load(f)]));f.close()' $(STACK_PARAMS_FILENAME)) # MozDef uses a nested CF stack, the mozdef-parent.yml will tie all child stacks together and load them from S3 # See also mozdef.infosec.mozilla.org bucket S3_BUCKET_NAME := mozdef.infosec.allizom.org @@ -28,9 +30,9 @@ create-stack: test ## Create everything you need for a fresh new stack! @echo "Make sure you have an environment variable OIDC_CLIENT_SECRET set." aws cloudformation create-stack --stack-name $(STACK_NAME) --template-url $(S3_STACK_URI)mozdef-parent.yml \ --capabilities CAPABILITY_IAM \ - --parameters $(STACK_PARAMS) \ --parameters ParameterKey=S3TemplateLocation,ParameterValue=$(S3_STACK_URI) \ - ParameterKey=OIDCClientSecret,ParameterValue=$(OIDC_CLIENT_SECRET) \ + $(OIDC_CLIENT_SECRET_PARAM_ARG) \ + $(STACK_PARAMS) \ --output text .PHONY: create-s3-bucket diff --git a/cloudy_mozdef/aws_parameters.example.json b/cloudy_mozdef/aws_parameters.example.json index 30c78dc2..3d6383ed 100644 --- a/cloudy_mozdef/aws_parameters.example.json +++ b/cloudy_mozdef/aws_parameters.example.json @@ -1,7 +1,52 @@ [ + { + "ParameterKey": "VpcId", + "ParameterValue": "vpc-abcdef12", + "UsePreviousValue": false + }, + { + "ParameterKey": "PublicSubnetIds", + "ParameterValue": "subnet-abcdef12,subnet-bcdef123", + "UsePreviousValue": false + }, + { + "ParameterKey": "InstanceType", + "ParameterValue": "m5.large", + "UsePreviousValue": false + }, + { + "ParameterKey": "KeyName", + "ParameterValue": "my-key-name", + "UsePreviousValue": false + }, + { + "ParameterKey": "AMIImageId", + "ParameterValue": "ami-0c3705bb3b43ad51f", + "UsePreviousValue": false + }, + { + "ParameterKey": "ACMCertArn", + "ParameterValue": "arn:aws:acm:us-west-2:123456789012:certificate/abcdef01-2345-6789-abcd-ef0123456789", + "UsePreviousValue": false + }, + { + "ParameterKey": "OIDCDiscoveryURL", + "ParameterValue": "https://auth.example.com/.well-known/openid-configuration", + "UsePreviousValue": false + }, + { + "ParameterKey": "OIDCClientId", + "ParameterValue": "abcdefghijklmnopqrstuvwxyz012345", + "UsePreviousValue": false + }, { "ParameterKey": "OIDCClientSecret", - "ParameterValue": "secret-goes-here", + "ParameterValue": "secret-value-goes-here", + "UsePreviousValue": false + }, + { + "ParameterKey": "S3TemplateLocation", + "ParameterValue": "https://s3-us-west-2.amazonaws.com/example-bucket-name/cloudformation/path/", "UsePreviousValue": false } -] \ No newline at end of file +] From 37aecb478d55dcdcef2a829c5bb76b944a7322b4 Mon Sep 17 00:00:00 2001 From: Gene Wood Date: Mon, 26 Nov 2018 10:31:39 -0800 Subject: [PATCH 42/89] Fix GuardDuty test indentation --- tests/alerts/test_guard_duty_probe.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/alerts/test_guard_duty_probe.py b/tests/alerts/test_guard_duty_probe.py index fed960d7..2e88a6d5 100644 --- a/tests/alerts/test_guard_duty_probe.py +++ b/tests/alerts/test_guard_duty_probe.py @@ -19,9 +19,11 @@ class TestGuardDutyProbe(AlertTestSuite): "source": "guardduty", "details": { "sourceipaddress": "1.2.3.4", - "finding": {"action": - {"actionType": "PORT_PROBE"} - } + "finding": { + "action": { + "actionType": "PORT_PROBE" + } + } } } } From 8831e8d50310e6f61545c91d801180e6a310c08b Mon Sep 17 00:00:00 2001 From: Gene Wood Date: Mon, 26 Nov 2018 10:32:01 -0800 Subject: [PATCH 43/89] Clarify "relative" reserved word --- cloudy_mozdef/cloudformation/mozdef-instance.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cloudy_mozdef/cloudformation/mozdef-instance.yml b/cloudy_mozdef/cloudformation/mozdef-instance.yml index 1532699e..e30ef4be 100644 --- a/cloudy_mozdef/cloudformation/mozdef-instance.yml +++ b/cloudy_mozdef/cloudformation/mozdef-instance.yml @@ -122,7 +122,9 @@ Resources: - content: | OPTIONS_ESSERVERS=${ESURL} OPTIONS_KIBANAURL=${KibanaURL} - OPTIONS_METEOR_KIBANAURL=https://relative:9090/_plugin/kibana/ # not a typo needed for meteor + # The OPTIONS_METEOR_KIBANAURL uses the reserved word "relative" which triggers MozDef + # to use relative links to Kibana : https://github.com/mozilla/MozDef/pull/956 + OPTIONS_METEOR_KIBANAURL=https://relative:9090/_plugin/kibana/ OPTIONS_METEOR_ROOTURL=https://${DomainName} # See https://github.com/mozilla-iam/mozilla.oidc.accessproxy/blob/master/README.md#setup client_id=${OIDCClientId} From 68ebe726790a7c29ac49c58a6d6cb6cee3c60f1e Mon Sep 17 00:00:00 2001 From: Gene Wood Date: Mon, 26 Nov 2018 10:33:54 -0800 Subject: [PATCH 44/89] Switch packer to pull from master --- cloudy_mozdef/packer/packer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudy_mozdef/packer/packer.json b/cloudy_mozdef/packer/packer.json index 71017ea9..0cbb1139 100644 --- a/cloudy_mozdef/packer/packer.json +++ b/cloudy_mozdef/packer/packer.json @@ -33,7 +33,7 @@ "sudo systemctl enable docker", "sudo mkdir -p /opt/mozdef/", "sudo git clone https://github.com/mozilla/MozDef /opt/mozdef", - "cd /opt/mozdef && sudo git checkout reinvent" + "cd /opt/mozdef && sudo git checkout master" ]} ] } From 6fc355ccb90ac9cddc4a174faa47cdb72c9af91e Mon Sep 17 00:00:00 2001 From: Michal Purzynski Date: Mon, 26 Nov 2018 13:50:14 -0800 Subject: [PATCH 45/89] Change the query window --- alerts/nsm_scan_port.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerts/nsm_scan_port.py b/alerts/nsm_scan_port.py index 06590f6b..43ae3d65 100644 --- a/alerts/nsm_scan_port.py +++ b/alerts/nsm_scan_port.py @@ -15,7 +15,7 @@ class NSMScanPort(AlertTask): self._config = self.parse_json_alert_config('nsm_scan_port.json') def main(self): - search_query = SearchQuery(minutes=1) + search_query = SearchQuery(minutes=2) search_query.add_must([ TermMatch('category', 'bro'), TermMatch('details.source', 'notice'), From cf88cee37422027ea2e1acb8f38482d5cce832f3 Mon Sep 17 00:00:00 2001 From: Gene Wood Date: Mon, 26 Nov 2018 14:45:54 -0800 Subject: [PATCH 46/89] Remove GitHub issue template Instead of trying to improve the issue template, let's remove it for now and if we start to see low quality issues, we can introduce a template designed to address those problems with issue quality Fixes #883 --- .github/ISSUE_TEMPLATE.md | 44 --------------------------------------- 1 file changed, 44 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index f38595f6..00000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,44 +0,0 @@ - - -#### ISSUE TYPE - - - Bug Report - - Feature Idea - - Documentation Report - -#### COMPONENT NAME - - -#### CONFIGURATION - - -#### OS / ENVIRONMENT - - -#### DESCRIPTION - - -#### STEPS TO REPRODUCE - - - - - - - -#### EXPECTED RESULTS - - -#### ACTUAL RESULTS - - From 24652048a1df69d2ca48e660cec9170176508168 Mon Sep 17 00:00:00 2001 From: Jon Moroney Date: Tue, 27 Nov 2018 11:53:55 -0800 Subject: [PATCH 47/89] Add option to do a rebuild when running tests. Very helpful when you don't know if your tests are correct --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index ec3ff072..be6abb5b 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,8 @@ run-test: run-tests: run-tests-resources ## Just run the tests (no build/get). Use `make TEST_CASE=tests/...` for specific tests only docker run -it --rm mozdef/mozdef_tester bash -c "source /opt/mozdef/envs/python/bin/activate && flake8 --config .flake8 ./" docker run -it --rm --network=test-mozdef_default mozdef/mozdef_tester bash -c "source /opt/mozdef/envs/python/bin/activate && py.test --delete_indexes --delete_queues $(TEST_CASE)" +rebuild-run-tests: build-tests run-tests + .PHONY: build build: ## Build local MozDef images (use make NO_CACHE=--no-cache build to disable caching) From 2975208615e54e50f44aedfdafa5354f6d416e5a Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Tue, 27 Nov 2018 14:04:01 -0600 Subject: [PATCH 48/89] Fix incorrect notify mozdefbot field in template --- tests/alerts/test_alert_template.template | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/alerts/test_alert_template.template b/tests/alerts/test_alert_template.template index 2c953ebd..c783fed5 100644 --- a/tests/alerts/test_alert_template.template +++ b/tests/alerts/test_alert_template.template @@ -29,7 +29,9 @@ class TEMPLATE_TEST_CLASSNAME(AlertTestSuite): "tags": ['hello', 'world'], "severity": "WARNING", "summary": 'My first alert!', - "notify_mozdefbot": False, + # Set to False if alert severity is less than a WARNING + # or if ircchannel is specifically set to None in the alert + "notify_mozdefbot": True, } test_cases = [] From c1a48d98a2a206d12218ad01f5fad7f2ccf8f1b6 Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Tue, 27 Nov 2018 16:30:02 -0600 Subject: [PATCH 49/89] Hide notify_mozdefbot field from alert specific tests --- tests/alerts/alert_test_suite.py | 17 ++-- tests/alerts/test_alert_template.template | 1 - tests/alerts/test_bruteforce_ssh.py | 1 - tests/alerts/test_feedback_events.py | 1 - tests/alerts/test_geomodel.py | 4 - tests/alerts/test_guard_duty_probe.py | 1 - tests/alerts/test_nsm_scan_port.py | 1 - tests/alerts/test_trace_audit.py | 107 +++++++++++----------- tests/alerts/test_write_audit.py | 1 - 9 files changed, 63 insertions(+), 71 deletions(-) diff --git a/tests/alerts/alert_test_suite.py b/tests/alerts/alert_test_suite.py index d8d94c67..40e616bc 100644 --- a/tests/alerts/alert_test_suite.py +++ b/tests/alerts/alert_test_suite.py @@ -136,13 +136,6 @@ class AlertTestSuite(UnitTestSuite): @freeze_time("2017-01-01 01:00:00", tz_offset=0) def test_alert_test_case(self, test_case): self.verify_starting_values(test_case) - if test_case.expected_test_result is True: - # if we dont set notify_mozdefbot field, autoset it to True - if 'notify_mozdefbot' not in test_case.expected_alert: - test_case.expected_alert['notify_mozdefbot'] = True - if 'ircchannel' not in test_case.expected_alert: - test_case.expected_alert['ircchannel'] = None - temp_events = test_case.events for event in temp_events: temp_event = self.dict_merge(self.generate_default_event(), self.default_event) @@ -202,8 +195,14 @@ class AlertTestSuite(UnitTestSuite): # Verify there is a utctimestamp field assert 'utctimestamp' in found_alert['_source'], 'Alert does not have utctimestamp specified' - # Verify notify_mozdefbot is set correctly - assert found_alert['_source']['notify_mozdefbot'] is test_case.expected_alert['notify_mozdefbot'], 'Alert notify_mozdefbot field is bad' + # Verify notify_mozdefbot is set correctly based on severity + expected_notify_mozdefbot = True + if test_case.expected_alert['severity'] == 'NOTICE' or test_case.expected_alert['severity'] == 'INFO': + expected_notify_mozdefbot = False + test_case.expected_alert['notify_mozdefbot'] = expected_notify_mozdefbot + + if 'ircchannel' not in test_case.expected_alert: + test_case.expected_alert['ircchannel'] = None # Verify ircchannel is set correctly assert found_alert['_source']['ircchannel'] == test_case.expected_alert['ircchannel'], 'Alert ircchannel field is bad' diff --git a/tests/alerts/test_alert_template.template b/tests/alerts/test_alert_template.template index 2c953ebd..050210a0 100644 --- a/tests/alerts/test_alert_template.template +++ b/tests/alerts/test_alert_template.template @@ -29,7 +29,6 @@ class TEMPLATE_TEST_CLASSNAME(AlertTestSuite): "tags": ['hello', 'world'], "severity": "WARNING", "summary": 'My first alert!', - "notify_mozdefbot": False, } test_cases = [] diff --git a/tests/alerts/test_bruteforce_ssh.py b/tests/alerts/test_bruteforce_ssh.py index 5e152d30..0304f07f 100644 --- a/tests/alerts/test_bruteforce_ssh.py +++ b/tests/alerts/test_bruteforce_ssh.py @@ -32,7 +32,6 @@ class TestAlertBruteforceSsh(AlertTestSuite): "severity": "NOTICE", "summary": "10 ssh bruteforce attempts by 1.2.3.4 exhostname (10 hits)", "tags": ['ssh'], - 'notify_mozdefbot': False, } test_cases = [] diff --git a/tests/alerts/test_feedback_events.py b/tests/alerts/test_feedback_events.py index a88d5372..71ca9d7f 100644 --- a/tests/alerts/test_feedback_events.py +++ b/tests/alerts/test_feedback_events.py @@ -62,7 +62,6 @@ class TestAlertFeedbackEvents(AlertTestSuite): "category": "user_feedback", "tags": ['user_feedback', 'customtag1'], "severity": "NOTICE", - "notify_mozdefbot": False, "summary": "ad|Mozilla|ttesterson escalated alert within single-sign on (SSO) dashboard. Event Date: 2012-06-15 Summary: \"Did you recently login from Montana, Tonga (109.117.1.33)?\"", } diff --git a/tests/alerts/test_geomodel.py b/tests/alerts/test_geomodel.py index 7a51117d..494e2cd2 100644 --- a/tests/alerts/test_geomodel.py +++ b/tests/alerts/test_geomodel.py @@ -36,7 +36,6 @@ class TestAlertGeomodel(AlertTestSuite): "category": "geomodel", "tags": ['geomodel'], "severity": "NOTICE", - "notify_mozdefbot": False, "summary": "ttesterson@mozilla.com NEWCOUNTRY Diamond Bar, United States access from 1.2.3.4 (duo) [deviation:12.07010770457331] last activity was from Ottawa, Canada (3763 km away) approx 23.43 hours before", "details": { "category": "NEWCOUNTRY", @@ -67,7 +66,6 @@ class TestAlertGeomodel(AlertTestSuite): event = AlertTestSuite.create_event(default_event) event['_source']['details']['severity'] = 3 alert = AlertTestSuite.create_alert(default_alert) - alert['notify_mozdefbot'] = True alert['severity'] = 'WARNING' test_cases.append( PositiveAlertTestCase( @@ -103,7 +101,6 @@ class TestAlertGeomodel(AlertTestSuite): "category": "geomodel", "tags": ['geomodel'], "severity": "NOTICE", - "notify_mozdefbot": False, "summary": "person1@mozilla.com MOVEMENT window violation (London, United Kingdom) -> (San Jose, United States) -> (Frankfurt am Main, Germany) within 4h window", } test_cases.append( @@ -130,7 +127,6 @@ class TestAlertGeomodel(AlertTestSuite): "category": "geomodel", "tags": ['geomodel'], "severity": "NOTICE", - "notify_mozdefbot": False, "summary": u"\xfcttesterson@mozilla.com NEWCOUNTRY \u0107Bar, \u0107United States access from 1.2.3.4 (duo) [deviation:12.07010770457331] last activity was from Ottawa, Canada (3763 km away) approx 23.43 hours before", "details": { "category": "NEWCOUNTRY", diff --git a/tests/alerts/test_guard_duty_probe.py b/tests/alerts/test_guard_duty_probe.py index 2e88a6d5..d29d144f 100644 --- a/tests/alerts/test_guard_duty_probe.py +++ b/tests/alerts/test_guard_duty_probe.py @@ -34,7 +34,6 @@ class TestGuardDutyProbe(AlertTestSuite): "tags": ['guardduty', 'bruteforce'], "severity": "INFO", "summary": 'Guard Duty Port Probe by 1.2.3.4', - "notify_mozdefbot": False, } test_cases = [] diff --git a/tests/alerts/test_nsm_scan_port.py b/tests/alerts/test_nsm_scan_port.py index a8c738bf..abea2dd7 100644 --- a/tests/alerts/test_nsm_scan_port.py +++ b/tests/alerts/test_nsm_scan_port.py @@ -35,7 +35,6 @@ class TestNSMScanPort(AlertTestSuite): "severity": "WARNING", "summary": "Port scan from 10.99.88.77 (mock.mozilla.org)", "tags": ['nsm', 'bro', 'portscan'], - "notify_mozdefbot": True, } test_cases = [] diff --git a/tests/alerts/test_trace_audit.py b/tests/alerts/test_trace_audit.py index 0a5453fc..dc403a35 100644 --- a/tests/alerts/test_trace_audit.py +++ b/tests/alerts/test_trace_audit.py @@ -33,71 +33,74 @@ class TestTraceAudit(AlertTestSuite): default_alert = { "category": "trace", "severity": "WARNING", - "summary": "5 instances of Strace or Ptrace executed on a system by randomjoe", + "summary": "8 instances of Strace or Ptrace executed by randomjoe on exhostname (5), exhostname2 (3)", "tags": ['audit'], - "notify_mozdefbot": True, } test_cases = [] + events = AlertTestSuite.create_events(default_event, 8) + events[5]['_source']['hostname'] = 'exhostname2' + events[6]['_source']['hostname'] = 'exhostname2' + events[7]['_source']['hostname'] = 'exhostname2' test_cases.append( PositiveAlertTestCase( description="Positive test with default event and default alert expected", - events=AlertTestSuite.create_events(default_event, 5), - expected_alert=default_alert - ) - ) - - events = AlertTestSuite.create_events(default_event, 5) - for event in events: - event['_source']['summary'] = 'Unknown' - test_cases.append( - PositiveAlertTestCase( - description="Positive test with events with a summary of 'Write: /etc/audit/rules.d/'", events=events, expected_alert=default_alert ) ) - events = AlertTestSuite.create_events(default_event, 5) - for event in events: - event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda(date_timedelta={'minutes': 1}) - event['_source']['receivedtimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda(date_timedelta={'minutes': 1}) - test_cases.append( - PositiveAlertTestCase( - description="Positive test with events a minute earlier", - events=events, - expected_alert=default_alert - ) - ) + # events = AlertTestSuite.create_events(default_event, 5) + # for event in events: + # event['_source']['summary'] = 'Unknown' + # test_cases.append( + # PositiveAlertTestCase( + # description="Positive test with events with a summary of 'Write: /etc/audit/rules.d/'", + # events=events, + # expected_alert=default_alert + # ) + # ) - events = AlertTestSuite.create_events(default_event, 5) - for event in events: - event['_source']['hostname'] = 'example.hostname.com' - test_cases.append( - NegativeAlertTestCase( - description="Negative test case with events with example hostname that matches exclusion of 'hostfilter'", - events=events, - ) - ) + # events = AlertTestSuite.create_events(default_event, 5) + # for event in events: + # event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda(date_timedelta={'minutes': 14}) + # event['_source']['receivedtimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda(date_timedelta={'minutes': 14}) + # test_cases.append( + # PositiveAlertTestCase( + # description="Positive test with events a minute earlier than window", + # events=events, + # expected_alert=default_alert + # ) + # ) - events = AlertTestSuite.create_events(default_event, 5) - for event in events: - event['_source']['details']['originaluser'] = None - test_cases.append( - NegativeAlertTestCase( - description="Negative test case aggregation key excluded", - events=events, - ) - ) + # events = AlertTestSuite.create_events(default_event, 5) + # for event in events: + # event['_source']['hostname'] = 'example.hostname.com' + # test_cases.append( + # NegativeAlertTestCase( + # description="Negative test case with events with example hostname that matches exclusion of 'hostfilter'", + # events=events, + # ) + # ) - events = AlertTestSuite.create_events(default_event, 5) - for event in events: - event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({'minutes': 15}) - event['_source']['receivedtimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({'minutes': 15}) - test_cases.append( - NegativeAlertTestCase( - description="Negative test case with old timestamp", - events=events, - ) - ) + # events = AlertTestSuite.create_events(default_event, 5) + # for event in events: + # event['_source']['details']['originaluser'] = None + # test_cases.append( + # NegativeAlertTestCase( + # description="Negative test case aggregation key excluded", + # events=events, + # ) + # ) + + # events = AlertTestSuite.create_events(default_event, 5) + # for event in events: + # event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({'minutes': 16}) + # event['_source']['receivedtimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({'minutes': 16}) + # test_cases.append( + # NegativeAlertTestCase( + # description="Negative test case with old timestamp", + # events=events, + # ) + # ) diff --git a/tests/alerts/test_write_audit.py b/tests/alerts/test_write_audit.py index a4ae4edb..ac84aa60 100644 --- a/tests/alerts/test_write_audit.py +++ b/tests/alerts/test_write_audit.py @@ -36,7 +36,6 @@ class TestWriteAudit(AlertTestSuite): "severity": "WARNING", "summary": "5 Filesystem write(s) to an auditd path by randomjoe", "tags": ['audit'], - "notify_mozdefbot": True, } test_cases = [] From 3b1194ba4d96c3d990c2cbbe40822cbef0907cd7 Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Tue, 27 Nov 2018 16:34:06 -0600 Subject: [PATCH 50/89] Revert c1a48d9 on trace audit test --- tests/alerts/test_trace_audit.py | 107 +++++++++++++++---------------- 1 file changed, 52 insertions(+), 55 deletions(-) diff --git a/tests/alerts/test_trace_audit.py b/tests/alerts/test_trace_audit.py index dc403a35..0a5453fc 100644 --- a/tests/alerts/test_trace_audit.py +++ b/tests/alerts/test_trace_audit.py @@ -33,74 +33,71 @@ class TestTraceAudit(AlertTestSuite): default_alert = { "category": "trace", "severity": "WARNING", - "summary": "8 instances of Strace or Ptrace executed by randomjoe on exhostname (5), exhostname2 (3)", + "summary": "5 instances of Strace or Ptrace executed on a system by randomjoe", "tags": ['audit'], + "notify_mozdefbot": True, } test_cases = [] - events = AlertTestSuite.create_events(default_event, 8) - events[5]['_source']['hostname'] = 'exhostname2' - events[6]['_source']['hostname'] = 'exhostname2' - events[7]['_source']['hostname'] = 'exhostname2' test_cases.append( PositiveAlertTestCase( description="Positive test with default event and default alert expected", + events=AlertTestSuite.create_events(default_event, 5), + expected_alert=default_alert + ) + ) + + events = AlertTestSuite.create_events(default_event, 5) + for event in events: + event['_source']['summary'] = 'Unknown' + test_cases.append( + PositiveAlertTestCase( + description="Positive test with events with a summary of 'Write: /etc/audit/rules.d/'", events=events, expected_alert=default_alert ) ) - # events = AlertTestSuite.create_events(default_event, 5) - # for event in events: - # event['_source']['summary'] = 'Unknown' - # test_cases.append( - # PositiveAlertTestCase( - # description="Positive test with events with a summary of 'Write: /etc/audit/rules.d/'", - # events=events, - # expected_alert=default_alert - # ) - # ) + events = AlertTestSuite.create_events(default_event, 5) + for event in events: + event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda(date_timedelta={'minutes': 1}) + event['_source']['receivedtimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda(date_timedelta={'minutes': 1}) + test_cases.append( + PositiveAlertTestCase( + description="Positive test with events a minute earlier", + events=events, + expected_alert=default_alert + ) + ) - # events = AlertTestSuite.create_events(default_event, 5) - # for event in events: - # event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda(date_timedelta={'minutes': 14}) - # event['_source']['receivedtimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda(date_timedelta={'minutes': 14}) - # test_cases.append( - # PositiveAlertTestCase( - # description="Positive test with events a minute earlier than window", - # events=events, - # expected_alert=default_alert - # ) - # ) + events = AlertTestSuite.create_events(default_event, 5) + for event in events: + event['_source']['hostname'] = 'example.hostname.com' + test_cases.append( + NegativeAlertTestCase( + description="Negative test case with events with example hostname that matches exclusion of 'hostfilter'", + events=events, + ) + ) - # events = AlertTestSuite.create_events(default_event, 5) - # for event in events: - # event['_source']['hostname'] = 'example.hostname.com' - # test_cases.append( - # NegativeAlertTestCase( - # description="Negative test case with events with example hostname that matches exclusion of 'hostfilter'", - # events=events, - # ) - # ) + events = AlertTestSuite.create_events(default_event, 5) + for event in events: + event['_source']['details']['originaluser'] = None + test_cases.append( + NegativeAlertTestCase( + description="Negative test case aggregation key excluded", + events=events, + ) + ) - # events = AlertTestSuite.create_events(default_event, 5) - # for event in events: - # event['_source']['details']['originaluser'] = None - # test_cases.append( - # NegativeAlertTestCase( - # description="Negative test case aggregation key excluded", - # events=events, - # ) - # ) - - # events = AlertTestSuite.create_events(default_event, 5) - # for event in events: - # event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({'minutes': 16}) - # event['_source']['receivedtimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({'minutes': 16}) - # test_cases.append( - # NegativeAlertTestCase( - # description="Negative test case with old timestamp", - # events=events, - # ) - # ) + events = AlertTestSuite.create_events(default_event, 5) + for event in events: + event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({'minutes': 15}) + event['_source']['receivedtimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({'minutes': 15}) + test_cases.append( + NegativeAlertTestCase( + description="Negative test case with old timestamp", + events=events, + ) + ) From 30f2cb19ccc36fef52191481b3de8059db6105bb Mon Sep 17 00:00:00 2001 From: Phrozyn Date: Tue, 27 Nov 2018 21:38:39 -0600 Subject: [PATCH 51/89] Updating documentation to add python2.7 dir and uwsgi version and node bundle instructions --- docs/source/installation.rst | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index f39e8197..f50b9f4c 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -61,6 +61,7 @@ Python Create a mozdef user:: adduser mozdef -d /opt/mozdef + cp /etc/skel/.bash* /opt/mozdef/ We need to install a python2.7 virtualenv. @@ -74,7 +75,8 @@ On APT-based systems:: Then:: - su - mozdef + sudo -i -u mozdef -g mozdef + mkdir /opt/mozdef/python2.7 wget https://www.python.org/ftp/python/2.7.11/Python-2.7.11.tgz tar xvzf Python-2.7.11.tgz cd Python-2.7.11 @@ -132,7 +134,7 @@ You can then install the rabbitmq server:: To start rabbitmq at startup:: - chkconfig rabbitmq-server on + systemctl enable rabbitmq-server On APT-based systems :: @@ -178,8 +180,11 @@ We have a mongod.conf in the config directory prepared for you. To use it simply For meteor installation follow these steps:: + sudo -i -u mozdef -g mozdef curl https://install.meteor.com/?release=1.8 | sh +For node you can exit from the mozdef user:: + wget https://nodejs.org/dist/v8.12.0/node-v8.12.0.tar.gz tar xvzf node-v8.12.0.tar.gz cd node-v8.12.0 @@ -187,8 +192,10 @@ For meteor installation follow these steps:: make sudo make install -Then from the meteor subdirectory of this git repository (/opt/mozdef/meteor) run:: +Then from the meteor subdirectory of this git repository (/opt/mozdef/meteor) run as the mozdef user with venv activated:: + sudo -i -u mozdef -g mozdef + source envs/python/bin/activate meteor add iron-router If you wish to use meteor as the authentication handler you'll also need to install the Accounts-Password pkg:: @@ -229,12 +236,12 @@ Alternatively you can run the meteor UI in 'deployment' mode using a native node First install node:: yum install bzip2 gcc gcc-c++ sqlite sqlite-devel - wget https://nodejs.org/dist/v4.7.0/node-v4.7.0.tar.gz - tar xvfz node-v4.7.0.tar.gz - cd node-v4.7.0 - python configure + wget https://nodejs.org/dist/v8.12.0/node-v8.12.0.tar.gz + tar xvzf node-v8.12.0.tar.gz + cd node-v8.12.0 + ./configure make - make install + sudo make install Then bundle the meteor portion of mozdef to deploy on another server:: @@ -317,9 +324,9 @@ UWSGI We use `uwsgi`_ to interface python and nginx, in your venv execute the following:: - wget https://projects.unbit.it/downloads/uwsgi-2.0.12.tar.gz - tar zxvf uwsgi-2.0.12.tar.gz - cd uwsgi-2.0.12 + wget https://projects.unbit.it/downloads/uwsgi-2.0.17.1.tar.gz + tar zxvf uwsgi-2.0.17.1.tar.gz + cd uwsgi-2.0.17 ~/python2.7/bin/python uwsgiconfig.py --build ~/python2.7/bin/python uwsgiconfig.py --plugin plugins/python core cp python_plugin.so ~/envs/python/bin/ From a872193add29fb0bfeb9124d5faa9669b26b9a88 Mon Sep 17 00:00:00 2001 From: Phrozyn Date: Tue, 27 Nov 2018 21:41:16 -0600 Subject: [PATCH 52/89] Updating with ownership instructions for mozdef files. --- docs/source/installation.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index f50b9f4c..c7b655be 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -62,6 +62,9 @@ Create a mozdef user:: adduser mozdef -d /opt/mozdef cp /etc/skel/.bash* /opt/mozdef/ + cd /opt/mozdef + chown mozdef: .bash* + chown -R mozdef: * We need to install a python2.7 virtualenv. From 4b0c3a6738426ec6d0bd897232671e37a7e8b02a Mon Sep 17 00:00:00 2001 From: Phrozyn Date: Tue, 27 Nov 2018 21:52:40 -0600 Subject: [PATCH 53/89] Updating nginx repo for centos v7 --- docs/source/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index c7b655be..4b2182e0 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -316,7 +316,7 @@ If you don't have this package in your repos, before installing create `/etc/yum [nginx] name=nginx repo - baseurl=http://nginx.org/packages/OS/OSRELEASE/$basearch/ + baseurl=http://nginx.org/packages/centos/7/$basearch/ gpgcheck=0 enabled=1 From b1915428626d4790599f0c947862eb457f83afa2 Mon Sep 17 00:00:00 2001 From: Phrozyn Date: Tue, 27 Nov 2018 21:54:54 -0600 Subject: [PATCH 54/89] Updating uwsgi directory --- docs/source/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 4b2182e0..ff544995 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -329,7 +329,7 @@ We use `uwsgi`_ to interface python and nginx, in your venv execute the followin wget https://projects.unbit.it/downloads/uwsgi-2.0.17.1.tar.gz tar zxvf uwsgi-2.0.17.1.tar.gz - cd uwsgi-2.0.17 + cd uwsgi-2.0.17.1 ~/python2.7/bin/python uwsgiconfig.py --build ~/python2.7/bin/python uwsgiconfig.py --plugin plugins/python core cp python_plugin.so ~/envs/python/bin/ From 80847d974797372d8576a1b0bbb05e98a6eb6e77 Mon Sep 17 00:00:00 2001 From: Jeff Bryner Date: Tue, 27 Nov 2018 21:11:33 -0800 Subject: [PATCH 55/89] user_id doesn't always exist --- cron/auth02mozdef.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cron/auth02mozdef.py b/cron/auth02mozdef.py index 8492e489..1f448abb 100644 --- a/cron/auth02mozdef.py +++ b/cron/auth02mozdef.py @@ -276,9 +276,13 @@ def process_msg(mozmsg, msg): # fields that should always exist mozmsg.timestamp = msg.date details['messageid'] = msg._id - details['userid'] = msg.user_id details['sourceipaddress'] = msg.ip + try: + details['userid'] = msg.user_id + except KeyError: + pass + try: details['username'] = msg.user_name except KeyError: From 26cbe04e9de3bebbb178819a404bea05a50c7cc2 Mon Sep 17 00:00:00 2001 From: Jon Moroney Date: Wed, 28 Nov 2018 10:31:47 -0800 Subject: [PATCH 56/89] Add initial alert guide --- docs/source/alert_development_guide.rst | 68 +++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/docs/source/alert_development_guide.rst b/docs/source/alert_development_guide.rst index 8ceaf442..91089faf 100644 --- a/docs/source/alert_development_guide.rst +++ b/docs/source/alert_development_guide.rst @@ -73,6 +73,74 @@ How to get the alert in MozDef? The best way to get your alert into MozDef (once it's completed) is to propose a pull request and ask for a review from a MozDef developer. They will be able to help you get the most out of the alert and help point out pitfalls. Once the alert is accepted into MozDef master, there is a process by which MozDef installations can make use or 'enable' that alert. It's best to work with that MozDef instance's maintainer to enable any new alerts. +Example first alert +------------------- +Let's step through creating a simple alert you might want to verify a working deployment. +For this sub-section it is assumed that you have a working MozDef instance which resides in some MozDefDir and is receiving logs. + +First move to to your MozDefDir and issue +:: + + make new-alert + +You will be asked for a string to name a new alert and the associated test. For this example we will use the string "foo" +:: + + make new-alert + Enter your alert name (Example: proxy drop executable): foo + Creating alerts/foo.py + Creating tests/alerts/test_foo.py + +These will be created as above in the alerts and tests/alerts directories. +There's a lot to the generated code, but a class called "AlertFoo" is of immediate interest and will define when and how to alert. +Here's the head of the auto generated class. +:: + + class AlertFoo(AlertTask): + def main(self): + # Create a query to look back the last 20 minutes + search_query = SearchQuery(minutes=20) + + # Add search terms to our query + search_query.add_must([ + TermMatch('category', 'helloworld'), + ExistsMatch('details.sourceipaddress'), + ]) + ... + +In essence this code will tell MozDef to query the collection of logs for messages timestamped within 20 minutes (from time of query execution) and to look for messages which are of category "helloworld" which also have a source IP address. +If you're pumping logs into MozDef odds are you don't have any which will be tagged as "helloworld". You can of course create those logs, but lets assume that you have logs tagged as "syslog" for the moment. +Change the TermMatch line to +:: + + TermMatch('category', 'syslog'), + +and you will get alerts for syslog labeled messages. +Ideally you should edit your test to match, but it's not strictly necessary. + +Next we will need to enable the log and to schedule it. At time of writing this is a bit annoying. +Open the file +:: + + docker/compose/mozdef_alerts/files/config.py + +or simply +:: + + mozdef_alerts/files/config.py + +if you are not working from the docker images +and add your new foo alert to the others with a crontab style schedule +:: + + ALERTS = { + 'foo.AlertFoo': {'schedule': crontab(minute='*/1')}, + 'bruteforce_ssh.AlertBruteforceSsh': {'schedule': crontab(minute='*/1')}, + 'unauth_ssh.AlertUnauthSSH': {'schedule': crontab(minute='*/1')}, + } + +Restart your MozDef instance and you should begin seeing alerts on the alerts page. + Questions? ---------- From 3be81b07b517eb4a1f39acfda42ec51b9a418a23 Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Wed, 28 Nov 2018 13:19:52 -0600 Subject: [PATCH 57/89] Update alert test suite to handle notify_mozdefbot correctly --- tests/alerts/alert_test_suite.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/alerts/alert_test_suite.py b/tests/alerts/alert_test_suite.py index 40e616bc..96bd9780 100644 --- a/tests/alerts/alert_test_suite.py +++ b/tests/alerts/alert_test_suite.py @@ -195,15 +195,15 @@ class AlertTestSuite(UnitTestSuite): # Verify there is a utctimestamp field assert 'utctimestamp' in found_alert['_source'], 'Alert does not have utctimestamp specified' - # Verify notify_mozdefbot is set correctly based on severity - expected_notify_mozdefbot = True - if test_case.expected_alert['severity'] == 'NOTICE' or test_case.expected_alert['severity'] == 'INFO': - expected_notify_mozdefbot = False - test_case.expected_alert['notify_mozdefbot'] = expected_notify_mozdefbot - if 'ircchannel' not in test_case.expected_alert: test_case.expected_alert['ircchannel'] = None + # Verify notify_mozdefbot is set correctly based on severity + expected_notify_mozdefbot = True + if (test_case.expected_alert['severity'] == 'NOTICE' or test_case.expected_alert['severity'] == 'INFO') and test_case.expected_alert['ircchannel'] is None: + expected_notify_mozdefbot = False + test_case.expected_alert['notify_mozdefbot'] = expected_notify_mozdefbot + # Verify ircchannel is set correctly assert found_alert['_source']['ircchannel'] == test_case.expected_alert['ircchannel'], 'Alert ircchannel field is bad' From 637d1c7a31eafdb7e60870855336377adc76ebb2 Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Wed, 28 Nov 2018 13:33:01 -0600 Subject: [PATCH 58/89] Add new make target for running test resources externally --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ec3ff072..f111e9d3 100644 --- a/Makefile +++ b/Makefile @@ -34,9 +34,12 @@ run-cloudy-mozdef: ## Run the MozDef containers necessary to run in AWS (`cloudy restart-cloudy-mozdef: docker-compose -f docker/compose/docker-compose-cloudy-mozdef.yml -p $(NAME) restart -.PHONY: tests run-tests-resources run-tests +.PHONY: tests run-tests-resources run-tests-resources-external run-tests test: build-tests run-tests tests: build-tests run-tests ## Run all tests (getting/building images as needed) +run-tests-resources-external: ## Just spin up external resources for tests and have them listen externally + docker-compose -f docker/compose/docker-compose-tests.yml -p test-$(NAME) run -p 9200:9200 -d elasticsearch + docker-compose -f docker/compose/docker-compose-tests.yml -p test-$(NAME) run -p 5672:5672 -d rabbitmq run-tests-resources: ## Just run the external resources required for tests docker-compose -f docker/compose/docker-compose-tests.yml -p test-$(NAME) up -d run-test: From d61876cae91a7810b5e33b76cbb279838f603f42 Mon Sep 17 00:00:00 2001 From: Jon Moroney Date: Wed, 28 Nov 2018 12:44:43 -0800 Subject: [PATCH 59/89] Update config location for non-docker version --- docs/source/alert_development_guide.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/alert_development_guide.rst b/docs/source/alert_development_guide.rst index 91089faf..432ed076 100644 --- a/docs/source/alert_development_guide.rst +++ b/docs/source/alert_development_guide.rst @@ -127,7 +127,7 @@ Open the file or simply :: - mozdef_alerts/files/config.py + alerts/files/config.py if you are not working from the docker images and add your new foo alert to the others with a crontab style schedule From 5c5ef6567805f0d06d472e8063aa260ae53b148f Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Wed, 28 Nov 2018 16:26:33 -0600 Subject: [PATCH 60/89] Add mozdef util documentation for classes --- docs/source/index.rst | 1 + docs/source/mozdef_util.rst | 9 + docs/source/mozdef_util/connect.rst | 8 + docs/source/mozdef_util/create.rst | 85 ++++++++++ .../mozdef_util/match_query_classes.rst | 145 +++++++++++++++++ docs/source/mozdef_util/search.rst | 154 ++++++++++++++++++ 6 files changed, 402 insertions(+) create mode 100644 docs/source/mozdef_util.rst create mode 100644 docs/source/mozdef_util/connect.rst create mode 100644 docs/source/mozdef_util/create.rst create mode 100644 docs/source/mozdef_util/match_query_classes.rst create mode 100644 docs/source/mozdef_util/search.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 35bfb41a..2c03670f 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -14,6 +14,7 @@ Table of Contents demo installation alert_development_guide + mozdef_util screenshots usage cloud_deployment diff --git a/docs/source/mozdef_util.rst b/docs/source/mozdef_util.rst new file mode 100644 index 00000000..3023c7f5 --- /dev/null +++ b/docs/source/mozdef_util.rst @@ -0,0 +1,9 @@ +Mozdef_util Library +=================== + +We provide a library used to interact with MozDef components. + +.. include:: mozdef_util/connect.rst +.. include:: mozdef_util/create.rst +.. include:: mozdef_util/search.rst +.. include:: mozdef_util/match_query_classes.rst diff --git a/docs/source/mozdef_util/connect.rst b/docs/source/mozdef_util/connect.rst new file mode 100644 index 00000000..ddd49807 --- /dev/null +++ b/docs/source/mozdef_util/connect.rst @@ -0,0 +1,8 @@ +Connecting to Elasticsearch +--------------------------- + +.. code-block:: python + :linenos: + + from mozdef_util.elasticsearch_client import ElasticsearchClient + es_client = ElasticsearchClient("http://127.0.0.1:9200") diff --git a/docs/source/mozdef_util/create.rst b/docs/source/mozdef_util/create.rst new file mode 100644 index 00000000..dbb10e56 --- /dev/null +++ b/docs/source/mozdef_util/create.rst @@ -0,0 +1,85 @@ +Creating/Updating Documents +--------------------------- + +Create a new Event +^^^^^^^^^^^^^^^^^^ + +.. code-block:: python + :linenos: + + event_dict = { + "example_key": "example value" + } + es_client.save_event(body=event_dict) + +Update an existing event +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: python + :linenos: + + event_dict = { + "example_key": "example new value" + } + # Assuming 12345 is the id of the existing entry + es_client.save_event(body=event_dict, doc_id="12345") + +Create a new alert +^^^^^^^^^^^^^^^^^^ + +.. code-block:: python + :linenos: + + alert_dict = { + "example_key": "example value" + } + es_client.save_alert(body=alert_dict) + +Update an existing alert +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: python + :linenos: + + alert_dict = { + "example_key": "example new value" + } + # Assuming 12345 is the id of the existing entry + es_client.save_alert(body=alert_dict, doc_id="12345") + +Create a new generic document +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: python + :linenos: + + document_dict = { + "example_key": "example value" + } + es_client.save_object(index='randomindex', doc_type='randomtype', body=document_dict) + +Update an existing document +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: python + :linenos: + + document_dict = { + "example_key": "example new value" + } + # Assuming 12345 is the id of the existing entry + es_client.save_object(index='randomindex', doc_type='randomtype', body=document_dict, doc_id="12345") + +Bulk Importing +^^^^^^^^^^^^^^ + +.. code-block:: python + :linenos: + + from mozdef_util.elasticsearch_client import ElasticsearchClient + es_client = ElasticsearchClient("http://127.0.0.1:9200", bulk_amount=30, bulk_refresh_time=5) + es_client.save_event(body={'key': 'value'}, bulk=True) + +- Line 2: bulk_amount (defaults to 100), specifies how many messages should sit in the bulk queue before they get written to elasticsearch +- Line 2: bulk_refresh_time (defaults to 30), is the amount of time that a bulk flush is forced +- Line 3: bulk (defaults to False) determines if an event should get added to a bulk queue diff --git a/docs/source/mozdef_util/match_query_classes.rst b/docs/source/mozdef_util/match_query_classes.rst new file mode 100644 index 00000000..a0662610 --- /dev/null +++ b/docs/source/mozdef_util/match_query_classes.rst @@ -0,0 +1,145 @@ +Match/Query Classes +------------------- + +ExistsMatch +^^^^^^^^^^^ + +Checks to see if a specific field exists in a document + +.. code-block:: python + :linenos: + + from mozdef_util.query_models import ExistsMatch + + ExistsMatch("randomfield") + + +TermMatch +^^^^^^^^^ + +Checks if a specific field matches the key + +.. code-block:: python + :linenos: + + from mozdef_util.query_models import TermMatch + + TermMatch("details.ip", "127.0.0.1") + + +TermsMatch +^^^^^^^^^^ + +Checks if a specific field matches any of the keys + +.. code-block:: python + :linenos: + + from mozdef_util.query_models import TermsMatch + + TermsMatch("details.ip", ["127.0.0.1", "1.2.3.4"]) + + +WildcardMatch +^^^^^^^^^^^^^ + +Allows regex to be used in looking for documents that a field contains all or part of a key + +.. code-block:: python + :linenos: + + from mozdef_util.query_models import WildcardMatch + + WildcardMatch('summary', 'test*') + + +PhraseMatch +^^^^^^^^^^^ + +Checks if a field contains a specific phrase (includes spaces) + +.. code-block:: python + :linenos: + + from mozdef_util.query_models import PhraseMatch + + PhraseMatch('summary', 'test run') + + +BooleanMatch +^^^^^^^^^^^^ + +Used to apply specific "matchers" to a query. This will unlikely be used outside of SearchQuery. + +.. code-block:: python + :linenos: + + from mozdef_util.query_models import ExistsMatch, TermMatch, BooleanMatch + + must = [ + ExistsMatch('details.ip') + ] + must_not = [ + TermMatch('_type', 'alert') + ] + + BooleanMatch(must=must, should=[], must_not=must_not) + + +MissingMatch +^^^^^^^^^^^^ + +Checks if a field does not exist in a document + +.. code-block:: python + :linenos: + + from mozdef_util.query_models import MissingMatch + + MissingMatch('summary') + + +RangeMatch +^^^^^^^^^^ + +Checks if a field value is within a specific range (mostly used to look for documents in a time frame) + +.. code-block:: python + :linenos: + + from mozdef_util.query_models import RangeMatch + + RangeMatch('utctimestamp', "2016-08-12T21:07:12.316450+00:00", "2016-08-13T21:07:12.316450+00:00") + + +QueryStringMatch +^^^^^^^^^^^^^^^^ + +Uses a custom query string to generate the "match" based on (Similar to what you would see in kibana) + +.. code-block:: python + :linenos: + + from mozdef_util.query_models import QueryStringMatch + + QueryStringMatch('summary: test') + + +Aggregation +^^^^^^^^^^^ + +Used to aggregate results based on a specific field + +.. code-block:: python + :linenos: + + from mozdef_util.query_models import Aggregation, SearchQuery, ExistsMatch + + search_query = SearchQuery(hours=24) + must = [ + ExistsMatch('seenindicator') + ] + search_query.add_must(must) + aggr = Aggregation('details.ip') + search_query.add_aggregation(aggr) + results = search_query.execute(es_client, indices=['events','events-previous']) diff --git a/docs/source/mozdef_util/search.rst b/docs/source/mozdef_util/search.rst new file mode 100644 index 00000000..1e6b36f0 --- /dev/null +++ b/docs/source/mozdef_util/search.rst @@ -0,0 +1,154 @@ +Searching for documents +----------------------- + +Simple search +^^^^^^^^^^^^^ + +.. code-block:: python + :linenos: + + from mozdef_util.query_models import SearchQuery, TermMatch, ExistsMatch + + search_query = SearchQuery(hours=24) + must = [ + TermMatch('category', 'brointel'), + ExistsMatch('seenindicator') + ] + search_query.add_must(must) + results = search_query.execute(es_client, indices=['events','events-previous']) + +SimpleResults + +When you perform a "simple" search (one without any aggregation), a SimpleResults object is returned. This object is a dict, with the following format: + +.. list-table:: + :widths: 25 50 + :header-rows: 1 + + * - Key + - Description + * - hits + - Contains an array of documents that matched the search query + * - meta + - Contains a hash of fields describing the search query (Ex: if the query timed out or not) + +Example simple result: + +.. code-block:: text + :linenos: + + { + 'hits': [ + { + '_id': u'cp5ZsOgLSu6tHQm5jAZW1Q', + '_index': 'events-20161005', + '_score': 1.0, + '_source': { + 'details': { + 'information': 'Example information' + }, + 'category': 'excategory', + 'summary': 'Test Summary' + }, + '_type': 'event' + } + ], + 'meta': {'timed_out': False} + } + + +Aggregate search +^^^^^^^^^^^^^^^^ + +.. code-block:: python + :linenos: + + from mozdef_util.query_models import SearchQuery, TermMatch, Aggregation + + search_query = SearchQuery(hours=24) + search_query.add_must(TermMatch('_type', 'event')) + search_query.add_must(TermMatch('category', 'brointel')) + search_query.add_aggregation(Aggregation('_type')) + results = search_query.execute(es_client) + +AggregatedResults + +When you perform an aggregated search (Ex: give me a count of all different ip addresses are in the documents that match a specific query), a AggregatedResults object is returned. This object is a dict, with the following format: + + +.. list-table:: + :widths: 25 50 + :header-rows: 1 + + * - Key + - Description + * - aggregations + - Contains the aggregation results, grouped by field name + * - hits + - Contains an array of documents that matched the search query + * - meta + - Contains a hash of fields describing the search query (Ex: if the query timed out or not) + +.. code-block:: text + :linenos: + + { + 'aggregations': { + 'ip': { + 'terms': [ + { + 'count': 2, + 'key': '1.2.3.4' + }, + { + 'count': 1, + 'key': '127.0.0.1' + } + ] + } + }, + 'hits': [ + { + '_id': u'LcdS2-koQWeICOpbOT__gA', + '_index': 'events-20161005', + '_score': 1.0, + '_source': { + 'details': { + 'information': 'Example information' + }, + 'ip': '1.2.3.4', + 'summary': 'Test Summary' + }, + '_type': 'event' + }, + { + '_id': u'F1dLS66DR_W3v7ZWlX4Jwg', + '_index': 'events-20161005', + '_score': 1.0, + '_source': { + 'details': { + 'information': 'Example information' + }, + 'ip': '1.2.3.4', + 'summary': 'Test Summary' + }, + '_type': 'event' + }, + { + '_id': u'G1nGdxqoT6eXkL5KIjLecA', + '_index': 'events-20161005', + '_score': 1.0, + '_source': { + 'details': { + 'information': 'Example information' + }, + 'ip': '127.0.0.1', + 'summary': 'Test Summary' + }, + '_type': 'event' + } + ], + 'meta': { + 'timed_out': False + } + } From dd78571f894c3ef666bd5244306320450965dcb1 Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Wed, 28 Nov 2018 16:27:04 -0600 Subject: [PATCH 61/89] Build html docs via readthedocs by default --- docs/requirements.txt | 2 ++ docs/source/conf.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..82133027 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +sphinx +sphinx_rtd_theme diff --git a/docs/source/conf.py b/docs/source/conf.py index 85f3b955..61df2e7e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -97,7 +97,7 @@ pygments_style = 'sphinx' # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the From ebfacbe1474b6c702af5b5d4490cb635fb4aba10 Mon Sep 17 00:00:00 2001 From: Michal Purzynski Date: Thu, 29 Nov 2018 13:53:43 -0800 Subject: [PATCH 62/89] Move the mapping configuration to a plugin directory --- mq/esworker_github_webhooks.conf | 10 ++++++++++ mq/{ => plugins}/github_mapping.yml | 0 mq/plugins/github_webhooks.py | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 mq/esworker_github_webhooks.conf rename mq/{ => plugins}/github_mapping.yml (100%) diff --git a/mq/esworker_github_webhooks.conf b/mq/esworker_github_webhooks.conf new file mode 100644 index 00000000..f88d78b7 --- /dev/null +++ b/mq/esworker_github_webhooks.conf @@ -0,0 +1,10 @@ +[options] +prefetch=10 +esbulksize=10 +esservers=http://elasticsearch:9200 +mqprotocol=sqs +taskexchange= +region = us-west-2 +accesskey = +secretkey = +mqack = False diff --git a/mq/github_mapping.yml b/mq/plugins/github_mapping.yml similarity index 100% rename from mq/github_mapping.yml rename to mq/plugins/github_mapping.yml diff --git a/mq/plugins/github_webhooks.py b/mq/plugins/github_webhooks.py index f9263914..e9312fd0 100644 --- a/mq/plugins/github_webhooks.py +++ b/mq/plugins/github_webhooks.py @@ -18,7 +18,7 @@ class message(object): self.registration = ['githubeventsqs'] self.priority = 10 - with open('mq/github_mapping.yml', 'r') as f: + with open('mq/plugins/github_mapping.yml', 'r') as f: map = f.read() yap = yaml.load(map) From 43f1fa2f535be03e31665ecfbe419663bf0222cd Mon Sep 17 00:00:00 2001 From: Michal Purzynski Date: Thu, 29 Nov 2018 18:06:36 -0800 Subject: [PATCH 63/89] Dynamically resolve path to the config file --- mq/plugins/github_webhooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mq/plugins/github_webhooks.py b/mq/plugins/github_webhooks.py index e9312fd0..50ea3f7e 100644 --- a/mq/plugins/github_webhooks.py +++ b/mq/plugins/github_webhooks.py @@ -18,7 +18,7 @@ class message(object): self.registration = ['githubeventsqs'] self.priority = 10 - with open('mq/plugins/github_mapping.yml', 'r') as f: + with open(os.path.join(os.path.dirname(__file__), 'github_mapping.yml'), 'r') as f: map = f.read() yap = yaml.load(map) From 410eb27e1bfcd729ec763e43a161728623d7f295 Mon Sep 17 00:00:00 2001 From: Jeff Bryner Date: Mon, 3 Dec 2018 15:38:24 -0500 Subject: [PATCH 64/89] explicitly accept/map 'source' field --- mq/esworker_cloudtrail.py | 2 +- mq/esworker_eventtask.py | 2 +- mq/esworker_papertrail.py | 4 ++-- mq/esworker_sqs.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mq/esworker_cloudtrail.py b/mq/esworker_cloudtrail.py index 1efb1867..b59bf73e 100755 --- a/mq/esworker_cloudtrail.py +++ b/mq/esworker_cloudtrail.py @@ -181,7 +181,7 @@ def keyMapping(aDict): elif k == 'sourceipaddress': returndict[u'details']['sourceipaddress'] = v - elif k == 'facility': + elif k in ('facility', 'source'): returndict[u'source'] = v elif k in ('eventsource'): diff --git a/mq/esworker_eventtask.py b/mq/esworker_eventtask.py index 41f45721..93a6c484 100755 --- a/mq/esworker_eventtask.py +++ b/mq/esworker_eventtask.py @@ -59,7 +59,7 @@ def keyMapping(aDict): if k == 'sourceip': returndict[u'details']['eventsourceipaddress'] = v - if k == 'facility': + if k in ('facility', 'source'): returndict[u'source'] = v if k in ('message', 'summary'): diff --git a/mq/esworker_papertrail.py b/mq/esworker_papertrail.py index 99db53d9..2fcacbf3 100755 --- a/mq/esworker_papertrail.py +++ b/mq/esworker_papertrail.py @@ -149,8 +149,8 @@ def keyMapping(aDict): if k in ('syslogseverity', 'severity', 'severityvalue', 'level'): returndict[u'severity'] = toUnicode(v).upper() - if k in ('facility', 'syslogfacility'): - returndict[u'facility'] = toUnicode(v) + if k in ('facility', 'syslogfacility','source'): + returndict[u'source'] = toUnicode(v) if k in ('pid', 'processid'): returndict[u'processid'] = toUnicode(v) diff --git a/mq/esworker_sqs.py b/mq/esworker_sqs.py index d1b0620c..b3fa1a6c 100755 --- a/mq/esworker_sqs.py +++ b/mq/esworker_sqs.py @@ -100,8 +100,8 @@ def keyMapping(aDict): if k in ('syslogseverity', 'severity', 'severityvalue', 'level'): returndict[u'severity'] = toUnicode(v).upper() - if k in ('facility', 'syslogfacility'): - returndict[u'facility'] = toUnicode(v) + if k in ('facility', 'syslogfacility','source'): + returndict[u'source'] = toUnicode(v) if k in ('pid', 'processid'): returndict[u'processid'] = toUnicode(v) From 3a7dcea53b1a109da6d193b3037d10f7ac81d57e Mon Sep 17 00:00:00 2001 From: Cag Date: Wed, 5 Dec 2018 17:59:43 -0500 Subject: [PATCH 65/89] Proposing fix and a change for proxy alert on exfil domains --- ...exfil_domains.py => proxy_exfil_domains.py} | 18 ++++++++++++------ ..._domains.py => test_proxy_exfil_domains.py} | 15 +++++++-------- 2 files changed, 19 insertions(+), 14 deletions(-) rename alerts/{proxy_drop_exfil_domains.py => proxy_exfil_domains.py} (78%) rename tests/alerts/{test_proxy_drop_exfil_domains.py => test_proxy_exfil_domains.py} (87%) diff --git a/alerts/proxy_drop_exfil_domains.py b/alerts/proxy_exfil_domains.py similarity index 78% rename from alerts/proxy_drop_exfil_domains.py rename to alerts/proxy_exfil_domains.py index b40cbdbf..59164025 100644 --- a/alerts/proxy_drop_exfil_domains.py +++ b/alerts/proxy_exfil_domains.py @@ -6,11 +6,12 @@ # Copyright (c) 2014 Mozilla Corporation +from urlparse import urlparse from lib.alerttask import AlertTask from mozdef_util.query_models import SearchQuery, TermMatch, QueryStringMatch, ExistsMatch, PhraseMatch, WildcardMatch -class AlertProxyDropExfilDomains(AlertTask): +class AlertProxyExfilDomains(AlertTask): def main(self): self.parse_config('proxy_drop_exfil_domains.conf', ['exfil_domains']) @@ -19,11 +20,10 @@ class AlertProxyDropExfilDomains(AlertTask): search_query.add_must([ TermMatch('category', 'squid'), TermMatch('tags', 'squid'), - TermMatch('details.proxyaction', "TCP_DENIED/-") ]) # Only notify on certain domains listed in the config - domain_regex = "/({0}).*/".format( + domain_regex = "/.*({0}).*/".format( self.config.exfil_domains.replace(',', '|')) search_query.add_must([ QueryStringMatch('details.destination: {}'.format(domain_regex)) @@ -49,10 +49,16 @@ class AlertProxyDropExfilDomains(AlertTask): exfil_domains = set() for event in aggreg['allevents']: - domain = event['_source']['details']['destination'].split(':') - exfil_domains.add(domain[0]) + try: + domain = urlparse(event['_source']['details']['destination']).netloc + except Exception: + # We already have a domain, not a URL + target = event['_source']['details']['destination'].split(':') + domain = target[0] - summary = 'Suspicious Proxy DROP event(s) detected from {0} to the following exfil domain(s): {1}'.format( + exfil_domains.add(domain) + + summary = 'Suspicious Proxy event(s) detected from {0} to the following exfil domain(s): {1}'.format( aggreg['value'], ",".join(sorted(exfil_domains)) ) diff --git a/tests/alerts/test_proxy_drop_exfil_domains.py b/tests/alerts/test_proxy_exfil_domains.py similarity index 87% rename from tests/alerts/test_proxy_drop_exfil_domains.py rename to tests/alerts/test_proxy_exfil_domains.py index f06fad04..1fb15e74 100644 --- a/tests/alerts/test_proxy_drop_exfil_domains.py +++ b/tests/alerts/test_proxy_exfil_domains.py @@ -7,9 +7,9 @@ from negative_alert_test_case import NegativeAlertTestCase from alert_test_suite import AlertTestSuite -class TestProxyDropExfilDomains(AlertTestSuite): - alert_filename = "proxy_drop_exfil_domains" - alert_classname = "AlertProxyDropExfilDomains" +class TestProxyExfilDomains(AlertTestSuite): + alert_filename = "proxy_exfil_domains" + alert_classname = "AlertProxyExfilDomains" # This event is the default positive event that will cause the # alert to trigger @@ -20,15 +20,14 @@ class TestProxyDropExfilDomains(AlertTestSuite): "tags": ["squid"], "details": { "sourceipaddress": "1.2.3.4", - "destination": "pastebin.com", - "proxyaction": "TCP_DENIED/-", + "destination": "https://pastebin.com", } } } # This event is an alternate destination that we'd want to aggregate default_event2 = AlertTestSuite.copy(default_event) - default_event2["_source"]["details"]["destination"] = "www.sendspace.com" + default_event2["_source"]["details"]["destination"] = "http://www.sendspace.com" # This event is the default negative event that will not cause the # alert to trigger @@ -40,13 +39,13 @@ class TestProxyDropExfilDomains(AlertTestSuite): "category": "squid", "tags": ['squid', 'proxy'], "severity": "WARNING", - "summary": 'Suspicious Proxy DROP event(s) detected from 1.2.3.4 to the following exfil domain(s): pastebin.com', + "summary": 'Suspicious Proxy event(s) detected from 1.2.3.4 to the following exfil domain(s): pastebin.com', } # This alert is the expected result from this task against multiple matching events default_alert_aggregated = AlertTestSuite.copy(default_alert) default_alert_aggregated[ - "summary"] = 'Suspicious Proxy DROP event(s) detected from 1.2.3.4 to the following exfil domain(s): pastebin.com,www.sendspace.com' + "summary"] = 'Suspicious Proxy event(s) detected from 1.2.3.4 to the following exfil domain(s): pastebin.com,www.sendspace.com' test_cases = [] From c39ac343dc88724616d335872b6a8edc996a0472 Mon Sep 17 00:00:00 2001 From: Cag <30646014+caggle@users.noreply.github.com> Date: Wed, 5 Dec 2018 18:14:53 -0500 Subject: [PATCH 66/89] Renaming the config file to reflect the change in alert name --- .../{proxy_drop_exfil_domains.conf => proxy_exfil_domains.conf} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename alerts/{proxy_drop_exfil_domains.conf => proxy_exfil_domains.conf} (100%) diff --git a/alerts/proxy_drop_exfil_domains.conf b/alerts/proxy_exfil_domains.conf similarity index 100% rename from alerts/proxy_drop_exfil_domains.conf rename to alerts/proxy_exfil_domains.conf From 43f9ae446348b076817d7da700ea866f9e02f44c Mon Sep 17 00:00:00 2001 From: Cag <30646014+caggle@users.noreply.github.com> Date: Thu, 6 Dec 2018 13:43:46 -0500 Subject: [PATCH 67/89] Minor fix to reflect change in the config file name --- alerts/proxy_exfil_domains.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alerts/proxy_exfil_domains.py b/alerts/proxy_exfil_domains.py index 59164025..0e15959e 100644 --- a/alerts/proxy_exfil_domains.py +++ b/alerts/proxy_exfil_domains.py @@ -13,7 +13,7 @@ from mozdef_util.query_models import SearchQuery, TermMatch, QueryStringMatch, E class AlertProxyExfilDomains(AlertTask): def main(self): - self.parse_config('proxy_drop_exfil_domains.conf', ['exfil_domains']) + self.parse_config('proxy_exfil_domains.conf', ['exfil_domains']) search_query = SearchQuery(minutes=20) From d7de9ee7af54830077b10e23beea28ecc7d569d4 Mon Sep 17 00:00:00 2001 From: Jeff Bryner Date: Tue, 11 Dec 2018 09:27:40 -0800 Subject: [PATCH 68/89] fix the test assert --- tests/mq/test_esworker_eventtask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mq/test_esworker_eventtask.py b/tests/mq/test_esworker_eventtask.py index 1efbc9af..07448224 100644 --- a/tests/mq/test_esworker_eventtask.py +++ b/tests/mq/test_esworker_eventtask.py @@ -60,7 +60,7 @@ class TestKeyMapping(): assert result['mozdefhostname'] == 'sample' assert result['hostname'] == 'ub_server' assert result['summary'] == 'Stopped Getty on tty1.' - assert result['source'] == 'daemon' + assert result['source'] == 'syslog_tcp' assert result['receivedtimestamp'] != result['utctimestamp'] expected_year = datetime.datetime.now().year assert result['utctimestamp'] == str(expected_year) + '-10-27T14:01:12+00:00' From c774d5921b479d7fc93b8c9367f6b1cd47e20958 Mon Sep 17 00:00:00 2001 From: Jeff Bryner Date: Tue, 11 Dec 2018 15:59:35 -0800 Subject: [PATCH 69/89] set details.success to match auth0, other auth sources --- cron/duo_logpull.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cron/duo_logpull.py b/cron/duo_logpull.py index 706355dd..76cd57f8 100644 --- a/cron/duo_logpull.py +++ b/cron/duo_logpull.py @@ -41,7 +41,9 @@ def normalize(details): continue if f == "result": if details[f] != "SUCCESS": - normalized["error"] = True + normalized["success"] = False + else: + normalized["success"] = True normalized[f] = details[f] return normalized @@ -79,6 +81,7 @@ def process_events(mozmsg, duo_events, etype, state): continue details[i] = e[i] + mozmsg.category= etype mozmsg.details = normalize(details) if etype == 'administration': mozmsg.summary = e['action'] @@ -107,11 +110,11 @@ def main(): duo = duo_client.Admin(ikey=options.IKEY, skey=options.SKEY, host=options.URL) mozmsg = mozdef.MozDefEvent(options.MOZDEF_URL) - mozmsg.tags=['duosecurity', 'logs'] + mozmsg.tags=['duosecurity'] if options.update_tags != '': mozmsg.tags.append(options.update_tags) - mozmsg.category = 'Authentication' - mozmsg.source = 'DuoSecurity API' + mozmsg.category = 'authentication' + mozmsg.source = 'DuoSecurityAPI' if options.DEBUG: mozmsg.debug = options.DEBUG mozmsg.set_send_to_syslog(True, only_syslog=True) From 72c51b64a936085a540022aa5be7bc603e660773 Mon Sep 17 00:00:00 2001 From: Jeff Bryner Date: Tue, 11 Dec 2018 16:03:33 -0800 Subject: [PATCH 70/89] success only on true, catch other ip field --- cron/duo_logpull.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cron/duo_logpull.py b/cron/duo_logpull.py index 76cd57f8..005af422 100644 --- a/cron/duo_logpull.py +++ b/cron/duo_logpull.py @@ -36,14 +36,14 @@ def normalize(details): normalized = {} for f in details: - if f in ("ip", "ip_address"): + if f in ("ip", "ip_address","client_ip"): normalized["sourceipaddress"] = details[f] continue if f == "result": - if details[f] != "SUCCESS": - normalized["success"] = False - else: + if details[f] == "SUCCESS": normalized["success"] = True + else: + normalized["success"] = False normalized[f] = details[f] return normalized From ef25be321ac937bf9228a1363c37d5604cfeb6ed Mon Sep 17 00:00:00 2001 From: Jeff Bryner Date: Tue, 11 Dec 2018 16:33:11 -0800 Subject: [PATCH 71/89] update docs for auth event sync --- docs/source/usage.rst | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 5c60446a..270819d3 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -152,14 +152,14 @@ Mandatory Fields +-----------------+-------------------------------------+-----------------------------------+ | Field | Purpose | Sample Value | +=================+=====================================+===================================+ -| category | General category/type of event | Authentication, Authorization, | -| | matching the 'what should I log' | Account Creation, Shutdown, | -| | section below | Startup, Account Deletion, | -| | | Account Unlock, brointel, | -| | | bronotice | +| category | General category/type of event | authentication, authorization, | +| | matching the 'what should I log' | account creation, shutdown, | +| | section below | atartup, account deletion, | +| | | account unlock, zeek | +| | | | +-----------------+-------------------------------------+-----------------------------------+ -| details | Additional, event-specific fields | "dn": "john@example.com,o=com, | -| | that you would like included with | dc=example", "facility": "daemon" | +| details | Additional, event-specific fields | | +| | that you would like included with | | | | the event. Please completely spell | | | | out a field rather an abbreviate: | | | | i.e. sourceipaddress instead of | | @@ -187,7 +187,7 @@ Mandatory Fields +-----------------+-------------------------------------+-----------------------------------+ | tags | An array or list of any tags you | vpn, audit | | | would like applied to the event | | -| | | nsm,bro,intel | +| | | nsm,zeek,intel | +-----------------+-------------------------------------+-----------------------------------+ | timestamp | Full date plus time timestamp of | 2014-01-30T19:24:43+06:00 | | | the event in ISO format including | | @@ -230,6 +230,9 @@ Details substructure (mandatory if such data is sent, otherwise optional) | error | Action resulted in an | true/false | | | error or failure | | +----------------------+--------------------------+---------------------------------+ +| success | Transaction failed/ | true/false | +| | or succeeded | | ++----------------------+--------------------------+---------------------------------+ | username | Username, email, login, | kang@mozilla.com | | | etc. | | +----------------------+--------------------------+---------------------------------+ @@ -259,7 +262,8 @@ Examples "details": { "username": "joe", "task": "access to admin page /admin_secret_radioactiv", - "result": "10 authentication failures in a row" + "result": "10 authentication failures in a row", + "success": false } } From 8983cd12ec484a521c1e062d0d0d21605d6eb77c Mon Sep 17 00:00:00 2001 From: Jeff Bryner Date: Wed, 12 Dec 2018 09:37:53 -0800 Subject: [PATCH 72/89] tabs/spaces fix --- docs/source/usage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 270819d3..7e2391c4 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -263,7 +263,7 @@ Examples "username": "joe", "task": "access to admin page /admin_secret_radioactiv", "result": "10 authentication failures in a row", - "success": false + "success": false } } From 9693dfa58e4aaa56389260b4bdba9b9031880804 Mon Sep 17 00:00:00 2001 From: Michal Purzynski Date: Wed, 12 Dec 2018 12:47:12 -0800 Subject: [PATCH 73/89] Address nits from the review - use mozdef_util instead of changing the path, remove unnecessary config file --- mq/esworker_github_webhooks.conf | 10 ---------- tests/mq/plugins/test_github_webhooks.py | 1 - 2 files changed, 11 deletions(-) delete mode 100644 mq/esworker_github_webhooks.conf diff --git a/mq/esworker_github_webhooks.conf b/mq/esworker_github_webhooks.conf deleted file mode 100644 index f88d78b7..00000000 --- a/mq/esworker_github_webhooks.conf +++ /dev/null @@ -1,10 +0,0 @@ -[options] -prefetch=10 -esbulksize=10 -esservers=http://elasticsearch:9200 -mqprotocol=sqs -taskexchange= -region = us-west-2 -accesskey = -secretkey = -mqack = False diff --git a/tests/mq/plugins/test_github_webhooks.py b/tests/mq/plugins/test_github_webhooks.py index 377ed4f2..211de579 100644 --- a/tests/mq/plugins/test_github_webhooks.py +++ b/tests/mq/plugins/test_github_webhooks.py @@ -7,7 +7,6 @@ import random import string sys.path.append(os.path.join(os.path.dirname(__file__), '../../../mq/plugins')) -sys.path.append('../../../mozdef_util') from mozdef_util.utilities.toUTC import toUTC from github_webhooks import message From e7cd202d7760406b5e14fb5699944bcc1f735afc Mon Sep 17 00:00:00 2001 From: Jeff Bryner Date: Thu, 13 Dec 2018 16:36:47 -0800 Subject: [PATCH 74/89] update category set use the setter in the mozdef_client object we are using, rather than set a dict key --- cron/duo_logpull.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cron/duo_logpull.py b/cron/duo_logpull.py index 005af422..defd2bcb 100644 --- a/cron/duo_logpull.py +++ b/cron/duo_logpull.py @@ -81,7 +81,7 @@ def process_events(mozmsg, duo_events, etype, state): continue details[i] = e[i] - mozmsg.category= etype + mozmsg.set_category(etype) mozmsg.details = normalize(details) if etype == 'administration': mozmsg.summary = e['action'] @@ -113,7 +113,7 @@ def main(): mozmsg.tags=['duosecurity'] if options.update_tags != '': mozmsg.tags.append(options.update_tags) - mozmsg.category = 'authentication' + mozmsg.set_category('authentication') mozmsg.source = 'DuoSecurityAPI' if options.DEBUG: mozmsg.debug = options.DEBUG From fc771bd531e62f1f33fadf77298ebb954bd5a329 Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Fri, 14 Dec 2018 11:34:42 -0600 Subject: [PATCH 75/89] Remove unused import statements --- .flake8 | 5 ++++- alerts/bugzilla_auth_bruteforce.py | 2 +- alerts/guard_duty_probe.py | 2 +- alerts/http_auth_bruteforce.py | 2 +- alerts/http_errors.py | 2 +- alerts/lib/alert_plugin_set.py | 3 --- alerts/nsm_scan_port.py | 2 +- alerts/plugins/dashboard_geomodel.py | 1 - alerts/proxy_exfil_domains.py | 2 +- alerts/ssh_bruteforce_bro.py | 2 +- alerts/trace_audit.py | 2 +- alerts/write_audit.py | 2 +- benchmarking/workers/json2Mozdef.py | 4 ---- bot/modules/roulette.py | 1 - bot/modules/zilla.py | 2 -- bot/mozdefbot.py | 7 ------- bot/mozdefbot_slack.py | 2 -- cron/collectAttackers.py | 2 -- cron/correlateUserMacAddress.py | 3 --- cron/createFDQNBlockList.py | 1 - cron/createIPBlockList.py | 1 - cron/duo_logpull.py | 1 - cron/esCacheMaint.py | 1 - cron/eventStats.py | 4 +--- cron/google2mozdef.py | 4 ---- cron/healthToMongo.py | 1 - cron/okta2mozdef.py | 5 +---- cron/pruneIndexes.py | 2 -- cron/setupIndexTemplates.py | 1 - cron/sqs_queue_status.py | 10 +--------- cron/syncAlertsToMongo.py | 2 -- cron/verify_event_fields.py | 1 - cron/vidyo2MozDef.py | 8 -------- .../compose/mozdef_bootstrap/files/initial_setup.py | 1 - examples/demo/sampleData2MozDef.py | 12 ------------ examples/es-docs/inject.py | 1 - loginput/index.py | 9 +++------ mq/esworker_eventtask.py | 1 - mq/esworker_papertrail.py | 1 - mq/esworker_sns_sqs.py | 1 - mq/esworker_sqs.py | 1 - mq/lib/plugins.py | 1 - mq/plugins/cloudtrail.py | 2 -- mq/plugins/complianceitems.py | 2 -- mq/plugins/fluentdSqsFixup.py | 2 -- mq/plugins/geoip.py | 1 - mq/plugins/github_webhooks.py | 1 - mq/plugins/guardDuty.py | 2 -- mq/plugins/parse_sshd.py | 3 --- mq/plugins/suricataFixup.py | 1 - mq/plugins/vulnerability.py | 3 +-- rest/index.py | 5 ++--- rest/plugins/vpc_blackhole.py | 2 -- tests/mozdef_util/query_models/test_exists_match.py | 2 -- .../mozdef_util/query_models/test_less_than_match.py | 2 -- tests/mozdef_util/query_models/test_phrase_match.py | 2 -- .../query_models/test_query_string_match.py | 2 -- tests/mozdef_util/query_models/test_range_match.py | 2 -- tests/mozdef_util/query_models/test_term_match.py | 2 -- tests/mozdef_util/query_models/test_terms_match.py | 2 -- .../mozdef_util/query_models/test_wildcard_match.py | 2 -- tests/mozdef_util/test_elasticsearch_client.py | 1 - tests/mozdef_util/test_event.py | 2 -- tests/mozdef_util/test_plugin_set.py | 2 +- tests/mozdef_util/test_state.py | 1 - tests/mozdef_util/utilities/test_key_exists.py | 3 --- tests/mq/plugins/test_github_webhooks.py | 6 ------ tests/mq/plugins/test_suricataFixup.py | 1 - tests/requirements_tests.txt | 1 + tests/suite_helper.py | 4 +--- tests/unit_test_suite.py | 3 --- 71 files changed, 25 insertions(+), 157 deletions(-) diff --git a/.flake8 b/.flake8 index 894ef0c2..18826e92 100644 --- a/.flake8 +++ b/.flake8 @@ -3,6 +3,10 @@ exclude = .flake8 .git *__init__.py +per-file-ignores = + # Ignore 'library imported but unused' for only the alert config files + # since we stub timedelta and crontab + config.py: F401 ignore = E123 # closing bracket does not match indentation of opening bracket's line E225 # missing whitespace around operator @@ -13,7 +17,6 @@ ignore = E402 # module level import not at top of file E501 # line too long E722 # do not use bare except' - F401 # library imported but unused F601 # dictionary key 'tags' repeated with different values F811 # redefinition of unused 'datetime' from line 10 F821 # undefined name 'SysLogHandler' diff --git a/alerts/bugzilla_auth_bruteforce.py b/alerts/bugzilla_auth_bruteforce.py index ff761483..eb891ed7 100644 --- a/alerts/bugzilla_auth_bruteforce.py +++ b/alerts/bugzilla_auth_bruteforce.py @@ -6,7 +6,7 @@ # Copyright (c) 2017 Mozilla Corporation from lib.alerttask import AlertTask -from mozdef_util.query_models import SearchQuery, TermMatch, ExistsMatch, PhraseMatch +from mozdef_util.query_models import SearchQuery, TermMatch, PhraseMatch class AlertBugzillaPBruteforce(AlertTask): diff --git a/alerts/guard_duty_probe.py b/alerts/guard_duty_probe.py index c9e63974..b452e612 100644 --- a/alerts/guard_duty_probe.py +++ b/alerts/guard_duty_probe.py @@ -7,7 +7,7 @@ from lib.alerttask import AlertTask -from mozdef_util.query_models import SearchQuery, TermMatch, QueryStringMatch, ExistsMatch, PhraseMatch, WildcardMatch +from mozdef_util.query_models import SearchQuery, TermMatch, ExistsMatch class AlertGuardDutyProbe(AlertTask): diff --git a/alerts/http_auth_bruteforce.py b/alerts/http_auth_bruteforce.py index 812a178d..2b85d183 100644 --- a/alerts/http_auth_bruteforce.py +++ b/alerts/http_auth_bruteforce.py @@ -6,7 +6,7 @@ # Copyright (c) 2017 Mozilla Corporation from lib.alerttask import AlertTask -from mozdef_util.query_models import SearchQuery, TermMatch, ExistsMatch, PhraseMatch +from mozdef_util.query_models import SearchQuery, TermMatch, PhraseMatch class AlertHTTPBruteforce(AlertTask): diff --git a/alerts/http_errors.py b/alerts/http_errors.py index 4bb1acb9..518fc38c 100644 --- a/alerts/http_errors.py +++ b/alerts/http_errors.py @@ -6,7 +6,7 @@ # Copyright (c) 2017 Mozilla Corporation from lib.alerttask import AlertTask -from mozdef_util.query_models import SearchQuery, TermMatch, ExistsMatch, PhraseMatch +from mozdef_util.query_models import SearchQuery, TermMatch, PhraseMatch class AlertHTTPErrors(AlertTask): diff --git a/alerts/lib/alert_plugin_set.py b/alerts/lib/alert_plugin_set.py index 119fba7f..aa8a32b9 100644 --- a/alerts/lib/alert_plugin_set.py +++ b/alerts/lib/alert_plugin_set.py @@ -1,6 +1,3 @@ -import os -import sys - from mozdef_util.plugin_set import PluginSet from mozdef_util.utilities.logger import logger diff --git a/alerts/nsm_scan_port.py b/alerts/nsm_scan_port.py index 43ae3d65..6192dbbd 100644 --- a/alerts/nsm_scan_port.py +++ b/alerts/nsm_scan_port.py @@ -6,7 +6,7 @@ # Copyright (c) 2018 Mozilla Corporation from lib.alerttask import AlertTask, add_hostname_to_ip -from mozdef_util.query_models import SearchQuery, TermMatch, QueryStringMatch, PhraseMatch, ExistsMatch +from mozdef_util.query_models import SearchQuery, TermMatch, QueryStringMatch, PhraseMatch class NSMScanPort(AlertTask): diff --git a/alerts/plugins/dashboard_geomodel.py b/alerts/plugins/dashboard_geomodel.py index 860060f4..793bd51d 100644 --- a/alerts/plugins/dashboard_geomodel.py +++ b/alerts/plugins/dashboard_geomodel.py @@ -5,7 +5,6 @@ import hjson import os -import sys from binascii import b2a_hex import boto3 import datetime diff --git a/alerts/proxy_exfil_domains.py b/alerts/proxy_exfil_domains.py index 0e15959e..e15772ca 100644 --- a/alerts/proxy_exfil_domains.py +++ b/alerts/proxy_exfil_domains.py @@ -8,7 +8,7 @@ from urlparse import urlparse from lib.alerttask import AlertTask -from mozdef_util.query_models import SearchQuery, TermMatch, QueryStringMatch, ExistsMatch, PhraseMatch, WildcardMatch +from mozdef_util.query_models import SearchQuery, TermMatch, QueryStringMatch class AlertProxyExfilDomains(AlertTask): diff --git a/alerts/ssh_bruteforce_bro.py b/alerts/ssh_bruteforce_bro.py index 99e4a129..de8a3f38 100644 --- a/alerts/ssh_bruteforce_bro.py +++ b/alerts/ssh_bruteforce_bro.py @@ -6,7 +6,7 @@ # Copyright (c) 2017 Mozilla Corporation from lib.alerttask import AlertTask -from mozdef_util.query_models import SearchQuery, TermMatch, ExistsMatch, PhraseMatch +from mozdef_util.query_models import SearchQuery, TermMatch, PhraseMatch class AlertSSHManyConns(AlertTask): diff --git a/alerts/trace_audit.py b/alerts/trace_audit.py index 26e61980..016ee76f 100644 --- a/alerts/trace_audit.py +++ b/alerts/trace_audit.py @@ -8,7 +8,7 @@ # This code alerts on every successfully opened session on any of the host from a given list from lib.alerttask import AlertTask -from mozdef_util.query_models import SearchQuery, TermMatch, QueryStringMatch, PhraseMatch +from mozdef_util.query_models import SearchQuery, TermMatch, PhraseMatch class TraceAudit(AlertTask): diff --git a/alerts/write_audit.py b/alerts/write_audit.py index f896fdc0..a04fd06e 100644 --- a/alerts/write_audit.py +++ b/alerts/write_audit.py @@ -8,7 +8,7 @@ # This code alerts on every successfully opened session on any of the host from a given list from lib.alerttask import AlertTask -from mozdef_util.query_models import SearchQuery, TermMatch, QueryStringMatch, PhraseMatch +from mozdef_util.query_models import SearchQuery, TermMatch, PhraseMatch class WriteAudit(AlertTask): diff --git a/benchmarking/workers/json2Mozdef.py b/benchmarking/workers/json2Mozdef.py index 6a05b294..e15fd2df 100755 --- a/benchmarking/workers/json2Mozdef.py +++ b/benchmarking/workers/json2Mozdef.py @@ -11,16 +11,12 @@ from datetime import datetime import pytz import json import socket -import json from optparse import OptionParser from requests_futures.sessions import FuturesSession from multiprocessing import Process, Queue -import random import logging -from logging.handlers import SysLogHandler from Queue import Empty from requests.packages.urllib3.exceptions import ClosedPoolError -import requests import time httpsession = FuturesSession(max_workers=5) diff --git a/bot/modules/roulette.py b/bot/modules/roulette.py index 162b9688..daddac61 100644 --- a/bot/modules/roulette.py +++ b/bot/modules/roulette.py @@ -10,7 +10,6 @@ import logging import random -from kitnirc.client import Channel from kitnirc.modular import Module from kitnirc.user import User diff --git a/bot/modules/zilla.py b/bot/modules/zilla.py index fe425557..12fc0250 100644 --- a/bot/modules/zilla.py +++ b/bot/modules/zilla.py @@ -7,9 +7,7 @@ # Copyright (c) 2014 Mozilla Corporation import logging -from kitnirc.client import Channel from kitnirc.modular import Module -from kitnirc.user import User import threading import time import json diff --git a/bot/mozdefbot.py b/bot/mozdefbot.py index 7ebdd36b..505986fd 100755 --- a/bot/mozdefbot.py +++ b/bot/mozdefbot.py @@ -8,25 +8,18 @@ import json import kitnirc.client import kitnirc.modular -import kombu import logging import netaddr import os -import pytz import random -import select import sys import time -import threading from configlib import getConfig, OptionParser from datetime import datetime -from dateutil.parser import parse from kombu import Connection, Queue, Exchange from kombu.mixins import ConsumerMixin from ipwhois import IPWhois -import sys -import os from mozdef_util.utilities.toUTC import toUTC from mozdef_util.geo_ip import GeoIP diff --git a/bot/mozdefbot_slack.py b/bot/mozdefbot_slack.py index 57ff8975..c87fb3a0 100644 --- a/bot/mozdefbot_slack.py +++ b/bot/mozdefbot_slack.py @@ -20,8 +20,6 @@ from kombu.mixins import ConsumerMixin from slackclient import SlackClient -import sys -import os from mozdef_util.utilities.toUTC import toUTC diff --git a/cron/collectAttackers.py b/cron/collectAttackers.py index 6769ef7e..2a9e8e73 100755 --- a/cron/collectAttackers.py +++ b/cron/collectAttackers.py @@ -19,8 +19,6 @@ from pymongo import MongoClient from collections import Counter from kombu import Connection, Exchange -import sys -import os from mozdef_util.utilities.toUTC import toUTC from mozdef_util.elasticsearch_client import ElasticsearchClient from mozdef_util.query_models import SearchQuery, PhraseMatch diff --git a/cron/correlateUserMacAddress.py b/cron/correlateUserMacAddress.py index 374b51ee..b2ac8699 100755 --- a/cron/correlateUserMacAddress.py +++ b/cron/correlateUserMacAddress.py @@ -7,7 +7,6 @@ import json import logging -import os import re import sys from datetime import datetime @@ -15,8 +14,6 @@ from configlib import getConfig, OptionParser from logging.handlers import SysLogHandler from hashlib import md5 -import sys -import os from mozdef_util.utilities.toUTC import toUTC from mozdef_util.elasticsearch_client import ElasticsearchClient, ElasticsearchBadServer from mozdef_util.query_models import SearchQuery, TermMatch, PhraseMatch diff --git a/cron/createFDQNBlockList.py b/cron/createFDQNBlockList.py index 4b636693..e04292c4 100644 --- a/cron/createFDQNBlockList.py +++ b/cron/createFDQNBlockList.py @@ -17,7 +17,6 @@ from configlib import getConfig, OptionParser from logging.handlers import SysLogHandler from pymongo import MongoClient -import os from mozdef_util.utilities.toUTC import toUTC diff --git a/cron/createIPBlockList.py b/cron/createIPBlockList.py index 3904ec69..e53553a1 100755 --- a/cron/createIPBlockList.py +++ b/cron/createIPBlockList.py @@ -17,7 +17,6 @@ from configlib import getConfig, OptionParser from logging.handlers import SysLogHandler from pymongo import MongoClient -import os from mozdef_util.utilities.toUTC import toUTC diff --git a/cron/duo_logpull.py b/cron/duo_logpull.py index defd2bcb..6acb7a31 100644 --- a/cron/duo_logpull.py +++ b/cron/duo_logpull.py @@ -5,7 +5,6 @@ # import sys -import os from datetime import datetime, timedelta, tzinfo try: from datetime import timezone diff --git a/cron/esCacheMaint.py b/cron/esCacheMaint.py index a220ed26..ca45ddbc 100755 --- a/cron/esCacheMaint.py +++ b/cron/esCacheMaint.py @@ -13,7 +13,6 @@ import logging from configlib import getConfig, OptionParser from datetime import datetime, date, timedelta -import os from mozdef_util.elasticsearch_client import ElasticsearchClient from mozdef_util.utilities.logger import logger diff --git a/cron/eventStats.py b/cron/eventStats.py index e64743ac..c4eb2dd4 100755 --- a/cron/eventStats.py +++ b/cron/eventStats.py @@ -15,10 +15,8 @@ from logging.handlers import SysLogHandler from time import sleep import socket -import sys -import os from mozdef_util.utilities.toUTC import toUTC -from mozdef_util.elasticsearch_client import ElasticsearchClient, ElasticsearchBadServer +from mozdef_util.elasticsearch_client import ElasticsearchClient from mozdef_util.query_models import SearchQuery, Aggregation logger = logging.getLogger(sys.argv[0]) diff --git a/cron/google2mozdef.py b/cron/google2mozdef.py index bbd72666..0636f3da 100755 --- a/cron/google2mozdef.py +++ b/cron/google2mozdef.py @@ -5,21 +5,17 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # Copyright (c) 2014 Mozilla Corporation -import os import sys import logging import requests import json from configlib import getConfig, OptionParser from datetime import datetime -from datetime import timedelta from logging.handlers import SysLogHandler from httplib2 import Http from oauth2client.client import SignedJwtAssertionCredentials from apiclient.discovery import build -import sys -import os from mozdef_util.utilities.toUTC import toUTC logger = logging.getLogger(sys.argv[0]) diff --git a/cron/healthToMongo.py b/cron/healthToMongo.py index 07a0c99b..7a15c857 100755 --- a/cron/healthToMongo.py +++ b/cron/healthToMongo.py @@ -14,7 +14,6 @@ from configlib import getConfig, OptionParser from logging.handlers import SysLogHandler from pymongo import MongoClient -import os from mozdef_util.utilities.toUTC import toUTC from mozdef_util.elasticsearch_client import ElasticsearchClient from mozdef_util.query_models import SearchQuery, TermMatch diff --git a/cron/okta2mozdef.py b/cron/okta2mozdef.py index 5cc88c5b..b869484f 100755 --- a/cron/okta2mozdef.py +++ b/cron/okta2mozdef.py @@ -5,9 +5,8 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # Copyright (c) 2014 Mozilla Corporation -import os import sys -from configlib import getConfig,OptionParser +from configlib import getConfig, OptionParser import logging from logging.handlers import SysLogHandler import json @@ -17,8 +16,6 @@ from datetime import date import requests import netaddr -import sys -import os from mozdef_util.utilities.toUTC import toUTC from mozdef_util.elasticsearch_client import ElasticsearchClient diff --git a/cron/pruneIndexes.py b/cron/pruneIndexes.py index 57e1a1f1..2b17e82c 100755 --- a/cron/pruneIndexes.py +++ b/cron/pruneIndexes.py @@ -18,8 +18,6 @@ from datetime import date from datetime import timedelta from configlib import getConfig, OptionParser -import sys -import os from mozdef_util.utilities.toUTC import toUTC from mozdef_util.elasticsearch_client import ElasticsearchClient diff --git a/cron/setupIndexTemplates.py b/cron/setupIndexTemplates.py index a74d93cd..e1631b43 100755 --- a/cron/setupIndexTemplates.py +++ b/cron/setupIndexTemplates.py @@ -9,7 +9,6 @@ # You only need to run it once, it will setup the templates # used as future indexes are created -import requests import sys import os from configlib import getConfig, OptionParser diff --git a/cron/sqs_queue_status.py b/cron/sqs_queue_status.py index d195e5d0..cd29d903 100644 --- a/cron/sqs_queue_status.py +++ b/cron/sqs_queue_status.py @@ -14,22 +14,14 @@ import json import os import sys -import socket -import time from configlib import getConfig, OptionParser from datetime import datetime from hashlib import md5 import boto.sqs -from boto.sqs.message import RawMessage -import base64 -import kombu from mozdef_util.utilities.toUTC import toUTC -from mozdef_util.utilities.to_unicode import toUnicode -from mozdef_util.utilities.remove_at import removeAt -from mozdef_util.utilities.is_cef import isCEF from mozdef_util.utilities.logger import logger, initLogger -from mozdef_util.elasticsearch_client import ElasticsearchClient, ElasticsearchBadServer, ElasticsearchInvalidIndex, ElasticsearchException +from mozdef_util.elasticsearch_client import ElasticsearchClient def getDocID(sqsregionidentifier): diff --git a/cron/syncAlertsToMongo.py b/cron/syncAlertsToMongo.py index 36ac1f1b..c3b4a729 100755 --- a/cron/syncAlertsToMongo.py +++ b/cron/syncAlertsToMongo.py @@ -14,8 +14,6 @@ from configlib import getConfig, OptionParser from logging.handlers import SysLogHandler from pymongo import MongoClient -import sys -import os from mozdef_util.utilities.toUTC import toUTC from mozdef_util.elasticsearch_client import ElasticsearchClient from mozdef_util.query_models import SearchQuery, TermMatch diff --git a/cron/verify_event_fields.py b/cron/verify_event_fields.py index 5688a177..931f2109 100644 --- a/cron/verify_event_fields.py +++ b/cron/verify_event_fields.py @@ -6,7 +6,6 @@ # Copyright (c) 2017 Mozilla Corporation import sys -import os from configlib import getConfig, OptionParser from mozdef_util.elasticsearch_client import ElasticsearchClient diff --git a/cron/vidyo2MozDef.py b/cron/vidyo2MozDef.py index 5d1f684d..1ef0a498 100755 --- a/cron/vidyo2MozDef.py +++ b/cron/vidyo2MozDef.py @@ -8,21 +8,13 @@ import copy import os import sys -import re import json -import csv -import string import ConfigParser -import tempfile -import logging import socket -import hashlib import MySQLdb from requests import Session from optparse import OptionParser from datetime import datetime -from os import stat -from os.path import exists, getsize class MozDefError(Exception): diff --git a/docker/compose/mozdef_bootstrap/files/initial_setup.py b/docker/compose/mozdef_bootstrap/files/initial_setup.py index a0bb2139..87a75d46 100644 --- a/docker/compose/mozdef_bootstrap/files/initial_setup.py +++ b/docker/compose/mozdef_bootstrap/files/initial_setup.py @@ -14,7 +14,6 @@ import json from elasticsearch.exceptions import ConnectionError -import sys import os from mozdef_util.elasticsearch_client import ElasticsearchClient diff --git a/examples/demo/sampleData2MozDef.py b/examples/demo/sampleData2MozDef.py index e2373648..019265eb 100755 --- a/examples/demo/sampleData2MozDef.py +++ b/examples/demo/sampleData2MozDef.py @@ -10,28 +10,16 @@ import sys from datetime import datetime import pytz import json -import socket -import json -from requests_futures.sessions import FuturesSession from multiprocessing import Process, Queue import random import logging -from logging.handlers import SysLogHandler from Queue import Empty -from requests.packages.urllib3.exceptions import ClosedPoolError import requests import time from configlib import getConfig, OptionParser import ConfigParser import glob -from datetime import datetime from datetime import timedelta -from dateutil.parser import parse -from datetime import date -import pytz - -import sys -import os from mozdef_util.utilities.toUTC import toUTC diff --git a/examples/es-docs/inject.py b/examples/es-docs/inject.py index 96b13ac2..433af3a4 100644 --- a/examples/es-docs/inject.py +++ b/examples/es-docs/inject.py @@ -7,7 +7,6 @@ import os import sys -import inspect from configlib import getConfig, OptionParser sys.path.insert(1, os.path.join(sys.path[0], '../..')) diff --git a/loginput/index.py b/loginput/index.py index d977523a..14b684fd 100644 --- a/loginput/index.py +++ b/loginput/index.py @@ -4,14 +4,11 @@ # Copyright (c) 2017 Mozilla Corporation import os -import sys -import bottle -from bottle import debug,route, run, template, response,request,post, default_app +from bottle import route, run, response, request, default_app from bottle import _stdout as bottlelog -import kombu -from kombu import Connection,Queue,Exchange +from kombu import Connection, Queue, Exchange import json -from configlib import getConfig,OptionParser +from configlib import getConfig, OptionParser @route('/status') diff --git a/mq/esworker_eventtask.py b/mq/esworker_eventtask.py index 93a6c484..d484cb05 100755 --- a/mq/esworker_eventtask.py +++ b/mq/esworker_eventtask.py @@ -8,7 +8,6 @@ import json import kombu -import os import sys import socket from configlib import getConfig, OptionParser diff --git a/mq/esworker_papertrail.py b/mq/esworker_papertrail.py index 2fcacbf3..4f622ec3 100755 --- a/mq/esworker_papertrail.py +++ b/mq/esworker_papertrail.py @@ -10,7 +10,6 @@ import json -import os import kombu import sys import socket diff --git a/mq/esworker_sns_sqs.py b/mq/esworker_sns_sqs.py index 374f0502..21e28249 100755 --- a/mq/esworker_sns_sqs.py +++ b/mq/esworker_sns_sqs.py @@ -7,7 +7,6 @@ import json -import os import sys import socket diff --git a/mq/esworker_sqs.py b/mq/esworker_sqs.py index b3fa1a6c..0cbcdab9 100755 --- a/mq/esworker_sqs.py +++ b/mq/esworker_sqs.py @@ -12,7 +12,6 @@ import json -import os import sys import socket import time diff --git a/mq/lib/plugins.py b/mq/lib/plugins.py index 1c7dcb45..751be546 100644 --- a/mq/lib/plugins.py +++ b/mq/lib/plugins.py @@ -6,7 +6,6 @@ # Copyright (c) 2017 Mozilla Corporation -import sys import os from operator import itemgetter from datetime import datetime diff --git a/mq/plugins/cloudtrail.py b/mq/plugins/cloudtrail.py index 0cb8f20a..88619fa6 100644 --- a/mq/plugins/cloudtrail.py +++ b/mq/plugins/cloudtrail.py @@ -3,8 +3,6 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # Copyright (c) 2017 Mozilla Corporation -import sys -import os from mozdef_util.utilities.key_exists import key_exists diff --git a/mq/plugins/complianceitems.py b/mq/plugins/complianceitems.py index 6737cf2f..d1ad3d43 100644 --- a/mq/plugins/complianceitems.py +++ b/mq/plugins/complianceitems.py @@ -4,8 +4,6 @@ # Copyright (c) 2014 Mozilla Corporation import hashlib -import sys -import os from mozdef_util.utilities.logger import logger diff --git a/mq/plugins/fluentdSqsFixup.py b/mq/plugins/fluentdSqsFixup.py index 2192d7a6..7d2d0297 100644 --- a/mq/plugins/fluentdSqsFixup.py +++ b/mq/plugins/fluentdSqsFixup.py @@ -6,8 +6,6 @@ # This script copies the format/handling mechanism of ipFixup.py (git f5734b0c7e412424b44a6d7af149de6250fc70a2) import netaddr -import sys -import os from mozdef_util.utilities.toUTC import toUTC diff --git a/mq/plugins/geoip.py b/mq/plugins/geoip.py index 34db7494..33360b07 100644 --- a/mq/plugins/geoip.py +++ b/mq/plugins/geoip.py @@ -6,7 +6,6 @@ import netaddr import os -import sys from mozdef_util.geo_ip import GeoIP diff --git a/mq/plugins/github_webhooks.py b/mq/plugins/github_webhooks.py index 50ea3f7e..82aae524 100644 --- a/mq/plugins/github_webhooks.py +++ b/mq/plugins/github_webhooks.py @@ -3,7 +3,6 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # Copyright (c) 2017 Mozilla Corporation -import sys import os import jmespath import yaml diff --git a/mq/plugins/guardDuty.py b/mq/plugins/guardDuty.py index b63f0976..dc6a263e 100644 --- a/mq/plugins/guardDuty.py +++ b/mq/plugins/guardDuty.py @@ -3,8 +3,6 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # Copyright (c) 2017 Mozilla Corporation -import sys -import os from mozdef_util.utilities.key_exists import key_exists from mozdef_util.utilities.toUTC import toUTC from mozdef_util.utilities.dot_dict import DotDict diff --git a/mq/plugins/parse_sshd.py b/mq/plugins/parse_sshd.py index e4715685..6c4ea72b 100644 --- a/mq/plugins/parse_sshd.py +++ b/mq/plugins/parse_sshd.py @@ -3,10 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # Copyright (c) 2017 Mozilla Corporation -import sys -import os import re -from mozdef_util.utilities.toUTC import toUTC class message(object): diff --git a/mq/plugins/suricataFixup.py b/mq/plugins/suricataFixup.py index 01104168..e792e90c 100644 --- a/mq/plugins/suricataFixup.py +++ b/mq/plugins/suricataFixup.py @@ -3,7 +3,6 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # Copyright (c) 2018 Mozilla Corporation -import netaddr import json from datetime import datetime from platform import node diff --git a/mq/plugins/vulnerability.py b/mq/plugins/vulnerability.py index 9fbaca18..7d76c731 100644 --- a/mq/plugins/vulnerability.py +++ b/mq/plugins/vulnerability.py @@ -4,8 +4,7 @@ # Copyright (c) 2015 Mozilla Corporation import hashlib -import sys -import os + from mozdef_util.utilities.logger import logger diff --git a/rest/index.py b/rest/index.py index 41f1c043..6d2a5c50 100644 --- a/rest/index.py +++ b/rest/index.py @@ -14,7 +14,7 @@ import requests import sys import socket from bottle import route, run, response, request, default_app, post -from datetime import datetime, timedelta +from datetime import datetime from configlib import getConfig, OptionParser from ipwhois import IPWhois from operator import itemgetter @@ -22,9 +22,8 @@ from pymongo import MongoClient from bson import json_util from mozdef_util.elasticsearch_client import ElasticsearchClient, ElasticsearchInvalidIndex -from mozdef_util.query_models import SearchQuery, TermMatch, RangeMatch, Aggregation +from mozdef_util.query_models import SearchQuery, TermMatch -from mozdef_util.utilities.toUTC import toUTC from mozdef_util.utilities.logger import logger, initLogger diff --git a/rest/plugins/vpc_blackhole.py b/rest/plugins/vpc_blackhole.py index 5a65acaa..6ecaa59f 100644 --- a/rest/plugins/vpc_blackhole.py +++ b/rest/plugins/vpc_blackhole.py @@ -6,8 +6,6 @@ import os import sys import ConfigParser -from datetime import datetime, timedelta -import json import netaddr from boto3.session import Session diff --git a/tests/mozdef_util/query_models/test_exists_match.py b/tests/mozdef_util/query_models/test_exists_match.py index d62200e8..4c679f74 100644 --- a/tests/mozdef_util/query_models/test_exists_match.py +++ b/tests/mozdef_util/query_models/test_exists_match.py @@ -1,8 +1,6 @@ from positive_test_suite import PositiveTestSuite from negative_test_suite import NegativeTestSuite -import os -import sys from mozdef_util.query_models import ExistsMatch diff --git a/tests/mozdef_util/query_models/test_less_than_match.py b/tests/mozdef_util/query_models/test_less_than_match.py index 99e26113..3f5de83f 100644 --- a/tests/mozdef_util/query_models/test_less_than_match.py +++ b/tests/mozdef_util/query_models/test_less_than_match.py @@ -1,8 +1,6 @@ from positive_test_suite import PositiveTestSuite from negative_test_suite import NegativeTestSuite -import os -import sys from mozdef_util.query_models import LessThanMatch diff --git a/tests/mozdef_util/query_models/test_phrase_match.py b/tests/mozdef_util/query_models/test_phrase_match.py index 8dc72890..7a948f67 100644 --- a/tests/mozdef_util/query_models/test_phrase_match.py +++ b/tests/mozdef_util/query_models/test_phrase_match.py @@ -1,8 +1,6 @@ from positive_test_suite import PositiveTestSuite from negative_test_suite import NegativeTestSuite -import os -import sys from mozdef_util.query_models import PhraseMatch diff --git a/tests/mozdef_util/query_models/test_query_string_match.py b/tests/mozdef_util/query_models/test_query_string_match.py index f7b1f286..7be4305c 100644 --- a/tests/mozdef_util/query_models/test_query_string_match.py +++ b/tests/mozdef_util/query_models/test_query_string_match.py @@ -1,8 +1,6 @@ from positive_test_suite import PositiveTestSuite from negative_test_suite import NegativeTestSuite -import os -import sys from mozdef_util.query_models import QueryStringMatch diff --git a/tests/mozdef_util/query_models/test_range_match.py b/tests/mozdef_util/query_models/test_range_match.py index c920f291..7a63e615 100644 --- a/tests/mozdef_util/query_models/test_range_match.py +++ b/tests/mozdef_util/query_models/test_range_match.py @@ -1,8 +1,6 @@ from positive_test_suite import PositiveTestSuite from negative_test_suite import NegativeTestSuite -import os -import sys from mozdef_util.query_models import RangeMatch diff --git a/tests/mozdef_util/query_models/test_term_match.py b/tests/mozdef_util/query_models/test_term_match.py index 270e363d..4beb848e 100644 --- a/tests/mozdef_util/query_models/test_term_match.py +++ b/tests/mozdef_util/query_models/test_term_match.py @@ -1,8 +1,6 @@ from positive_test_suite import PositiveTestSuite from negative_test_suite import NegativeTestSuite -import os -import sys from mozdef_util.query_models import TermMatch diff --git a/tests/mozdef_util/query_models/test_terms_match.py b/tests/mozdef_util/query_models/test_terms_match.py index fb224385..75236410 100644 --- a/tests/mozdef_util/query_models/test_terms_match.py +++ b/tests/mozdef_util/query_models/test_terms_match.py @@ -1,8 +1,6 @@ from positive_test_suite import PositiveTestSuite from negative_test_suite import NegativeTestSuite -import os -import sys from mozdef_util.query_models import TermsMatch diff --git a/tests/mozdef_util/query_models/test_wildcard_match.py b/tests/mozdef_util/query_models/test_wildcard_match.py index 97607d8f..77a450b5 100644 --- a/tests/mozdef_util/query_models/test_wildcard_match.py +++ b/tests/mozdef_util/query_models/test_wildcard_match.py @@ -1,8 +1,6 @@ from positive_test_suite import PositiveTestSuite from negative_test_suite import NegativeTestSuite -import os -import sys from mozdef_util.query_models import WildcardMatch diff --git a/tests/mozdef_util/test_elasticsearch_client.py b/tests/mozdef_util/test_elasticsearch_client.py index 01e45a4d..beb6f0f6 100644 --- a/tests/mozdef_util/test_elasticsearch_client.py +++ b/tests/mozdef_util/test_elasticsearch_client.py @@ -9,7 +9,6 @@ import time import json import pytest -from elasticsearch.exceptions import ConnectionTimeout import os import sys diff --git a/tests/mozdef_util/test_event.py b/tests/mozdef_util/test_event.py index b1a4b79e..89640edc 100644 --- a/tests/mozdef_util/test_event.py +++ b/tests/mozdef_util/test_event.py @@ -1,5 +1,3 @@ -import os -import sys from mozdef_util.event import Event from mozdef_util.utilities.toUTC import toUTC import socket diff --git a/tests/mozdef_util/test_plugin_set.py b/tests/mozdef_util/test_plugin_set.py index 75a46ee5..b8927b99 100644 --- a/tests/mozdef_util/test_plugin_set.py +++ b/tests/mozdef_util/test_plugin_set.py @@ -1,5 +1,5 @@ import os -import sys + from mozdef_util.plugin_set import PluginSet diff --git a/tests/mozdef_util/test_state.py b/tests/mozdef_util/test_state.py index 73131a9c..be9d9bb1 100644 --- a/tests/mozdef_util/test_state.py +++ b/tests/mozdef_util/test_state.py @@ -1,7 +1,6 @@ import pytest import os -import sys from mozdef_util.state import State, StateParsingError states_directory_location = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'states') diff --git a/tests/mozdef_util/utilities/test_key_exists.py b/tests/mozdef_util/utilities/test_key_exists.py index 25ae4c28..2c0700e6 100644 --- a/tests/mozdef_util/utilities/test_key_exists.py +++ b/tests/mozdef_util/utilities/test_key_exists.py @@ -5,9 +5,6 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. # Copyright (c) 2017 Mozilla Corporation - -import os -import sys from mozdef_util.utilities.key_exists import key_exists diff --git a/tests/mq/plugins/test_github_webhooks.py b/tests/mq/plugins/test_github_webhooks.py index 211de579..6fc56c37 100644 --- a/tests/mq/plugins/test_github_webhooks.py +++ b/tests/mq/plugins/test_github_webhooks.py @@ -1,13 +1,7 @@ import os import sys -import mock -import json -import random -import string - sys.path.append(os.path.join(os.path.dirname(__file__), '../../../mq/plugins')) -from mozdef_util.utilities.toUTC import toUTC from github_webhooks import message diff --git a/tests/mq/plugins/test_suricataFixup.py b/tests/mq/plugins/test_suricataFixup.py index dd76f0d7..43e08a9b 100644 --- a/tests/mq/plugins/test_suricataFixup.py +++ b/tests/mq/plugins/test_suricataFixup.py @@ -2,7 +2,6 @@ import os import sys from mozdef_util.utilities.toUTC import toUTC -import mock import json import random import string diff --git a/tests/requirements_tests.txt b/tests/requirements_tests.txt index bf5488a5..093d85d6 100644 --- a/tests/requirements_tests.txt +++ b/tests/requirements_tests.txt @@ -1,5 +1,6 @@ freezegun==0.3.9 flake8==3.5.0 +flake8-per-file-ignores==0.6 mock==2.0.0 pytest==3.1.1 WebTest==2.0.27 diff --git a/tests/suite_helper.py b/tests/suite_helper.py index 84e5a3f9..79966c3b 100644 --- a/tests/suite_helper.py +++ b/tests/suite_helper.py @@ -9,11 +9,9 @@ from configlib import getConfig from kombu import Connection, Queue, Exchange - import os -import sys -from mozdef_util.elasticsearch_client import ElasticsearchClient +from mozdef_util.elasticsearch_client import ElasticsearchClient from mozdef_util.utilities.dot_dict import DotDict diff --git a/tests/unit_test_suite.py b/tests/unit_test_suite.py index 38784259..b7dde0d9 100644 --- a/tests/unit_test_suite.py +++ b/tests/unit_test_suite.py @@ -11,9 +11,6 @@ from dateutil.parser import parse import random import pytest -import os -import sys - from mozdef_util.utilities import toUTC from suite_helper import parse_config_file, parse_mapping_file, setup_es_client, setup_rabbitmq_client From 00c6715ddfd0d27b82940b94cc5f70e2540c739e Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Fri, 14 Dec 2018 11:36:11 -0600 Subject: [PATCH 76/89] More specific ignore in flake8 config --- .flake8 | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.flake8 b/.flake8 index 18826e92..5c79564c 100644 --- a/.flake8 +++ b/.flake8 @@ -1,12 +1,11 @@ [flake8] -exclude = - .flake8 - .git - *__init__.py per-file-ignores = # Ignore 'library imported but unused' for only the alert config files # since we stub timedelta and crontab - config.py: F401 + alerts/lib/config.py: F401 + docker/compose/mozdef_alerts/files/config.py: F401 + # Ignore any import statements in __init__ files + mozdef_util/mozdef_util/query_models/__init__.py: F401 ignore = E123 # closing bracket does not match indentation of opening bracket's line E225 # missing whitespace around operator From d04485c8505b6fabb50cc8806840447bbd28f6bf Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Fri, 14 Dec 2018 12:27:57 -0600 Subject: [PATCH 77/89] Fixup pep8 undefined library --- .flake8 | 1 - bot/mozdefbot.py | 9 --------- cron/pruneIndexes.py | 2 ++ mq/esworker_cloudtrail.py | 1 + 4 files changed, 3 insertions(+), 10 deletions(-) diff --git a/.flake8 b/.flake8 index 5c79564c..7663b043 100644 --- a/.flake8 +++ b/.flake8 @@ -18,6 +18,5 @@ ignore = E722 # do not use bare except' F601 # dictionary key 'tags' repeated with different values F811 # redefinition of unused 'datetime' from line 10 - F821 # undefined name 'SysLogHandler' F841 # local variable 'CIDR' is assigned to but never used W503 # line break before binary operator diff --git a/bot/mozdefbot.py b/bot/mozdefbot.py index 505986fd..21070a93 100755 --- a/bot/mozdefbot.py +++ b/bot/mozdefbot.py @@ -190,15 +190,6 @@ class mozdefBot(): # start the mq consumer consumeAlerts(self) - @self.client.handle('LINE') - def line_handler(client, *params): - try: - self.root_logger.debug('linegot:' + line) - except AttributeError as e: - # catch error in kitnrc : chan.remove(actor) where channel - # object has no attribute remove - pass - @self.client.handle('PRIVMSG') def priv_handler(client, actor, recipient, message): self.root_logger.debug( diff --git a/cron/pruneIndexes.py b/cron/pruneIndexes.py index 2b17e82c..c66db742 100755 --- a/cron/pruneIndexes.py +++ b/cron/pruneIndexes.py @@ -18,6 +18,8 @@ from datetime import date from datetime import timedelta from configlib import getConfig, OptionParser +from logging.handlers import SysLogHandler + from mozdef_util.utilities.toUTC import toUTC from mozdef_util.elasticsearch_client import ElasticsearchClient diff --git a/mq/esworker_cloudtrail.py b/mq/esworker_cloudtrail.py index b59bf73e..1fe0a4f6 100755 --- a/mq/esworker_cloudtrail.py +++ b/mq/esworker_cloudtrail.py @@ -20,6 +20,7 @@ from StringIO import StringIO from threading import Timer import re import time +import kombu from ssl import SSLEOFError, SSLError from mozdef_util.utilities.toUTC import toUTC From 07f5072946f44d9f61c0dfc72445caf6caeb7c9d Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Fri, 14 Dec 2018 12:29:04 -0600 Subject: [PATCH 78/89] Remove duplicate keyname in dict --- .flake8 | 1 - tests/mozdef_util/test_event.py | 1 - 2 files changed, 2 deletions(-) diff --git a/.flake8 b/.flake8 index 7663b043..cedc1ee3 100644 --- a/.flake8 +++ b/.flake8 @@ -16,7 +16,6 @@ ignore = E402 # module level import not at top of file E501 # line too long E722 # do not use bare except' - F601 # dictionary key 'tags' repeated with different values F811 # redefinition of unused 'datetime' from line 10 F841 # local variable 'CIDR' is assigned to but never used W503 # line break before binary operator diff --git a/tests/mozdef_util/test_event.py b/tests/mozdef_util/test_event.py index 89640edc..4b83a1fb 100644 --- a/tests/mozdef_util/test_event.py +++ b/tests/mozdef_util/test_event.py @@ -58,7 +58,6 @@ class TestEvent(object): 'utctimestamp': '2017-09-14T20:05:20.299387+00:00', 'timestamp': '2017-09-14T20:05:19.116195+00:00', 'mozdefhostname': 'randomhostname', - 'tags': [], 'category': 'Authentication', 'hostname': 'host.domain.com', 'processid': 12345, From 0d7ece3d05517e17acb6f71175d319ae7a07dad6 Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Fri, 14 Dec 2018 12:31:08 -0600 Subject: [PATCH 79/89] Fixup W503 line break before binary operation --- .flake8 | 1 - bot/mozdefbot.py | 9 +++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.flake8 b/.flake8 index cedc1ee3..f61cf94a 100644 --- a/.flake8 +++ b/.flake8 @@ -18,4 +18,3 @@ ignore = E722 # do not use bare except' F811 # redefinition of unused 'datetime' from line 10 F841 # local variable 'CIDR' is assigned to but never used - W503 # line break before binary operator diff --git a/bot/mozdefbot.py b/bot/mozdefbot.py index 21070a93..b7d2facd 100755 --- a/bot/mozdefbot.py +++ b/bot/mozdefbot.py @@ -144,10 +144,11 @@ def formatAlert(jsonDictIn): if 'category' in jsonDictIn.keys(): category = jsonDictIn['category'] - return colorify('{0}: {1} {2}'.format(severity, colors['blue'] - + category - + colors['normal'], - summary.encode('ascii', 'replace'))) + return colorify('{0}: {1} {2}'.format( + severity, + colors['blue'] + category + colors['normal'], + summary.encode('ascii', 'replace') + )) class mozdefBot(): From feaa882e99bd64e60768c64c15d764bf62e47ebf Mon Sep 17 00:00:00 2001 From: Jeff Bryner Date: Fri, 14 Dec 2018 10:33:04 -0800 Subject: [PATCH 80/89] match mozdef_client's expectation for set_category --- cron/auth02mozdef.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cron/auth02mozdef.py b/cron/auth02mozdef.py index 1f448abb..825a4aae 100644 --- a/cron/auth02mozdef.py +++ b/cron/auth02mozdef.py @@ -271,7 +271,7 @@ def process_msg(mozmsg, msg): failed_words = ['Failed'] # default category (might be modified below to be more specific) - mozmsg.category = 'iam' + mozmsg.set_category('iam') mozmsg.source = 'auth0' # fields that should always exist mozmsg.timestamp = msg.date @@ -307,9 +307,9 @@ def process_msg(mozmsg, msg): details['eventname'] = log_types[msg.type].event # determine the event category if any(authword in details['eventname'] for authword in authentication_words): - mozmsg.category = "authentication" + mozmsg.set_category("authentication") if any(authword in details['eventname'] for authword in authorization_words): - mozmsg.category = "authorization" + mozmsg.set_category("authorization") # determine success/failure if any(failword in details['eventname'] for failword in failed_words): details.success = False @@ -337,7 +337,7 @@ def process_msg(mozmsg, msg): details['description'] = "" # set the summary - if 'auth' in mozmsg.category: + if 'auth' in mozmsg._category: # make summary be action/username (success login user@place.com) mozmsg.summary = "{event} {desc}".format( event=details.eventname, From 4d5f70295c5ed3f6be66ba44406063bafacddd39 Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Fri, 14 Dec 2018 12:35:18 -0600 Subject: [PATCH 81/89] Fixup redefinition of unused import statements --- .flake8 | 6 +++++- cron/healthAndStatus.py | 2 -- cron/rotateIndexes.py | 1 - tests/mozdef_util/utilities/test_toUTC.py | 3 --- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.flake8 b/.flake8 index f61cf94a..e93ac1ea 100644 --- a/.flake8 +++ b/.flake8 @@ -4,8 +4,13 @@ per-file-ignores = # since we stub timedelta and crontab alerts/lib/config.py: F401 docker/compose/mozdef_alerts/files/config.py: F401 + # Ignore any import statements in __init__ files mozdef_util/mozdef_util/query_models/__init__.py: F401 + + # Ignore redefinition of index name + rest/index.py: F811 + ignore = E123 # closing bracket does not match indentation of opening bracket's line E225 # missing whitespace around operator @@ -16,5 +21,4 @@ ignore = E402 # module level import not at top of file E501 # line too long E722 # do not use bare except' - F811 # redefinition of unused 'datetime' from line 10 F841 # local variable 'CIDR' is assigned to but never used diff --git a/cron/healthAndStatus.py b/cron/healthAndStatus.py index 9e508e55..c77fdb0b 100755 --- a/cron/healthAndStatus.py +++ b/cron/healthAndStatus.py @@ -16,8 +16,6 @@ from requests.auth import HTTPBasicAuth from configlib import getConfig, OptionParser from logging.handlers import SysLogHandler -import sys -import os from mozdef_util.utilities.toUTC import toUTC from mozdef_util.elasticsearch_client import ElasticsearchClient diff --git a/cron/rotateIndexes.py b/cron/rotateIndexes.py index 6ab1d0d6..08cf1bb4 100755 --- a/cron/rotateIndexes.py +++ b/cron/rotateIndexes.py @@ -19,7 +19,6 @@ from datetime import timedelta from configlib import getConfig, OptionParser import json -import sys import os from mozdef_util.utilities.toUTC import toUTC from mozdef_util.elasticsearch_client import ElasticsearchClient diff --git a/tests/mozdef_util/utilities/test_toUTC.py b/tests/mozdef_util/utilities/test_toUTC.py index 374f2473..f32fd8e1 100644 --- a/tests/mozdef_util/utilities/test_toUTC.py +++ b/tests/mozdef_util/utilities/test_toUTC.py @@ -22,12 +22,9 @@ def utc_timezone(): tzlocal.get_localzone = utc_timezone -import sys if 'mozdef_util.utilities.toUTC' in sys.modules: reload(sys.modules['mozdef_util.utilities.toUTC']) -from mozdef_util.utilities.toUTC import toUTC - class TestToUTC(): From 09989706a04571389371e3086256e821d8fdd0ab Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Fri, 14 Dec 2018 12:39:23 -0600 Subject: [PATCH 82/89] Fixup closing bracket indentation not matching original --- .flake8 | 1 - alerts/ssh_key.py | 2 +- cron/backupSnapshot.py | 24 ++++++++++++------------ cron/pruneIndexes.py | 16 ++++++++-------- cron/rotateIndexes.py | 18 +++++++++--------- cron/setupIndexTemplates.py | 6 +++--- mq/plugins/broFixup.py | 2 +- tests/mq/plugins/test_broFixup.py | 2 +- 8 files changed, 35 insertions(+), 36 deletions(-) diff --git a/.flake8 b/.flake8 index e93ac1ea..c0059044 100644 --- a/.flake8 +++ b/.flake8 @@ -12,7 +12,6 @@ per-file-ignores = rest/index.py: F811 ignore = - E123 # closing bracket does not match indentation of opening bracket's line E225 # missing whitespace around operator E226 # missing whitespace around arithmetic operator E228 # missing whitespace around modulo operator diff --git a/alerts/ssh_key.py b/alerts/ssh_key.py index 1291af79..5ddc229b 100644 --- a/alerts/ssh_key.py +++ b/alerts/ssh_key.py @@ -56,7 +56,7 @@ class SSHKey(AlertTask): self._whitelist.append({ 'hostre': entry[:pindex], 'path': entry[pindex + 1:] - }) + }) # Return false if the key path is present in the whitelist, otherwise return # true diff --git a/cron/backupSnapshot.py b/cron/backupSnapshot.py index 02eb59c0..4d39f7d4 100755 --- a/cron/backupSnapshot.py +++ b/cron/backupSnapshot.py @@ -129,65 +129,65 @@ def initConfig(): 'output', 'stdout', options.configfile - ) + ) # syslog hostname options.sysloghostname = getConfig( 'sysloghostname', 'localhost', options.configfile - ) + ) options.syslogport = getConfig( 'syslogport', 514, options.configfile - ) + ) options.esservers = list(getConfig( 'esservers', 'http://localhost:9200', options.configfile).split(',') - ) + ) options.indices = list(getConfig( 'backup_indices', 'events,alerts,.kibana', options.configfile).split(',') - ) + ) options.dobackup = list(getConfig( 'backup_dobackup', '1,1,1', options.configfile).split(',') - ) + ) options.rotation = list(getConfig( 'backup_rotation', 'daily,monthly,none', options.configfile).split(',') - ) + ) options.pruning = list(getConfig( 'backup_pruning', '20,0,0', options.configfile).split(',') - ) + ) # aws credentials to use to send files to s3 options.aws_access_key_id = getConfig( 'aws_access_key_id', '', options.configfile - ) + ) options.aws_secret_access_key = getConfig( 'aws_secret_access_key', '', options.configfile - ) + ) options.aws_region = getConfig( 'aws_region', 'us-west-1', options.configfile - ) + ) options.aws_bucket = getConfig( 'aws_bucket', '', options.configfile - ) + ) if __name__ == '__main__': diff --git a/cron/pruneIndexes.py b/cron/pruneIndexes.py index c66db742..f86ff244 100755 --- a/cron/pruneIndexes.py +++ b/cron/pruneIndexes.py @@ -71,43 +71,43 @@ def initConfig(): 'output', 'stdout', options.configfile - ) + ) # syslog hostname options.sysloghostname = getConfig( 'sysloghostname', 'localhost', options.configfile - ) + ) options.syslogport = getConfig( 'syslogport', 514, options.configfile - ) + ) options.esservers = list(getConfig( 'esservers', 'http://localhost:9200', options.configfile).split(',') - ) + ) options.indices = list(getConfig( 'backup_indices', 'events,alerts,.kibana', options.configfile).split(',') - ) + ) options.dobackup = list(getConfig( 'backup_dobackup', '1,1,1', options.configfile).split(',') - ) + ) options.rotation = list(getConfig( 'backup_rotation', 'daily,monthly,none', options.configfile).split(',') - ) + ) options.pruning = list(getConfig( 'backup_pruning', '20,0,0', options.configfile).split(',') - ) + ) if __name__ == '__main__': diff --git a/cron/rotateIndexes.py b/cron/rotateIndexes.py index 08cf1bb4..821d1a7d 100755 --- a/cron/rotateIndexes.py +++ b/cron/rotateIndexes.py @@ -132,48 +132,48 @@ def initConfig(): 'output', 'stdout', options.configfile - ) + ) # syslog hostname options.sysloghostname = getConfig( 'sysloghostname', 'localhost', options.configfile - ) + ) options.syslogport = getConfig( 'syslogport', 514, options.configfile - ) + ) options.esservers = list(getConfig( 'esservers', 'http://localhost:9200', options.configfile).split(',') - ) + ) options.indices = list(getConfig( 'backup_indices', 'events,alerts,.kibana', options.configfile).split(',') - ) + ) options.dobackup = list(getConfig( 'backup_dobackup', '1,1,1', options.configfile).split(',') - ) + ) options.rotation = list(getConfig( 'backup_rotation', 'daily,monthly,none', options.configfile).split(',') - ) + ) options.pruning = list(getConfig( 'backup_pruning', '20,0,0', options.configfile).split(',') - ) + ) options.weekly_rotation_indices = list(getConfig( 'weekly_rotation_indices', 'events', options.configfile).split(',') - ) + ) default_mapping_location = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'defaultMappingTemplate.json') options.default_mapping_file = getConfig('default_mapping_file', default_mapping_location, options.configfile) diff --git a/cron/setupIndexTemplates.py b/cron/setupIndexTemplates.py index e1631b43..0bbd885e 100755 --- a/cron/setupIndexTemplates.py +++ b/cron/setupIndexTemplates.py @@ -22,17 +22,17 @@ def initConfig(): 'esservers', 'http://localhost:9200', options.configfile).split(',') - ) + ) options.templatenames = list(getConfig( 'templatenames', 'defaulttemplate', options.configfile).split(',') - ) + ) options.templatefiles = list(getConfig( 'templatefiles', '', options.configfile).split(',') - ) + ) if __name__ == '__main__': diff --git a/mq/plugins/broFixup.py b/mq/plugins/broFixup.py index df37bacc..e295015d 100644 --- a/mq/plugins/broFixup.py +++ b/mq/plugins/broFixup.py @@ -419,7 +419,7 @@ class message(object): u'source {src} ' u'destination {dst} ' u'port {p}' - ).format(**sumstruct) + ).format(**sumstruct) # Thank you for your service return (newmessage, metadata) diff --git a/tests/mq/plugins/test_broFixup.py b/tests/mq/plugins/test_broFixup.py index 6c63bef2..6b976069 100644 --- a/tests/mq/plugins/test_broFixup.py +++ b/tests/mq/plugins/test_broFixup.py @@ -157,7 +157,7 @@ class TestBroFixup(object): "audisp-json", "2.1.1", "audit" - ], + ], "summary": "Execve: sh -c sudo bro nsm /usr/lib64/nagios/plugins/custom/check_auditd.sh", "processname": "audisp-json", "details": { From be7788089dcb8a0e550a71d075a6cf5bba96eada Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Fri, 14 Dec 2018 12:49:25 -0600 Subject: [PATCH 83/89] Fixup missing whitespace around arithmetic operator --- .flake8 | 1 - bot/modules/zilla.py | 4 +- cron/backupSnapshot.py | 4 +- cron/createFDQNBlockList.py | 22 +++++------ cron/createIPBlockList.py | 44 +++++++++++----------- cron/duo_logpull.py | 14 +++---- cron/okta2mozdef.py | 9 +---- cron/pruneIndexes.py | 8 ++-- cron/vidyo2MozDef.py | 2 +- examples/demo/sampleData2MozDef.py | 6 +-- mozdef_util/mozdef_util/utilities/toUTC.py | 2 +- mq/plugins/nagioshostname.py | 4 +- mq/plugins/vidyoCallID.py | 4 +- 13 files changed, 59 insertions(+), 65 deletions(-) diff --git a/.flake8 b/.flake8 index c0059044..7f5dba23 100644 --- a/.flake8 +++ b/.flake8 @@ -13,7 +13,6 @@ per-file-ignores = ignore = E225 # missing whitespace around operator - E226 # missing whitespace around arithmetic operator E228 # missing whitespace around modulo operator E231 # missing whitespace after ',' E265 # block comment should start with '# ' diff --git a/bot/modules/zilla.py b/bot/modules/zilla.py index 12fc0250..71a66eda 100644 --- a/bot/modules/zilla.py +++ b/bot/modules/zilla.py @@ -39,7 +39,7 @@ class Zilla(Module): self.interval = 9999999 self.channel = '#test' - self._bugzilla = bugzilla.Bugzilla(url=self.url+'rest/', api_key=self.api_key) + self._bugzilla = bugzilla.Bugzilla(url=self.url + 'rest/', api_key=self.api_key) _log.info("zilla module initialized for {}, pooling every {} seconds.".format(self.url, self.interval)) @@ -47,7 +47,7 @@ class Zilla(Module): last = 0 while not self._stop: now = time.time() - if ((now-last) > self.interval): + if ((now - last) > self.interval): #Add all the actions you want to do with bugzilla here ;) self.bugzilla_search() last = now diff --git a/cron/backupSnapshot.py b/cron/backupSnapshot.py index 4d39f7d4..0f06704d 100755 --- a/cron/backupSnapshot.py +++ b/cron/backupSnapshot.py @@ -50,8 +50,8 @@ def main(): aws_access_key_id=options.aws_access_key_id, aws_secret_access_key=options.aws_secret_access_key ) - idate = date.strftime(datetime.utcnow()-timedelta(days=1),'%Y%m%d') - bucketdate = date.strftime(datetime.utcnow()-timedelta(days=1),'%Y-%m') + idate = date.strftime(datetime.utcnow() - timedelta(days=1), '%Y%m%d') + bucketdate = date.strftime(datetime.utcnow() - timedelta(days=1), '%Y-%m') hostname = socket.gethostname() # Create or update snapshot configuration diff --git a/cron/createFDQNBlockList.py b/cron/createFDQNBlockList.py index e04292c4..2c03da3c 100644 --- a/cron/createFDQNBlockList.py +++ b/cron/createFDQNBlockList.py @@ -61,7 +61,7 @@ def parse_fqdn_whitelist(fqdn_whitelist_location): fqdns = [] with open(fqdn_whitelist_location, "r") as text_file: for line in text_file: - line=line.strip().strip("'").strip('"') + line = line.strip().strip("'").strip('"') if isFQDN(line): fqdns.append(line) return fqdns @@ -76,10 +76,10 @@ def main(): mozdefdb = client.meteor fqdnblocklist = mozdefdb['fqdnblocklist'] # ensure indexes - fqdnblocklist.create_index([('dateExpiring',-1)]) + fqdnblocklist.create_index([('dateExpiring', -1)]) # delete any that expired - fqdnblocklist.delete_many({'dateExpiring': {"$lte": datetime.utcnow()-timedelta(days=options.expireage)}}) + fqdnblocklist.delete_many({'dateExpiring': {"$lte": datetime.utcnow() - timedelta(days=options.expireage)}}) # Lastly, export the combined blocklist fqdnCursor = mozdefdb['fqdnblocklist'].aggregate([ @@ -94,7 +94,7 @@ def main(): {"$project": {"address": 1}}, {"$limit": options.fqdnlimit} ]) - FQDNList=[] + FQDNList = [] for fqdn in fqdnCursor: if fqdn not in options.fqdnwhitelist: FQDNList.append(fqdn['address']) @@ -104,7 +104,7 @@ def main(): outputfile.write("{0}\n".format(fqdn)) outputfile.close() # to s3? - if len(options.aws_bucket_name)>0: + if len(options.aws_bucket_name) > 0: s3_upload_file(options.outputfile, options.aws_bucket_name, options.aws_document_key_name) except ValueError as e: @@ -133,23 +133,23 @@ def initConfig(): options.outputfile = getConfig('outputfile', 'fqdnblocklist.txt', options.configfile) # Days after expiration that we purge an fqdnblocklist entry (from the ui, they don't end up in the export after expiring) - options.expireage = getConfig('expireage',1,options.configfile) + options.expireage = getConfig('expireage', 1, options.configfile) # Max FQDNs to emit options.fqdnlimit = getConfig('fqdnlimit', 1000, options.configfile) # AWS creds - options.aws_access_key_id=getConfig('aws_access_key_id','',options.configfile) # aws credentials to use to connect to mozilla_infosec_blocklist - options.aws_secret_access_key=getConfig('aws_secret_access_key','',options.configfile) - options.aws_bucket_name=getConfig('aws_bucket_name','',options.configfile) - options.aws_document_key_name=getConfig('aws_document_key_name','',options.configfile) + options.aws_access_key_id = getConfig('aws_access_key_id', '', options.configfile) # aws credentials to use to connect to mozilla_infosec_blocklist + options.aws_secret_access_key = getConfig('aws_secret_access_key', '', options.configfile) + options.aws_bucket_name = getConfig('aws_bucket_name', '', options.configfile) + options.aws_document_key_name = getConfig('aws_document_key_name', '', options.configfile) def s3_upload_file(file_path, bucket_name, key_name): """ Upload a file to the given s3 bucket and return a template url. """ - conn = boto.connect_s3(aws_access_key_id=options.aws_access_key_id,aws_secret_access_key=options.aws_secret_access_key) + conn = boto.connect_s3(aws_access_key_id=options.aws_access_key_id, aws_secret_access_key=options.aws_secret_access_key) try: bucket = conn.get_bucket(bucket_name, validate=False) except boto.exception.S3ResponseError as e: diff --git a/cron/createIPBlockList.py b/cron/createIPBlockList.py index e53553a1..ec7cc7c7 100755 --- a/cron/createIPBlockList.py +++ b/cron/createIPBlockList.py @@ -51,7 +51,7 @@ def isIPv4(ip): # netaddr on it's own considers 1 and 0 to be valid_ipv4 # so a little sanity check prior to netaddr. # Use IPNetwork instead of valid_ipv4 to allow CIDR - if '.' in ip and len(ip.split('.'))==4: + if '.' in ip and len(ip.split('.')) == 4: # some ips are quoted netaddr.IPNetwork(ip.strip("'").strip('"')) return True @@ -88,7 +88,7 @@ def aggregateAttackerIPs(attackers): whitelisted = False logger.debug('working {0}'.format(i)) ip = i['_id']['ipv4address'] - ipcidr=netaddr.IPNetwork(ip) + ipcidr = netaddr.IPNetwork(ip) if not ipcidr.ip.is_loopback() and not ipcidr.ip.is_private() and not ipcidr.ip.is_reserved(): for whitelist_range in options.ipwhitelist: whitelist_network = netaddr.IPNetwork(whitelist_range) @@ -96,8 +96,8 @@ def aggregateAttackerIPs(attackers): logger.debug(str(ipcidr) + " is whitelisted as part of " + str(whitelist_network)) whitelisted = True - #strip any host bits 192.168.10/24 -> 192.168.0/24 - ipcidrnet=str(ipcidr.cidr) + # strip any host bits 192.168.10/24 -> 192.168.0/24 + ipcidrnet = str(ipcidr.cidr) if ipcidrnet not in iplist and not whitelisted: iplist.append(ipcidrnet) else: @@ -109,7 +109,7 @@ def parse_network_whitelist(network_whitelist_location): networks = [] with open(network_whitelist_location, "r") as text_file: for line in text_file: - line=line.strip().strip("'").strip('"') + line = line.strip().strip("'").strip('"') if isIPv4(line) or isIPv6(line): networks.append(line) return networks @@ -123,29 +123,29 @@ def main(): client = MongoClient(options.mongohost, options.mongoport) mozdefdb = client.meteor ipblocklist = mozdefdb['ipblocklist'] - attackers=mozdefdb['attackers'] + attackers = mozdefdb['attackers'] # ensure indexes - ipblocklist.create_index([('dateExpiring',-1)]) - attackers.create_index([('lastseentimestamp',-1)]) - attackers.create_index([('category',1)]) + ipblocklist.create_index([('dateExpiring', -1)]) + attackers.create_index([('lastseentimestamp', -1)]) + attackers.create_index([('category', 1)]) # First, gather IP addresses from recent attackers and add to the block list attackerIPList = aggregateAttackerIPs(attackers) # add attacker IPs to the blocklist # first delete ones we've created from an attacker - ipblocklist.delete_many({'creator': 'mozdef','reference':'attacker'}) + ipblocklist.delete_many({'creator': 'mozdef', 'reference': 'attacker'}) # delete any that expired - ipblocklist.delete_many({'dateExpiring': {"$lte": datetime.utcnow()-timedelta(days=options.expireage)}}) + ipblocklist.delete_many({'dateExpiring': {"$lte": datetime.utcnow() - timedelta(days=options.expireage)}}) # add the aggregations we've found recently for ip in attackerIPList: ipblocklist.insert_one( {'_id': genMeteorID(), - 'address':ip, + 'address': ip, 'reference': 'attacker', - 'creator':'mozdef', + 'creator': 'mozdef', 'dateAdded': datetime.utcnow()}) # Lastly, export the combined blocklist @@ -161,7 +161,7 @@ def main(): {"$project": {"address": 1}}, {"$limit": options.iplimit} ]) - IPList=[] + IPList = [] for ip in ipCursor: IPList.append(ip['address']) # to text @@ -170,7 +170,7 @@ def main(): outputfile.write("{0}\n".format(ip)) outputfile.close() # to s3? - if len(options.aws_bucket_name)>0: + if len(options.aws_bucket_name) > 0: s3_upload_file(options.outputfile, options.aws_bucket_name, options.aws_document_key_name) except ValueError as e: @@ -202,26 +202,26 @@ def initConfig(): options.category = getConfig('category', 'bruteforcer', options.configfile) # Max days to look back for attackers - options.attackerage = getConfig('attackerage',90,options.configfile) + options.attackerage = getConfig('attackerage', 90, options.configfile) # Days after expiration that we purge an ipblocklist entry (from the ui, they don't end up in the export after expiring) - options.expireage = getConfig('expireage',1,options.configfile) + options.expireage = getConfig('expireage', 1, options.configfile) # Max IPs to emit options.iplimit = getConfig('iplimit', 1000, options.configfile) # AWS creds - options.aws_access_key_id=getConfig('aws_access_key_id','',options.configfile) # aws credentials to use to connect to mozilla_infosec_blocklist - options.aws_secret_access_key=getConfig('aws_secret_access_key','',options.configfile) - options.aws_bucket_name=getConfig('aws_bucket_name','',options.configfile) - options.aws_document_key_name=getConfig('aws_document_key_name','',options.configfile) + options.aws_access_key_id = getConfig('aws_access_key_id', '', options.configfile) # aws credentials to use to connect to mozilla_infosec_blocklist + options.aws_secret_access_key = getConfig('aws_secret_access_key', '', options.configfile) + options.aws_bucket_name = getConfig('aws_bucket_name', '', options.configfile) + options.aws_document_key_name = getConfig('aws_document_key_name', '', options.configfile) def s3_upload_file(file_path, bucket_name, key_name): """ Upload a file to the given s3 bucket and return a template url. """ - conn = boto.connect_s3(aws_access_key_id=options.aws_access_key_id,aws_secret_access_key=options.aws_secret_access_key) + conn = boto.connect_s3(aws_access_key_id=options.aws_access_key_id, aws_secret_access_key=options.aws_secret_access_key) try: bucket = conn.get_bucket(bucket_name, validate=False) except boto.exception.S3ResponseError as e: diff --git a/cron/duo_logpull.py b/cron/duo_logpull.py index 6acb7a31..72c95d81 100644 --- a/cron/duo_logpull.py +++ b/cron/duo_logpull.py @@ -10,7 +10,7 @@ try: from datetime import timezone utc = timezone.utc except ImportError: - #Hi there python2 user + # Hi there python2 user class UTC(tzinfo): def utcoffset(self, dt): return timedelta(0) @@ -35,7 +35,7 @@ def normalize(details): normalized = {} for f in details: - if f in ("ip", "ip_address","client_ip"): + if f in ("ip", "ip_address", "client_ip"): normalized["sourceipaddress"] = details[f] continue if f == "result": @@ -87,7 +87,7 @@ def process_events(mozmsg, duo_events, etype, state): elif etype == 'telephony': mozmsg.summary = e['context'] elif etype == 'authentication': - mozmsg.summary = e['eventtype']+' '+e['result']+' for '+e['username'] + mozmsg.summary = e['eventtype'] + ' ' + e['result'] + ' for ' + e['username'] mozmsg.send() @@ -109,7 +109,7 @@ def main(): duo = duo_client.Admin(ikey=options.IKEY, skey=options.SKEY, host=options.URL) mozmsg = mozdef.MozDefEvent(options.MOZDEF_URL) - mozmsg.tags=['duosecurity'] + mozmsg.tags = ['duosecurity'] if options.update_tags != '': mozmsg.tags.append(options.update_tags) mozmsg.set_category('authentication') @@ -120,9 +120,9 @@ def main(): # This will process events for all 3 log types and send them to MozDef. the state stores the last position in the # log when this script was last called. - state = process_events(mozmsg, duo.get_administrator_log(mintime=state['administration']+1), 'administration', state) - state = process_events(mozmsg, duo.get_authentication_log(mintime=state['authentication']+1), 'authentication', state) - state = process_events(mozmsg, duo.get_telephony_log(mintime=state['telephony']+1), 'telephony', state) + state = process_events(mozmsg, duo.get_administrator_log(mintime=state['administration'] + 1), 'administration', state) + state = process_events(mozmsg, duo.get_authentication_log(mintime=state['authentication'] + 1), 'authentication', state) + state = process_events(mozmsg, duo.get_telephony_log(mintime=state['telephony'] + 1), 'telephony', state) pickle.dump(state, open(options.statepath, 'wb')) diff --git a/cron/okta2mozdef.py b/cron/okta2mozdef.py index b869484f..de3387dc 100755 --- a/cron/okta2mozdef.py +++ b/cron/okta2mozdef.py @@ -11,8 +11,6 @@ import logging from logging.handlers import SysLogHandler import json from datetime import datetime -from datetime import timedelta -from datetime import date import requests import netaddr @@ -21,7 +19,7 @@ from mozdef_util.elasticsearch_client import ElasticsearchClient logger = logging.getLogger(sys.argv[0]) -logger.level=logging.INFO +logger.level = logging.INFO formatter = logging.Formatter('%(asctime)s %(name)s %(levelname)s %(message)s') @@ -72,14 +70,11 @@ def main(): s = requests.Session() s.headers.update({'Accept': 'application/json'}) s.headers.update({'Content-type': 'application/json'}) - s.headers.update({'Authorization':'SSWS {0}'.format(options.apikey)}) + s.headers.update({'Authorization': 'SSWS {0}'.format(options.apikey)}) #capture the time we start running so next time we catch any events created while we run. state = State(options.state_file) lastrun = toUTC(datetime.now()).isoformat() - #in case we don't archive files..only look at today and yesterday's files. - yesterday=date.strftime(datetime.utcnow()-timedelta(days=1),'%Y/%m/%d') - today = date.strftime(datetime.utcnow(),'%Y/%m/%d') r = s.get('https://{0}/api/v1/events?startDate={1}&limit={2}'.format( options.oktadomain, diff --git a/cron/pruneIndexes.py b/cron/pruneIndexes.py index f86ff244..0a45942d 100755 --- a/cron/pruneIndexes.py +++ b/cron/pruneIndexes.py @@ -25,7 +25,7 @@ from mozdef_util.elasticsearch_client import ElasticsearchClient logger = logging.getLogger(sys.argv[0]) -logger.level=logging.WARNING +logger.level = logging.WARNING formatter = logging.Formatter('%(asctime)s %(name)s %(levelname)s %(message)s') @@ -47,10 +47,10 @@ def esPruneIndexes(): if pruning != '0': index_to_prune = index if rotation == 'daily': - idate = date.strftime(toUTC(datetime.now()) - timedelta(days=int(pruning)),'%Y%m%d') + idate = date.strftime(toUTC(datetime.now()) - timedelta(days=int(pruning)), '%Y%m%d') index_to_prune += '-%s' % idate elif rotation == 'monthly': - idate = date.strftime(datetime.utcnow() - timedelta(days=31*int(pruning)),'%Y%m') + idate = date.strftime(datetime.utcnow() - timedelta(days=31 * int(pruning)), '%Y%m') index_to_prune += '-%s' % idate if index_to_prune in indices: @@ -62,7 +62,7 @@ def esPruneIndexes(): logger.error("Unhandled exception while deleting %s, terminating: %r" % (index_to_prune, e)) except Exception as e: - logger.error("Unhandled exception, terminating: %r"%e) + logger.error("Unhandled exception, terminating: %r" % e) def initConfig(): diff --git a/cron/vidyo2MozDef.py b/cron/vidyo2MozDef.py index 1ef0a498..5746d4ba 100755 --- a/cron/vidyo2MozDef.py +++ b/cron/vidyo2MozDef.py @@ -149,7 +149,7 @@ def main(): call[k] = call[k].decode('utf-8','ignore').encode('ascii','ignore') mdEvent.send(timestamp=call['JoinTime'], - summary='Vidyo call status for '+call['UniqueCallID'].encode('ascii', 'ignore'), + summary='Vidyo call status for ' + call['UniqueCallID'].encode('ascii', 'ignore'), tags=['vidyo'], details=call, category='vidyo', diff --git a/examples/demo/sampleData2MozDef.py b/examples/demo/sampleData2MozDef.py index 019265eb..03299d30 100755 --- a/examples/demo/sampleData2MozDef.py +++ b/examples/demo/sampleData2MozDef.py @@ -115,7 +115,7 @@ def makeEvents(): #print(eventfile) events = json.load(open(eventfile)) target = random.randint(0, len(events)) - for event in events[target:target+1]: + for event in events[target:target + 1]: event['timestamp'] = pytz.timezone('UTC').localize(datetime.utcnow()).isoformat() #remove stored times if 'utctimestamp' in event.keys(): @@ -179,7 +179,7 @@ def makeAlerts(): # if there's only one event in the file..use it. if len(events) == 1 and target == 1: target = 0 - for event in events[target:target+1]: + for event in events[target:target + 1]: event['timestamp'] = pytz.timezone('UTC').localize(datetime.utcnow()).isoformat() #remove stored times if 'utctimestamp' in event.keys(): @@ -249,7 +249,7 @@ def makeAttackers(): # if there's only one event in the file..use it. if len(events) == 1 and target == 1: target = 0 - for event in events[target:target+1]: + for event in events[target:target + 1]: event['timestamp'] = pytz.timezone('UTC').localize(datetime.utcnow()).isoformat() #remove stored times if 'utctimestamp' in event.keys(): diff --git a/mozdef_util/mozdef_util/utilities/toUTC.py b/mozdef_util/mozdef_util/utilities/toUTC.py index 05fa12dc..9e8ee447 100644 --- a/mozdef_util/mozdef_util/utilities/toUTC.py +++ b/mozdef_util/mozdef_util/utilities/toUTC.py @@ -29,7 +29,7 @@ def toUTC(suspectedDate): else: # epoch? but seconds/milliseconds/nanoseconds (lookin at you heka) epochDivisor = int(str(1) + '0' * (len(str(suspectedDate)) % 10)) - objDate = datetime.fromtimestamp(float(suspectedDate/epochDivisor), LOCAL_TIMEZONE) + objDate = datetime.fromtimestamp(float(suspectedDate / epochDivisor), LOCAL_TIMEZONE) elif type(suspectedDate) in (str, unicode): # try to parse float or negative number from string: objDate = None diff --git a/mq/plugins/nagioshostname.py b/mq/plugins/nagioshostname.py index e5a2b815..65bcc4a9 100644 --- a/mq/plugins/nagioshostname.py +++ b/mq/plugins/nagioshostname.py @@ -21,6 +21,6 @@ class message(object): self.priority = 5 def onMessage(self, message, metadata): - docid=hashlib.md5('nagiosstatus'+message['details']['nagios_hostname']).hexdigest() - metadata['id']=docid + docid = hashlib.md5('nagiosstatus' + message['details']['nagios_hostname']).hexdigest() + metadata['id'] = docid return (message, metadata) diff --git a/mq/plugins/vidyoCallID.py b/mq/plugins/vidyoCallID.py index 8bc01220..23d48937 100644 --- a/mq/plugins/vidyoCallID.py +++ b/mq/plugins/vidyoCallID.py @@ -21,7 +21,7 @@ class message(object): self.priority = 5 def onMessage(self, message, metadata): - docid=hashlib.md5('vidyoUniqueCallID'+message['details']['UniqueCallID']).hexdigest() - metadata['id']=docid + docid = hashlib.md5('vidyoUniqueCallID' + message['details']['UniqueCallID']).hexdigest() + metadata['id'] = docid metadata['doc_type'] = 'vidyo' return (message, metadata) From 77e93f3fd8db4ff953339aa60e82d6f4e8a0de9e Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Fri, 14 Dec 2018 12:53:25 -0600 Subject: [PATCH 84/89] Fixup missing whitespace around modulo operator --- .flake8 | 1 - benchmarking/workers/json2Mozdef.py | 16 ++++++++-------- cron/backupSnapshot.py | 2 +- cron/google2mozdef.py | 2 +- cron/healthAndStatus.py | 2 +- cron/okta2mozdef.py | 2 +- examples/demo/sampleData2MozDef.py | 18 +++++++++--------- 7 files changed, 21 insertions(+), 22 deletions(-) diff --git a/.flake8 b/.flake8 index 7f5dba23..f2af52e9 100644 --- a/.flake8 +++ b/.flake8 @@ -13,7 +13,6 @@ per-file-ignores = ignore = E225 # missing whitespace around operator - E228 # missing whitespace around modulo operator E231 # missing whitespace after ',' E265 # block comment should start with '# ' E402 # module level import not at top of file diff --git a/benchmarking/workers/json2Mozdef.py b/benchmarking/workers/json2Mozdef.py index e15fd2df..a2cdd39b 100755 --- a/benchmarking/workers/json2Mozdef.py +++ b/benchmarking/workers/json2Mozdef.py @@ -38,7 +38,7 @@ def postLogs(logcache): try: while not logcache.empty(): postdata=logcache.get_nowait() - if len(postdata)>0: + if len(postdata) > 0: url=options.url a=httpsession.get_adapter(url) a.max_retries=3 @@ -48,15 +48,15 @@ def postLogs(logcache): pass for p,postdata,url in posts: try: - if p.result().status_code >=500: - logger.error("exception posting to %s %r [will retry]\n"%(url,p.result().status_code)) + if p.result().status_code >= 500: + logger.error("exception posting to %s %r [will retry]\n" % (url, p.result().status_code)) #try again later when the next message in forces other attempts at posting. logcache.put(postdata) except ClosedPoolError as e: #logger.fatal("Closed Pool Error exception posting to %s %r %r [will retry]\n"%(url,e,postdata)) logcache.put(postdata) except Exception as e: - logger.fatal("exception posting to %s %r %r [will not retry]\n"%(url,e,postdata)) + logger.fatal("exception posting to %s %r %r [will not retry]\n" % (url, e, postdata)) sys.exit(1) @@ -94,21 +94,21 @@ if __name__ == '__main__': postingProcess=Process(target=postLogs,args=(logcache,),name="json2MozdefStressTest") postingProcess.start() except OSError as e: - if e.errno==35: # resource temporarily unavailable. + if e.errno == 35: # resource temporarily unavailable. print(e) pass else: - logger.error('%r'%e) + logger.error('%r' % e) while not logcache.empty(): try: postingProcess=Process(target=postLogs,args=(logcache,),name="json2MozdefStressTest") postingProcess.start() except OSError as e: - if e.errno==35: # resource temporarily unavailable. + if e.errno == 35: # resource temporarily unavailable. print(e) pass else: - logger.error('%r'%e) + logger.error('%r' % e) except KeyboardInterrupt as e: sys.exit(1) diff --git a/cron/backupSnapshot.py b/cron/backupSnapshot.py index 0f06704d..8d2adac4 100755 --- a/cron/backupSnapshot.py +++ b/cron/backupSnapshot.py @@ -120,7 +120,7 @@ echo "DONE!" except boto.exception.NoAuthHandlerFound: logger.error("No auth handler found, check your credentials") except Exception as e: - logger.error("Unhandled exception, terminating: %r"%e) + logger.error("Unhandled exception, terminating: %r" % e) def initConfig(): diff --git a/cron/google2mozdef.py b/cron/google2mozdef.py index 0636f3da..af45bd50 100755 --- a/cron/google2mozdef.py +++ b/cron/google2mozdef.py @@ -183,7 +183,7 @@ def main(): state.data['lastrun'] = lastrun state.write_state_file() except Exception as e: - logger.error("Unhandled exception, terminating: %r"%e) + logger.error("Unhandled exception, terminating: %r" % e) def initConfig(): diff --git a/cron/healthAndStatus.py b/cron/healthAndStatus.py index c77fdb0b..759e1655 100755 --- a/cron/healthAndStatus.py +++ b/cron/healthAndStatus.py @@ -69,7 +69,7 @@ def main(): logger.debug('Creating %s index' % index) es.create_index(index, default_mapping_contents) except Exception as e: - logger.error("Unhandled exception, terminating: %r"%e) + logger.error("Unhandled exception, terminating: %r" % e) auth = HTTPBasicAuth(options.mquser, options.mqpassword) diff --git a/cron/okta2mozdef.py b/cron/okta2mozdef.py index de3387dc..a9630d05 100755 --- a/cron/okta2mozdef.py +++ b/cron/okta2mozdef.py @@ -130,7 +130,7 @@ def main(): else: logger.error('Could not get Okta events HTTP error code {} reason {}'.format(r.status_code, r.reason)) except Exception as e: - logger.error("Unhandled exception, terminating: %r"%e) + logger.error("Unhandled exception, terminating: %r" % e) def initConfig(): diff --git a/examples/demo/sampleData2MozDef.py b/examples/demo/sampleData2MozDef.py index 03299d30..463d8a89 100755 --- a/examples/demo/sampleData2MozDef.py +++ b/examples/demo/sampleData2MozDef.py @@ -144,14 +144,14 @@ def makeEvents(): if not logcache.empty(): time.sleep(.01) try: - postingProcess=Process(target=postLogs,args=(logcache,),name="json2MozdefDemoData") + postingProcess = Process(target=postLogs, args=(logcache,), name="json2MozdefDemoData") postingProcess.start() except OSError as e: - if e.errno==35: # resource temporarily unavailable. + if e.errno == 35: # resource temporarily unavailable. print(e) pass else: - logger.error('%r'%e) + logger.error('%r' % e) except KeyboardInterrupt as e: sys.exit(1) @@ -217,11 +217,11 @@ def makeAlerts(): postingProcess=Process(target=postLogs,args=(logcache,),name="json2MozdefDemoData") postingProcess.start() except OSError as e: - if e.errno==35: # resource temporarily unavailable. + if e.errno == 35: # resource temporarily unavailable. print(e) pass else: - logger.error('%r'%e) + logger.error('%r' % e) except KeyboardInterrupt as e: sys.exit(1) @@ -287,11 +287,11 @@ def makeAttackers(): postingProcess=Process(target=postLogs,args=(logcache,),name="json2MozdefDemoData") postingProcess.start() except OSError as e: - if e.errno==35: # resource temporarily unavailable. + if e.errno == 35: # resource temporarily unavailable. print(e) pass else: - logger.error('%r'%e) + logger.error('%r' % e) except KeyboardInterrupt as e: sys.exit(1) @@ -337,8 +337,8 @@ if __name__ == '__main__': postingProcess=Process(target=postLogs,args=(logcache,),name="json2MozdefDemoData") postingProcess.start() except OSError as e: - if e.errno==35: # resource temporarily unavailable. + if e.errno == 35: # resource temporarily unavailable. print(e) pass else: - logger.error('%r'%e) + logger.error('%r' % e) From df84a1942d129e2d6c9d4cc4129a44a8c0a698f9 Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Fri, 14 Dec 2018 13:40:07 -0600 Subject: [PATCH 85/89] Fixup block comments not having a space after hash --- .flake8 | 1 - alerts/fxa_alerts.py | 6 +- alerts/trace_audit.py | 6 +- alerts/write_audit.py | 6 +- benchmarking/workers/json2Mozdef.py | 14 ++-- bot/modules/zilla.py | 2 +- cron/collectAttackers.py | 4 +- cron/correlateUserMacAddress.py | 2 +- cron/google2mozdef.py | 4 +- cron/okta2mozdef.py | 4 +- cron/vidyo2MozDef.py | 4 +- examples/demo/sampleData2MozDef.py | 84 +++++++++++------------ examples/es-docs/simpleDefaultTemplate.py | 4 +- loginput/index.py | 36 +++++----- mq/plugins/broFixup.py | 2 +- mq/plugins/dropMessage.py | 10 +-- mq/plugins/fxaFixup.py | 12 ++-- rest/index.py | 2 +- rest/plugins/vpc_blackhole.py | 2 +- 19 files changed, 102 insertions(+), 103 deletions(-) diff --git a/.flake8 b/.flake8 index f2af52e9..82945fc1 100644 --- a/.flake8 +++ b/.flake8 @@ -14,7 +14,6 @@ per-file-ignores = ignore = E225 # missing whitespace around operator E231 # missing whitespace after ',' - E265 # block comment should start with '# ' E402 # module level import not at top of file E501 # line too long E722 # do not use bare except' diff --git a/alerts/fxa_alerts.py b/alerts/fxa_alerts.py index a4df948e..ef2b62bd 100644 --- a/alerts/fxa_alerts.py +++ b/alerts/fxa_alerts.py @@ -40,9 +40,9 @@ class AlertAccountCreations(AlertTask): severity = 'INFO' summary = ('{0} fxa account creation attempts by {1}'.format(aggreg['count'], aggreg['value'])) - emails = self.mostCommon(aggreg['allevents'],'_source.details.email') - #did they try to create more than one email account? - #or just retry an existing one + emails = self.mostCommon(aggreg['allevents'], '_source.details.email') + # did they try to create more than one email account? + # or just retry an existing one if len(emails) > 1: for i in emails[:5]: summary += ' {0} ({1} hits)'.format(i[0], i[1]) diff --git a/alerts/trace_audit.py b/alerts/trace_audit.py index 016ee76f..4e56d5a3 100644 --- a/alerts/trace_audit.py +++ b/alerts/trace_audit.py @@ -33,9 +33,9 @@ class TraceAudit(AlertTask): tags = ['audit'] summary = ('{0} instances of Strace or Ptrace executed on a system by {1}'.format(aggreg['count'], aggreg['value'], )) - hostnames = self.mostCommon(aggreg['allevents'],'_source.hostname') - #did they modify more than one host? - #or just modify an existing configuration more than once? + hostnames = self.mostCommon(aggreg['allevents'], '_source.hostname') + # did they modify more than one host? + # or just modify an existing configuration more than once? if len(hostnames) > 1: for i in hostnames[:5]: summary += ' on {0} ({1} hosts)'.format(i[0], i[1]) diff --git a/alerts/write_audit.py b/alerts/write_audit.py index a04fd06e..adaa96a4 100644 --- a/alerts/write_audit.py +++ b/alerts/write_audit.py @@ -34,9 +34,9 @@ class WriteAudit(AlertTask): tags = ['audit'] summary = ('{0} Filesystem write(s) to an auditd path by {1}'.format(aggreg['count'], aggreg['value'], )) - hostnames = self.mostCommon(aggreg['allevents'],'_source.hostname') - #did they modify more than one host? - #or just modify an existing configuration more than once? + hostnames = self.mostCommon(aggreg['allevents'], '_source.hostname') + # did they modify more than one host? + # or just modify an existing configuration more than once? if len(hostnames) > 1: for i in hostnames[:5]: summary += ' on {0} ({1} hosts)'.format(i[0], i[1]) diff --git a/benchmarking/workers/json2Mozdef.py b/benchmarking/workers/json2Mozdef.py index a2cdd39b..c531b629 100755 --- a/benchmarking/workers/json2Mozdef.py +++ b/benchmarking/workers/json2Mozdef.py @@ -21,8 +21,8 @@ import time httpsession = FuturesSession(max_workers=5) httpsession.trust_env=False # turns of needless .netrc check for creds -#a = requests.adapters.HTTPAdapter(max_retries=2) -#httpsession.mount('http://', a) +# a = requests.adapters.HTTPAdapter(max_retries=2) +# httpsession.mount('http://', a) logger = logging.getLogger(sys.argv[0]) @@ -32,8 +32,8 @@ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(messag def postLogs(logcache): - #post logs asynchronously with requests workers and check on the results - #expects a queue object from the multiprocessing library + # post logs asynchronously with requests workers and check on the results + # expects a queue object from the multiprocessing library posts=[] try: while not logcache.empty(): @@ -50,10 +50,10 @@ def postLogs(logcache): try: if p.result().status_code >= 500: logger.error("exception posting to %s %r [will retry]\n" % (url, p.result().status_code)) - #try again later when the next message in forces other attempts at posting. + # try again later when the next message in forces other attempts at posting. logcache.put(postdata) except ClosedPoolError as e: - #logger.fatal("Closed Pool Error exception posting to %s %r %r [will retry]\n"%(url,e,postdata)) + # logger.fatal("Closed Pool Error exception posting to %s %r %r [will retry]\n"%(url,e,postdata)) logcache.put(postdata) except Exception as e: logger.fatal("exception posting to %s %r %r [will not retry]\n" % (url, e, postdata)) @@ -67,7 +67,7 @@ if __name__ == '__main__': sh=logging.StreamHandler(sys.stdout) sh.setFormatter(formatter) logger.addHandler(sh) - #create a list of logs we can append json to and call for a post when we want. + # create a list of logs we can append json to and call for a post when we want. logcache=Queue() try: for i in range(0,10): diff --git a/bot/modules/zilla.py b/bot/modules/zilla.py index 71a66eda..55742987 100644 --- a/bot/modules/zilla.py +++ b/bot/modules/zilla.py @@ -48,7 +48,7 @@ class Zilla(Module): while not self._stop: now = time.time() if ((now - last) > self.interval): - #Add all the actions you want to do with bugzilla here ;) + # Add all the actions you want to do with bugzilla here ;) self.bugzilla_search() last = now time.sleep(1) diff --git a/cron/collectAttackers.py b/cron/collectAttackers.py index 2a9e8e73..4f714288 100755 --- a/cron/collectAttackers.py +++ b/cron/collectAttackers.py @@ -258,10 +258,10 @@ def searchMongoAlerts(mozdefdb): # summarize the alert categories # returns list of tuples: [(u'bruteforce', 8)] categoryCounts= mostCommon(matchingalerts,'category') - #are the alerts all the same category? + # are the alerts all the same category? if len(categoryCounts) == 1: - #is the alert category mapped to an attacker category? + # is the alert category mapped to an attacker category? for category in options.categorymapping: if category.keys()[0] == categoryCounts[0][0]: attacker['category'] = category[category.keys()[0]] diff --git a/cron/correlateUserMacAddress.py b/cron/correlateUserMacAddress.py index b2ac8699..f6f3afd9 100755 --- a/cron/correlateUserMacAddress.py +++ b/cron/correlateUserMacAddress.py @@ -60,7 +60,7 @@ def readOUIFile(ouifilename): for i in ouifile.readlines()[0::]: i=i.strip() if '(hex)' in i: - #print(i) + # print(i) fields=i.split('\t') macprefix=fields[0][0:8].replace('-',':').lower() entity=fields[2] diff --git a/cron/google2mozdef.py b/cron/google2mozdef.py index af45bd50..f8fc02eb 100755 --- a/cron/google2mozdef.py +++ b/cron/google2mozdef.py @@ -199,10 +199,10 @@ def initConfig(): # for detailed information on delegating a service account for use in gathering google admin sdk reports # - #google's json credential file exported from the project/admin console + # google's json credential file exported from the project/admin console options.jsoncredentialfile=getConfig('jsoncredentialfile','/path/to/filename.json',options.configfile) - #email of admin to impersonate as a service account + # email of admin to impersonate as a service account options.impersonate = getConfig('impersonate', 'someone@yourcompany.com', options.configfile) diff --git a/cron/okta2mozdef.py b/cron/okta2mozdef.py index a9630d05..3a1968ee 100755 --- a/cron/okta2mozdef.py +++ b/cron/okta2mozdef.py @@ -64,7 +64,7 @@ def main(): logger.addHandler(sh) logger.debug('started') - #logger.debug(options) + # logger.debug(options) try: es = ElasticsearchClient((list('{0}'.format(s) for s in options.esservers))) s = requests.Session() @@ -72,7 +72,7 @@ def main(): s.headers.update({'Content-type': 'application/json'}) s.headers.update({'Authorization': 'SSWS {0}'.format(options.apikey)}) - #capture the time we start running so next time we catch any events created while we run. + # capture the time we start running so next time we catch any events created while we run. state = State(options.state_file) lastrun = toUTC(datetime.now()).isoformat() diff --git a/cron/vidyo2MozDef.py b/cron/vidyo2MozDef.py index 5746d4ba..fa41ea4a 100755 --- a/cron/vidyo2MozDef.py +++ b/cron/vidyo2MozDef.py @@ -115,7 +115,7 @@ def main(): mdEvent.debug = True mdEvent.fire_and_forget_mode = False - #connect to mysql + # connect to mysql db=MySQLdb.connect(host=options.hostname, user=options.username,passwd=options.password,db=options.database) c=db.cursor(MySQLdb.cursors.DictCursor) @@ -138,7 +138,7 @@ def main(): duration = call['LeaveTime'] - call['JoinTime'] call['CallDuration'] = duration.seconds - #fix up the data for json + # fix up the data for json for k in call.keys(): # convert datetime objects to isoformat for json serialization if isinstance(call[k], datetime): diff --git a/examples/demo/sampleData2MozDef.py b/examples/demo/sampleData2MozDef.py index 463d8a89..20401473 100755 --- a/examples/demo/sampleData2MozDef.py +++ b/examples/demo/sampleData2MozDef.py @@ -23,12 +23,12 @@ from datetime import timedelta from mozdef_util.utilities.toUTC import toUTC -#use futures to run in the background -#httpsession = FuturesSession(max_workers=5) +# use futures to run in the background +# httpsession = FuturesSession(max_workers=5) httpsession = requests.session() httpsession.trust_env=False # turns of needless .netrc check for creds -#a = requests.adapters.HTTPAdapter(max_retries=2) -#httpsession.mount('http://', a) +# a = requests.adapters.HTTPAdapter(max_retries=2) +# httpsession.mount('http://', a) logger = logging.getLogger(sys.argv[0]) @@ -36,7 +36,7 @@ logger.level=logging.INFO formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') -#create a list of logs we can append json to and call for a post when we want. +# create a list of logs we can append json to and call for a post when we want. logcache=Queue() @@ -55,8 +55,8 @@ def setConfig(option,value,configfile): def postLogs(logcache): - #post logs asynchronously with requests workers and check on the results - #expects a queue object from the multiprocessing library + # post logs asynchronously with requests workers and check on the results + # expects a queue object from the multiprocessing library posts=[] try: while not logcache.empty(): @@ -67,9 +67,9 @@ def postLogs(logcache): a.max_retries=3 r=httpsession.post(url,data=postdata) print(r, postdata) - #append to posts if this is long running and you want - #events to try again later. - #posts.append((r,postdata,url)) + # append to posts if this is long running and you want + # events to try again later. + # posts.append((r,postdata,url)) except Empty as e: pass # for p, postdata, url in posts: @@ -87,18 +87,18 @@ def postLogs(logcache): def genRandomIPv4(): - #random, IPs + # random, IPs return '.'.join("%d" % (random.randint(0,254)) for x in range(4)) def genAttackerIPv4(): - #random, but not too random as to allow for alerting about attacks from - #the same IP. + # random, but not too random as to allow for alerting about attacks from + # the same IP. coreIPs=['1.93.25.', '222.73.115.', '116.10.191.', '144.0.0.'] - #change this to non zero according to taste for semi-random-ness + # change this to non zero according to taste for semi-random-ness if random.randint(0,10)>= 0: return '{0}{1}'.format(random.choice(coreIPs), random.randint(1,2)) else: @@ -108,28 +108,28 @@ def genAttackerIPv4(): def makeEvents(): try: eventfiles = glob.glob(options.eventsglob) - #pick a random number of events to send + # pick a random number of events to send for i in range(1, random.randrange(20, 100)): - #pick a random type of event to send + # pick a random type of event to send eventfile = random.choice(eventfiles) - #print(eventfile) + # print(eventfile) events = json.load(open(eventfile)) target = random.randint(0, len(events)) for event in events[target:target + 1]: event['timestamp'] = pytz.timezone('UTC').localize(datetime.utcnow()).isoformat() - #remove stored times + # remove stored times if 'utctimestamp' in event.keys(): del event['utctimestamp'] if 'receivedtimestamp' in event.keys(): del event['receivedtimestamp'] - #add demo to the tags so it's clear it's not real data. + # add demo to the tags so it's clear it's not real data. if 'tags' not in event.keys(): event['tags'] = list() event['tags'].append('demodata') - #replace potential with a random ip address + # replace potential with a random ip address if 'summary' in event.keys() and '' in event['summary']: randomIP = genRandomIPv4() event['summary'] = event['summary'].replace("", randomIP) @@ -138,7 +138,7 @@ def makeEvents(): event['details']['sourceipaddress'] = randomIP event['details']['sourceipv4address'] = randomIP - #print(event['timestamp'], event['tags'], event['summary']) + # print(event['timestamp'], event['tags'], event['summary']) logcache.put(json.dumps(event)) if not logcache.empty(): @@ -162,17 +162,17 @@ def makeAlerts(): send events that will be correlated into alerts ''' try: - #time for us to run? - timetoRun=toUTC(options.lastalert) + timedelta(minutes=options.alertsminutesinterval) + # time for us to run? + timetoRun = toUTC(options.lastalert) + timedelta(minutes=options.alertsminutesinterval) if timetoRun > toUTC(datetime.now()): - #print(timetoRun) + # print(timetoRun) return - #print(timetoRun, options.lastalert) + # print(timetoRun, options.lastalert) eventfiles = glob.glob(options.alertsglob) - #pick a random number of events to send + # pick a random number of events to send for i in range(0, options.alertscount): - #pick a random type of event to send + # pick a random type of event to send eventfile = random.choice(eventfiles) events = json.load(open(eventfile)) target = random.randint(0, len(events)) @@ -181,20 +181,20 @@ def makeAlerts(): target = 0 for event in events[target:target + 1]: event['timestamp'] = pytz.timezone('UTC').localize(datetime.utcnow()).isoformat() - #remove stored times + # remove stored times if 'utctimestamp' in event.keys(): del event['utctimestamp'] if 'receivedtimestamp' in event.keys(): del event['receivedtimestamp'] - #add demo to the tags so it's clear it's not real data. + # add demo to the tags so it's clear it's not real data. if 'tags' not in event.keys(): event['tags'] = list() event['tags'].append('demodata') event['tags'].append('demoalert') - #replace potential with a random ip address + # replace potential with a random ip address if 'summary' in event.keys() and '' in event['summary']: randomIP = genRandomIPv4() event['summary'] = event['summary'].replace("", randomIP) @@ -232,17 +232,17 @@ def makeAttackers(): send events that will be correlated into attackers using pre-defined IPs ''' try: - #time for us to run? + # time for us to run? timetoRun=toUTC(options.lastattacker) + timedelta(minutes=options.attackersminutesinterval) if timetoRun > toUTC(datetime.now()): - #print(timetoRun) + # print(timetoRun) return - #print(timetoRun, options.lastalert) + # print(timetoRun, options.lastalert) eventfiles = glob.glob(options.alertsglob) - #pick a random number of events to send + # pick a random number of events to send for i in range(0, options.alertscount): - #pick a random type of event to send + # pick a random type of event to send eventfile = random.choice(eventfiles) events = json.load(open(eventfile)) target = random.randint(0, len(events)) @@ -251,20 +251,20 @@ def makeAttackers(): target = 0 for event in events[target:target + 1]: event['timestamp'] = pytz.timezone('UTC').localize(datetime.utcnow()).isoformat() - #remove stored times + # remove stored times if 'utctimestamp' in event.keys(): del event['utctimestamp'] if 'receivedtimestamp' in event.keys(): del event['receivedtimestamp'] - #add demo to the tags so it's clear it's not real data. + # add demo to the tags so it's clear it's not real data. if 'tags' not in event.keys(): event['tags'] = list() event['tags'].append('demodata') event['tags'].append('demoalert') - #replace potential with a random ip address + # replace potential with a random ip address if 'summary' in event.keys() and '' in event['summary']: randomIP = genAttackerIPv4() event['summary'] = event['summary'].replace("", randomIP) @@ -302,15 +302,15 @@ def initConfig(): options.eventsglob = getConfig('eventsglob', './sampleevents/events*json', options.configfile) options.alertsglob = getConfig('alertsglob', './sampleevents/alert*json', options.configfile) options.attackersglob = getConfig('attackersglob', './sampleevents/attacker*json', options.configfile) - #how many alerts to create + # how many alerts to create options.alertscount = getConfig('alertscount', 2, options.configfile) - #how many minutes to wait between creating ^ alerts + # how many minutes to wait between creating ^ alerts options.alertsminutesinterval = getConfig('alertsminutesinterval', 5, options.configfile) options.lastalert = getConfig('lastalert', datetime.now() - timedelta(hours=1), options.configfile) - #how many attackers to create + # how many attackers to create options.attackerscount = getConfig('attackers', 1, options.configfile) - #how many minutes to wait between creating ^ attackers + # how many minutes to wait between creating ^ attackers options.attackersminutesinterval = getConfig('attackersminutesinterval', 5, options.configfile) options.lastattacker = getConfig('lastattacker', datetime.now() - timedelta(hours=1), options.configfile) diff --git a/examples/es-docs/simpleDefaultTemplate.py b/examples/es-docs/simpleDefaultTemplate.py index 1d380943..7b35eca9 100644 --- a/examples/es-docs/simpleDefaultTemplate.py +++ b/examples/es-docs/simpleDefaultTemplate.py @@ -67,9 +67,9 @@ defaultTemplate = r''' } ''' -#valid json? +# valid json? templateJson = json.loads(defaultTemplate) -#post it: +# post it r = requests.put(url="http://servername:9200/_template/defaulttemplate", data=defaultTemplate) print(r) diff --git a/loginput/index.py b/loginput/index.py index 14b684fd..a6d9c7e2 100644 --- a/loginput/index.py +++ b/loginput/index.py @@ -28,10 +28,10 @@ def status(): @route('/test/') def testindex(): ip = request.environ.get('REMOTE_ADDR') - #response.headers['X-IP'] = '{0}'.format(ip) + # response.headers['X-IP'] = '{0}'.format(ip) response.status=200 -#act like elastic search bulk index +# act like elastic search bulk index @route('/_bulk',method='POST') @@ -39,10 +39,10 @@ def testindex(): def bulkindex(): if request.body: bulkpost=request.body.read() - #bottlelog('request:{0}\n'.format(bulkpost)) + # bottlelog('request:{0}\n'.format(bulkpost)) request.body.close() if len(bulkpost)>10: # TODO Check for bulk format. - #iterate on messages and post to event message queue + # iterate on messages and post to event message queue eventlist=[] for i in bulkpost.splitlines(): @@ -50,7 +50,7 @@ def bulkindex(): for i in eventlist: try: - #valid json? + # valid json? try: eventDict=json.loads(i) except ValueError as e: @@ -74,17 +74,17 @@ def bulkindex(): def eventsindex(): if request.body: anevent=request.body.read() - #bottlelog('request:{0}\n'.format(anevent)) + # bottlelog('request:{0}\n'.format(anevent)) request.body.close() - #valid json? + # valid json? try: eventDict=json.loads(anevent) except ValueError as e: response.status=500 return - #let the message queue worker who gets this know where it was posted + # let the message queue worker who gets this know where it was posted eventDict['endpoint']='events' - #post to event message queue + # post to event message queue ensurePublish=mqConn.ensure(mqproducer,mqproducer.publish,max_retries=10) ensurePublish(eventDict,exchange=eventTaskExchange,routing_key=options.taskexchange) @@ -93,21 +93,21 @@ def eventsindex(): @route('/cef', method=['POST','PUT']) @route('/cef/',method=['POST','PUT']) -#debug(True) +# debug(True) def cefindex(): if request.body: anevent=request.body.read() request.body.close() - #valid json? + # valid json? try: cefDict=json.loads(anevent) except ValueError as e: response.status=500 return - #let the message queue worker who gets this know where it was posted + # let the message queue worker who gets this know where it was posted cefDict['endpoint']='cef' - #post to eventtask exchange + # post to eventtask exchange ensurePublish=mqConn.ensure(mqproducer,mqproducer.publish,max_retries=10) ensurePublish(cefDict,exchange=eventTaskExchange,routing_key=options.taskexchange) return @@ -126,17 +126,17 @@ def customindex(application): if request.body: anevent=request.body.read() request.body.close() - #valid json? + # valid json? try: customDict=json.loads(anevent) except ValueError as e: response.status=500 return - #let the message queue worker who gets this know where it was posted + # let the message queue worker who gets this know where it was posted customDict['endpoint']= application customDict['customendpoint'] = True - #post to eventtask exchange + # post to eventtask exchange ensurePublish=mqConn.ensure(mqproducer,mqproducer.publish,max_retries=10) ensurePublish(customDict,exchange=eventTaskExchange,routing_key=options.taskexchange) return @@ -151,13 +151,13 @@ def initConfig(): options.listen_host=getConfig('listen_host', '127.0.0.1', options.configfile) -#get config info: +# get config info: parser=OptionParser() parser.add_option("-c", dest='configfile', default=os.path.join(os.path.dirname(__file__), __file__).replace('.py', '.conf'), help="configuration file to use") (options,args) = parser.parse_args() initConfig() -#connect and declare the message queue/kombu objects. +# connect and declare the message queue/kombu objects. connString='amqp://{0}:{1}@{2}:{3}//'.format(options.mquser,options.mqpassword,options.mqserver,options.mqport) mqConn=Connection(connString) diff --git a/mq/plugins/broFixup.py b/mq/plugins/broFixup.py index e295015d..533a0429 100644 --- a/mq/plugins/broFixup.py +++ b/mq/plugins/broFixup.py @@ -101,7 +101,7 @@ class message(object): if 'ts' in newmessage['details']: newmessage[u'utctimestamp'] = toUTC(float(newmessage['details']['ts'])).isoformat() newmessage[u'timestamp'] = toUTC(float(newmessage['details']['ts'])).isoformat() - #del(newmessage['details']['ts']) + # del(newmessage['details']['ts']) else: # a malformed message somehow managed to crawl to us, let's put it somewhat together newmessage[u'utctimestamp'] = toUTC(datetime.now()).isoformat() diff --git a/mq/plugins/dropMessage.py b/mq/plugins/dropMessage.py index a45b643f..5918e731 100644 --- a/mq/plugins/dropMessage.py +++ b/mq/plugins/dropMessage.py @@ -28,15 +28,15 @@ class message(object): # early exit by setting message = None and return if 'details' in message.keys(): # drop disabled for now - #if 'signatureid' in message['details']: - #if message['details'].lower() == 'execve' and \ - #'command' not in message['details']: + # if 'signatureid' in message['details']: + # if message['details'].lower() == 'execve' and \ + # 'command' not in message['details']: # auditd entry without a command # likely a result of another command (java starting a job, etc.) # signal a drop - #message = None - #return message + # message = None + # return message if 'http_user_agent' in message['details']: if message['details']['http_user_agent'] == 'ELB-HealthChecker/1.0': message = None diff --git a/mq/plugins/fxaFixup.py b/mq/plugins/fxaFixup.py index c850c5ac..7bf689e3 100644 --- a/mq/plugins/fxaFixup.py +++ b/mq/plugins/fxaFixup.py @@ -38,17 +38,17 @@ class message(object): if 'eventsource' not in message: return (message, metadata) - #drop non-relevant messages + # drop non-relevant messages if message['eventsource'] in ('Fxa-customsMozSvc', 'FxaContentWebserver', 'FxaAuthWebserver', 'FxaOauthWebserver', 'FxaAuth', 'fxa-auth-server'): if 'details' in message.keys(): if 'status' in message['details']: if message['details']['status'] == 200: - #normal 200 returns for web content + # normal 200 returns for web content return(None, metadata) # FxaAuth sends http status as 'code' if 'code' in message['details']: if message['details']['code'] == 200: - #normal 200 returns for web content + # normal 200 returns for web content return(None, metadata) if 'op' in message['details']: if message['details']['op'] == 'mailer.send.1': @@ -93,15 +93,15 @@ class message(object): message['details']['remoteAddressChain'][0] == '[' and message['details']['remoteAddressChain'][-1] == ']'): # remove the brackets and double quotes - for i in ['[',']','"']: - message['details']['remoteAddressChain']=message['details']['remoteAddressChain'].replace(i,'') + for i in ['[', ']', '"']: + message['details']['remoteAddressChain'] = message['details']['remoteAddressChain'].replace(i, '') # make sure it's still a list if ',' in message['details']['remoteAddressChain']: sourceIP = message['details']['remoteAddressChain'].split(',')[0] if isIP(sourceIP): message['details']['sourceipaddress'] = sourceIP - #fxacustoms sends source ip as just 'ip' + # fxacustoms sends source ip as just 'ip' if 'ip' in message['details'].keys(): if isIP(message['details']['ip']): message['details']['sourceipaddress'] = message['details']['ip'] diff --git a/rest/index.py b/rest/index.py index 6d2a5c50..6c88abf9 100644 --- a/rest/index.py +++ b/rest/index.py @@ -163,7 +163,7 @@ def ipintel(): '''send an IP address through plugins for intel enhancement''' if request.body: arequest = request.body.read() - #request.body.close() + # request.body.close() # valid json? try: requestDict = json.loads(arequest) diff --git a/rest/plugins/vpc_blackhole.py b/rest/plugins/vpc_blackhole.py index 6ecaa59f..0b21fb1f 100644 --- a/rest/plugins/vpc_blackhole.py +++ b/rest/plugins/vpc_blackhole.py @@ -177,7 +177,7 @@ class message(object): sendToBHVPC = False if sendToBHVPC and ipaddress is not None: - #figure out the CIDR mask + # figure out the CIDR mask if isIPv4(ipaddress) or isIPv6(ipaddress): ipcidr=netaddr.IPNetwork(ipaddress) if not ipcidr.ip.is_loopback() \ From 46be867d2fa69da6ce208930e9ab299c978b18b4 Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Fri, 14 Dec 2018 13:46:27 -0600 Subject: [PATCH 86/89] Fixup unused variables check --- .flake8 | 1 - alerts/plugins/pagerDutyTriggerEvent.py | 2 +- bot/mozdefbot.py | 2 +- cron/createFDQNBlockList.py | 2 +- cron/createIPBlockList.py | 2 +- cron/vidyo2MozDef.py | 2 +- examples/demo/sampleData2MozDef.py | 4 ++-- loginput/index.py | 10 +++++----- mq/esworker_cloudtrail.py | 4 +--- mq/plugins/geoip.py | 2 +- rest/index.py | 8 ++++---- rest/plugins/cymon.py | 2 +- rest/plugins/vpc_blackhole.py | 1 - tests/mozdef_util/test_elasticsearch_client.py | 2 +- 14 files changed, 20 insertions(+), 24 deletions(-) diff --git a/.flake8 b/.flake8 index 82945fc1..4f2ea83b 100644 --- a/.flake8 +++ b/.flake8 @@ -17,4 +17,3 @@ ignore = E402 # module level import not at top of file E501 # line too long E722 # do not use bare except' - F841 # local variable 'CIDR' is assigned to but never used diff --git a/alerts/plugins/pagerDutyTriggerEvent.py b/alerts/plugins/pagerDutyTriggerEvent.py index 65ae3521..f9f10e8c 100644 --- a/alerts/plugins/pagerDutyTriggerEvent.py +++ b/alerts/plugins/pagerDutyTriggerEvent.py @@ -65,7 +65,7 @@ class message(object): } ] }) - r = requests.post( + requests.post( 'https://events.pagerduty.com/generic/2010-04-15/create_event.json', headers=headers, data=payload, diff --git a/bot/mozdefbot.py b/bot/mozdefbot.py index b7d2facd..11d825ce 100755 --- a/bot/mozdefbot.py +++ b/bot/mozdefbot.py @@ -127,7 +127,7 @@ def ipLocation(ip): if geoDict['country_code'] in ('US'): if geoDict['metro_code']: location = location + '/{0}'.format(geoDict['metro_code']) - except Exception as e: + except Exception: location = "" return location diff --git a/cron/createFDQNBlockList.py b/cron/createFDQNBlockList.py index 2c03da3c..f6d3b71a 100644 --- a/cron/createFDQNBlockList.py +++ b/cron/createFDQNBlockList.py @@ -152,7 +152,7 @@ def s3_upload_file(file_path, bucket_name, key_name): conn = boto.connect_s3(aws_access_key_id=options.aws_access_key_id, aws_secret_access_key=options.aws_secret_access_key) try: bucket = conn.get_bucket(bucket_name, validate=False) - except boto.exception.S3ResponseError as e: + except boto.exception.S3ResponseError: conn.create_bucket(bucket_name) bucket = conn.get_bucket(bucket_name, validate=False) diff --git a/cron/createIPBlockList.py b/cron/createIPBlockList.py index ec7cc7c7..b9963ce2 100755 --- a/cron/createIPBlockList.py +++ b/cron/createIPBlockList.py @@ -224,7 +224,7 @@ def s3_upload_file(file_path, bucket_name, key_name): conn = boto.connect_s3(aws_access_key_id=options.aws_access_key_id, aws_secret_access_key=options.aws_secret_access_key) try: bucket = conn.get_bucket(bucket_name, validate=False) - except boto.exception.S3ResponseError as e: + except boto.exception.S3ResponseError: conn.create_bucket(bucket_name) bucket = conn.get_bucket(bucket_name, validate=False) diff --git a/cron/vidyo2MozDef.py b/cron/vidyo2MozDef.py index fa41ea4a..6258e740 100755 --- a/cron/vidyo2MozDef.py +++ b/cron/vidyo2MozDef.py @@ -100,7 +100,7 @@ class MozDefEvent(): raise MozDefError('Summary is a required field') try: - r = self.httpsession.post(self.url, json.dumps(log_msg, encoding='utf-8'), verify=self.verify_certificate) + self.httpsession.post(self.url, json.dumps(log_msg, encoding='utf-8'), verify=self.verify_certificate) except Exception as e: if not self.fire_and_forget_mode: diff --git a/examples/demo/sampleData2MozDef.py b/examples/demo/sampleData2MozDef.py index 20401473..4323fd1c 100755 --- a/examples/demo/sampleData2MozDef.py +++ b/examples/demo/sampleData2MozDef.py @@ -57,7 +57,7 @@ def setConfig(option,value,configfile): def postLogs(logcache): # post logs asynchronously with requests workers and check on the results # expects a queue object from the multiprocessing library - posts=[] + # posts=[] try: while not logcache.empty(): postdata=logcache.get_nowait() @@ -70,7 +70,7 @@ def postLogs(logcache): # append to posts if this is long running and you want # events to try again later. # posts.append((r,postdata,url)) - except Empty as e: + except Empty: pass # for p, postdata, url in posts: # try: diff --git a/loginput/index.py b/loginput/index.py index a6d9c7e2..c3cf7005 100644 --- a/loginput/index.py +++ b/loginput/index.py @@ -27,7 +27,7 @@ def status(): @route('/test') @route('/test/') def testindex(): - ip = request.environ.get('REMOTE_ADDR') + # ip = request.environ.get('REMOTE_ADDR') # response.headers['X-IP'] = '{0}'.format(ip) response.status=200 @@ -53,7 +53,7 @@ def bulkindex(): # valid json? try: eventDict=json.loads(i) - except ValueError as e: + except ValueError: response.status=500 return # don't post the items telling us where to post things.. @@ -79,7 +79,7 @@ def eventsindex(): # valid json? try: eventDict=json.loads(anevent) - except ValueError as e: + except ValueError: response.status=500 return # let the message queue worker who gets this know where it was posted @@ -101,7 +101,7 @@ def cefindex(): # valid json? try: cefDict=json.loads(anevent) - except ValueError as e: + except ValueError: response.status=500 return # let the message queue worker who gets this know where it was posted @@ -129,7 +129,7 @@ def customindex(application): # valid json? try: customDict=json.loads(anevent) - except ValueError as e: + except ValueError: response.status=500 return # let the message queue worker who gets this know where it was posted diff --git a/mq/esworker_cloudtrail.py b/mq/esworker_cloudtrail.py index 1fe0a4f6..f2278855 100755 --- a/mq/esworker_cloudtrail.py +++ b/mq/esworker_cloudtrail.py @@ -308,8 +308,6 @@ class taskConsumer(object): def authenticate(self): if options.cloudtrail_arn not in ['', 'cloudtrail_arn']: - role_manager_args = {} - role_manager = RoleManager(**get_aws_credentials( options.region, options.accesskey, @@ -369,7 +367,7 @@ class taskConsumer(object): self.on_message(event) self.taskQueue.delete_message(msg) - except (SSLEOFError, SSLError, socket.error) as e: + except (SSLEOFError, SSLError, socket.error): logger.info('Received network related error...reconnecting') time.sleep(5) self.connection, self.taskQueue = connect_sqs( diff --git a/mq/plugins/geoip.py b/mq/plugins/geoip.py index 33360b07..b21afc8f 100644 --- a/mq/plugins/geoip.py +++ b/mq/plugins/geoip.py @@ -37,7 +37,7 @@ class message(object): return geoDict else: location['location'] = 'unknown' - except ValueError as e: + except ValueError: pass return location diff --git a/rest/index.py b/rest/index.py index 6c88abf9..e4e35a27 100644 --- a/rest/index.py +++ b/rest/index.py @@ -50,7 +50,7 @@ def enable_cors(fn): @route('/test/') def test(): '''test endpoint for..testing''' - ip = request.environ.get('REMOTE_ADDR') + # ip = request.environ.get('REMOTE_ADDR') # response.headers['X-IP'] = '{0}'.format(ip) response.status = 200 @@ -143,7 +143,7 @@ def index(): # valid json? try: requestDict = json.loads(arequest) - except ValueError as e: + except ValueError: response.status = 500 if 'ipaddress' in requestDict.keys() and isIPv4(requestDict['ipaddress']): @@ -167,7 +167,7 @@ def ipintel(): # valid json? try: requestDict = json.loads(arequest) - except ValueError as e: + except ValueError: response.status = 500 if 'ipaddress' in requestDict.keys() and isIPv4(requestDict['ipaddress']): response.content_type = "application/json" @@ -192,7 +192,7 @@ def index(): # valid json? try: requestDict = json.loads(arequest) - except ValueError as e: + except ValueError: response.status = 500 return if 'ipaddress' in requestDict.keys() and isIPv4(requestDict['ipaddress']): diff --git a/rest/plugins/cymon.py b/rest/plugins/cymon.py index d3bda909..6e3b1796 100644 --- a/rest/plugins/cymon.py +++ b/rest/plugins/cymon.py @@ -55,7 +55,7 @@ class message(object): request.body.close() try: requestDict = json.loads(arequest) - except ValueError as e: + except ValueError: response.status = 500 print(requestDict, requestDict.keys()) diff --git a/rest/plugins/vpc_blackhole.py b/rest/plugins/vpc_blackhole.py index 0b21fb1f..7a938f41 100644 --- a/rest/plugins/vpc_blackhole.py +++ b/rest/plugins/vpc_blackhole.py @@ -158,7 +158,6 @@ class message(object): ''' # format/validate request.json: ipaddress = None - CIDR = None sendToBHVPC = False # loop through the fields of the form diff --git a/tests/mozdef_util/test_elasticsearch_client.py b/tests/mozdef_util/test_elasticsearch_client.py index beb6f0f6..99ec0726 100644 --- a/tests/mozdef_util/test_elasticsearch_client.py +++ b/tests/mozdef_util/test_elasticsearch_client.py @@ -357,7 +357,7 @@ class TestWriteWithIDExists(ElasticsearchClientTest): saved_event = self.es_client.save_event(body=event, doc_id=event_id) assert saved_event['_id'] == event_id self.flush(self.event_index_name) - fetched_event = self.es_client.get_event_by_id(event_id) + self.es_client.get_event_by_id(event_id) class TestGetIndices(ElasticsearchClientTest): From 2b47045a4b8027dfaa2ca1cc27e4b2ca3c5b4612 Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Fri, 14 Dec 2018 14:19:50 -0600 Subject: [PATCH 87/89] Remove unused cookie cutter files --- mozdef_util/.editorconfig | 21 ---- mozdef_util/.github/ISSUE_TEMPLATE.md | 15 --- mozdef_util/.travis.yml | 16 --- mozdef_util/CONTRIBUTING.rst | 128 --------------------- mozdef_util/MANIFEST.in | 10 -- mozdef_util/Makefile | 49 +------- mozdef_util/README.rst | 17 +-- mozdef_util/docs/Makefile | 20 ---- mozdef_util/docs/conf.py | 160 -------------------------- mozdef_util/docs/contributing.rst | 1 - mozdef_util/docs/history.rst | 1 - mozdef_util/docs/index.rst | 19 --- mozdef_util/docs/installation.rst | 52 --------- mozdef_util/docs/make.bat | 36 ------ mozdef_util/docs/readme.rst | 1 - mozdef_util/docs/usage.rst | 7 -- mozdef_util/setup.cfg | 22 ---- mozdef_util/tox.ini | 21 ---- 18 files changed, 4 insertions(+), 592 deletions(-) delete mode 100644 mozdef_util/.editorconfig delete mode 100644 mozdef_util/.github/ISSUE_TEMPLATE.md delete mode 100644 mozdef_util/.travis.yml delete mode 100644 mozdef_util/CONTRIBUTING.rst delete mode 100644 mozdef_util/MANIFEST.in delete mode 100644 mozdef_util/docs/Makefile delete mode 100755 mozdef_util/docs/conf.py delete mode 100644 mozdef_util/docs/contributing.rst delete mode 100644 mozdef_util/docs/history.rst delete mode 100644 mozdef_util/docs/index.rst delete mode 100644 mozdef_util/docs/installation.rst delete mode 100644 mozdef_util/docs/make.bat delete mode 100644 mozdef_util/docs/readme.rst delete mode 100644 mozdef_util/docs/usage.rst delete mode 100644 mozdef_util/setup.cfg delete mode 100644 mozdef_util/tox.ini diff --git a/mozdef_util/.editorconfig b/mozdef_util/.editorconfig deleted file mode 100644 index d4a2c440..00000000 --- a/mozdef_util/.editorconfig +++ /dev/null @@ -1,21 +0,0 @@ -# http://editorconfig.org - -root = true - -[*] -indent_style = space -indent_size = 4 -trim_trailing_whitespace = true -insert_final_newline = true -charset = utf-8 -end_of_line = lf - -[*.bat] -indent_style = tab -end_of_line = crlf - -[LICENSE] -insert_final_newline = false - -[Makefile] -indent_style = tab diff --git a/mozdef_util/.github/ISSUE_TEMPLATE.md b/mozdef_util/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index bc0309c8..00000000 --- a/mozdef_util/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,15 +0,0 @@ -* MozDef Util version: -* Python version: -* Operating System: - -### Description - -Describe what you were trying to get done. -Tell us what happened, what went wrong, and what you expected to happen. - -### What I Did - -``` -Paste the command(s) you ran and the output. -If there was a crash, please include the traceback here. -``` diff --git a/mozdef_util/.travis.yml b/mozdef_util/.travis.yml deleted file mode 100644 index 3c59f7e7..00000000 --- a/mozdef_util/.travis.yml +++ /dev/null @@ -1,16 +0,0 @@ -# Config file for automatic testing at travis-ci.org - -language: python -python: - - 3.6 - - 3.5 - - 3.4 - - 2.7 - -# Command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors -install: pip install -U tox-travis - -# Command to run tests, e.g. python setup.py test -script: tox - - diff --git a/mozdef_util/CONTRIBUTING.rst b/mozdef_util/CONTRIBUTING.rst deleted file mode 100644 index 22024d3a..00000000 --- a/mozdef_util/CONTRIBUTING.rst +++ /dev/null @@ -1,128 +0,0 @@ -.. highlight:: shell - -============ -Contributing -============ - -Contributions are welcome, and they are greatly appreciated! Every little bit -helps, and credit will always be given. - -You can contribute in many ways: - -Types of Contributions ----------------------- - -Report Bugs -~~~~~~~~~~~ - -Report bugs at https://github.com/jeffbryner/mozdef_util/issues. - -If you are reporting a bug, please include: - -* Your operating system name and version. -* Any details about your local setup that might be helpful in troubleshooting. -* Detailed steps to reproduce the bug. - -Fix Bugs -~~~~~~~~ - -Look through the GitHub issues for bugs. Anything tagged with "bug" and "help -wanted" is open to whoever wants to implement it. - -Implement Features -~~~~~~~~~~~~~~~~~~ - -Look through the GitHub issues for features. Anything tagged with "enhancement" -and "help wanted" is open to whoever wants to implement it. - -Write Documentation -~~~~~~~~~~~~~~~~~~~ - -MozDef Util could always use more documentation, whether as part of the -official MozDef Util docs, in docstrings, or even on the web in blog posts, -articles, and such. - -Submit Feedback -~~~~~~~~~~~~~~~ - -The best way to send feedback is to file an issue at https://github.com/jeffbryner/mozdef_util/issues. - -If you are proposing a feature: - -* Explain in detail how it would work. -* Keep the scope as narrow as possible, to make it easier to implement. -* Remember that this is a volunteer-driven project, and that contributions - are welcome :) - -Get Started! ------------- - -Ready to contribute? Here's how to set up `mozdef_util` for local development. - -1. Fork the `mozdef_util` repo on GitHub. -2. Clone your fork locally:: - - $ git clone git@github.com:your_name_here/mozdef_util.git - -3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: - - $ mkvirtualenv mozdef_util - $ cd mozdef_util/ - $ python setup.py develop - -4. Create a branch for local development:: - - $ git checkout -b name-of-your-bugfix-or-feature - - Now you can make your changes locally. - -5. When you're done making changes, check that your changes pass flake8 and the - tests, including testing other Python versions with tox:: - - $ flake8 mozdef_util tests - $ python setup.py test or py.test - $ tox - - To get flake8 and tox, just pip install them into your virtualenv. - -6. Commit your changes and push your branch to GitHub:: - - $ git add . - $ git commit -m "Your detailed description of your changes." - $ git push origin name-of-your-bugfix-or-feature - -7. Submit a pull request through the GitHub website. - -Pull Request Guidelines ------------------------ - -Before you submit a pull request, check that it meets these guidelines: - -1. The pull request should include tests. -2. If the pull request adds functionality, the docs should be updated. Put - your new functionality into a function with a docstring, and add the - feature to the list in README.rst. -3. The pull request should work for Python 2.7, 3.4, 3.5 and 3.6, and for PyPy. Check - https://travis-ci.org/jeffbryner/mozdef_util/pull_requests - and make sure that the tests pass for all supported Python versions. - -Tips ----- - -To run a subset of tests:: - - - $ python -m unittest tests.test_mozdef_util - -Deploying ---------- - -A reminder for the maintainers on how to deploy. -Make sure all your changes are committed (including an entry in HISTORY.rst). -Then run:: - -$ bumpversion patch # possible: major / minor / patch -$ git push -$ git push --tags - -Travis will then deploy to PyPI if tests pass. diff --git a/mozdef_util/MANIFEST.in b/mozdef_util/MANIFEST.in deleted file mode 100644 index 292d6dd9..00000000 --- a/mozdef_util/MANIFEST.in +++ /dev/null @@ -1,10 +0,0 @@ -include CONTRIBUTING.rst -include HISTORY.rst -include LICENSE -include README.rst - -recursive-include tests * -recursive-exclude * __pycache__ -recursive-exclude * *.py[co] - -recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif diff --git a/mozdef_util/Makefile b/mozdef_util/Makefile index 07f6d7de..7ab5b9fa 100644 --- a/mozdef_util/Makefile +++ b/mozdef_util/Makefile @@ -1,18 +1,6 @@ -.PHONY: clean clean-test clean-pyc clean-build docs help +.PHONY: clean clean-pyc clean-build help .DEFAULT_GOAL := help -define BROWSER_PYSCRIPT -import os, webbrowser, sys - -try: - from urllib import pathname2url -except: - from urllib.request import pathname2url - -webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) -endef -export BROWSER_PYSCRIPT - define PRINT_HELP_PYSCRIPT import re, sys @@ -24,12 +12,10 @@ for line in sys.stdin: endef export PRINT_HELP_PYSCRIPT -BROWSER := python -c "$$BROWSER_PYSCRIPT" - help: @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) -clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts +clean: clean-build clean-pyc ## remove all build and Python artifacts clean-build: ## remove build artifacts rm -fr build/ @@ -44,37 +30,8 @@ clean-pyc: ## remove Python file artifacts find . -name '*~' -exec rm -f {} + find . -name '__pycache__' -exec rm -fr {} + -clean-test: ## remove test and coverage artifacts - rm -fr .tox/ - rm -f .coverage - rm -fr htmlcov/ - rm -fr .pytest_cache - lint: ## check style with flake8 - flake8 mozdef_util tests - -test: ## run tests quickly with the default Python - python setup.py test - -test-all: ## run tests on every Python version with tox - tox - -coverage: ## check code coverage quickly with the default Python - coverage run --source mozdef_util setup.py test - coverage report -m - coverage html - $(BROWSER) htmlcov/index.html - -docs: ## generate Sphinx HTML documentation, including API docs - rm -f docs/mozdef_util.rst - rm -f docs/modules.rst - sphinx-apidoc -o docs/ mozdef_util - $(MAKE) -C docs clean - $(MAKE) -C docs html - $(BROWSER) docs/_build/html/index.html - -servedocs: docs ## compile the docs watching for changes - watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . + flake8 mozdef_util release: dist ## package and upload a release twine upload dist/* diff --git a/mozdef_util/README.rst b/mozdef_util/README.rst index 19f1bc05..194ce925 100644 --- a/mozdef_util/README.rst +++ b/mozdef_util/README.rst @@ -6,19 +6,4 @@ MozDef Util Utilities shared throughout the MozDef codebase -* Free software: Mozilla Public License 2.0 -* Documentation: https://mozdef-util.readthedocs.io. - - -Features --------- - -* TODO - -Credits -------- - -This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template. - -.. _Cookiecutter: https://github.com/audreyr/cookiecutter -.. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage +* Documentation: https://mozdef.readthedocs.io/en/latest/mozdef_util.html diff --git a/mozdef_util/docs/Makefile b/mozdef_util/docs/Makefile deleted file mode 100644 index 0ada2e2a..00000000 --- a/mozdef_util/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = python -msphinx -SPHINXPROJ = mozdef_util -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/mozdef_util/docs/conf.py b/mozdef_util/docs/conf.py deleted file mode 100755 index cce4f09b..00000000 --- a/mozdef_util/docs/conf.py +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# mozdef_util documentation build configuration file, created by -# sphinx-quickstart on Fri Jun 9 13:47:02 2017. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -# If extensions (or modules to document with autodoc) are in another -# directory, add these directories to sys.path here. If the directory is -# relative to the documentation root, use os.path.abspath to make it -# absolute, like shown here. -# -import os -import sys -sys.path.insert(0, os.path.abspath('..')) - -import mozdef_util - -# -- General configuration --------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'MozDef Util' -copyright = u"2018, Mozilla Infosec" -author = u"Mozilla Infosec" - -# The version info for the project you're documenting, acts as replacement -# for |version| and |release|, also used in various other places throughout -# the built documents. -# -# The short X.Y version. -version = mozdef_util.__version__ -# The full version, including alpha/beta/rc tags. -release = mozdef_util.__version__ - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = False - - -# -- Options for HTML output ------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'alabaster' - -# Theme options are theme-specific and customize the look and feel of a -# theme further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - - -# -- Options for HTMLHelp output --------------------------------------- - -# Output file base name for HTML help builder. -htmlhelp_basename = 'mozdef_utildoc' - - -# -- Options for LaTeX output ------------------------------------------ - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass -# [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'mozdef_util.tex', - u'MozDef Util Documentation', - u'Mozilla Infosec', 'manual'), -] - - -# -- Options for manual page output ------------------------------------ - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'mozdef_util', - u'MozDef Util Documentation', - [author], 1) -] - - -# -- Options for Texinfo output ---------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, 'mozdef_util', - u'MozDef Util Documentation', - author, - 'mozdef_util', - 'One line description of project.', - 'Miscellaneous'), -] diff --git a/mozdef_util/docs/contributing.rst b/mozdef_util/docs/contributing.rst deleted file mode 100644 index e582053e..00000000 --- a/mozdef_util/docs/contributing.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../CONTRIBUTING.rst diff --git a/mozdef_util/docs/history.rst b/mozdef_util/docs/history.rst deleted file mode 100644 index 25064996..00000000 --- a/mozdef_util/docs/history.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../HISTORY.rst diff --git a/mozdef_util/docs/index.rst b/mozdef_util/docs/index.rst deleted file mode 100644 index 1293b615..00000000 --- a/mozdef_util/docs/index.rst +++ /dev/null @@ -1,19 +0,0 @@ -Welcome to MozDef Util's documentation! -====================================== - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - readme - installation - usage - modules - contributing - history - -Indices and tables -================== -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/mozdef_util/docs/installation.rst b/mozdef_util/docs/installation.rst deleted file mode 100644 index ca5e638f..00000000 --- a/mozdef_util/docs/installation.rst +++ /dev/null @@ -1,52 +0,0 @@ -.. highlight:: shell - -============ -Installation -============ - - -Stable release --------------- - -To install MozDef Util, run this command in your terminal: - -.. code-block:: console - - $ pip install mozdef_util - -This is the preferred method to install MozDef Util, as it will always install the most recent stable release. - -If you don't have `pip`_ installed, this `Python installation guide`_ can guide -you through the process. - -.. _pip: https://pip.pypa.io -.. _Python installation guide: http://docs.python-guide.org/en/latest/starting/installation/ - - -From sources ------------- - -The sources for MozDef Util can be downloaded from the `Github repo`_. - -You can either clone the public repository: - -.. code-block:: console - - $ git clone git://github.com/mozilla/mozdef - -Or download the `tarball`_: - -.. code-block:: console - - $ curl -OL https://github.com/mozilla/mozdef/tarball/master - -Once you have a copy of the source, you can install it with: - -.. code-block:: console - - $ cd mozdef_util - $ python setup.py install - - -.. _Github repo: https://github.com/mozilla/mozdef -.. _tarball: https://github.com/mozilla/mozdef/tarball/master diff --git a/mozdef_util/docs/make.bat b/mozdef_util/docs/make.bat deleted file mode 100644 index 0b7196dd..00000000 --- a/mozdef_util/docs/make.bat +++ /dev/null @@ -1,36 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=python -msphinx -) -set SOURCEDIR=. -set BUILDDIR=_build -set SPHINXPROJ=mozdef_util - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The Sphinx module was not found. Make sure you have Sphinx installed, - echo.then set the SPHINXBUILD environment variable to point to the full - echo.path of the 'sphinx-build' executable. Alternatively you may add the - echo.Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% - -:end -popd diff --git a/mozdef_util/docs/readme.rst b/mozdef_util/docs/readme.rst deleted file mode 100644 index 72a33558..00000000 --- a/mozdef_util/docs/readme.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../README.rst diff --git a/mozdef_util/docs/usage.rst b/mozdef_util/docs/usage.rst deleted file mode 100644 index 7d21d026..00000000 --- a/mozdef_util/docs/usage.rst +++ /dev/null @@ -1,7 +0,0 @@ -===== -Usage -===== - -To use MozDef Util in a project:: - - import mozdef_util diff --git a/mozdef_util/setup.cfg b/mozdef_util/setup.cfg deleted file mode 100644 index 6c5f8ef3..00000000 --- a/mozdef_util/setup.cfg +++ /dev/null @@ -1,22 +0,0 @@ -[bumpversion] -current_version = 1.0.0 -commit = True -tag = True - -[bumpversion:file:setup.py] -search = version='{current_version}' -replace = version='{new_version}' - -[bumpversion:file:mozdef_util/__init__.py] -search = __version__ = '{current_version}' -replace = __version__ = '{new_version}' - -[bdist_wheel] -universal = 1 - -[flake8] -exclude = docs - -[aliases] -# Define setup.py command aliases here - diff --git a/mozdef_util/tox.ini b/mozdef_util/tox.ini deleted file mode 100644 index 7d4704dc..00000000 --- a/mozdef_util/tox.ini +++ /dev/null @@ -1,21 +0,0 @@ -[tox] -envlist = py27, py34, py35, py36, flake8 - -[travis] -python = - 3.6: py36 - 3.5: py35 - 3.4: py34 - 2.7: py27 - -[testenv:flake8] -basepython = python -deps = flake8 -commands = flake8 mozdef_util - -[testenv] -setenv = - PYTHONPATH = {toxinidir} - -commands = python setup.py test - From 9c35aa02a95cd7e5ea6647864e9ac3334bec1cf6 Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Fri, 14 Dec 2018 15:18:21 -0600 Subject: [PATCH 88/89] Update urllib3 version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index caebd2ba..f0673b09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -58,6 +58,6 @@ six==1.11.0 supervisor==3.3.1 tzlocal==1.4 uritemplate==0.6 -urllib3==1.22 +urllib3==1.23 uwsgi==2.0.17.1 virtualenv==1.11.4 From e1643cf3e26b2358d0fef655b12717c9a162b5e7 Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Fri, 14 Dec 2018 15:21:07 -0600 Subject: [PATCH 89/89] Update requests version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index caebd2ba..4f2275b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,7 +48,7 @@ pynsive==0.2.6 pyparsing==2.1.10 python-dateutil==2.6.1 pytz==2017.3 -requests==2.18.4 +requests==2.20.0 requests-futures==0.9.7 rsa==3.1.4 s3cmd==1.0.1