From 893c44c3fa18df7d7db42522e296ef060eb5dce0 Mon Sep 17 00:00:00 2001
From: andrewkrug <akrug@mozilla.com>
Date: Wed, 21 Nov 2018 05:44:57 -0800
Subject: [PATCH] add container for generic SQS ingest

---
 .../cloudformation/mozdef-instance.yml          |  9 ++++++++-
 cloudy_mozdef/cloudformation/mozdef-parent.yml  |  5 +++--
 docker/compose/docker-compose-cloudy-mozdef.yml | 17 +++++++++++++++++
 3 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/cloudy_mozdef/cloudformation/mozdef-instance.yml b/cloudy_mozdef/cloudformation/mozdef-instance.yml
index e5cf6762..09cbdf26 100644
--- a/cloudy_mozdef/cloudformation/mozdef-instance.yml
+++ b/cloudy_mozdef/cloudformation/mozdef-instance.yml
@@ -68,7 +68,10 @@ Parameters:
     Description: The URL of your OIDC provider's well-known discovery URL
   CloudTrailSQSNotificationQueueName:
     Type: String
-    Description: The URL of your OIDC provider's well-known discovery URL
+    Description: The name of the SQS used for CloudTrail notifications.
+  MozDefSQSQueueName:
+    Type: String
+    Description: The name of the generic SQS queue used to pickup events.
 Resources:
   MozDefElasticLoadBalancingV2TargetGroup:
     Type: AWS::ElasticLoadBalancingV2::TargetGroup
@@ -147,9 +150,13 @@ Resources:
             - content: |
                 OPTIONS_TASKEXCHANGE=${CloudTrailSQSNotificationQueueName}
               path: /opt/mozdef/docker/compose/cloudy_mozdef_mq_cloudtrail.env
+            - content: |
+                OPTIONS_TASKEXCHANGE=${MozDefSQSQueName}
+              path: /opt/mozdef/docker/compose/cloudy_mozdef_mq_sqs.env
           runcmd:
             - chmod --verbose 600 /opt/mozdef/docker/compose/cloudy_mozdef.env
             - chmod --verbose 600 /opt/mozdef/docker/compose/cloudy_mozdef_kibana.env
+            - chmod --verbose 600 /opt/mozdef/docker/compose/cloudy_mozdef_mq_sqs.env
             - mkdir --verbose --parents ${EFSMountPoint}
             - echo '*.* @@127.0.0.1:514' >> /etc/rsyslog.conf
             - systemctl enable rsyslog
diff --git a/cloudy_mozdef/cloudformation/mozdef-parent.yml b/cloudy_mozdef/cloudformation/mozdef-parent.yml
index 9227c840..7f737cde 100644
--- a/cloudy_mozdef/cloudformation/mozdef-parent.yml
+++ b/cloudy_mozdef/cloudformation/mozdef-parent.yml
@@ -98,6 +98,7 @@ Resources:
         OIDCClientSecret: !Ref OIDCClientSecret
         OIDCDiscoveryURL: !Ref OIDCDiscoveryURL
         CloudTrailSQSNotificationQueueName: !GetAtt MozDefCloudTrail.Outputs.CloudTrailSQSQueueName
+        MozDefSQSQueueName: !GetAtt MozDefSQS.Outputs.SQSQueueName
       Tags:
         - Key: application
           Value: mozdef
@@ -142,7 +143,7 @@ Resources:
         Value: mozdef
       - Key: stack
         Value: !Ref AWS::StackName
-      TemplateURL: !Join [ '', [ !Ref S3TemplateLocation, mozdef-sqs.yml ] ]
+      TemplateURL: !Join [ '', [ !Ref S3TemplateLocation, mozdef-sqs.yml ]
   MozDefCloudTrail:
     Type: AWS::CloudFormation::Stack
     Properties:
@@ -245,4 +246,4 @@ Resources:
     Properties:
       RoleName: AWSServiceRoleForAmazonElasticsearchService
       PathPrefix: '/aws-service-role/es.amazonaws.com/'
-      ServiceToken: !GetAtt DoesRoleExistLambdaFunction.Arn
\ No newline at end of file
+      ServiceToken: !GetAtt DoesRoleExistLambdaFunction.Arn
diff --git a/docker/compose/docker-compose-cloudy-mozdef.yml b/docker/compose/docker-compose-cloudy-mozdef.yml
index 8a520927..72bb16f4 100644
--- a/docker/compose/docker-compose-cloudy-mozdef.yml
+++ b/docker/compose/docker-compose-cloudy-mozdef.yml
@@ -186,6 +186,23 @@ services:
       - default
     volumes:
       - geolite_db:/opt/mozdef/envs/mozdef/data/
+  mq_sqs:
+    image: mozdef/mozdef_mq_worker
+    env_file:
+      - cloudy_mozdef.env
+      - cloudy_mozdef_mq_sqs.env
+    restart: always
+    command: bash -c 'source /opt/mozdef/envs/python/bin/activate && python esworker_sqs.py -c esworker_sqs.conf'
+    scale: 1
+    depends_on:
+      - base
+      - rabbitmq
+      - loginput
+      - bootstrap
+    networks:
+      - default
+    volumes:
+      - geolite_db:/opt/mozdef/envs/mozdef/data/
 volumes:
   cron:
   geolite_db: