add container for generic SQS ingest

cloudy_mozdef/cloudformation/mozdef-instance.yml

@@ -68,7 +68,10 @@ Parameters:
     Description: The URL of your OIDC provider's well-known discovery URL
   CloudTrailSQSNotificationQueueName:
     Type: String
-    Description: The URL of your OIDC provider's well-known discovery URL
+    Description: The name of the SQS used for CloudTrail notifications.
+  MozDefSQSQueueName:
+    Type: String
+    Description: The name of the generic SQS queue used to pickup events.
 Resources:
   MozDefElasticLoadBalancingV2TargetGroup:
     Type: AWS::ElasticLoadBalancingV2::TargetGroup
@@ -147,9 +150,13 @@ Resources:
             - content: |
                 OPTIONS_TASKEXCHANGE=${CloudTrailSQSNotificationQueueName}
               path: /opt/mozdef/docker/compose/cloudy_mozdef_mq_cloudtrail.env
+            - content: |
+                OPTIONS_TASKEXCHANGE=${MozDefSQSQueName}
+              path: /opt/mozdef/docker/compose/cloudy_mozdef_mq_sqs.env
           runcmd:
             - chmod --verbose 600 /opt/mozdef/docker/compose/cloudy_mozdef.env
             - chmod --verbose 600 /opt/mozdef/docker/compose/cloudy_mozdef_kibana.env
+            - chmod --verbose 600 /opt/mozdef/docker/compose/cloudy_mozdef_mq_sqs.env
             - mkdir --verbose --parents ${EFSMountPoint}
             - echo '*.* @@127.0.0.1:514' >> /etc/rsyslog.conf
             - systemctl enable rsyslog

cloudy_mozdef/cloudformation/mozdef-parent.yml

@@ -98,6 +98,7 @@ Resources:
         OIDCClientSecret: !Ref OIDCClientSecret
         OIDCDiscoveryURL: !Ref OIDCDiscoveryURL
         CloudTrailSQSNotificationQueueName: !GetAtt MozDefCloudTrail.Outputs.CloudTrailSQSQueueName
+        MozDefSQSQueueName: !GetAtt MozDefSQS.Outputs.SQSQueueName
       Tags:
         - Key: application
           Value: mozdef
@@ -142,7 +143,7 @@ Resources:
         Value: mozdef
       - Key: stack
         Value: !Ref AWS::StackName
-      TemplateURL: !Join [ '', [ !Ref S3TemplateLocation, mozdef-sqs.yml ] ]
+      TemplateURL: !Join [ '', [ !Ref S3TemplateLocation, mozdef-sqs.yml ]
   MozDefCloudTrail:
     Type: AWS::CloudFormation::Stack
     Properties:
@@ -245,4 +246,4 @@ Resources:
     Properties:
       RoleName: AWSServiceRoleForAmazonElasticsearchService
       PathPrefix: '/aws-service-role/es.amazonaws.com/'
-      ServiceToken: !GetAtt DoesRoleExistLambdaFunction.Arn
+      ServiceToken: !GetAtt DoesRoleExistLambdaFunction.Arn

docker/compose/docker-compose-cloudy-mozdef.yml

@@ -186,6 +186,23 @@ services:
       - default
     volumes:
       - geolite_db:/opt/mozdef/envs/mozdef/data/
+  mq_sqs:
+    image: mozdef/mozdef_mq_worker
+    env_file:
+      - cloudy_mozdef.env
+      - cloudy_mozdef_mq_sqs.env
+    restart: always
+    command: bash -c 'source /opt/mozdef/envs/python/bin/activate && python esworker_sqs.py -c esworker_sqs.conf'
+    scale: 1
+    depends_on:
+      - base
+      - rabbitmq
+      - loginput
+      - bootstrap
+    networks:
+      - default
+    volumes:
+      - geolite_db:/opt/mozdef/envs/mozdef/data/
 volumes:
   cron:
   geolite_db: