Add ansible orchestration around current provisioning logic

.gitignore

@@ -18,3 +18,4 @@ htmlcov/
 histogram_tools.py
 .DS_Store
 CMakeFiles
+provisioning/ansible/envs/dev_secrets.yml

provisioning/ansible/README.md

@@ -0,0 +1,67 @@
+Deploying telemetry-analysis
+
+Manual setup tasks:
+
+1. In the AWS SES console, make sure that the email address "telemetry-alerts@mozilla.com" is verified.
+
+2. Make sure cross IAM s3 permissions are set up if cross-IAM access is required. Edit bucket policies for relevant buckets to look something like this:
+
+    {
+        "Version": "2008-10-17",
+        "Statement": [
+            {
+                "Sid": "ListAccess",
+                "Effect": "Allow",
+                "Principal": {
+                    "AWS": [
+                        "arn:aws:iam::XXXXXXXXXXXX:root"
+                    ]
+                },
+                "Action": "S3:ListBucket",
+                "Resource": "arn:aws:s3:::telemetry-published-v2"
+            },
+            {
+                "Sid": "GetAccess",
+                "Effect": "Allow",
+                "Principal": {
+                    "AWS": [
+                        "arn:aws:iam::XXXXXXXXXXXX:root"
+                    ]
+                },
+                "Action": "S3:GetObject",
+                "Resource": "arn:aws:s3:::telemetry-published-v2/*"
+            }
+        ]
+    }
+
+Automated deployment tasks:
+
+1. Build an AMI for telemetry workers:
+
+    ansible-playbook -i hosts -v --extra-vars "@envs/dev.yml" playbooks/build_ami.yml
+
+2. Set `worker_ami_id` in envs/dev.yml to the value output by (1). This a git-managed file.
+
+3. Set the RDS password in `envs/dev_secrets.yml`.
+
+   See `envs/dev_secrets.example.yml` for an example. This is an un-managed
+   file. If the telemetry-analysis resources stack has already been created,
+   the value you should set this to is the password portion of the URL.
+
+4. Create the static resources Cloudformation template (only needs to be run once):
+
+    ansible-playbook -i hosts -v --extra-vars "@envs/dev.yml" --extra-vars "@envs/dev_secrets.yml" playbooks/resources.yml
+
+To update / deploy the application servers:
+
+5. Create a new code package to use by updating `sources_version` in envs/dev.yml and running:
+
+    ansible-playbook -i hosts -v --extra-vars "@envs/dev.yml" playbooks/make_code_package.yml
+
+6. Deploy the Cloudformation template by running:
+
+    ansible-playbook -i hosts -v --extra-vars "@envs/dev.yml" playbooks/app.yml
+
+7. Deploy user-facing DNS with (only needs to be run once):
+
+    ansible-playbook -i hosts -v --extra-vars "@envs/dev.yml" playbooks/route53.yaml

provisioning/ansible/envs/dev.yml

@@ -0,0 +1,17 @@
+stack_name: telemetry-analysis
+region: us-west-2
+env: dev
+key_name: "20130730-svcops-base-key-dev"
+ssl_cert_arn: arn:aws:iam::927034868273:server-certificate/exp20180829_star_telemetry_mozilla_org
+
+# code version
+sources_version: 24
+
+dns_name: "telemetry-analysis.dev.mozaws.net"
+dns_zone_name: "dev.mozaws.net."
+
+instance_type: t2.micro
+server_ami_id: ami-a40bea97
+
+# this value can be updated using the build_ami playbook
+worker_ami_id: ami-0057b733

provisioning/ansible/envs/dev_secrets.example.yml

@@ -0,0 +1,4 @@
+# This value should be populated from the output of
+# the telemetry-analysis-resources if it already exists.
+database_password: 'but does it achieve the scale of the web?'
+server_secret: 'yes'

provisioning/ansible/hosts

@@ -0,0 +1 @@
+localhost

provisioning/ansible/playbooks/app.yml

@@ -0,0 +1,43 @@
+---
+- include: resources.yml
+
+- hosts: localhost
+  connection: local
+  tasks:
+
+    - name: get top level git dir
+      command: 'git rev-parse --show-toplevel'
+      register: top_level_dir
+
+    - name: make resources template
+      command: make analysis-service-stack.json
+      args:
+        chdir: "{{ top_level_dir.stdout }}/http/analysis-service"
+
+    - name: resources
+      cloudformation: stack_name="{{stack_name}}-app" region={{region}} state=present
+      args:
+        template: "{{ top_level_dir.stdout }}/http/analysis-service/analysis-service-stack.json"
+        tags:
+          App: "telemetry"
+          Env: "{{env}}"
+          Stack: "{{stack_name}}"
+        template_parameters:
+          ServerInstanceType: "{{instance_type}}"
+          AnalysisSourcesBucket: "{{resources_cfn.stack_outputs.AnalysisSourcesBucket}}"
+          AnalysisPrivateDataBucket: "{{resources_cfn.stack_outputs.AnalysisPrivateDataBucket}}"
+          AnalysisDBSecurityGroup: "{{resources_cfn.stack_outputs.AnalysisDBSecurityGroup}}"
+          AnalysisDatabaseURL: "{{resources_cfn.stack_outputs.AnalysisDatabaseURL}}"
+          AnalysisPublicDataBucket: "{{resources_cfn.stack_outputs.AnalysisPublicDataBucket}}"
+          AnalysisTemporaryBucket: "{{resources_cfn.stack_outputs.AnalysisTemporaryBucket}}"
+          AnalysisLoadBalancer: "{{resources_cfn.stack_outputs.AnalysisLoadBalancer}}"
+          ServerSecret: "{{server_secret}}"
+          KeyName: "{{ key_name }}"
+          WorkerAMI: "{{worker_ami_id}}"
+          ServerAMI: "{{server_ami_id}}"
+          SourcesVersion: "{{ sources_version }}"
+          AnalysisPublicWorkerProfile: "{{resources_cfn.stack_outputs.AnalysisPublicWorkerProfile}}"
+          AnalysisPrivateWorkerProfile: "{{resources_cfn.stack_outputs.AnalysisPrivateWorkerProfile}}"
+          # ignore roles
+          # AnalysisPublicWorkerRole: "{{resources_cfn.stack_outputs.AnalysisPublicWorkerRole}}"
+          # AnalysisPrivateWorkerRole: "{{resources_cfn.stack_outputs.AnalysisPrivateWorkerRole}}"

provisioning/ansible/playbooks/build_ami.yml

@@ -0,0 +1,14 @@
+---
+- hosts: localhost
+  connection: local
+  gather_facts: false
+
+  tasks:
+  - name: get top level git dir
+    command: 'git rev-parse --show-toplevel'
+    register: top_level_dir
+
+  - name: build telemetry AMI
+    command: 'time python -u -m provisioning.aws.create_telemetry_worker_ami provisioning/aws/telemetry_worker.hvm.json'
+    args:
+      chdir: '{{ top_level_dir.stdout }}'

provisioning/ansible/playbooks/make_code_package.yml

@@ -0,0 +1,16 @@
+---
+- include: resources.yml
+
+- hosts: localhost
+  connection: local
+  gather_facts: false
+
+  tasks:
+  - name: get top level git dir
+    command: 'git rev-parse --show-toplevel'
+    register: top_level_dir
+
+  - name: build and upload telemetry code
+    command: make put SOURCES_BUCKET={{resources_cfn.stack_outputs.AnalysisSourcesBucket}} VERSION={{ sources_version }}
+    args:
+      chdir: "{{ top_level_dir.stdout }}/http/analysis-service"

provisioning/ansible/playbooks/resources.yml

@@ -0,0 +1,27 @@
+---
+- hosts: localhost
+  connection: local
+  tasks:
+    - include_vars: ../envs/dev_secrets.yml
+
+    - name: get top level git dir
+      command: 'git rev-parse --show-toplevel'
+      register: top_level_dir
+
+    - name: make resources template
+      command: make analysis-resources.json
+      args:
+        chdir: "{{ top_level_dir.stdout }}/http/analysis-service"
+
+    - name: create resources CFN
+      cloudformation: stack_name="{{stack_name}}-resources" region={{region}} state=present
+      args:
+        template: "{{ top_level_dir.stdout }}/http/analysis-service/analysis-resources.json"
+        tags:
+          App: "telemetry"
+          Env: "{{env}}"
+          Stack: "{{stack_name}}"
+        template_parameters:
+          SSLCert: "{{ ssl_cert_arn }}"
+          AnalysisDatabasePassword : "{{ database_password }}"
+      register: resources_cfn

provisioning/ansible/playbooks/route53.yml

@@ -0,0 +1,24 @@
+---
+- include: resources.yml
+
+- hosts: localhost
+  connection: local
+  tasks:
+    - name: promote stack
+      cloudformation: stack_name="{{stack_name}}-route53" region={{region}} state=present
+      args:
+        template: ../templates/route53.json
+        template_parameters:
+          DNSName: "{{dns_name}}"
+          DNSZoneName: "{{dns_zone_name}}"
+          ELBDNSName: "{{resources_cfn.stack_outputs.AnalysisLoadBalancerDNSName}}"
+          ELBZoneNameID: "{{resources_cfn.stack_outputs.AnalysisLoadBalancerZoneID}}"
+          Environment: "{{env}}"
+          Region: "{{region}}"
+        tags:
+          App: "telemetry"
+          Env: "{{env}}"
+          Stack: "{{stack_name}}"
+      register: promote
+
+    - debug: var=promote

provisioning/ansible/templates/route53.json

@@ -0,0 +1,53 @@
+{
+    "AWSTemplateFormatVersion": "2010-09-09",
+    "Description": "Telemetry analysis Route53",
+    "Parameters": {
+        "DNSName": {
+            "Type": "String"
+        },
+        "DNSZoneName": {
+            "Default": "dev.mozaws.net.",
+            "Type": "String"
+        },
+        "ELBZoneNameID": {
+            "Description": "From app stack.",
+            "Type": "String"
+        },
+        "ELBDNSName": {
+            "Description": "From app stack.",
+            "Type": "String"
+        },
+        "Environment": {
+            "Description": "Environment",
+            "Type": "String",
+            "Default": "dev"
+        },
+        "Region": {
+            "Description": "Environment",
+            "Type": "String",
+            "Default": "us-west-2"
+        }
+    },
+    "Resources": {
+        "R53AliasRecord" : {
+            "Type":"AWS::Route53::RecordSet",
+            "Properties" : {
+                "HostedZoneName": {"Ref": "DNSZoneName"},
+                "Name": {"Ref": "DNSName"},
+                "Type":"A",
+                "Region": {"Ref": "Region"},
+                "AliasTarget" : {
+                    "EvaluateTargetHealth": true,
+                    "HostedZoneId": {"Ref": "ELBZoneNameID"},
+                    "DNSName": {"Ref": "ELBDNSName"}
+                },
+                "SetIdentifier": {"Ref": "Region"}
+            }
+        }
+    },
+    "Outputs": {
+        "Domain": {
+            "Value": {"Ref": "R53AliasRecord"}
+        }
+    }
+}