Improve admin services discoverability.

* Setup haproxy as reverse proxy to admin services
* Use an alternative port for jenkins (was port 80)
* Backend services
 * marathon
 * chronos
 * jenkins
 * uchiwa

roles/admin/files/haproxy.ctmpl

@@ -0,0 +1,58 @@
+{{- define "SERVICES_FQDN" -}}
+    ops.mozilla.community
+{{- end -}}
+
+global
+        log /dev/log    local0
+        log /dev/log    local1 notice
+        chroot /var/lib/haproxy
+        maxconn 4000
+        stats socket /run/haproxy/admin.sock mode 660 level admin
+        stats timeout 30s
+        user haproxy
+        group haproxy
+        daemon
+
+defaults
+        log     global
+        mode    http
+        option  httplog
+        option  dontlognull
+        timeout connect 5000
+        timeout client  50000
+        timeout server  50000
+        monitor-uri /index.html
+
+        errorfile 400 /etc/haproxy/errors/400.http
+        errorfile 403 /etc/haproxy/errors/403.http
+        errorfile 408 /etc/haproxy/errors/408.http
+        errorfile 500 /etc/haproxy/errors/500.http
+        errorfile 502 /etc/haproxy/errors/502.http
+        errorfile 503 /etc/haproxy/errors/503.http
+        errorfile 504 /etc/haproxy/errors/504.http
+
+frontend http-in
+        bind *:80
+        # Redirect http to https
+        acl is_http hdr(X-Forwarded-Proto) http
+        redirect scheme https code 301 if is_http
+
+        # Services
+        {{- range services -}}
+        {{- if .Tags.Contains "admin" }}
+        acl {{ .Name }}-aclrule hdr(host) -i {{.Name}}.{{ template "SERVICES_FQDN" }}
+        use_backend {{ .Name }}-backend if {{ .Name }}-aclrule
+        {{ end -}}
+        {{ end -}}
+
+{{ range services -}}
+{{- if .Tags.Contains "admin" }}
+backend {{ .Name }}-backend
+        balance leastconn
+        option httpclose
+        option forwardfor
+        {{ range service .Name }}
+        server {{ .ID }} {{ .NodeAddress }}:{{ .Port }} check inter 3000
+        {{- end }}
+{{ end -}}
+{{- end -}}

roles/admin/handlers/main.yml

@@ -0,0 +1,11 @@
+---
+
+- name: restart haproxy
+  service:
+    name: haproxy
+    state: restarted
+
+- name: restart consul-template
+  service:
+    name: consul-template
+    state: restarted

roles/admin/tasks/main.yml

@@ -0,0 +1,128 @@
+---
+
+- name: Enable haproxy ppa
+  apt_repository:
+    repo: "ppa:vbernat/haproxy-1.5"
+    state: present
+  tags:
+    - haproxy
+    - admin
+
+- name: Update repository cache
+  apt:
+    update_cache: yes
+    cache_valid_time: 3600
+  tags:
+    - admin
+
+- stat:
+    path: "/tmp/consul_template_{{ consul_template_version }}.zip"
+  register: consul_template_ver_local
+  tags:
+    - admin
+    - consul-template
+
+- name: Install unzip
+  apt:
+    name: unzip
+    state: present
+  tags:
+    - admin
+    - consul-template
+
+- name: Download consul-template
+  get_url:
+    url: 'https://releases.hashicorp.com/consul-template/{{ consul_template_version }}/consul-template_{{ consul_template_version}}_linux_amd64.zip'
+    dest: '/tmp/consul_template_{{ consul_template_version }}.zip'
+    owner: root
+    group: root
+    mode: 0644
+  when: consul_template_ver_local.stat.exists == False
+  tags:
+    - admin
+    - consul-template
+
+- name: Unpack consul-template
+  unarchive:
+    src: '/tmp/consul_template_{{ consul_template_version }}.zip'
+    dest: '/usr/local/bin'
+    copy: no
+    owner: root
+    group: root
+    mode: 0755
+  when: consul_template_ver_local.stat.exists == False
+  tags:
+    - admin
+    - consul-template
+
+- name: Create required directories for consul-template
+  file:
+    path: "{{ item }}"
+    state: directory
+  with_items:
+    - "/etc/consul-template.d/templates"
+  tags:
+    - admin
+    - consul-template
+
+- name: Install haproxy
+  apt:
+    name: "haproxy"
+    state: present
+  tags:
+    - admin
+    - haproxy
+
+- name: Make sure haproxy folder exists
+  file:
+    path: /var/run/haproxy
+    state: directory
+  tags:
+    - admin
+    - haproxy
+
+- name: Ensure haproxy starts on boot
+  service:
+    name: haproxy
+    enabled: yes
+    state: started
+  tags:
+    - admin
+    - haproxy
+
+- name: Add consul-template upstart script
+  template:
+    src: consul_template_upstart.j2
+    dest: /etc/init/consul-template.conf
+    owner: root
+    group: root
+    mode: 0644
+  tags:
+    - admin
+    - consul-template
+
+- name: Add consul-template configuration
+  template:
+    src: consul_template_config.j2
+    dest: /etc/consul-template.d/config
+    owner: root
+    group: root
+    mode: 0644
+  notify:
+    - restart consul-template
+  tags:
+    - admin
+    - consul-template
+
+- name: Add haproxy config template
+  copy:
+    src: haproxy.ctmpl
+    dest: /etc/consul-template.d/templates/haproxy.ctmpl
+    owner: root
+    group: root
+    mode: 0644
+  notify:
+    - restart consul-template
+  tags:
+    - admin
+    - consul-template

roles/admin/templates/consul_template_config.j2

@@ -0,0 +1,7 @@
+consul = "{{ ansible_default_ipv4['address'] }}:8500"
+
+template {
+  source = "/etc/consul-template.d/templates/haproxy.ctmpl"
+  destination = "/etc/haproxy/haproxy.cfg"
+  command = "service haproxy restart"
+}

roles/admin/templates/consul_template_upstart.j2

@@ -0,0 +1,15 @@
+description "Consul-template server process"
+
+start on (local-filesystems and net-device-up IFACE=eth0)
+stop on runlevel [!12345]
+
+respawn
+
+setuid root
+setgid root
+
+script
+    . /etc/environment
+    export AWS_ENV
+    exec consul-template -config /etc/consul-template.d/config
+end script

roles/admin/vars/main.yml

@@ -0,0 +1,3 @@
+---
+
+consul_template_version: '0.14.0'

roles/jenkins/tasks/main.yml

@@ -108,7 +108,7 @@
     state: reloaded
     pull: always
     ports:
-      - "80:8080"
+      - "8081:8080"
       - "5000:5000"
     volumes:
       - "/opt/jenkins:/var/jenkins_home"

roles/jenkins/templates/consul.json.j2

@@ -4,13 +4,13 @@
     "name": "jenkins",
     "id": "jenkins-{{ ec2_id }}",
     "tags": ["jenkins", "admin", "{{ ec2_tag_env }}"],
-    "port": 80,
+    "port": 8081,
     "enableTagOverride": false,
     "checks": [
       {
       "id": "jenkins-{{ ec2_id }}",
       "name": "Jenkins on port 80",
-      "tcp": "{{ ansible_default_ipv4['address'] }}:80",
+      "http": "http://{{ ansible_default_ipv4['address'] }}:8081",
       "interval": "10s",
       "timeout": "4s"
       }