Revert "Drop ytdl servers"

This reverts commit 8d198bfcc3.

ansible/roles/ytdl/tasks/main.yml

@@ -0,0 +1,33 @@
+---
+
+- block:
+  - name: Include main vars
+    include_vars:
+      file: "{{ secrets_path }}/roles/ytdl/vars/main.yml"
+
+  - name: Include environment specific vars
+    include_vars:
+      file: "{{ secrets_path }}/roles/ytdl/vars/{{ env }}.yml"
+
+  - name: Install moz hab-butterfly (needed until https://github.com/habitat-sh/habitat/issues/5257 is closed)
+    shell: "hab pkg install mozillareality/hab-butterfly"
+
+  - name: Create work directory
+    tempfile:
+      state: directory
+      suffix: deploy
+    register: work_dir
+
+  - name: Write YT-DL config file
+    template:
+      src: ytdl.toml.j2
+      dest: "{{ work_dir.path }}/ytdl.toml"
+
+  - name: Deploy YT-DL configs
+    shell: "cat {{ work_dir.path }}/ytdl.toml | /hab/pkgs/mozillareality/hab-butterfly/0.39.1/20171118004554/bin/hab-butterfly config apply --peer $(curl -s http://169.254.169.254/latest/meta-data/local-ipv4) --org mozillareality --ring mr youtube-dl-api-server.default $(date +%s)"
+
+  always:
+  - name: Remove work directory
+    file:
+      path: "{{ work_dir.path }}"
+      state: absent

ansible/roles/ytdl/templates/dd-agent.toml.j2

@@ -0,0 +1,5 @@
+[general]
+api_key = "{{ dd_api_key }}"
+
+[aws]
+collect_ec2_tags = "yes"

ansible/roles/ytdl/templates/ytdl.toml.j2

@@ -0,0 +1,2 @@
+[general]
+bind_ip = "0.0.0.0"

ansible/ytdl-config.yml

@@ -0,0 +1,7 @@
+---
+
+- hosts: all
+  gather_facts: false
+  become: true
+  roles:
+  - role: ytdl

terraform/live/dev/ret/terraform.tfvars

@@ -26,5 +26,3 @@ max_ret_servers = 2
 reticulum_restart_strategy = "at-once"
 public_domain_enabled = false
 public_domain = "hubs.mozilla.com"
-ytdl_restart_strategy = "at-once"
-ytdl_channel = "stable"

terraform/live/dev/ytdl/terraform.tfvars

@@ -0,0 +1,22 @@
+terragrunt = {
+  terraform {
+    source = "git::git@github.com:mozilla/mr-ops.git//terraform/modules/ytdl"
+  }
+
+  include {
+    path = "${find_in_parent_folders()}"
+  }
+
+  dependencies {
+    paths = ["../vpc", "../base", "../bastion", "../hab", "../ret"]
+  }
+}
+
+ytdl_domain = "reticulum.io"
+ytdl_instance_type = "m3.medium"
+ytdl_dns_prefix = "ytdl-dev."
+ytdl_http_port = 8080
+min_ytdl_servers = 1
+max_ytdl_servers = 1
+ytdl_restart_strategy = "at-once"
+ytdl_channel = "stable"

terraform/live/prod/ret/terraform.tfvars

@@ -26,5 +26,3 @@ max_ret_servers = 2
 reticulum_restart_strategy = "at-once"
 public_domain_enabled = true
 public_domain = "hubs.mozilla.com"
-ytdl_restart_strategy = "at-once"
-ytdl_channel = "stable"

terraform/live/prod/ytdl/terraform.tfvars

@@ -0,0 +1,22 @@
+terragrunt = {
+  terraform {
+    source = "git::git@github.com:mozilla/mr-ops.git//terraform/modules/ytdl"
+  }
+
+  include {
+    path = "${find_in_parent_folders()}"
+  }
+
+  dependencies {
+    paths = ["../vpc", "../base", "../bastion", "../hab", "../ret"]
+  }
+}
+
+ytdl_domain = "reticulum.io"
+ytdl_instance_type = "m3.medium"
+ytdl_dns_prefix = "ytdl."
+ytdl_http_port = 8080
+min_ytdl_servers = 2
+max_ytdl_servers = 2
+ytdl_restart_strategy = "at-once"
+ytdl_channel = "stable"

terraform/modules/ret/main.tf

@@ -619,7 +619,6 @@ popd
 sudo /usr/bin/hab svc load mozillareality/reticulum --strategy ${var.reticulum_restart_strategy} --url https://bldr.habitat.sh --channel ${var.ret_pools[count.index]}
 sudo /usr/bin/hab svc load mozillareality/telegraf --strategy at-once --url https://bldr.habitat.sh --channel stable
 sudo /usr/bin/hab svc load mozillareality/hubs-docs --strategy at-once --url https://bldr.habitat.sh --channel stable
-sudo /usr/bin/hab svc load mozillareality/youtube-dl-api-server --strategy ${var.ytdl_restart_strategy} --url https://bldr.habitat.sh --channel ${var.ytdl_channel}
 EOF
 }
 
@@ -696,7 +695,6 @@ popd
 sudo /usr/bin/hab svc load mozillareality/reticulum --strategy ${var.reticulum_restart_strategy} --url https://bldr.habitat.sh --channel ${var.ret_pools[count.index]}
 sudo /usr/bin/hab svc load mozillareality/telegraf --strategy at-once --url https://bldr.habitat.sh --channel stable
 sudo /usr/bin/hab svc load mozillareality/hubs-docs --strategy at-once --url https://bldr.habitat.sh --channel stable
-sudo /usr/bin/hab svc load mozillareality/youtube-dl-api-server --strategy ${var.ytdl_restart_strategy} --url https://bldr.habitat.sh --channel ${var.ytdl_channel}
 EOF
 }
 

terraform/modules/ret/vars.tf

@@ -38,10 +38,3 @@ variable "ret_pools" {
   default = ["earth", "arbre"]
 }
 
-variable "ytdl_channel" {
-  description = "Distribution channel for YT-DL servers"
-}
-
-variable "ytdl_restart_strategy" {
-  description = "Habitat restart strategy for YT-DL"
-}

terraform/modules/ytdl/main.tf

@@ -0,0 +1,220 @@
+variable "shared" { type = "map" }
+terraform { backend "s3" {} }
+provider "aws" { region = "${var.shared["region"]}", version = "~> 1.15" }
+provider "aws" { alias = "east", region = "us-east-1", version = "~> 1.15" }
+data "aws_availability_zones" "all" {}
+
+data "terraform_remote_state" "vpc" { backend = "s3", config = { key = "vpc/terraform.tfstate", bucket = "${var.shared["state_bucket"]}", region = "${var.shared["region"]}", dynamodb_table = "${var.shared["dynamodb_table"]}", encrypt = "true" } }
+data "terraform_remote_state" "base" { backend = "s3", config = { key = "base/terraform.tfstate", bucket = "${var.shared["state_bucket"]}", region = "${var.shared["region"]}", dynamodb_table = "${var.shared["dynamodb_table"]}", encrypt = "true" } }
+data "terraform_remote_state" "bastion" { backend = "s3", config = { key = "bastion/terraform.tfstate", bucket = "${var.shared["state_bucket"]}", region = "${var.shared["region"]}", dynamodb_table = "${var.shared["dynamodb_table"]}", encrypt = "true" } }
+data "terraform_remote_state" "hab" { backend = "s3", config = { key = "hab/terraform.tfstate", bucket = "${var.shared["state_bucket"]}", region = "${var.shared["region"]}", dynamodb_table = "${var.shared["dynamodb_table"]}", encrypt = "true" } }
+data "terraform_remote_state" "ret" { backend = "s3", config = { key = "ret/terraform.tfstate", bucket = "${var.shared["state_bucket"]}", region = "${var.shared["region"]}", dynamodb_table = "${var.shared["dynamodb_table"]}", encrypt = "true" } }
+
+data "aws_route53_zone" "ytdl-zone" {
+  name = "${var.ytdl_domain}."
+}
+
+data "aws_acm_certificate" "ytdl-alb-listener-cert" {
+  domain = "*.${var.ytdl_domain}"
+  statuses = ["ISSUED"]
+  most_recent = true
+}
+
+data "aws_acm_certificate" "ytdl-alb-listener-cert-east" {
+  provider = "aws.east"
+  domain = "*.${var.ytdl_domain}"
+  statuses = ["ISSUED"]
+  most_recent = true
+}
+
+data "aws_ami" "hab-base-ami" {
+  most_recent = true
+  owners = ["self"]
+
+  filter {
+    name = "name"
+    values = ["hab-base-*"]
+  }
+}
+
+resource "aws_security_group" "ytdl-alb" {
+  name = "${var.shared["env"]}-ytdl-alb"
+  vpc_id = "${data.terraform_remote_state.vpc.vpc_id}"
+
+  ingress {
+    from_port = "443"
+    to_port = "443"
+    protocol = "tcp"
+    security_groups = ["${data.terraform_remote_state.ret.ret_security_group_id}"]
+  }
+}
+
+resource "aws_security_group_rule" "ytdl-alb-egress" {
+  type = "egress"
+  from_port = "${var.ytdl_http_port}"
+  to_port = "${var.ytdl_http_port}"
+  protocol = "tcp"
+  security_group_id = "${aws_security_group.ytdl-alb.id}"
+  source_security_group_id = "${aws_security_group.ytdl.id}"
+}
+
+resource "aws_alb" "ytdl-alb" {
+  name = "${var.shared["env"]}-ytdl-alb"
+
+  security_groups = [
+    "${aws_security_group.ytdl-alb.id}"
+  ]
+
+  subnets = ["${data.terraform_remote_state.vpc.private_subnet_ids}"]
+  internal = true
+
+  lifecycle { create_before_destroy = true }
+}
+
+resource "aws_alb_target_group" "ytdl-alb-group-http" {
+  name = "${var.shared["env"]}-ytdl-alb-group-http"
+  vpc_id = "${data.terraform_remote_state.vpc.vpc_id}"
+  port = "${var.ytdl_http_port}"
+  protocol = "HTTP"
+  deregistration_delay = 0
+
+  health_check {
+    path = "/api/version"
+    healthy_threshold = 2
+    unhealthy_threshold = 2
+    interval = 10
+    timeout = 5
+  }
+}
+
+resource "aws_alb_listener" "ytdl-ssl-alb-listener" {
+  load_balancer_arn = "${aws_alb.ytdl-alb.arn}"
+  port = 443
+
+  protocol = "HTTPS"
+  ssl_policy = "ELBSecurityPolicy-2015-05"
+
+  certificate_arn = "${data.aws_acm_certificate.ytdl-alb-listener-cert.arn}"
+
+  default_action {
+    target_group_arn = "${aws_alb_target_group.ytdl-alb-group-http.arn}"
+    type = "forward"
+  }
+}
+
+resource "aws_security_group" "ytdl" {
+  name = "${var.shared["env"]}-ytdl"
+  vpc_id = "${data.terraform_remote_state.vpc.vpc_id}"
+
+  egress {
+    from_port = "80"
+    to_port = "80"
+    protocol = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    from_port = "443"
+    to_port = "443"
+    protocol = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  # YT-DL HTTP
+  ingress {
+    from_port = "${var.ytdl_http_port}"
+    to_port = "${var.ytdl_http_port}"
+    protocol = "tcp"
+    security_groups = ["${aws_security_group.ytdl-alb.id}"]
+  }
+
+  # SSH
+  ingress {
+    from_port = "22"
+    to_port = "22"
+    protocol = "tcp"
+    security_groups = ["${data.terraform_remote_state.bastion.bastion_security_group_id}"]
+  }
+
+  # NTP
+  egress {
+    from_port = "123"
+    to_port = "123"
+    protocol = "udp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  # InfluxDB
+  egress {
+    from_port = "8086"
+    to_port = "8086"
+    protocol = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_iam_role" "ytdl" {
+  name = "${var.shared["env"]}-ytdl"
+  assume_role_policy = "${var.shared["ec2_role_policy"]}"
+}
+
+resource "aws_iam_role_policy_attachment" "bastion-base-policy" {
+  role = "${aws_iam_role.ytdl.name}"
+  policy_arn = "${data.terraform_remote_state.base.base_policy_arn}"
+}
+
+resource "aws_iam_instance_profile" "ytdl" {
+  name = "${var.shared["env"]}-ytdl"
+  role = "${aws_iam_role.ytdl.id}"
+}
+
+resource "aws_launch_configuration" "ytdl" {
+  image_id = "${data.aws_ami.hab-base-ami.id}"
+  instance_type = "${var.ytdl_instance_type}"
+  security_groups = [
+    "${aws_security_group.ytdl.id}",
+    "${data.terraform_remote_state.hab.hab_ring_security_group_id}",
+  ]
+  key_name = "${data.terraform_remote_state.base.mr_ssh_key_id}"
+  iam_instance_profile = "${aws_iam_instance_profile.ytdl.id}"
+  associate_public_ip_address = false
+  lifecycle { create_before_destroy = true }
+  root_block_device { volume_size = 64 }
+  user_data = <<EOF
+#!/usr/bin/env bash
+while ! nc -z localhost 9632 ; do sleep 1; done
+systemctl restart systemd-sysctl.service
+
+sudo /usr/bin/hab svc load mozillareality/youtube-dl-api-server --strategy ${var.ytdl_restart_strategy} --url https://bldr.habitat.sh --channel ${var.ytdl_channel}
+sudo /usr/bin/hab svc load mozillareality/telegraf --strategy at-once --url https://bldr.habitat.sh --channel stable
+EOF
+}
+
+resource "aws_autoscaling_group" "ytdl" {
+  name = "${var.shared["env"]}-ytdl"
+  launch_configuration = "${aws_launch_configuration.ytdl.id}"
+  availability_zones = ["${data.aws_availability_zones.all.names}"]
+  vpc_zone_identifier = ["${data.terraform_remote_state.vpc.private_subnet_ids}"]
+
+  min_size = "${var.min_ytdl_servers}"
+  max_size = "${var.max_ytdl_servers}"
+
+  target_group_arns = ["${aws_alb_target_group.ytdl-alb-group-http.arn}"]
+
+  lifecycle { create_before_destroy = true }
+  tag { key = "env", value = "${var.shared["env"]}", propagate_at_launch = true }
+  tag { key = "host-type", value = "${var.shared["env"]}-ytdl", propagate_at_launch = true }
+  tag { key = "hab-ring", value = "${var.shared["env"]}", propagate_at_launch = true }
+}
+
+resource "aws_route53_record" "ytdl-dns" {
+  zone_id = "${data.aws_route53_zone.ytdl-zone.zone_id}"
+  name = "${var.ytdl_dns_prefix}${data.aws_route53_zone.ytdl-zone.name}"
+  type = "A"
+
+  alias {
+    name = "${aws_alb.ytdl-alb.dns_name}"
+    zone_id = "${aws_alb.ytdl-alb.zone_id}"
+    evaluate_target_health = true
+  }
+}

terraform/modules/ytdl/output.tf

@@ -0,0 +1,11 @@
+output "ytdl_target_group_id" {
+  value = "${aws_alb_target_group.ytdl-alb-group-http.arn}"
+}
+
+output "ytdl_security_group_id" {
+  value = "${aws_security_group.ytdl.id}"
+}
+
+output "ytdl_alb_id" {
+  value = "${aws_alb.ytdl-alb.id}"
+}

terraform/modules/ytdl/vars.tf

@@ -0,0 +1,31 @@
+variable "ytdl_instance_type" {
+  description = "YT-DL server instance type"
+}
+
+variable "ytdl_dns_prefix" {
+  description = "Prefix before domain for DNS entry"
+}
+
+variable "ytdl_http_port" {
+  description = "YT-DL HTTP service listener port"
+}
+
+variable "min_ytdl_servers" {
+  description = "Minimum number of YT-DL servers to run"
+}
+
+variable "max_ytdl_servers" {
+  description = "Maximum number of YT-DL servers to run"
+}
+
+variable "ytdl_domain" {
+  description = "Domain name being used for YT-DL server (ex reticulum.io)"
+}
+
+variable "ytdl_channel" {
+  description = "Distribution channel for YT-DL servers"
+}
+
+variable "ytdl_restart_strategy" {
+  description = "Habitat restart strategy for YT-DL"
+}