From 9ea68c27ba81861d1ed0617315a8487693921645 Mon Sep 17 00:00:00 2001 From: Anthony Howe Date: Sat, 17 Jul 2021 10:57:51 -0400 Subject: [PATCH] update the cachewarmer prepare module (#1280) --- .../modules/cachewarmer_build/README.md | 34 ++++++-- .../cachewarmer_build/cachewarmer_build.sh | 36 ++++++++ .../cachewarmer_prepare_bootstrap.sh | 85 +++++++++++++++++++ .../modules/cachewarmer_build/main.tf | 63 +++++--------- .../modules/cachewarmer_build/outputs.tf | 4 +- .../modules/cachewarmer_build/variables.tf | 28 +++--- 6 files changed, 187 insertions(+), 63 deletions(-) create mode 100644 src/terraform/modules/cachewarmer_build/cachewarmer_build.sh create mode 100644 src/terraform/modules/cachewarmer_build/cachewarmer_prepare_bootstrap.sh diff --git a/src/terraform/modules/cachewarmer_build/README.md b/src/terraform/modules/cachewarmer_build/README.md index dcc2ac72..b7779bf0 100644 --- a/src/terraform/modules/cachewarmer_build/README.md +++ b/src/terraform/modules/cachewarmer_build/README.md @@ -1,8 +1,30 @@ -# CacheWarmer Bootstrap Setup Module +# CacheWarmer Bootstrap Preparation Module -This module installs the CacheWarmer bootstrap script for the [CacheWarmer](../../../go/cmd/cachewarmer). The [CacheWarmer for HPC Cache](../../examples/HPC%20Cache/cachewarmer) or the [CacheWarmer for Avere vFXT for Azure](../../examples/vfxt/cachewarmer) examples demonstrate how to use this module. +This module prepares the CacheWarmer bootstrap folder on a target NFS file server for the [CacheWarmer](../../../go/cmd/cachewarmer). The [CacheWarmer for HPC Cache](../../examples/HPC%20Cache/cachewarmer) or the [CacheWarmer for Avere vFXT for Azure](../../examples/vfxt/cachewarmer) examples demonstrate how to use this module. -It requires the following: -1. SSH access to a node with mountable access to the target NFS Server. Both the [controller](../controller3) and [jumpbox](../jumpbox) modules achieve this. -2. the node must have internet access. -3. the node must have a service principal in the environment defined. +If the controller or jumpbox does not have internet access, manually prepare the folder using one of the following approaches: + +1. Use the cachewarmer release binaries: + ```bash + # prepare the bootstrap directory, updating the env vars with your own vars + export LOCAL_MOUNT_DIR=/b + export BOOTSTRAP_MOUNT_ADDRESS=192.168.254.244 + export BOOTSTRAP_MOUNT_EXPORT=/data + export BOOTSTRAP_SUBDIR=/bootstrap + curl --retry 5 --retry-delay 5 -o /tmp/cachewarmer_prepare_bootstrap.sh https://raw.githubusercontent.com/Azure/Avere/main/src/terraform/modules/cachewarmer_build/cachewarmer_prepare_bootstrap.sh + source /tmp/cachewarmer_prepare_bootstrap.sh + ``` + +1. Build your own cachewarmer: + ```bash + # build the cachewarmer, it will correctly set the env vars for the paths + curl --retry 5 --retry-delay 5 -o /tmp/cachewarmer_build.sh https://raw.githubusercontent.com/Azure/Avere/main/src/terraform/modules/cachewarmer_build/cachewarmer_build.sh + source /tmp/cachewarmer_build.sh + # prepare the bootstrap directory, updating the env vars with your own vars + export LOCAL_MOUNT_DIR=/b + export BOOTSTRAP_MOUNT_ADDRESS=192.168.254.244 + export BOOTSTRAP_MOUNT_EXPORT=/data + export BOOTSTRAP_SUBDIR=/bootstrap + curl --retry 5 --retry-delay 5 -o /tmp/cachewarmer_prepare_bootstrap.sh https://raw.githubusercontent.com/Azure/Avere/main/src/terraform/modules/cachewarmer_build/cachewarmer_prepare_bootstrap.sh + source /tmp/cachewarmer_prepare_bootstrap.sh + ``` diff --git a/src/terraform/modules/cachewarmer_build/cachewarmer_build.sh b/src/terraform/modules/cachewarmer_build/cachewarmer_build.sh new file mode 100644 index 00000000..70cc3d2c --- /dev/null +++ b/src/terraform/modules/cachewarmer_build/cachewarmer_build.sh @@ -0,0 +1,36 @@ +#!/bin/bash +set -x + +# install golang +if ! command -v go &> /dev/null ; then + GO_DL_FILE=go1.16.6.linux-amd64.tar.gz + wget --tries=12 --wait=5 https://dl.google.com/go/$GO_DL_FILE + sudo tar -C /usr/local -xzf $GO_DL_FILE + rm -f $GO_DL_FILE + echo "export PATH=$PATH:/usr/local/go/bin" >> $HOME/.profile + source $HOME/.profile +fi + +# checkout and build CacheWarmer +cd +RELEASE_DIR=$HOME/release +mkdir -p $RELEASE_DIR +git clone https://github.com/Azure/Avere.git +# build the cache warmer +cd $HOME/Avere/src/go/cmd/cachewarmer/cachewarmer-jobsubmitter +go build +mv cachewarmer-jobsubmitter $RELEASE_DIR/. +cd $HOME/Avere/src/go/cmd/cachewarmer/cachewarmer-manager +go build +mv cachewarmer-manager $RELEASE_DIR/. +cd $HOME/Avere/src/go/cmd/cachewarmer/cachewarmer-worker +go build +mv cachewarmer-worker $RELEASE_DIR/. +cd + +# set the path +export CACHEWARMER_MANAGER_PATH=$RELEASE_DIR/cachewarmer-manager +export CACHEWARMER_WORKER_PATH=$RELEASE_DIR/cachewarmer-worker +export CACHEWARMER_JOBSUBMITTER_PATH=$RELEASE_DIR/cachewarmer-jobsubmitter + +set +x \ No newline at end of file diff --git a/src/terraform/modules/cachewarmer_build/cachewarmer_prepare_bootstrap.sh b/src/terraform/modules/cachewarmer_build/cachewarmer_prepare_bootstrap.sh new file mode 100644 index 00000000..6def7388 --- /dev/null +++ b/src/terraform/modules/cachewarmer_build/cachewarmer_prepare_bootstrap.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +# before running this script define the following env vars +# LOCAL_MOUNT_DIR=/b +# BOOTSTRAP_MOUNT_ADDRESS=FILE_ADDRESS +# BOOTSTRAP_MOUNT_EXPORT=/data +# BOOTSTRAP_SUBDIR=/bootstrap +# +# # leave CACHEWARMER_MANAGER_PATH empty if you did not build your own cachewarmer binaries +# CACHEWARMER_MANAGER_PATH="" +# CACHEWARMER_WORKER_PATH="" +# CACHEWARMER_JOBSUBMITTER_PATH="" + +if [ -z $LOCAL_MOUNT_DIR ]; then + echo "LOCAL_MOUNT_DIR is not set" + return +fi +if [ -z $BOOTSTRAP_MOUNT_ADDRESS ]; then + echo "BOOTSTRAP_MOUNT_ADDRESS is not set" + return +fi +if [ -z $BOOTSTRAP_MOUNT_EXPORT ]; then + echo "BOOTSTRAP_MOUNT_EXPORT is not set" + return +fi +if [ -z $BOOTSTRAP_SUBDIR ]; then + echo "BOOTSTRAP_SUBDIR is not set" + return +fi + +set -x + +# go home +cd + +# remove a previous install if one exists +sudo rm -f $LOCAL_MOUNT_DIR/$BOOTSTRAP_SUBDIR/bootstrap.cachewarmer-manager.sh +sudo rm -f $LOCAL_MOUNT_DIR/$BOOTSTRAP_SUBDIR/bootstrap.cachewarmer-worker.sh +sudo rm -f $LOCAL_MOUNT_DIR/$BOOTSTRAP_SUBDIR/rsyslog/35-cachewarmer-manager.conf +sudo rm -f $LOCAL_MOUNT_DIR/$BOOTSTRAP_SUBDIR/rsyslog/36-cachewarmer-worker.conf +sudo rm -f $LOCAL_MOUNT_DIR/$BOOTSTRAP_SUBDIR/systemd/cachewarmer-manager.service +sudo rm -f $LOCAL_MOUNT_DIR/$BOOTSTRAP_SUBDIR/systemd/cachewarmer-worker.service + +# create the bootstrap directory +sudo mkdir -p $LOCAL_MOUNT_DIR +sudo mount -o 'hard,nointr,proto=tcp,mountproto=tcp,retry=30' $BOOTSTRAP_MOUNT_ADDRESS:$BOOTSTRAP_MOUNT_EXPORT $LOCAL_MOUNT_DIR +sudo mkdir -p $LOCAL_MOUNT_DIR/$BOOTSTRAP_SUBDIR/cachewarmerbin +sudo mkdir -p $LOCAL_MOUNT_DIR/$BOOTSTRAP_SUBDIR/rsyslog +sudo mkdir -p $LOCAL_MOUNT_DIR/$BOOTSTRAP_SUBDIR/systemd + +# download the content +sudo curl --retry 5 --retry-delay 5 -o $LOCAL_MOUNT_DIR/$BOOTSTRAP_SUBDIR/bootstrap.cachewarmer-manager.sh https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/cachewarmer/deploymentartifacts/bootstrap/bootstrap.cachewarmer-manager.sh +sudo curl --retry 5 --retry-delay 5 -o $LOCAL_MOUNT_DIR/$BOOTSTRAP_SUBDIR/bootstrap.cachewarmer-worker.sh https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/cachewarmer/deploymentartifacts/bootstrap/bootstrap.cachewarmer-worker.sh +sudo curl --retry 5 --retry-delay 5 -o $LOCAL_MOUNT_DIR/$BOOTSTRAP_SUBDIR/rsyslog/35-cachewarmer-manager.conf https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/cachewarmer/deploymentartifacts/bootstrap/rsyslog/35-cachewarmer-manager.conf +sudo curl --retry 5 --retry-delay 5 -o $LOCAL_MOUNT_DIR/$BOOTSTRAP_SUBDIR/rsyslog/36-cachewarmer-worker.conf https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/cachewarmer/deploymentartifacts/bootstrap/rsyslog/36-cachewarmer-worker.conf +sudo curl --retry 5 --retry-delay 5 -o $LOCAL_MOUNT_DIR/$BOOTSTRAP_SUBDIR/systemd/cachewarmer-manager.service https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/cachewarmer/deploymentartifacts/bootstrap/systemd/cachewarmer-manager.service +sudo curl --retry 5 --retry-delay 5 -o $LOCAL_MOUNT_DIR/$BOOTSTRAP_SUBDIR/systemd/cachewarmer-worker.service https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/cachewarmer/deploymentartifacts/bootstrap/systemd/cachewarmer-worker.service + +TARGET_PATH=$LOCAL_MOUNT_DIR/$BOOTSTRAP_SUBDIR/cachewarmerbin/cachewarmer-manager +if [ -z "$CACHEWARMER_MANAGER_PATH" ]; then + browser_download_url=$(curl -s https://api.github.com/repos/Azure/Avere/releases/latest | jq -r .assets[].browser_download_url | grep -e "cachewarmer-manager$") + sudo curl --retry 5 --retry-delay 5 -o $TARGET_PATH $browser_download_url +else + sudo cp $CACHEWARMER_MANAGER_PATH $TARGET_PATH +fi +TARGET_PATH=$LOCAL_MOUNT_DIR/$BOOTSTRAP_SUBDIR/cachewarmerbin/cachewarmer-worker +if [ -z "$CACHEWARMER_WORKER_PATH" ]; then + browser_download_url=$(curl -s https://api.github.com/repos/Azure/Avere/releases/latest | jq -r .assets[].browser_download_url | grep -e "cachewarmer-worker$") + sudo curl --retry 5 --retry-delay 5 -o $TARGET_PATH $browser_download_url +else + sudo cp $CACHEWARMER_WORKER_PATH $TARGET_PATH +fi +TARGET_PATH=$LOCAL_MOUNT_DIR/$BOOTSTRAP_SUBDIR/cachewarmerbin/cachewarmer-jobsubmitter +if [ -z "$CACHEWARMER_JOBSUBMITTER_PATH" ]; then + browser_download_url=$(curl -s https://api.github.com/repos/Azure/Avere/releases/latest | jq -r .assets[].browser_download_url | grep -e "cachewarmer-jobsubmitter$") + sudo curl --retry 5 --retry-delay 5 -o $TARGET_PATH $browser_download_url +else + sudo cp $CACHEWARMER_JOBSUBMITTER_PATH $TARGET_PATH +fi + +# umount and remove the local mount directory +sudo umount $LOCAL_MOUNT_DIR +sudo rmdir $LOCAL_MOUNT_DIR + +set +x \ No newline at end of file diff --git a/src/terraform/modules/cachewarmer_build/main.tf b/src/terraform/modules/cachewarmer_build/main.tf index ade7484f..2f41a1fa 100644 --- a/src/terraform/modules/cachewarmer_build/main.tf +++ b/src/terraform/modules/cachewarmer_build/main.tf @@ -1,8 +1,17 @@ locals { - mount_dir = "/b" - bootstrap_dir = "bootstrap" - manager_bootstrap_path = "/${local.bootstrap_dir}/bootstrap.cachewarmer-manager.sh" - worker_bootstrap_path = "/${local.bootstrap_dir}/bootstrap.cachewarmer-worker.sh" + build_cachewarmer_lines = [ + "curl --retry 5 --retry-delay 5 -o /tmp/cachewarmer_build.sh https://raw.githubusercontent.com/Azure/Avere/main/src/terraform/modules/cachewarmer_build/cachewarmer_build.sh", + "source /tmp/cachewarmer_build.sh", + ] + + env_vars = "LOCAL_MOUNT_DIR=/b BOOTSTRAP_MOUNT_ADDRESS=${var.bootstrap_mount_address} BOOTSTRAP_MOUNT_EXPORT=${var.bootstrap_export_path} BOOTSTRAP_SUBDIR=${var.bootstrap_subdir}" + + prepare_cachewarmer_bootstrap_lines = [ + "curl --retry 5 --retry-delay 5 -o /tmp/cachewarmer_prepare_bootstrap.sh https://raw.githubusercontent.com/Azure/Avere/main/src/terraform/modules/cachewarmer_build/cachewarmer_prepare_bootstrap.sh", + "${local.env_vars} source /tmp/cachewarmer_prepare_bootstrap.sh" + ] + + provisioner_lines = var.build_cachewarmer ? concat(build_cachewarmer_lines, prepare_cachewarmer_bootstrap_lines) : prepare_cachewarmer_bootstrap_lines } resource "null_resource" "build_cachewarmer_bootstrap" { @@ -10,48 +19,14 @@ resource "null_resource" "build_cachewarmer_bootstrap" { connection { type = "ssh" - port = var.ssh_port - host = var.node_address - user = var.admin_username - password = var.ssh_key_data != null && var.ssh_key_data != "" ? "" : var.admin_password - private_key = var.ssh_key_data != null && var.ssh_key_data != "" ? file("~/.ssh/id_rsa") : null + port = var.jumpbox_ssh_port + host = var.jumpbox_address + user = var.jumpbox_username + password = var.jumpbox_ssh_key_data != null && var.jumpbox_ssh_key_data != "" ? "" : var.jumpbox_password + private_key = var.jumpbox_ssh_key_data != null && var.jumpbox_ssh_key_data != "" ? file("~/.ssh/id_rsa") : null } provisioner "remote-exec" { - inline = [ - "set -x", - "if [ -f '/etc/centos-release' ]; then sudo yum -y install git nfs-utils ; else sudo apt-get install -y nfs-common ; fi", - "wget -O ~/go.tgz https://golang.org/dl/go1.14.4.linux-amd64.tar.gz", - "tar zxf ~/go.tgz", - "sudo chown -R root:root ~/go", - "sudo rm -rf /usr/local/go", - "sudo mv go /usr/local", - "rm ~/go.tgz", - "rm -rf ~/gopath", - "mkdir ~/gopath", - "sudo mkdir -p ${local.mount_dir}", - "sudo mount -o 'hard,nointr,proto=tcp,mountproto=tcp,retry=30' ${var.bootstrap_mount_address}:${var.bootstrap_export_path} ${local.mount_dir}", - "sudo mkdir -p ${local.mount_dir}/${local.bootstrap_dir}", - "sudo mkdir -p ${local.mount_dir}/${local.bootstrap_dir}/cachewarmerbin", - "echo \"export GOPATH=$HOME/gopath\" >> ~/.profile", - "echo \"export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin\" >> ~/.profile", - "/bin/bash -c 'source ~/.profile && cd $GOPATH && go get github.com/Azure/Avere/src/go/... && sudo cp $GOPATH/bin/cachewarmer-* ${local.mount_dir}/${local.bootstrap_dir}/cachewarmerbin'", - "sudo mkdir -p ${local.mount_dir}/${local.bootstrap_dir}/rsyslog", - "sudo mkdir -p ${local.mount_dir}/${local.bootstrap_dir}/systemd", - "sudo rm -f ${local.mount_dir}${local.manager_bootstrap_path}", - "sudo rm -f ${local.mount_dir}${local.worker_bootstrap_path}", - "sudo rm -f ${local.mount_dir}/${local.bootstrap_dir}/rsyslog/35-cachewarmer-manager.conf", - "sudo rm -f ${local.mount_dir}/${local.bootstrap_dir}/rsyslog/36-cachewarmer-worker.conf", - "sudo rm -f ${local.mount_dir}/${local.bootstrap_dir}/systemd/cachewarmer-manager.service", - "sudo rm -f ${local.mount_dir}/${local.bootstrap_dir}/systemd/cachewarmer-worker.service", - "sudo curl --retry 5 --retry-delay 5 -o ${local.mount_dir}${local.manager_bootstrap_path} https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/cachewarmer/deploymentartifacts/bootstrap/bootstrap.cachewarmer-manager.sh", - "sudo curl --retry 5 --retry-delay 5 -o ${local.mount_dir}${local.worker_bootstrap_path} https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/cachewarmer/deploymentartifacts/bootstrap/bootstrap.cachewarmer-worker.sh", - "sudo curl --retry 5 --retry-delay 5 -o ${local.mount_dir}/${local.bootstrap_dir}/rsyslog/35-cachewarmer-manager.conf https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/cachewarmer/deploymentartifacts/bootstrap/rsyslog/35-cachewarmer-manager.conf", - "sudo curl --retry 5 --retry-delay 5 -o ${local.mount_dir}/${local.bootstrap_dir}/rsyslog/36-cachewarmer-worker.conf https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/cachewarmer/deploymentartifacts/bootstrap/rsyslog/36-cachewarmer-worker.conf", - "sudo curl --retry 5 --retry-delay 5 -o ${local.mount_dir}/${local.bootstrap_dir}/systemd/cachewarmer-manager.service https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/cachewarmer/deploymentartifacts/bootstrap/systemd/cachewarmer-manager.service", - "sudo curl --retry 5 --retry-delay 5 -o ${local.mount_dir}/${local.bootstrap_dir}/systemd/cachewarmer-worker.service https://raw.githubusercontent.com/Azure/Avere/main/src/go/cmd/cachewarmer/deploymentartifacts/bootstrap/systemd/cachewarmer-worker.service", - "sudo umount ${local.mount_dir}", - "sudo rmdir ${local.mount_dir}", - ] + inline = local.provisioner_lines } } diff --git a/src/terraform/modules/cachewarmer_build/outputs.tf b/src/terraform/modules/cachewarmer_build/outputs.tf index 8f8df28d..db0a39a9 100644 --- a/src/terraform/modules/cachewarmer_build/outputs.tf +++ b/src/terraform/modules/cachewarmer_build/outputs.tf @@ -10,10 +10,10 @@ output "bootstrap_export_path" { output "cachewarmer_worker_bootstrap_script_path" { description = "The path of the cachewarmer worker on the NFS share." - value = local.worker_bootstrap_path + value = "${var.bootstrap_subdir}/bootstrap.cachewarmer-worker.sh" } output "cachewarmer_manager_bootstrap_script_path" { description = "The path of the cachewarmer manager on the NFS share." - value = local.manager_bootstrap_path + value = "${var.bootstrap_subdir}/bootstrap.cachewarmer-manager.sh" } diff --git a/src/terraform/modules/cachewarmer_build/variables.tf b/src/terraform/modules/cachewarmer_build/variables.tf index 2ceab634..60633886 100644 --- a/src/terraform/modules/cachewarmer_build/variables.tf +++ b/src/terraform/modules/cachewarmer_build/variables.tf @@ -1,28 +1,23 @@ -variable "deploy_cachewarmer" { - description = "specifies to create the cachewarmer or not" - default = true -} - -variable "node_address" { +variable "jumpbox_address" { description = "The address of controller or jumpbox" } -variable "admin_username" { - description = "Admin username on the controller or jumpbox" +variable "jumpbox_username" { + description = "The username on the controller or jumpbox" default = "azureuser" } -variable "admin_password" { +variable "jumpbox_password" { description = "(optional) The password used for access to the controller or jumpbox. If not specified, ssh_key_data needs to be set." default = null } -variable "ssh_key_data" { +variable "jumpbox_ssh_key_data" { description = "(optional) The public SSH key used for access to the controller or jumpbox. If not specified, the password needs to be set. The ssh_key_data takes precedence over the password, and if set, the password will be ignored." default = null } -variable "ssh_port" { +variable "jumpbox_ssh_port" { description = "specifies the tcp port to use for ssh" default = 22 } @@ -34,3 +29,14 @@ variable "bootstrap_mount_address" { variable "bootstrap_export_path" { description = "the export path that hosts the worker bootstrap script" } + +variable "bootstrap_subdir" { + description = "the subdirectory containing the cachewarmer bootstrap scripts" + default = "/bootstrap" +} + +variable "build_cachewarmer" { + description = "specify to build the cachewarmer, otherwise it will be downloaded from the release site" + type = bool + default = false +}