update cachewarmer modules (#1295)

src/go/cmd/cachewarmer/deploymentartifacts/bootstrap/bootstrap.cachewarmer-manager.sh

@@ -17,6 +17,7 @@ set -x
 # (OPT)VMSS_SUBNET
 # (OPT)VMSS_SSHPUBLICKEY
 # (OPT)VMSS_PASSWORD
+# (OPT)VMSS_WORKER_COUNT
 
 SERVICE_USER=root
 RSYSLOG_FILE="35-cachewarmer-manager.conf"
@@ -66,6 +67,12 @@ function write_system_files() {
     else
         sed -i "s/VMSS_SUBNET_NAME_REPLACE/-vmssSubnetName $VMSS_SUBNET/g" $DST_FILE
     fi
+    
+    if [[ -z "${VMSS_WORKER_COUNT}" ]]; then
+        sed -i "s/VMSS_WORKER_COUNT_REPLACE//g" $DST_FILE
+    else
+        sed -i "s/VMSS_WORKER_COUNT_REPLACE/-workerCount $VMSS_WORKER_COUNT/g" $DST_FILE
+    fi
 
     if [ -f '/etc/centos-release' ]; then 
         sed -i "s/chown syslog:adm/chown root:root/g" $DST_FILE

src/go/cmd/cachewarmer/deploymentartifacts/bootstrap/systemd/cachewarmer-manager.service

@@ -12,7 +12,7 @@ LimitNOFILE=16384
 Restart=always
 RestartSec=2
 
-ExecStart=/usr/local/bin/cachewarmer-manager -storageAccountResourceGroup "STORAGE_RG_REPLACE"  -storageAccountName "STORAGE_ACCOUNT_REPLACE" -queueNamePrefix "QUEUE_PREFIX_REPLACE" -bootstrapExportPath "BOOTSTRAP_EXPORT_PATH_REPLACE" -bootstrapMountAddress "BOOTSTRAP_MOUNT_ADDRESS_REPLACE" -bootstrapScriptPath "BOOTSTRAP_SCRIPT_PATH_REPLACE" -vmssUserName "VMSS_USERNAME_REPLACE" VMSS_SSH_PUBLIC_KEY_REPLACE VMSS_PASSWORD_REPLACE VMSS_SUBNET_NAME_REPLACE
+ExecStart=/usr/local/bin/cachewarmer-manager -storageAccountResourceGroup "STORAGE_RG_REPLACE"  -storageAccountName "STORAGE_ACCOUNT_REPLACE" -queueNamePrefix "QUEUE_PREFIX_REPLACE" -bootstrapExportPath "BOOTSTRAP_EXPORT_PATH_REPLACE" -bootstrapMountAddress "BOOTSTRAP_MOUNT_ADDRESS_REPLACE" -bootstrapScriptPath "BOOTSTRAP_SCRIPT_PATH_REPLACE" -vmssUserName "VMSS_USERNAME_REPLACE" VMSS_SSH_PUBLIC_KEY_REPLACE VMSS_PASSWORD_REPLACE VMSS_SUBNET_NAME_REPLACE VMSS_WORKER_COUNT_REPLACE
 
 # make sure log directory exists and owned by syslog
 PermissionsStartOnly=true

src/terraform/modules/cachewarmer_manager_install/main.tf

@@ -3,7 +3,7 @@ locals {
   vmss_password_str       = var.vmss_password == null ? "" : var.vmss_password
   vmss_ssh_public_key_str = var.vmss_ssh_public_key == null ? "" : var.vmss_ssh_public_key
   vmss_subnet_name_str    = var.vmss_subnet_name == null ? "" : var.vmss_subnet_name
-  env_vars                = "export BOOTSTRAP_PATH=${local.mount_dir} && export STORAGE_ACCOUNT_RESOURCE_GROUP='${var.storage_account_rg}' && export STORAGE_ACCOUNT=${var.storage_account} && export QUEUE_PREFIX=${var.queue_name_prefix} && export BOOTSTRAP_EXPORT_PATH=${var.bootstrap_export_path} && export BOOTSTRAP_MOUNT_ADDRESS=${var.bootstrap_mount_address} && export BOOTSTRAP_SCRIPT=${var.bootstrap_worker_script_path} && export VMSS_USERNAME=${var.vmss_user_name} && export VMSS_SSHPUBLICKEY='${local.vmss_ssh_public_key_str}' && export VMSS_PASSWORD='${local.vmss_password_str}' && export VMSS_SUBNET=${local.vmss_subnet_name_str}"
+  env_vars                = "export BOOTSTRAP_PATH=${local.mount_dir} && export STORAGE_ACCOUNT_RESOURCE_GROUP='${var.storage_account_rg}' && export STORAGE_ACCOUNT=${var.storage_account} && export QUEUE_PREFIX=${var.queue_name_prefix} && export BOOTSTRAP_EXPORT_PATH=${var.bootstrap_export_path} && export BOOTSTRAP_MOUNT_ADDRESS=${var.bootstrap_mount_address} && export BOOTSTRAP_SCRIPT=${var.bootstrap_worker_script_path} && export VMSS_USERNAME=${var.vmss_user_name} && export VMSS_SSHPUBLICKEY='${local.vmss_ssh_public_key_str}' && export VMSS_PASSWORD='${local.vmss_password_str}' && export VMSS_SUBNET=${local.vmss_subnet_name_str} && export VMSS_WORKER_COUNT=${var.vmss_worker_count}"
 }
 
 resource "null_resource" "install_cachewarmer_manager" {

src/terraform/modules/cachewarmer_manager_install/variables.tf

@@ -69,3 +69,9 @@ variable "vmss_subnet_name" {
   description = "(optional) the subnet to use for the VMSS, if not specified use the same subnet as the controller"
   default     = null
 }
+
+variable "vmss_worker_count" {
+  description = "(optional) the maximum workers to create, defaults to 12"
+  default     = 12
+  type        = number
+}

src/terraform/modules/cachewarmer_submitmultiplejobs/main.tf

@@ -2,25 +2,27 @@ locals {
   mount_dir        = "/b"
   block_flag       = " -blockUntilWarm "
   no_block_flag    = ""
-  warm_paths_array = [for i, z in var.warm_paths : setproduct([i], z)][0]
+  warm_paths_array = [for i, z in var.warm_paths : setproduct([i], z)]
+  warm_paths_sets  = flatten([for i in local.warm_paths_array : [for j in i : { export = j[0], path = j[1] }]])
+  maxFileSizeBytes = var.maxFileSizeBytes == 0 ? "" : " -maxFileSizeBytes ${var.maxFileSizeBytes} "
 }
 
 resource "null_resource" "cachewarmer_submitmultiplejobs" {
-  count = length(local.warm_paths_array)
+  count = length(local.warm_paths_sets)
 
   connection {
     type        = "ssh"
-    port        = var.ssh_port
-    host        = var.node_address
-    user        = var.admin_username
-    password    = var.ssh_key_data != null && var.ssh_key_data != "" ? "" : var.admin_password
-    private_key = var.ssh_key_data != null && var.ssh_key_data != "" ? file("~/.ssh/id_rsa") : null
+    port        = var.jumpbox_ssh_port
+    host        = var.jumpbox_address
+    user        = var.jumpbox_username
+    password    = var.jumpbox_ssh_key_data != null && var.jumpbox_ssh_key_data != "" ? "" : var.jumpbox_password
+    private_key = var.jumpbox_ssh_key_data != null && var.jumpbox_ssh_key_data != "" ? file("~/.ssh/id_rsa") : null
   }
 
   provisioner "remote-exec" {
     inline = [
       "set -x",
-      join("", ["sudo /usr/local/bin/cachewarmer-jobsubmitter -storageAccountName ", var.storage_account, " -storageKey '", var.storage_key, "' -queueNamePrefix ", var.queue_name_prefix, " -warmTargetExportPath ", local.warm_paths_array[count.index][0], " -warmTargetMountAddresses ", var.warm_mount_addresses, " -warmTargetPath ", local.warm_paths_array[count.index][1], " ", var.block_until_warm && count.index == 0 ? local.block_flag : local.no_block_flag]),
+      "sudo /usr/local/bin/cachewarmer-jobsubmitter -storageAccountName ${var.storage_account} -storageAccountResourceGroup ${var.storage_account_rg} -queueNamePrefix ${var.queue_name_prefix} -warmTargetExportPath \"${local.warm_paths_sets[count.index].export}\" -warmTargetMountAddresses \"${var.warm_mount_addresses}\" -warmTargetPath \"${local.warm_paths_sets[count.index].path}\" -inclusionCsv \"${var.inclusion_csv}\" -exclusionCsv \"${var.exclusion_csv}\" ${local.maxFileSizeBytes} ${var.block_until_warm && count.index == 0 ? local.block_flag : local.no_block_flag}",
     ]
   }
 }

src/terraform/modules/cachewarmer_submitmultiplejobs/variables.tf

@@ -1,32 +1,33 @@
-variable "node_address" {
+variable "jumpbox_address" {
   description = "The address of controller or jumpbox"
 }
 
-variable "admin_username" {
-  description = "Admin username on the controller or jumpbox"
+variable "jumpbox_username" {
+  description = "The username on the controller or jumpbox"
   default     = "azureuser"
 }
 
-variable "admin_password" {
+variable "jumpbox_password" {
   description = "(optional) The password used for access to the controller or jumpbox.  If not specified, ssh_key_data needs to be set."
   default     = null
 }
 
-variable "ssh_key_data" {
+variable "jumpbox_ssh_key_data" {
   description = "(optional) The public SSH key used for access to the controller or jumpbox.  If not specified, the password needs to be set.  The ssh_key_data takes precedence over the password, and if set, the password will be ignored."
+  default     = null
 }
 
-variable "ssh_port" {
+variable "jumpbox_ssh_port" {
   description = "specifies the tcp port to use for ssh"
   default     = 22
 }
 
-variable "storage_account" {
-  description = "the storage account holding the queue"
+variable "storage_account_rg" {
+  description = "the storage account resource group"
 }
 
-variable "storage_key" {
-  description = "the storage key"
+variable "storage_account" {
+  description = "the storage account holding the queue"
 }
 
 variable "queue_name_prefix" {
@@ -42,6 +43,20 @@ variable "warm_paths" {
   default     = {}
 }
 
+variable "inclusion_csv" {
+  description = "the inclusion list of file match strings per https://golang.org/pkg/path/filepath/#Match.  Leave blank to include everything."
+}
+
+variable "exclusion_csv" {
+  description = "the exclusion list of file match strings per https://golang.org/pkg/path/filepath/#Match.  Leave blank to not exlude anything."
+}
+
+variable "maxFileSizeBytes" {
+  description = "the maximum file size in bytes to warm."
+  type        = number
+  default     = 0
+}
+
 variable "block_until_warm" {
   description = "block the operation until the cache warming has finished"
   default     = true