diff --git a/src/go/cmd/cachewarmer/deploymentartifacts/bootstrap/bootstrap.cachewarmer-manager.sh b/src/go/cmd/cachewarmer/deploymentartifacts/bootstrap/bootstrap.cachewarmer-manager.sh index 226be090..dc8dca96 100644 --- a/src/go/cmd/cachewarmer/deploymentartifacts/bootstrap/bootstrap.cachewarmer-manager.sh +++ b/src/go/cmd/cachewarmer/deploymentartifacts/bootstrap/bootstrap.cachewarmer-manager.sh @@ -17,6 +17,7 @@ set -x # (OPT)VMSS_SUBNET # (OPT)VMSS_SSHPUBLICKEY # (OPT)VMSS_PASSWORD +# (OPT)VMSS_WORKER_COUNT SERVICE_USER=root RSYSLOG_FILE="35-cachewarmer-manager.conf" @@ -66,6 +67,12 @@ function write_system_files() { else sed -i "s/VMSS_SUBNET_NAME_REPLACE/-vmssSubnetName $VMSS_SUBNET/g" $DST_FILE fi + + if [[ -z "${VMSS_WORKER_COUNT}" ]]; then + sed -i "s/VMSS_WORKER_COUNT_REPLACE//g" $DST_FILE + else + sed -i "s/VMSS_WORKER_COUNT_REPLACE/-workerCount $VMSS_WORKER_COUNT/g" $DST_FILE + fi if [ -f '/etc/centos-release' ]; then sed -i "s/chown syslog:adm/chown root:root/g" $DST_FILE diff --git a/src/go/cmd/cachewarmer/deploymentartifacts/bootstrap/systemd/cachewarmer-manager.service b/src/go/cmd/cachewarmer/deploymentartifacts/bootstrap/systemd/cachewarmer-manager.service index c9c2778e..68008ab7 100644 --- a/src/go/cmd/cachewarmer/deploymentartifacts/bootstrap/systemd/cachewarmer-manager.service +++ b/src/go/cmd/cachewarmer/deploymentartifacts/bootstrap/systemd/cachewarmer-manager.service @@ -12,7 +12,7 @@ LimitNOFILE=16384 Restart=always RestartSec=2 -ExecStart=/usr/local/bin/cachewarmer-manager -storageAccountResourceGroup "STORAGE_RG_REPLACE" -storageAccountName "STORAGE_ACCOUNT_REPLACE" -queueNamePrefix "QUEUE_PREFIX_REPLACE" -bootstrapExportPath "BOOTSTRAP_EXPORT_PATH_REPLACE" -bootstrapMountAddress "BOOTSTRAP_MOUNT_ADDRESS_REPLACE" -bootstrapScriptPath "BOOTSTRAP_SCRIPT_PATH_REPLACE" -vmssUserName "VMSS_USERNAME_REPLACE" VMSS_SSH_PUBLIC_KEY_REPLACE VMSS_PASSWORD_REPLACE VMSS_SUBNET_NAME_REPLACE +ExecStart=/usr/local/bin/cachewarmer-manager -storageAccountResourceGroup "STORAGE_RG_REPLACE" -storageAccountName "STORAGE_ACCOUNT_REPLACE" -queueNamePrefix "QUEUE_PREFIX_REPLACE" -bootstrapExportPath "BOOTSTRAP_EXPORT_PATH_REPLACE" -bootstrapMountAddress "BOOTSTRAP_MOUNT_ADDRESS_REPLACE" -bootstrapScriptPath "BOOTSTRAP_SCRIPT_PATH_REPLACE" -vmssUserName "VMSS_USERNAME_REPLACE" VMSS_SSH_PUBLIC_KEY_REPLACE VMSS_PASSWORD_REPLACE VMSS_SUBNET_NAME_REPLACE VMSS_WORKER_COUNT_REPLACE # make sure log directory exists and owned by syslog PermissionsStartOnly=true diff --git a/src/terraform/modules/cachewarmer_manager_install/main.tf b/src/terraform/modules/cachewarmer_manager_install/main.tf index 7ff8a78a..8fbf341e 100644 --- a/src/terraform/modules/cachewarmer_manager_install/main.tf +++ b/src/terraform/modules/cachewarmer_manager_install/main.tf @@ -3,7 +3,7 @@ locals { vmss_password_str = var.vmss_password == null ? "" : var.vmss_password vmss_ssh_public_key_str = var.vmss_ssh_public_key == null ? "" : var.vmss_ssh_public_key vmss_subnet_name_str = var.vmss_subnet_name == null ? "" : var.vmss_subnet_name - env_vars = "export BOOTSTRAP_PATH=${local.mount_dir} && export STORAGE_ACCOUNT_RESOURCE_GROUP='${var.storage_account_rg}' && export STORAGE_ACCOUNT=${var.storage_account} && export QUEUE_PREFIX=${var.queue_name_prefix} && export BOOTSTRAP_EXPORT_PATH=${var.bootstrap_export_path} && export BOOTSTRAP_MOUNT_ADDRESS=${var.bootstrap_mount_address} && export BOOTSTRAP_SCRIPT=${var.bootstrap_worker_script_path} && export VMSS_USERNAME=${var.vmss_user_name} && export VMSS_SSHPUBLICKEY='${local.vmss_ssh_public_key_str}' && export VMSS_PASSWORD='${local.vmss_password_str}' && export VMSS_SUBNET=${local.vmss_subnet_name_str}" + env_vars = "export BOOTSTRAP_PATH=${local.mount_dir} && export STORAGE_ACCOUNT_RESOURCE_GROUP='${var.storage_account_rg}' && export STORAGE_ACCOUNT=${var.storage_account} && export QUEUE_PREFIX=${var.queue_name_prefix} && export BOOTSTRAP_EXPORT_PATH=${var.bootstrap_export_path} && export BOOTSTRAP_MOUNT_ADDRESS=${var.bootstrap_mount_address} && export BOOTSTRAP_SCRIPT=${var.bootstrap_worker_script_path} && export VMSS_USERNAME=${var.vmss_user_name} && export VMSS_SSHPUBLICKEY='${local.vmss_ssh_public_key_str}' && export VMSS_PASSWORD='${local.vmss_password_str}' && export VMSS_SUBNET=${local.vmss_subnet_name_str} && export VMSS_WORKER_COUNT=${var.vmss_worker_count}" } resource "null_resource" "install_cachewarmer_manager" { diff --git a/src/terraform/modules/cachewarmer_manager_install/variables.tf b/src/terraform/modules/cachewarmer_manager_install/variables.tf index 32a2e4e2..4ce9b0de 100644 --- a/src/terraform/modules/cachewarmer_manager_install/variables.tf +++ b/src/terraform/modules/cachewarmer_manager_install/variables.tf @@ -69,3 +69,9 @@ variable "vmss_subnet_name" { description = "(optional) the subnet to use for the VMSS, if not specified use the same subnet as the controller" default = null } + +variable "vmss_worker_count" { + description = "(optional) the maximum workers to create, defaults to 12" + default = 12 + type = number +} diff --git a/src/terraform/modules/cachewarmer_submitmultiplejobs/main.tf b/src/terraform/modules/cachewarmer_submitmultiplejobs/main.tf index 45185694..c1879032 100644 --- a/src/terraform/modules/cachewarmer_submitmultiplejobs/main.tf +++ b/src/terraform/modules/cachewarmer_submitmultiplejobs/main.tf @@ -2,25 +2,27 @@ locals { mount_dir = "/b" block_flag = " -blockUntilWarm " no_block_flag = "" - warm_paths_array = [for i, z in var.warm_paths : setproduct([i], z)][0] + warm_paths_array = [for i, z in var.warm_paths : setproduct([i], z)] + warm_paths_sets = flatten([for i in local.warm_paths_array : [for j in i : { export = j[0], path = j[1] }]]) + maxFileSizeBytes = var.maxFileSizeBytes == 0 ? "" : " -maxFileSizeBytes ${var.maxFileSizeBytes} " } resource "null_resource" "cachewarmer_submitmultiplejobs" { - count = length(local.warm_paths_array) + count = length(local.warm_paths_sets) connection { type = "ssh" - port = var.ssh_port - host = var.node_address - user = var.admin_username - password = var.ssh_key_data != null && var.ssh_key_data != "" ? "" : var.admin_password - private_key = var.ssh_key_data != null && var.ssh_key_data != "" ? file("~/.ssh/id_rsa") : null + port = var.jumpbox_ssh_port + host = var.jumpbox_address + user = var.jumpbox_username + password = var.jumpbox_ssh_key_data != null && var.jumpbox_ssh_key_data != "" ? "" : var.jumpbox_password + private_key = var.jumpbox_ssh_key_data != null && var.jumpbox_ssh_key_data != "" ? file("~/.ssh/id_rsa") : null } provisioner "remote-exec" { inline = [ "set -x", - join("", ["sudo /usr/local/bin/cachewarmer-jobsubmitter -storageAccountName ", var.storage_account, " -storageKey '", var.storage_key, "' -queueNamePrefix ", var.queue_name_prefix, " -warmTargetExportPath ", local.warm_paths_array[count.index][0], " -warmTargetMountAddresses ", var.warm_mount_addresses, " -warmTargetPath ", local.warm_paths_array[count.index][1], " ", var.block_until_warm && count.index == 0 ? local.block_flag : local.no_block_flag]), + "sudo /usr/local/bin/cachewarmer-jobsubmitter -storageAccountName ${var.storage_account} -storageAccountResourceGroup ${var.storage_account_rg} -queueNamePrefix ${var.queue_name_prefix} -warmTargetExportPath \"${local.warm_paths_sets[count.index].export}\" -warmTargetMountAddresses \"${var.warm_mount_addresses}\" -warmTargetPath \"${local.warm_paths_sets[count.index].path}\" -inclusionCsv \"${var.inclusion_csv}\" -exclusionCsv \"${var.exclusion_csv}\" ${local.maxFileSizeBytes} ${var.block_until_warm && count.index == 0 ? local.block_flag : local.no_block_flag}", ] } } diff --git a/src/terraform/modules/cachewarmer_submitmultiplejobs/variables.tf b/src/terraform/modules/cachewarmer_submitmultiplejobs/variables.tf index 94f42666..0000b894 100644 --- a/src/terraform/modules/cachewarmer_submitmultiplejobs/variables.tf +++ b/src/terraform/modules/cachewarmer_submitmultiplejobs/variables.tf @@ -1,32 +1,33 @@ -variable "node_address" { +variable "jumpbox_address" { description = "The address of controller or jumpbox" } -variable "admin_username" { - description = "Admin username on the controller or jumpbox" +variable "jumpbox_username" { + description = "The username on the controller or jumpbox" default = "azureuser" } -variable "admin_password" { +variable "jumpbox_password" { description = "(optional) The password used for access to the controller or jumpbox. If not specified, ssh_key_data needs to be set." default = null } -variable "ssh_key_data" { +variable "jumpbox_ssh_key_data" { description = "(optional) The public SSH key used for access to the controller or jumpbox. If not specified, the password needs to be set. The ssh_key_data takes precedence over the password, and if set, the password will be ignored." + default = null } -variable "ssh_port" { +variable "jumpbox_ssh_port" { description = "specifies the tcp port to use for ssh" default = 22 } -variable "storage_account" { - description = "the storage account holding the queue" +variable "storage_account_rg" { + description = "the storage account resource group" } -variable "storage_key" { - description = "the storage key" +variable "storage_account" { + description = "the storage account holding the queue" } variable "queue_name_prefix" { @@ -42,6 +43,20 @@ variable "warm_paths" { default = {} } +variable "inclusion_csv" { + description = "the inclusion list of file match strings per https://golang.org/pkg/path/filepath/#Match. Leave blank to include everything." +} + +variable "exclusion_csv" { + description = "the exclusion list of file match strings per https://golang.org/pkg/path/filepath/#Match. Leave blank to not exlude anything." +} + +variable "maxFileSizeBytes" { + description = "the maximum file size in bytes to warm." + type = number + default = 0 +} + variable "block_until_warm" { description = "block the operation until the cache warming has finished" default = true