update in static templates installation and configuration
This commit is contained in:
nidhi0622
Родитель
182ac04383
Коммит
16227aa2f4
|
|
@ -1,5 +1,4 @@
|
|||
#!/bin/bash -e
|
||||
#TODO: Change path so it does not mess with permissions
|
||||
export HF_LOGDIR=/tmp/log
|
||||
export HF_CONFDIR=$HF_TOP/conf
|
||||
export HF_WORKDIR=/tmp/work
|
||||
|
|
@ -13,12 +12,12 @@ export PRO_CONF_DIR=${HF_CONFDIR}/providers/azurecc
|
|||
export PRO_DATA_DIR=${HF_WORKDIR}
|
||||
|
||||
env > /tmp/invoke2.env
|
||||
venv_path=/opt/ibm/spectrumcomputing/hostfactory/1.1/providerplugins/azurecc/venv/bin
|
||||
|
||||
venv_path=$HF_TOP/$HF_VERSION/providerplugins/azurecc/venv/bin
|
||||
scriptDir=`dirname $0`
|
||||
export PYTHONPATH=$PYTHONPATH:$scriptDir/src
|
||||
# $scriptDir/./invoke_provider.sh generate_template $@
|
||||
. $venv_path/activate
|
||||
$venv_path/python3 -m cyclecloud_provider generate_template -f /tmp/input.json 2>> /tmp/template_generate.out
|
||||
$venv_path/python3 -m cyclecloud_provider generate_templates -f /tmp/input.json 2>> /tmp/template_generate.out
|
||||
if [ $? != 0 ]; then
|
||||
echo "Template generation failed check logs in /tmp/template_generate.out"
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -12,14 +12,14 @@ export PYTHONPATH=$PYTHONPATH:$scriptDir/src
|
|||
env > /tmp/invoke.env
|
||||
|
||||
embedded_python=/opt/cycle/jetpack/system/embedded/bin/python
|
||||
venv_path=/opt/ibm/spectrumcomputing/hostfactory/1.2/providerplugins/azurecc/venv/bin
|
||||
venv_path=$HF_TOP/$HF_VERSION/providerplugins/azurecc/venv/bin
|
||||
|
||||
|
||||
if [ -e $venv_path ]; then
|
||||
# Check group membership
|
||||
touch /opt/cycle/jetpack/logs/jetpack.log 1>&2 2> /dev/null
|
||||
|
||||
if [ $? == 0 ]; then
|
||||
if [ $? != 0 ]; then
|
||||
. $venv_path/activate
|
||||
$venv_path/python3 -m cyclecloud_provider $@ 2>>$STDERR_FILE
|
||||
exit $?
|
||||
|
|
|
|||
|
|
@ -59,8 +59,10 @@ class Cluster:
|
|||
self.node_mgr.add_default_resource(selection={}, resource_name="template_id",
|
||||
default_value=lambda node: "{node.nodearray + node.vm_size.replace('_', '')}".lower())
|
||||
self.node_mgr.add_default_resource(selection={}, resource_name="weight", default_value=1)
|
||||
|
||||
result = self.node_mgr.allocate({"weight": 1, "template_id": template_id, "capacity-failure-backoff": 300},
|
||||
|
||||
# Time in seconds to check waiting period after last capacity failure
|
||||
capacity_failure_backoff = self.provider_config.get("capacity-failure-backoff", 300)
|
||||
result = self.node_mgr.allocate({"weight": 1, "template_id": template_id, "capacity-failure-backoff": capacity_failure_backoff},
|
||||
slot_count=request['sets'][0]['count'],
|
||||
allow_existing=False)
|
||||
self.logger.debug("Result of allocation %s", result)
|
||||
|
|
|
|||
|
|
@ -983,10 +983,10 @@ def main(argv=sys.argv, json_writer=simple_json_writer): # pragma: no cover
|
|||
logger.info("BEGIN - %s %s %s", cmd, ignore, input_json_path)
|
||||
logger.debug("Input: %s", json.dumps(input_json))
|
||||
|
||||
if input_json.get("dry-run"):
|
||||
if cmd == "validate_templates" or input_json.get("dry-run"):
|
||||
provider.validate_template()
|
||||
provider.dry_run = True
|
||||
if cmd == "generate_template":
|
||||
if cmd == "generate_templates":
|
||||
provider.generate_sample_template()
|
||||
if cmd == "templates":
|
||||
provider.templates()
|
||||
|
|
|
|||
|
|
@ -303,11 +303,9 @@ class ProviderConfig:
|
|||
|
||||
def provider_config_from_environment(pro_conf_dir=os.getenv('PRO_CONF_DIR', os.getcwd())):
|
||||
config_file = os.path.join(pro_conf_dir, "conf", "azureccprov_config.json")
|
||||
templates_file = os.path.join(pro_conf_dir, "conf", "azureccprov_templates.json")
|
||||
if os.name == 'nt':
|
||||
# TODO: Why does the path matter? Can we use one or the other for both OSs?
|
||||
config_file = os.path.join(pro_conf_dir, "azureccprov_config.json")
|
||||
templates_file = os.path.join(pro_conf_dir, "azureccprov_templates.json")
|
||||
|
||||
|
||||
hf_conf_dir = os.getenv('HF_CONFDIR', os.path.join(pro_conf_dir, "..", "..", ".."))
|
||||
|
|
@ -353,43 +351,7 @@ def provider_config_from_environment(pro_conf_dir=os.getenv('PRO_CONF_DIR', os.g
|
|||
logger = init_logging(log_levels[log_level_name.lower()])
|
||||
|
||||
for level, message in delayed_log_statements:
|
||||
logger.log(level, message)
|
||||
|
||||
# on disk per-nodearray template override
|
||||
customer_templates = {}
|
||||
if os.path.exists(templates_file):
|
||||
logger.debug("Loading template overrides: %s" % templates_file)
|
||||
customer_templates = load_json(templates_file)
|
||||
else:
|
||||
try:
|
||||
with open(templates_file, "w") as fw:
|
||||
json.dump({}, fw)
|
||||
logger.info("Template overrides file does not exist, wrote an empty one: %s" % templates_file)
|
||||
except IOError:
|
||||
logger.debug("Template overrides file does not exist and can't write a default one: %s" % templates_file)
|
||||
|
||||
# don't let the user define these in two places
|
||||
if config.pop("templates", {}):
|
||||
logger.warning("Please define template overrides in %s, not the azureccprov_config.json" % templates_file)
|
||||
|
||||
# and merge them so it is transparent to the code
|
||||
flattened_templates = {}
|
||||
for template in customer_templates.get("templates", []):
|
||||
|
||||
if "templateId" not in template:
|
||||
logger.warning("Skipping template because templateId is not defined: %s", template)
|
||||
continue
|
||||
|
||||
nodearray = template.pop("templateId") # definitely don't want to rename them as machineId
|
||||
|
||||
if nodearray in flattened_templates:
|
||||
logger.warning("Ignoring redefinition of templateId %s", nodearray)
|
||||
continue
|
||||
|
||||
flattened_templates[nodearray] = template
|
||||
|
||||
config["templates"] = flattened_templates
|
||||
|
||||
logger.log(level, message)
|
||||
return ProviderConfig(config), logger, fine
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -126,7 +126,6 @@ function Update-Requestors-Config
|
|||
then
|
||||
mkdir -p $requestorConfPath
|
||||
fi
|
||||
echo "Expected default host requestors conf file! Will generate, but this may indicate a failure..."
|
||||
hostRequestorsJson="{
|
||||
\"version\": 2,
|
||||
\"requestors\":[
|
||||
|
|
@ -163,12 +162,15 @@ echo "$hostRequestorsJson" > "$requestorConfPath/hostRequestors.json"
|
|||
}
|
||||
function Install-Python-Packages
|
||||
{
|
||||
echo "Installing python packages..."
|
||||
PKG_NAME=$( jetpack config symphony.pkg_plugin )
|
||||
cd /tmp
|
||||
pluginSrcPath=$HF_TOP/$HF_VERSION/providerplugins/azurecc
|
||||
mkdir -p $pluginSrcPath
|
||||
VENV=$pluginSrcPath/venv
|
||||
# export PATH=$(echo $PATH | sed -e 's/\/opt\/cycle\/jetpack\/system\/embedded\/bin://g' | sed -e 's/:\/opt\/cycle\/jetpack\/system\/embedded\/bin//g')
|
||||
# export PATH=$PATH:/root/bin:/usr/bin
|
||||
# remove jetpack python from PATH so default python3 is used.
|
||||
export PATH=$(echo $PATH | sed -e 's/\/opt\/cycle\/jetpack\/system\/embedded\/bin://g' | sed -e 's/:\/opt\/cycle\/jetpack\/system\/embedded\/bin//g')
|
||||
export PATH=$PATH:/root/bin:/usr/bin
|
||||
|
||||
python3 -m virtualenv --version 2>&1 > /dev/null
|
||||
if [ $? != 0 ]; then
|
||||
|
|
@ -186,8 +188,17 @@ function Install-Python-Packages
|
|||
rm -rf packages
|
||||
rm -rf hostfactory
|
||||
fi
|
||||
echo "Python plugin virtualenv created at $VENV"
|
||||
}
|
||||
Generate-Provider-Config
|
||||
Generate-Provider-Plugins-Config
|
||||
Update-Requestors-Config
|
||||
Install-Python-Packages
|
||||
#check for command line argument for generate_config
|
||||
if [ $# -eq 1 ]; then
|
||||
if [ $1 == "generate_config" ]; then
|
||||
Generate-Provider-Config
|
||||
Generate-Provider-Plugins-Config
|
||||
Update-Requestors-Config
|
||||
else
|
||||
echo "Argument $1 is invalid"
|
||||
fi
|
||||
else
|
||||
Install-Python-Packages
|
||||
fi
|
||||
14
package.py
14
package.py
|
|
@ -11,8 +11,8 @@ from argparse import Namespace
|
|||
from subprocess import check_call
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
SCALELIB_VERSION = "1.0.0"
|
||||
CYCLECLOUD_API_VERSION = "8.1.0"
|
||||
SCALELIB_VERSION = "1.0.3"
|
||||
CYCLECLOUD_API_VERSION = "8.6.0"
|
||||
CONCURRENT_HANDLER_VERSION = "0.9.21"
|
||||
|
||||
def get_cycle_libs(args: Namespace) -> List[str]:
|
||||
|
|
@ -23,11 +23,13 @@ def get_cycle_libs(args: Namespace) -> List[str]:
|
|||
CYCLECLOUD_API_VERSION
|
||||
)
|
||||
concurrent_handler_file ="concurrent-log-handler-{}.tar.gz".format(CONCURRENT_HANDLER_VERSION)
|
||||
scalelib_url = "https://github.com/Azure/cyclecloud-scalelib/archive/{}.tar.gz".format(
|
||||
SCALELIB_VERSION
|
||||
)
|
||||
# scalelib_url = "https://github.com/Azure/cyclecloud-scalelib/archive/{}.tar.gz".format(
|
||||
# SCALELIB_VERSION
|
||||
# )
|
||||
# TODO: Switch back to pulling it from scalelib release when 1.0.3 is released
|
||||
scalelib_url ="https://github.com/Azure/cyclecloud-symphony/releases/download/2024-03-01-bins/cyclecloud-scalelib-1.0.3.tar.gz"
|
||||
# TODO RDH!!!
|
||||
cyclecloud_api_url = "https://github.com/Azure/cyclecloud-gridengine/releases/download/2.0.0/cyclecloud_api-8.0.1-py2.py3-none-any.whl"
|
||||
cyclecloud_api_url = "https://github.com/Azure/cyclecloud-symphony/releases/download/2024-03-01-bins/cyclecloud_api-{}-py2.py3-none-any.whl".format(CYCLECLOUD_API_VERSION)
|
||||
concurrent_handler_url = "https://github.com/Preston-Landers/concurrent-log-handler/archive/refs/tags/{}.tar.gz".format(CONCURRENT_HANDLER_VERSION)
|
||||
to_download = {
|
||||
scalelib_file: (args.scalelib, scalelib_url),
|
||||
|
|
|
|||
|
|
@ -130,7 +130,16 @@ bash 'Unzipping HostFactory Package...' do
|
|||
code <<-EOH
|
||||
cd #{node['jetpack']['downloads']}
|
||||
unzip #{node['symphony']['pkg_plugin']} -d /tmp
|
||||
EOH
|
||||
user "root"
|
||||
group "root"
|
||||
end
|
||||
|
||||
bash 'Installing HostFactory Package...' do
|
||||
code <<-EOH
|
||||
cd /tmp/hostfactory
|
||||
chmod +x install.sh
|
||||
./install.sh
|
||||
EOH
|
||||
user "root"
|
||||
group "root"
|
||||
|
|
|
|||
|
|
@ -24,9 +24,7 @@ if [ -z "${HF_TOP}" ]; then
|
|||
fi
|
||||
|
||||
HF_VERSION=$( jetpack config symphony.hostfactory.version )
|
||||
if [ -z "${HF_VERSION}" ]; then
|
||||
HF_VERSION="1.2"
|
||||
fi
|
||||
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
|
@ -48,9 +46,7 @@ else
|
|||
fi
|
||||
|
||||
set +e
|
||||
# for jetpack log access
|
||||
usermod -a -G cyclecloud egoadmin
|
||||
chown root:cyclecloud /opt/cycle/jetpack/logs/jetpack.log
|
||||
set -e
|
||||
|
||||
# echo "TEMPORARY: Patching symA Requestor..."
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче