Allow cron to initiate azpbs autoscale

install.sh

@@ -11,6 +11,8 @@ SCHEDULER=pbspro
 INSTALL_PYTHON3=0
 INSTALL_VIRTUALENV=0
 VENV=/opt/cycle/${SCHEDULER}/venv
+CRON_METHOD=pbs_hook
+
 mkdir -p /opt/cycle/${SCHEDULER}/server_dyn_res
 cp server_dyn_res_wrapper.sh /opt/cycle/${SCHEDULER}/
 chmod +x /opt/cycle/${SCHEDULER}/server_dyn_res_wrapper.sh
@@ -32,6 +34,10 @@ while (( "$#" )); do
             VENV=$2
             shift 2
             ;;
+        --cron-method)
+            CRON_METHOD=$2
+            shift 2
+            ;;
         -*|--*=)
             echo "Unknown option $1" >&2
             exit 1
@@ -113,11 +119,22 @@ EOF
 
 cp autoscale_hook.py $INSTALL_DIR/
 cp logging.conf $INSTALL_DIR/
-/opt/pbs/bin/qmgr -c "list hook autoscale" 1>&2 2>/dev/null || /opt/pbs/bin/qmgr -c "create hook autoscale" 1>&2
-/opt/pbs/bin/qmgr -c "import hook autoscale application/x-python default $INSTALL_DIR/autoscale_hook.py"
-/opt/pbs/bin/qmgr -c "import hook autoscale application/x-config default $INSTALL_DIR/autoscale_hook_config.json"
-/opt/pbs/bin/qmgr -c "set hook autoscale event = periodic"
-/opt/pbs/bin/qmgr -c "set hook autoscale freq = 15"
+
+if [ "$CRON_METHOD" == "pbs_hook" ]; then
+    /opt/pbs/bin/qmgr -c "list hook autoscale" 1>&2 2>/dev/null || /opt/pbs/bin/qmgr -c "create hook autoscale" 1>&2
+    /opt/pbs/bin/qmgr -c "import hook autoscale application/x-python default $INSTALL_DIR/autoscale_hook.py"
+    /opt/pbs/bin/qmgr -c "import hook autoscale application/x-config default $INSTALL_DIR/autoscale_hook_config.json"
+    /opt/pbs/bin/qmgr -c "set hook autoscale event = periodic"
+    /opt/pbs/bin/qmgr -c "set hook autoscale freq = 15"
+else
+    echo Installing cron job
+    cat > /etc/cron.d/azpbs_autoscale<<EOF
+* * * * * root /opt/cycle/jetpack/system/bootstrap/cron_wrapper.sh $VENV/bin/azpbs autoscale --config $INSTALL_DIR/autoscale.json 2>$INSTALL_DIR/last_cron.log
+* * * * * root sleep 15 && /opt/cycle/jetpack/system/bootstrap/cron_wrapper.sh $VENV/bin/azpbs autoscale --config $INSTALL_DIR/autoscale.json 2>$INSTALL_DIR/last_cron.log
+* * * * * root sleep 30 && /opt/cycle/jetpack/system/bootstrap/cron_wrapper.sh $VENV/bin/azpbs autoscale --config $INSTALL_DIR/autoscale.json 2>$INSTALL_DIR/last_cron.log
+* * * * * root sleep 45 && /opt/cycle/jetpack/system/bootstrap/cron_wrapper.sh $VENV/bin/azpbs autoscale --config $INSTALL_DIR/autoscale.json 2>$INSTALL_DIR/last_cron.log
+EOF
+fi
 
 if [ -e /etc/profile.d ]; then
     cat > /etc/profile.d/azpbs_autocomplete.sh<<EOF

pbspro/src/pbspro/pbscmd.py

@@ -1,4 +1,5 @@
 import json
+import os
 from json.decoder import JSONDecodeError
 from shutil import which
 from subprocess import PIPE, CalledProcessError, check_output
@@ -11,14 +12,14 @@ from pbspro.parser import PBSProParser
 QSTAT_BIN = which("qstat") or ""
 QMGR_BIN = which("qmgr") or ""
 PBSNODES_BIN = which("pbsnodes") or ""
-# assert QSTAT_BIN
-# assert QMGR_BIN
 
 
 class PBSCMD:
     def __init__(self, parser: PBSProParser) -> None:
         super().__init__()
         self.parser = parser
+        if not QSTAT_BIN or not QMGR_BIN or not PBSNODES_BIN:
+            raise RuntimeError(f"Could not find qstat, qmgr and pbsnodes in the PATH. Current path is {os.environ['PATH']}")
 
     def qstat(self, *args: str) -> str:
         cmd = [QSTAT_BIN] + list(args)

specs/default/chef/site-cookbooks/pbspro/attributes/default.rb

@@ -5,13 +5,13 @@
 
 default[:pbspro][:autoscale_version] = "2.0.19"
 default[:pbspro][:autoscale_installer] = "cyclecloud-pbspro-pkg-#{node[:pbspro][:autoscale_version]}.tar.gz"
+default[:pbspro][:cron_method] = "pbs_cron"
 default[:pbspro][:version] = "20.0.1-0"
 default[:pbspro][:slots] = nil
 default[:pbspro][:idle_timeout] = 300
 default[:pbspro][:boot_timeout] = 3600
 default[:pbspro][:hwlocs_lib_el8] = "hwloc-libs-1.11.9-3.el8.x86_64.rpm"
 
-
 default[:pbspro][:is_grouped] = true
 
 default[:pbspro][:autoscale_hook][:__comment__] = "This file was generated by serializing node[:cyclecloud][:pbspro][autoscale_hook]."

specs/default/chef/site-cookbooks/pbspro/recipes/server.rb

@@ -7,6 +7,7 @@ include_recipe 'pbspro::default'
 pbsprover = node[:pbspro][:version]
 plat_ver = node['platform_version'].to_i
 pbsdist = "el#{plat_ver}"
+cron_method = node[:pbspro][:cron_method] || "pbs_cron"
 
 if pbsprover.to_i < 20 
   package_name = "pbspro-server-#{pbsprover}.x86_64.rpm"
@@ -92,7 +93,7 @@ bash 'setup cyclecloud-pbspro' do
 
   ./initialize_default_queues.sh
 
-  ./install.sh --install-python3 --venv $INSTALLDIR/venv
+  ./install.sh --install-python3 --venv $INSTALLDIR/venv --cron-method #{cron_method}
   
   ./generate_autoscale_json.sh --install-dir $INSTALLDIR \
                                 --username #{node[:cyclecloud][:config][:username]} \

templates/openpbs.txt

@@ -65,6 +65,7 @@ Autoscale = $Autoscale
         [[[configuration]]]
         cyclecloud.mounts.nfs_sched.disabled = true
         cyclecloud.mounts.nfs_shared.disabled = ${NFSType != "External"}
+        pbspro.cron_method = $AzpbsCronMethod
 
 
         [[[cluster-init cyclecloud/pbspro:server]]]
@@ -182,6 +183,13 @@ Order = 10
         Widget.Plugin = pico.form.BooleanCheckBox
         Widget.Label = Start and stop execute instances automatically
 
+        [[[parameter AzpbsCronMethod]]]
+        Label = Cron Method
+        DefaultValue = cron
+        Widget.Plugin = pico.form.SelectBox
+        Config.Options = cron,pbs_hook
+        Description = The method used to run the azpbs cron job.  Cron is the default and will run the azpbs cron job 15 seconds.  The pbs_hook method will do the same but use PBS built-in timer.
+
         [[[parameter MaxExecuteCoreCount]]]
         Label = Max Cores
         Description = The total number of execute cores to start