Add rebalance based on preempted node count

convoy/autoscale.py

@@ -79,6 +79,18 @@ def _formula_tasks(pool):
             ),
             'reqVMs = {}TaskAvg / maxTasksPerNode'.format(task_type),
         ]
+        if pool.autoscale.scenario.rebalance_preemption_percentage is not None:
+            req_vms.extend([
+                'preemptsamplepercent = '
+                '$PreemptedNodeCount.GetSamplePercent(sli)',
+                'lastpreemptsample = val($PreemptedNodeCount.GetSample(1), 0)',
+                'preemptedavg = avg($PreemptedNodeCount.GetSample(sli))',
+                ('preemptcount = preemptsamplepercent < {} ? '
+                 'max(0, lastpreemptsample) : (lastpreemptsample > '
+                 'preemptedavg ? avg(lastpreemptsample, preemptedavg) : '
+                 'min(lastpreemptsample, preemptedavg))').format(
+                     pool.autoscale.scenario.required_sample_percentage),
+            ])
     else:
         req_vms = [
             'sli = TimeInterval_Second * {}'.format(
@@ -91,17 +103,41 @@ def _formula_tasks(pool):
             'reqVMs = ({}TaskAvg > 0 && reqVMs < 1) ? 1 : reqVMs'.format(
                 task_type),
         ]
+        if pool.autoscale.scenario.rebalance_preemption_percentage is not None:
+            req_vms.extend([
+                'preemptcount = avg($PreemptedNodeCount.GetSample('
+                'sli, {}))'.format(
+                    pool.autoscale.scenario.required_sample_percentage),
+            ])
+    if pool.autoscale.scenario.rebalance_preemption_percentage is not None:
+        req_vms.extend([
+            'currenttotal = $CurrentDedicatedNodes + '
+            '$CurrentLowPriorityNodes',
+            'preemptedpercent = currenttotal > 0 ? '
+            'preemptcount / currenttotal : 0',
+            'rebalance = preemptedpercent >= {}'.format(
+                pool.autoscale.scenario.rebalance_preemption_percentage),
+        ])
+    else:
+        req_vms.extend([
+            'preemptcount = 0',
+            'rebalance = 0 == 1',
+        ])
     req_vms = ';\n'.join(req_vms)
-    if pool.autoscale.scenario.bias_node_type is None:
+    if pool.autoscale.scenario.bias_node_type == 'auto':
         target_vms = [
             'divisor = (maxTargetDedicated == 0 || '
             'maxTargetLowPriority == 0) ? 1 : 2',
             'dedicatedVMs = max(minTargetDedicated, reqVMs / divisor)',
             'dedicatedVMs = min(maxTargetDedicated, '
             '(dedicatedVMs > 0 && dedicatedVMs < 1) ? 1 : dedicatedVMs)',
+            'remainingVMs = reqVMs - dedicatedVMs',
+            'redistVMs = rebalance ? '
+            'min(preemptcount, remainingVMs) : 0',
+            'dedicatedVMs = min(maxTargetDedicated, dedicatedVMs + redistVMs)',
+            'lowPriVMs = min(maxTargetLowPriority, reqVMs - dedicatedVMs)',
             '$TargetDedicatedNodes = dedicatedVMs',
-            '$TargetLowPriorityNodes = max(minTargetLowPriority, '
-            'min(maxTargetLowPriority, reqVMs - dedicatedVMs))',
+            '$TargetLowPriorityNodes = max(minTargetLowPriority, lowPriVMs)',
         ]
     elif pool.autoscale.scenario.bias_node_type == 'dedicated':
         target_vms = [
@@ -113,11 +149,16 @@ def _formula_tasks(pool):
         ]
     elif pool.autoscale.scenario.bias_node_type == 'low_priority':
         target_vms = [
-            'lowPriVms = min(maxTargetLowPriority, '
+            'lowPriVMs = min(maxTargetLowPriority, '
             'max(minTargetLowPriority, reqVMs))',
-            '$TargetLowPriorityNodes = lowPriVms',
-            '$TargetDedicatedNodes = max(minTargetDedicated, '
-            'min(maxTargetDedicated, reqVMs - lowPriVms))',
+            'remainingVMs = min(maxTargetDedicated, reqVMs - lowPriVMs)',
+            'redistVMs = rebalance ? '
+            'min(preemptcount, lowPriVMs) : 0',
+            'lowPriVMs = min(maxTargetLowPriority, '
+            'max(minTargetLowPriority, max(0, reqVMs - redistVMs)))',
+            'remainingVMs = min(maxTargetDedicated, reqVMs - lowPriVMs)',
+            '$TargetLowPriorityNodes = lowPriVMs',
+            '$TargetDedicatedNodes = max(minTargetDedicated, remainingVMs)',
         ]
     else:
         raise ValueError(
@@ -182,7 +223,7 @@ def _formula_day_of_week(pool):
         raise ValueError('autoscale scenario name invalid: {}'.format(
             pool.autoscale.scenario.name))
     if pool.autoscale.scenario.name != 'workday_with_offpeak_max_low_priority':
-        if pool.autoscale.scenario.bias_node_type is None:
+        if pool.autoscale.scenario.bias_node_type == 'auto':
             target_vms.append(
                 '$TargetDedicatedNodes = isPeakTime ? '
                 'maxTargetDedicated : minTargetDedicated')

convoy/settings.py

@@ -103,6 +103,7 @@ PoolAutoscaleScenarioSettings = collections.namedtuple(
         'node_deallocation_option',
         'sample_lookback_interval',
         'required_sample_percentage',
+        'rebalance_preemption_percentage',
         'bias_last_sample',
         'bias_node_type',
     ]
@@ -640,10 +641,12 @@ def pool_autoscale_settings(config):
             sample_lookback_interval=sli,
             required_sample_percentage=_kv_read(
                 scenconf, 'required_sample_percentage', 70),
+            rebalance_preemption_percentage=_kv_read(
+                scenconf, 'rebalance_preemption_percentage', None),
             bias_last_sample=_kv_read(
                 scenconf, 'bias_last_sample', True),
             bias_node_type=_kv_read_checked(
-                scenconf, 'bias_node_type'),
+                scenconf, 'bias_node_type', 'auto').lower(),
         )
     else:
         scenario = None

docs/13-batch-shipyard-configuration-pool.md

@@ -203,10 +203,14 @@ each node type for `scenario` based autoscale.
       to averages. The default is `true`.
     * (optional) `bias_node_type` will bias the the autoscale scenario, if
       applicable, to favor one type of node over the other when making a
-      decision on how many of each node to allocate. The default is `null`
+      decision on how many of each node to allocate. The default is `auto`
       or equal weight to both `dedicated` and `low_priority` nodes. Valid
       values are `null` (or omitting the property), `dedicated`, or
       `low_priority`.
+    * (optional) `rebalance_preemption_percentage` will rebalance the compute
+      nodes to bias for dedicated nodes when the pre-empted node count reaches
+      the indicated threshold percentage of the total current dedicated and
+      low priority nodes. The default is `null` or no rebalancing is performed.
   * (optional) `formula` is a custom autoscale formula to apply to the pool.
     If both `formula` and `scenario` are specified, then `formula` is used.
 * (optional) `inter_node_communication_enabled` designates if this pool is set

docs/30-batch-shipyard-autoscale.md

@@ -82,6 +82,10 @@ applies only to `active_tasks` and `pending_tasks` scenarios.
 node over the other when making a decision on how many of each node to
 allocate. By default, allocation is equal-weighted but can be selected to
 favor either `dedicated` or `low_priority`. This applies to all scenarios.
+* `rebalance_preemption_percentage` will rebalance the compute nodes to bias
+for dedicated nodes when the pre-empted node count reaches the indicated
+threshold percentage of the total current dedicated and low priority nodes.
+This applies only to `active_tasks` and `pending_tasks` scenarios.
 
 An example autoscale specification in the pool configuration may be:
 ```json