Add more Slurm partition settings

- Partition preemption settings
- Partition other options

config_templates/slurm.yaml

@@ -94,6 +94,11 @@ slurm:
             reclaim_exclude_num_nodes: 0
         max_runtime_limit: null
         default: true
+        preempt_type: preempt/partition_prio
+        preempt_mode: requeue
+        over_subscribe: no
+        priority_tier: 10
+        other_options: []
       partition_2:
         batch_pools:
           mypool3:

convoy/settings.py

@@ -475,7 +475,8 @@ SlurmBatchPoolSettings = collections.namedtuple(
 )
 SlurmPartitionSettings = collections.namedtuple(
     'SlurmPartitionSettings', [
-        'batch_pools', 'max_runtime_limit', 'default',
+        'batch_pools', 'max_runtime_limit', 'default', 'preempt_type',
+        'preempt_mode', 'over_subscribe', 'priority_tier', 'other_options',
     ]
 )
 SlurmUnmanagedPartitionSettings = collections.namedtuple(
@@ -5047,6 +5048,11 @@ def slurm_options_settings(config):
             batch_pools=batch_pools,
             max_runtime_limit=max_runtime_limit,
             default=_kv_read(part, 'default'),
+            preempt_type=_kv_read_checked(part, 'preempt_type'),
+            preempt_mode=_kv_read_checked(part, 'preempt_mode'),
+            over_subscribe=_kv_read_checked(part, 'over_subscribe'),
+            priority_tier=_kv_read(part, 'priority_tier'),
+            other_options=_kv_read_checked(part, 'other_options', default=[]),
         )
         partitions[key] = partition
     unmanaged_partitions = []

convoy/slurm.py

@@ -540,10 +540,28 @@ def create_slurm_controller(
                 table_client, queue_client, config, cluster_id, partname,
                 bpool.batch_service_url, pool_id, bpool.compute_node_type,
                 bpool.max_compute_nodes, nodes)
+        if util.is_not_empty(part.preempt_type):
+            preempt_type = ' PreemptType={}'.format(part.preempt_type)
+        else:
+            preempt_type = ''
+        if util.is_not_empty(part.preempt_mode):
+            preempt_mode = ' PreemptMode={}'.format(part.preempt_mode)
+        else:
+            preempt_mode = ''
+        if util.is_not_empty(part.over_subscribe):
+            over_subscribe = ' OverSubscribe={}'.format(part.over_subscribe)
+        else:
+            over_subscribe = ''
+        if util.is_not_empty(part.priority_tier):
+            priority_tier = ' PriorityTier={}'.format(part.priority_tier)
+        else:
+            priority_tier = ''
         slurmpartinfo.append(
-            'PartitionName={} Default={} MaxTime={} Nodes={}\n'.format(
+            'PartitionName={} Default={} MaxTime={}{}{}{}{} {} '
+            'Nodes={}\n'.format(
                 partname, part.default, part.max_runtime_limit,
-                ','.join(partnodes)))
+                preempt_type, preempt_mode, over_subscribe, priority_tier,
+                ' '.join(part.other_options), ','.join(partnodes)))
         del partnodes
     # configure files and write to resources
     with slurm_files['slurm'][1].open('r') as f:

docs/18-batch-shipyard-configuration-slurm.md

@@ -102,6 +102,11 @@ slurm:
             reclaim_exclude_num_nodes: 0
         max_runtime_limit: null
         default: true
+        preempty_type: preempt/partition_prio
+        preempt_mode: requeue
+        over_subscribe: no
+        priority_tier: 10
+        other_options: []
       partition_2:
         batch_pools:
           mypool3:
@@ -278,6 +283,13 @@ representation of "d.HH:mm:ss". "HH:mm:ss" is required but "d" is optional.
           required but "d" is optional.
         * (required) `default` designates this partition as the default
           partition.
+        * (optional) `preempt_type` is the PreemptType setting for preemption
+        * (optional) `preempt_mode` is the PreemptMode setting for preemption
+        * (opation) `over_subscribe` is the OverSubscribe setting associated
+          with preemption
+        * (optional) `priority_tier` is the PriorityTier setting for preemption
+        * (optional) `other_options` is a sequence of other options to
+          specify on the partition
 * (optional) `unmanaged_partitions` specifies partitions which are not
 managed by Batch Shipyard but those that you wish to join to the Slurm
 controller. This is useful for joining on-premises nodes within the same

schemas/slurm.yaml

@@ -253,6 +253,16 @@ mapping:
                   default:
                     type: bool
                     required: true
+                  preempt_type:
+                    type: str
+                  preempt_mode:
+                    type: str
+                  over_subscribe:
+                    type: str
+                  priority_tier:
+                    type: int
+                  other_options:
+                    type: str
           unmanaged_partitions:
             type: seq
             sequence: