From df185eecde3b23a0cb780e6ce5b4828acc98db0f Mon Sep 17 00:00:00 2001 From: Yifan Xiong Date: Thu, 20 Aug 2020 15:07:58 +0800 Subject: [PATCH] Update feature demo examples (#30) * Update feature demo examples. * Add defaulting for `ignoreK8sSuggestedNodes`. * Fix sort in `getUsablePhysicalCells`. --- README.md | 1 - example/feature/README.md | 43 ++++++------------ example/feature/file/hived-config-1.yaml | 4 +- example/feature/file/hived-config-2.yaml | 2 +- example/feature/file/hived-config-3.yaml | 2 +- example/feature/file/hived-config-33.yaml | 2 +- example/feature/file/hived-config-4.yaml | 2 +- example/feature/file/hived-config-5.yaml | 2 +- example/feature/file/hived-config-6.yaml | 2 +- example/feature/file/hived-config-7.yaml | 2 +- example/feature/file/hived-config-8.yaml | 4 +- example/feature/file/itc-badnode50.yaml | 10 ++-- example/feature/file/itc-buddy.yaml | 5 +- example/feature/file/itc-dtf.yaml | 2 - example/feature/file/itc-elastic.yaml | 3 +- example/feature/file/itc-gang.yaml | 3 +- example/feature/file/itc-gang4.yaml | 5 +- .../feature/file/itc-inter-preempt-oppo.yaml | 5 +- .../feature/file/itc-inter-preempt-prod.yaml | 3 +- .../file/itc-intra-imd-preempt-prod.yaml | 5 +- .../file/itc-intra-imd-preempt-test.yaml | 5 +- .../file/itc-intra-lazy-preempt-prod.yaml | 5 +- .../file/itc-intra-lazy-preempt-prod2.yaml | 3 +- .../file/itc-intra-lazy-preempt-test.yaml | 6 +-- example/feature/file/itc-k80-type.yaml | 4 +- example/feature/file/itc-no-type.yaml | 5 +- .../feature/file/itc-no-worker-label-1.png | Bin 19030 -> 0 bytes .../feature/file/itc-no-worker-label-2.png | Bin 21698 -> 0 bytes example/feature/file/itc-no-worker-label.yaml | 33 -------------- example/feature/file/itc-oppo.yaml | 3 +- example/feature/file/itc-pin.yaml | 2 +- example/feature/file/itc-reconfig-1.yaml | 4 +- example/feature/file/itc-reconfig-2.yaml | 3 +- example/feature/file/itc-reconfig-3.yaml | 2 +- example/feature/file/itc-reconfig-4.yaml | 5 +- example/feature/file/itc-reconfig-5.yaml | 5 +- example/feature/file/itc-safety-1.yaml | 2 +- example/feature/file/itc-safety-2.yaml | 2 +- pkg/algorithm/cell_allocation.go | 6 +-- pkg/api/types.go | 2 +- pkg/internal/utils.go | 2 +- 41 files changed, 70 insertions(+), 136 deletions(-) delete mode 100644 example/feature/file/itc-no-worker-label-1.png delete mode 100644 example/feature/file/itc-no-worker-label-2.png delete mode 100644 example/feature/file/itc-no-worker-label.yaml diff --git a/README.md b/README.md index a710e03..a096e69 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,6 @@ HiveD supports multiple job **priorities**. Higher-priority jobs can **[preempt] 5. [Priorities](example/feature/README.md#Guaranteed-Job), [Overuse with Low Priority](example/feature/README.md#Opportunistic-Job), and [Inter-](example/feature/README.md#Inter-VC-Preemption)/[Intra-VC Preemption](example/feature/README.md#Intra-VC-Preemption) 6. [Job (Full/Partial) Gang Scheduling/Preemption](example/feature/README.md#Gang-Scheduling) 7. Fault-Tolerance, [Bad Hardware Awareness](example/feature/README.md#Bad-Hardware-Awareness), [Work-Preserving Reconfiguration](example/feature/README.md#Work-Preserving-Reconfiguration) -8. [Leverage K8S Default Scheduler](example/feature/README.md#Leverage-K8S-Default-Scheduler) ## Prerequisite 1. A Kubernetes cluster, v1.14.2 or above, on-cloud or on-premise. diff --git a/example/feature/README.md b/example/feature/README.md index 5f04b69..866a229 100644 --- a/example/feature/README.md +++ b/example/feature/README.md @@ -11,7 +11,7 @@ HiveD guarantees **quota safety for all VCs**, in the sense that the requests to VC's cells can be described by Hardware Quantity, [Topology](#VC-Safety), [Type](#SKU-Type), [Pinned Cells](#Pinned-Cells), etc. To guarantee safety, HiveD never allows a VC to "invade" other VCs' cells. For example, to guarantee all VCs' topology, one VC's [guaranteed jobs](#Guaranteed-Job) should never make fragmentation inside other VCs: -Two DGX-2s, two VCs each owns one DGX-2 node. For a traditional scheduler, this will translate into two VCs each owning 16 GPUs. When a user submits 16 1-GPU jobs to VC1, the user in VC2 might not be able to run a 16-GPU job, due to possible fragmentation issue caused by VC1. While HiveD can guarantee each VC always has one entire node available for its dedicated use. +Two DGX-2s, two VCs each owns one DGX-2 node. For a traditional scheduler, this will translate into two VCs each owning 16 GPUs. When a user submits 16 1-GPU jobs to vc1, the user in vc2 might not be able to run a 16-GPU job, due to possible fragmentation issue caused by vc1. While HiveD can guarantee each VC always has one entire node available for its dedicated use. ### Reproduce Steps 1. Use [hived-config-1](file/hived-config-1.yaml). @@ -27,7 +27,7 @@ This is similar to [K8S Taints and Tolerations](https://kubernetes.io/docs/conce ### Reproduce Steps 1. Use [hived-config-8](file/hived-config-8.yaml). -2. Submit job [itc-pin](file/itc-pin.yaml) to VC1, all tasks in task role vc1pinned will be on node 10.151.41.25 (which is pinned), all tasks in task role vc1nopinned will NOT be on node 10.151.41.25. +2. Submit job [itc-pin](file/itc-pin.yaml) to vc1, all tasks in task role vc1pinned will be on node 10.151.41.25 (which is pinned), all tasks in task role vc1nopinned will NOT be on node 10.151.41.25. ## SKU Type @@ -68,8 +68,8 @@ This is useful for jobs that cannot perform any useful work, such as making prog #### TensorFlow Distributed Training -1. Use [hived-config-1](file/hived-config-1.yaml). -2. Submit job [itc-dtf](file/itc-dtf.yaml) to VC2, it will success. +1. Use [hived-config-2](file/hived-config-2.yaml). +2. Submit job [itc-dtf](file/itc-dtf.yaml) to default VC, it will success. ## Incremental Scheduling @@ -110,27 +110,28 @@ Within one VC, a high-priority job can preempt low-priority jobs. ### Reproduce Steps #### Immediate Preemption 1. Use [hived-config-3](file/hived-config-3.yaml). -2. Submit [itc-intra-imd-preempt-test](file/itc-intra-imd-preempt-test.yaml), which requests for 4 M60 GPUs for VC1 with test (0) priority. -3. Submit [itc-intra-imd-preempt-prod](file/itc-intra-imd-preempt-prod.yaml), which also requests for 4 M60 GPUs for VC1 with prod (100) priority. The job will preempt the test job immediately, so the test job is retried and waiting for resource. +2. Submit [itc-intra-imd-preempt-test](file/itc-intra-imd-preempt-test.yaml), which requests for 4 M60 GPUs for vc1 with test (0) priority. +3. Submit [itc-intra-imd-preempt-prod](file/itc-intra-imd-preempt-prod.yaml), which also requests for 4 M60 GPUs for vc1 with prod (100) priority. The job will preempt the test job immediately, so the test job is retried and waiting for resource. #### Lazy Preemption 1. Use [hived-config-3](file/hived-config-3.yaml). -2. Submit [itc-intra-lazy-preempt-test](file/itc-intra-lazy-preempt-test.yaml), which requests for 4 K80 GPUs for VC1 with test (0) priority. -3. Submit [itc-intra-lazy-preempt-prod](file/itc-intra-lazy-preempt-prod.yaml), which also requests for 4 K80 GPUs for VC1 with prod (100) priority. The job will just downgrade the test job to be [Opportunistic Job](#Opportunistic-Job), instead of preempting it immediately, because all jobs can still fit into the whole physical cluster. +2. Submit [itc-intra-lazy-preempt-test](file/itc-intra-lazy-preempt-test.yaml), which requests for 4 K80 GPUs for vc1 with test (0) priority. +3. Submit [itc-intra-lazy-preempt-prod](file/itc-intra-lazy-preempt-prod.yaml), which also requests for 4 K80 GPUs for vc1 with prod (100) priority. The job will just downgrade the test job to be [Opportunistic Job](#Opportunistic-Job), instead of preempting it immediately, because all jobs can still fit into the whole physical cluster. 4. Submit [itc-intra-lazy-preempt-prod2](file/itc-intra-lazy-preempt-prod2.yaml), which also requests for 3 * 4 K80 GPUs for default VC with prod (100) priority. The job will preempt the test job immediately, because all jobs cannot fit into the whole physical cluster. +> NOTE: `lazyPreemptionEnable` option is disabled by default, becasue earlier job may be downgraded to low priority job and get preempted by later jobs, which may be confusing. ## Inter-VC Preemption ### Description One VC's [Guaranteed Job](#Guaranteed-Job) can preempt other VCs' [Opportunistic Jobs](#Opportunistic-Job). ### Reproduce Steps -1. Use [hived-config-2](file/hived-config-2.yaml). -2. Submit [itc-inter-preempt-oppo](file/itc-inter-preempt-oppo.yaml), which requests for 2 * 4 K80 GPUs for VC1 with oppo (-1) priority. +1. Use [hived-config-3](file/hived-config-3.yaml). +2. Submit [itc-inter-preempt-oppo](file/itc-inter-preempt-oppo.yaml), which requests for 2 * 4 K80 GPUs for vc1 with oppo (-1) priority. 3. Submit [itc-inter-preempt-prod](file/itc-inter-preempt-prod.yaml), which also requests for 3 * 4 K80 GPUs for default VC with prod (100) priority. The job will preempt the oppo job immediately. @@ -190,20 +191,20 @@ HiveD can be reconfigured without unnecessary user impacts, such as add/update/d #### VirtualCluster Reconfig - Delete VirtualCluster 1. Use [hived-config-2](file/hived-config-2.yaml). 2. Submit job [itc-reconfig-3](file/itc-reconfig-3.yaml) to default VC. Wait until it is running. -3. Delete the default VC and move its quota to VC1, then becomes [hived-config-5](file/hived-config-5.yaml). +3. Delete the default VC and move its quota to vc1, then becomes [hived-config-5](file/hived-config-5.yaml). 4. Use [hived-config-5](file/hived-config-5.yaml), and restart HiveD. 5. The job will still run without any interruption but [lazy preempted](#Lazy-Preemption) by HiveD. -6. To confirm it is [lazy preempted](#Lazy-Preemption), submit job [itc-reconfig-4](file/itc-reconfig-4.yaml) to VC1 which requests all K80 nodes. The job will immediately preempt [itc-reconfig-3](file/itc-reconfig-3.yaml). +6. To confirm it is [lazy preempted](#Lazy-Preemption), submit job [itc-reconfig-4](file/itc-reconfig-4.yaml) to vc1 which requests all K80 nodes. The job will immediately preempt [itc-reconfig-3](file/itc-reconfig-3.yaml). #### VirtualCluster Reconfig - Update VirtualCluster 1. Use [hived-config-2](file/hived-config-2.yaml). 2. Submit job [itc-reconfig-3](file/itc-reconfig-3.yaml) to default VC. Wait until it is running. -3. Move one K80-NODE cell from default VC to VC1, then becomes [hived-config-6](file/hived-config-6.yaml). +3. Move one K80-NODE cell from default VC to vc1, then becomes [hived-config-6](file/hived-config-6.yaml). 4. Use [hived-config-6](file/hived-config-6.yaml), and restart HiveD. 5. The job will still run without any interruption but [lazy preempted](#Lazy-Preemption) by HiveD. -6. To confirm it is [lazy preempted](#Lazy-Preemption), submit job [itc-reconfig-5](file/itc-reconfig-5.yaml) to VC1 which requests all K80 nodes. The job will immediately preempt [itc-reconfig-3](file/itc-reconfig-3.yaml). +6. To confirm it is [lazy preempted](#Lazy-Preemption), submit job [itc-reconfig-5](file/itc-reconfig-5.yaml) to vc1 which requests all K80 nodes. The job will immediately preempt [itc-reconfig-3](file/itc-reconfig-3.yaml). ## Bad Hardware Awareness @@ -219,17 +220,3 @@ Avoid scheduling pods to bad hardware. 4. Bring back 10.151.41.26 by `sudo systemctl start kubelet`. Wait until this is detected by K8S. 5. The waiting job will start running, without any retries. - -## Leverage K8S Default Scheduler -### Description -You can still leverage almost all scheduling features provided by your existing [K8S Default Scheduler](https://kubernetes.io/docs/concepts/scheduling/kube-scheduler) with HiveD, such as these [Filtering Policies](https://kubernetes.io/docs/concepts/scheduling/kube-scheduler/#filtering). - -### Reproduce Steps -#### Leverage [Labels and Selectors](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels) -1. Use [hived-config-2](file/hived-config-2.yaml). -2. Remove PAI worker label for 10.151.41.26 (the only M60 node). -3. Submit job [itc-no-worker-label](file/itc-no-worker-label.yaml), which requests M60 node, it will be waiting without IP associated. - -4. Add back PAI worker label for 10.151.41.26. -5. The waiting job will start running, without any retries. - diff --git a/example/feature/file/hived-config-1.yaml b/example/feature/file/hived-config-1.yaml index f873049..6fa12a9 100644 --- a/example/feature/file/hived-config-1.yaml +++ b/example/feature/file/hived-config-1.yaml @@ -34,11 +34,11 @@ physicalCluster: - cellAddress: 10.151.41.24 virtualClusters: - VC1: + vc1: virtualCells: - cellType: 3-K80-NODE.K80-NODE cellNumber: 1 - VC2: + vc2: virtualCells: - cellType: 3-K80-NODE.K80-NODE cellNumber: 1 diff --git a/example/feature/file/hived-config-2.yaml b/example/feature/file/hived-config-2.yaml index c6751ae..8caceac 100644 --- a/example/feature/file/hived-config-2.yaml +++ b/example/feature/file/hived-config-2.yaml @@ -45,7 +45,7 @@ physicalCluster: - cellAddress: 10.151.41.26 virtualClusters: - VC1: + vc1: virtualCells: - cellType: K80-NODE-POOL.K80-NODE cellNumber: 1 diff --git a/example/feature/file/hived-config-3.yaml b/example/feature/file/hived-config-3.yaml index 65ec79f..378adde 100644 --- a/example/feature/file/hived-config-3.yaml +++ b/example/feature/file/hived-config-3.yaml @@ -45,7 +45,7 @@ physicalCluster: - cellAddress: 10.151.41.26 virtualClusters: - VC1: + vc1: virtualCells: - cellType: K80-NODE-POOL.K80-NODE cellNumber: 1 diff --git a/example/feature/file/hived-config-33.yaml b/example/feature/file/hived-config-33.yaml index 950d74c..8183c0c 100644 --- a/example/feature/file/hived-config-33.yaml +++ b/example/feature/file/hived-config-33.yaml @@ -42,7 +42,7 @@ physicalCluster: # - cellAddress: 10.151.41.25 virtualClusters: - VC1: + vc1: virtualCells: - cellType: K80-NODE-POOL.K80-NODE cellNumber: 1 diff --git a/example/feature/file/hived-config-4.yaml b/example/feature/file/hived-config-4.yaml index 727a061..8021ee4 100644 --- a/example/feature/file/hived-config-4.yaml +++ b/example/feature/file/hived-config-4.yaml @@ -45,7 +45,7 @@ physicalCluster: - cellAddress: 10.151.41.25 virtualClusters: - VC1: + vc1: virtualCells: - cellType: K80-NODE-POOL.K80-NODE cellNumber: 1 diff --git a/example/feature/file/hived-config-5.yaml b/example/feature/file/hived-config-5.yaml index 3cb7f6e..68934b6 100644 --- a/example/feature/file/hived-config-5.yaml +++ b/example/feature/file/hived-config-5.yaml @@ -45,7 +45,7 @@ physicalCluster: - cellAddress: 10.151.41.26 virtualClusters: - VC1: + vc1: virtualCells: - cellType: K80-NODE-POOL.K80-NODE cellNumber: 4 diff --git a/example/feature/file/hived-config-6.yaml b/example/feature/file/hived-config-6.yaml index 81060c5..167b7b4 100644 --- a/example/feature/file/hived-config-6.yaml +++ b/example/feature/file/hived-config-6.yaml @@ -45,7 +45,7 @@ physicalCluster: - cellAddress: 10.151.41.26 virtualClusters: - VC1: + vc1: virtualCells: - cellType: K80-NODE-POOL.K80-NODE cellNumber: 3 diff --git a/example/feature/file/hived-config-7.yaml b/example/feature/file/hived-config-7.yaml index 647b04e..6f0dd08 100644 --- a/example/feature/file/hived-config-7.yaml +++ b/example/feature/file/hived-config-7.yaml @@ -44,7 +44,7 @@ physicalCluster: - cellAddress: 10.151.41.26 virtualClusters: - VC1: + vc1: virtualCells: - cellType: K80-NODE-POOL.K80-NODE cellNumber: 1 diff --git a/example/feature/file/hived-config-8.yaml b/example/feature/file/hived-config-8.yaml index 8c4d648..25f6949 100644 --- a/example/feature/file/hived-config-8.yaml +++ b/example/feature/file/hived-config-8.yaml @@ -34,13 +34,13 @@ physicalCluster: - cellAddress: 10.151.41.24 virtualClusters: - VC1: + vc1: virtualCells: - cellType: 3-K80-NODE.K80-NODE cellNumber: 1 pinnedCells: - pinnedCellId: VC1-K80 - VC2: + vc2: virtualCells: - cellType: 3-K80-NODE.K80-NODE cellNumber: 1 diff --git a/example/feature/file/itc-badnode50.yaml b/example/feature/file/itc-badnode50.yaml index cb50042..790735c 100644 --- a/example/feature/file/itc-badnode50.yaml +++ b/example/feature/file/itc-badnode50.yaml @@ -11,18 +11,17 @@ taskRoles: instances: 1 completion: minFailedInstances: 1 - minSucceededInstances: 6 + minSucceededInstances: 1 dockerImage: keras_tensorflow_example resourcePerInstance: cpu: 4 memoryMB: 8192 gpu: 1 commands: - - nvidia-smi -L - - printenv - - sleep 10000 + - rm /usr/local/cuda/lib64/stubs/libcuda.so.1 + - python mnist_cnn.py defaults: - virtualCluster: VC1 + virtualCluster: vc1 extras: gangAllocation: true hivedScheduler: @@ -30,4 +29,3 @@ extras: taskRoles: train: skuType: M60 - submitFrom: submit-job-v2 diff --git a/example/feature/file/itc-buddy.yaml b/example/feature/file/itc-buddy.yaml index a097621..21156ac 100644 --- a/example/feature/file/itc-buddy.yaml +++ b/example/feature/file/itc-buddy.yaml @@ -20,9 +20,9 @@ taskRoles: commands: - nvidia-smi -L - printenv - - sleep 10000 + - sleep 10m defaults: - virtualCluster: VC1 + virtualCluster: vc1 extras: gangAllocation: true hivedScheduler: @@ -30,4 +30,3 @@ extras: taskRoles: train: skuType: K80 - submitFrom: submit-job-v2 diff --git a/example/feature/file/itc-dtf.yaml b/example/feature/file/itc-dtf.yaml index 6507287..43c408d 100644 --- a/example/feature/file/itc-dtf.yaml +++ b/example/feature/file/itc-dtf.yaml @@ -100,5 +100,3 @@ deployments: - echo "Uploading data ..." defaults: deployment: tf_example -extras: - submitFrom: submit-job-v2 diff --git a/example/feature/file/itc-elastic.yaml b/example/feature/file/itc-elastic.yaml index 2ccc928..13a6b4f 100644 --- a/example/feature/file/itc-elastic.yaml +++ b/example/feature/file/itc-elastic.yaml @@ -21,7 +21,7 @@ taskRoles: - rm /usr/local/cuda/lib64/stubs/libcuda.so.1 - python mnist_cnn.py defaults: - virtualCluster: VC1 + virtualCluster: vc1 extras: gangAllocation: false hivedScheduler: @@ -29,4 +29,3 @@ extras: taskRoles: train: skuType: K80 - submitFrom: submit-job-v2 diff --git a/example/feature/file/itc-gang.yaml b/example/feature/file/itc-gang.yaml index 7abc5e0..ba7e12e 100644 --- a/example/feature/file/itc-gang.yaml +++ b/example/feature/file/itc-gang.yaml @@ -21,7 +21,7 @@ taskRoles: - rm /usr/local/cuda/lib64/stubs/libcuda.so.1 - python mnist_cnn.py defaults: - virtualCluster: VC1 + virtualCluster: vc1 extras: gangAllocation: true hivedScheduler: @@ -29,4 +29,3 @@ extras: taskRoles: train: skuType: K80 - submitFrom: submit-job-v2 diff --git a/example/feature/file/itc-gang4.yaml b/example/feature/file/itc-gang4.yaml index cf917ca..797bd15 100644 --- a/example/feature/file/itc-gang4.yaml +++ b/example/feature/file/itc-gang4.yaml @@ -11,7 +11,7 @@ taskRoles: instances: 4 completion: minFailedInstances: 1 - minSucceededInstances: 6 + minSucceededInstances: 4 dockerImage: keras_tensorflow_example resourcePerInstance: cpu: 4 @@ -21,7 +21,7 @@ taskRoles: - rm /usr/local/cuda/lib64/stubs/libcuda.so.1 - python mnist_cnn.py defaults: - virtualCluster: VC1 + virtualCluster: vc1 extras: gangAllocation: true hivedScheduler: @@ -29,4 +29,3 @@ extras: taskRoles: train: skuType: K80 - submitFrom: submit-job-v2 diff --git a/example/feature/file/itc-inter-preempt-oppo.yaml b/example/feature/file/itc-inter-preempt-oppo.yaml index 6846062..e8143cf 100644 --- a/example/feature/file/itc-inter-preempt-oppo.yaml +++ b/example/feature/file/itc-inter-preempt-oppo.yaml @@ -20,13 +20,12 @@ taskRoles: commands: - rm /usr/local/cuda/lib64/stubs/libcuda.so.1 - python mnist_cnn.py - - sleep 10000 + - sleep 10m defaults: - virtualCluster: VC1 + virtualCluster: vc1 extras: hivedScheduler: jobPriorityClass: oppo taskRoles: train: skuType: K80 - submitFrom: submit-job-v2 diff --git a/example/feature/file/itc-inter-preempt-prod.yaml b/example/feature/file/itc-inter-preempt-prod.yaml index e057af1..61444ae 100644 --- a/example/feature/file/itc-inter-preempt-prod.yaml +++ b/example/feature/file/itc-inter-preempt-prod.yaml @@ -20,7 +20,7 @@ taskRoles: commands: - rm /usr/local/cuda/lib64/stubs/libcuda.so.1 - python mnist_cnn.py - - sleep 10000 + - sleep 10m defaults: virtualCluster: default extras: @@ -29,4 +29,3 @@ extras: taskRoles: train: skuType: K80 - submitFrom: submit-job-v2 diff --git a/example/feature/file/itc-intra-imd-preempt-prod.yaml b/example/feature/file/itc-intra-imd-preempt-prod.yaml index e07a939..19d1149 100644 --- a/example/feature/file/itc-intra-imd-preempt-prod.yaml +++ b/example/feature/file/itc-intra-imd-preempt-prod.yaml @@ -20,13 +20,12 @@ taskRoles: commands: - rm /usr/local/cuda/lib64/stubs/libcuda.so.1 - python mnist_cnn.py - - sleep 10000 + - sleep 10m defaults: - virtualCluster: VC1 + virtualCluster: vc1 extras: hivedScheduler: jobPriorityClass: prod taskRoles: train: skuType: M60 - submitFrom: submit-job-v2 diff --git a/example/feature/file/itc-intra-imd-preempt-test.yaml b/example/feature/file/itc-intra-imd-preempt-test.yaml index 53dce4d..7b8eef9 100644 --- a/example/feature/file/itc-intra-imd-preempt-test.yaml +++ b/example/feature/file/itc-intra-imd-preempt-test.yaml @@ -20,13 +20,12 @@ taskRoles: commands: - rm /usr/local/cuda/lib64/stubs/libcuda.so.1 - python mnist_cnn.py - - sleep 10000 + - sleep 10m defaults: - virtualCluster: VC1 + virtualCluster: vc1 extras: hivedScheduler: jobPriorityClass: test taskRoles: train: skuType: M60 - submitFrom: submit-job-v2 diff --git a/example/feature/file/itc-intra-lazy-preempt-prod.yaml b/example/feature/file/itc-intra-lazy-preempt-prod.yaml index 23ffd8a..59ecf78 100644 --- a/example/feature/file/itc-intra-lazy-preempt-prod.yaml +++ b/example/feature/file/itc-intra-lazy-preempt-prod.yaml @@ -20,13 +20,12 @@ taskRoles: commands: - rm /usr/local/cuda/lib64/stubs/libcuda.so.1 - python mnist_cnn.py - - sleep 10000 + - sleep 10m defaults: - virtualCluster: VC1 + virtualCluster: vc1 extras: hivedScheduler: jobPriorityClass: prod taskRoles: train: skuType: K80 - submitFrom: submit-job-v2 diff --git a/example/feature/file/itc-intra-lazy-preempt-prod2.yaml b/example/feature/file/itc-intra-lazy-preempt-prod2.yaml index 47590f8..c28b221 100644 --- a/example/feature/file/itc-intra-lazy-preempt-prod2.yaml +++ b/example/feature/file/itc-intra-lazy-preempt-prod2.yaml @@ -20,7 +20,7 @@ taskRoles: commands: - rm /usr/local/cuda/lib64/stubs/libcuda.so.1 - python mnist_cnn.py - - sleep 10000 + - sleep 10m defaults: virtualCluster: default extras: @@ -29,4 +29,3 @@ extras: taskRoles: train: skuType: K80 - submitFrom: submit-job-v2 diff --git a/example/feature/file/itc-intra-lazy-preempt-test.yaml b/example/feature/file/itc-intra-lazy-preempt-test.yaml index df8ac1a..112c263 100644 --- a/example/feature/file/itc-intra-lazy-preempt-test.yaml +++ b/example/feature/file/itc-intra-lazy-preempt-test.yaml @@ -20,13 +20,13 @@ taskRoles: commands: - rm /usr/local/cuda/lib64/stubs/libcuda.so.1 - python mnist_cnn.py - - sleep 10000 + - sleep 10m defaults: - virtualCluster: VC1 + virtualCluster: vc1 extras: hivedScheduler: jobPriorityClass: test taskRoles: train: skuType: K80 - submitFrom: submit-job-v2 + lazyPreemptionEnable: true diff --git a/example/feature/file/itc-k80-type.yaml b/example/feature/file/itc-k80-type.yaml index 59d2751..bffa077 100644 --- a/example/feature/file/itc-k80-type.yaml +++ b/example/feature/file/itc-k80-type.yaml @@ -20,8 +20,9 @@ taskRoles: commands: - rm /usr/local/cuda/lib64/stubs/libcuda.so.1 - python mnist_cnn.py + - sleep 10m defaults: - virtualCluster: VC1 + virtualCluster: vc1 extras: gangAllocation: false hivedScheduler: @@ -29,4 +30,3 @@ extras: taskRoles: train: skuType: K80 - submitFrom: submit-job-v2 diff --git a/example/feature/file/itc-no-type.yaml b/example/feature/file/itc-no-type.yaml index ef757fc..5095703 100644 --- a/example/feature/file/itc-no-type.yaml +++ b/example/feature/file/itc-no-type.yaml @@ -1,5 +1,5 @@ protocolVersion: 2 -name: itc-k80-type +name: itc-no-type type: job prerequisites: - protocolVersion: 2 @@ -21,9 +21,8 @@ taskRoles: - rm /usr/local/cuda/lib64/stubs/libcuda.so.1 - python mnist_cnn.py defaults: - virtualCluster: VC1 + virtualCluster: vc1 extras: gangAllocation: false hivedScheduler: jobPriorityClass: prod - submitFrom: submit-job-v2 diff --git a/example/feature/file/itc-no-worker-label-1.png b/example/feature/file/itc-no-worker-label-1.png deleted file mode 100644 index f0d4d52caf0e12c5e78a4a5fa1d204f014f6c002..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19030 zcmd42bx<79_b)gRLIMGT2M-Y3lHeXZID%b7G$sq?l!<6gDiaC zd;7=MZoRi#?^m@`HFa*^>f3!z_dWjUFa)VWFiBZDm_;&` z08`7H#i;pjDbvr6Akl-=@baSuzs00D3Nz-yUkIoIP_ljb(LvYbc-dp;4-(7p?g?Vg zkkw&lYcz$`G?RO0IU;7vMCpko06?Z7@lNo0;Oq7&{FzI@xeyNOGu;lL#e1fs$Uw$t z`Wqux{h59t6RjCz;UuS^pdeQp*J0c;&!eOeFG?KIkZ04!=7hue3pl|=EyGm;GG4c5 zuT|C^Y_<~ThcR9}t=(U9!_n&a>}=ax5f3fJa9FQ#uLlr0|#BRKdX<&!!WQnjx(E#s?w286}Dib?d?8SAxo7>=V^*rH{VITKd^DQ zCB!|>sd7IH{BzTP(LlPvNB6;}=xa14VF}DePtOSv;I&D7qw=S&hTL8g$g}LBHyGgDG9I_~?MEyN}p_SDYeJo#_CWi^RC)7D4A>ylqn06!BK8abbqnc;)7rjj$vr0`{4Y3#nAl5eex$yeB zb@4Fpq{W%ToHvbNP1$0@XL61>#XprNKVr_wo}UkI>H;27T!NamcDg~%P<9xDd7M#i z=tQ|5OBf09)nal+QCss3*zO%zfoBVJoo05A$gtsxVW5F>&%*|td8Md@jzW66v{SG+%1=`sOI~WYqo}Z!D zfGNQGPbMHZ1>^Mid%(9u?b*9BU7&M;-!4n+_G|MeGMT@?>j)wxd%A&#`pc-_b3TuT zcfT*HwgyT!vINeqYgYs_i*&LCX_DQn9$OAq=A(U%+hvy^1UPMi_xQ-F_xi%qCPku} zBhv%&chx<4C);(S1+(s|_aLBWdC%^S&1xFo8U{o3asmu;8QE;AH~Ui`e9YOS$I@OOwkg~4zwyiF_JPNTj{T-|2bF}zq8{0vFN@FiZ~5FgU!l&v zFKI@AQ`RGueTG94`dhLkOGv>#?~RM174#@x)(;ZI66vuU*Y@mBN0`0p}vwGA` zoBc_QOG-qL|M&R1O?@1H=kdOmy-~nhFH}(jEN0#AGLk%yc;Lfch?-V1tbK2zwUiSk zzcMJObhCQh+Vmi`(m7x0$hBD$?prdljkcx5gP;bD+NhD7)#ZDp>tpEC9Z!r_DEs&G z@#~5Xru&R$c;RkM?quz{c<=V2nV$$8t{r^uX?fpKl4npx>qH2*^%}LI+|2*L=R`cR zBtIYV#aa4|^l_8vz=2%%<7zBpv(Lokm&o>;*cxviA2+9hbd_VTwJ~U!Y?V_h^x;&n z)ARHJ-JbM$oB1~8Eih%Bs3v#=ul8SlLpJzckvXI1y1ziVY`26SUX@EM=&n3^bPNBL z8;Jgqaoh3E=721p7P(`UZs+E^BJ#f81PTG^6p;I23l9^&&bNM}>bS$SiRuU4qbD}T z-%=E3g>;TMl#^I>u`B5Rpvv=2t?*)yuDs(Kr5K%%}fX9+2+IU!j~i6uECp_AfK8q zdq=`damM3`6>=$Os*+&%#C5oV90rTf5tzjSQjB2-SSP`2~;Xmu4z`$jo2 zxt7`a3ejlZBWCz*^MRefa8m;FO591A-UX-Hxd+zvXSggcmq}`HQj2H!td>2kgI2Aw zd5aWN_4vNQPbAafW6omD63b>no{wpVwxi~rquP1`seM@K<|ick0SB^p{bGw@`jG4B zz3GagbVY*nf~D9uP{y!!%Kh$ydMXlOYqG41)$kaOPQh3TdDDylj}f|7%e$o!Iw@=J zf`iwOWi9S@l|!H|7BU$h{oz?87V6J@4;Se>Sy_)(2ExMYOU9j^_Ct3bv(VUQm9Xs@ z0a^{RsB0U6E;wWOaHs2t?hm{ly-Ak)CwP*0-!!NUcP=RBDSN>tq&kIMs`2C+e#Io$ zTi>^d?zu{UiN8g4+@BKz(n~+=nVWjX zAKo7%3(`yNRy3YfIlp7qOC*ku=NT9tPM-j~s+Htwli2ZM06laD;*F$Bn*MItn;Mb_3r9x761L z`q<&Vg(H+FqjlFL8kdM6^ECQiSZbAh99GiD#}~fh*a#Lq9!SSysjPWCnT{^*-M=)J zFjWC(i0Lmw3o&pda6;w z4TANIlzE^1)7pcN@!QGD^lQx^z=As9PnxJk_XBOP@&d1|{edGyRmm+6CsVg&ig=Ez zwoGVDEi0wuUHJF0MlwDgx-y>07@ys2P-`BUjQhpCjmD~`B|Qhz_2mffgwEr?+@inT zFCV`0HJ6e4d;y)i@HNk+mR508Z|?gjJ&T^rt)otJJh-D5cJipSY6uLVWNf4v>ZBI) zTQudKShn*%K)srQg}xmbEK<#A*>moJW?G;m-^a*8r#9AH?A6Aarp|q*A6NCgaJ4e9 zzDiwzXdh!;0vD{D1USXfa>TLm zUZ<9OHT6Pk~T_ z=P>&JPZzjeu;So`_(qisX_eh5msOib6b94#NN0FmdvoY#Gdhg#ALz<-XpN^z=Z!1d z81AQ^ocI7cGriP=|DFDQ!5iP`wB6di4wA{ZzrPk!>iOHesq;!v%OoWSmR@ovE=@l+ zF|J(5yx)nhGE`vCD4WE27!_K_$iRz7oA88B zY-xu(^yN&CzUqvfHY(3ApHuAcv2{>bTa=o}5di?MK}Db%BBe(99;KiW-k0b5DXSTy zSFDWHDhtXSLNSV(fgk3qgegT5YLATV<>(j~Pq-48*L8+U#;{qK5&E<&T#-93>{#QC zf6JiKX(gq%0GVw{w}<%oQFK#E^l2K{zSWOqaXc9<+l~pr$F}rxd5yIqUir%=haaKo zrXXCcQqW`Ld0oZh-dBU~nEWp10+d<{%a7l77#Zs4yssdpeNHaubn>!HWE33Bzj<1# z&qH6#*P6tE(`CT|T0%>v5tGHN_bH00+=zCa5P`8Fh~mYBtcQHGKgF!D5R#`Y}I$lru?%VdbF8` zKWa~4^jcGQ!217k2MICq-yZVR^SCjvl;?ip-pW_qj4db7teH}NL28gB>0aP&2| z$9>-6uU@tiS#uRZ`M0Zu!fzQC=JVDJnu902ATRd_+veXFm$4rg>C6}z{hVfBk7QECGw9TG3h$`dFZ(nm3eZt63N8=t$I)__G!Evz z7+b=!ZvLJ|7g*lt?$LEP00b5GC&|9CO5g0O1=o&_)_&075$$xKo*)`NA7AQKDPg$x zGaeUX;?s}CLBfNf*4Tl}2mIu8CcjBejA^nlmE&5zbY%TT;kZxX&FnbUNT91aw@xHH z0821*2A+ZO|3oD?D1RGLX5TT2J#Pzbak}dI&TZV~?Z;O_z){GoB(JY7QA#Y@5V{nO zhm=tHK%_TqR1jn=W%fCv_8hd8PRhOlx70>3WcqTqIG=$=U? z#TYWuD7oytLJQVr{6wo$7!<}NYdFDXYo=uQaxpH5fjmQNsHF8x=%lZ3O3Qx5A=l88 z_8)j(N3*nl3q)Tl5JA)CI4-Yb(CK#&9ywRl--t5o-Ofj&UuY`hvGsL$z2O$eQ3#7a zn=^JAW+ss`)iI|QR5vkS89KRfeXB8d^uEiu zHL%$4(z!7E;(pukR+nd{(6FMjbfj^1fHrQ!2J@A3?KHVR5_e1NnJ}e6e2r^OTlJw0 zV<^btgH95+sg~(#Eg#e9^&INaYRz$oJ@>}DW7zr%OR(!u z3SQxYmIING7WY=pyM1PWFN4HJuDW3XH0$(eR3a3#T<|E|QR+_P)0SjnJ>xU%u`f^T z9EqjX6g>V{OPyA!G+MpMGsw|erCqjzCIPrNDHhUZnO?uTH{aZQo&7RNW@qiSbw+k^ z^1aW7#f|X|jI5lb%FxeML%;d%wym!97=yRjsbPt=aAWwvWEo z?H;N|xZ&*DNAr-bu()8n{W~qm3*@=_7q2T~xAP^)r?u>?F)@vg&2R+?V9O2r63#S| z9?xi;6!Z|>c(BZ#ZMavv$(>@XuN3koR1T^WZe{l_>OFUj3Y*$#(Kp2DgjDghvhz1; z8XF+9)T2or8(ik{n6yiPoiJol-GxdX)sSFNr{O+1M+qM-{wAfZoX?SvEJnRBDY`4c~smL`#!4?6#l+{)KPcoczZf1Hd?|AA!$)7GP^f{=Z)RdtUHg=$X%I zAXfI%uNqF~?@k6=AJn{NQ*)s$a)KFY7fOxX^V&3m|EW_paS>wnWiH^(%w3Jl*~`uB zft}pCeEt4Ynge9txbfuvTh4?oiY%NfS^N>;vC|zTwmd=nyv&R@=wDa`Pvy-FZ$;*A zQj6}Ia87Es+>7uo(o^UMqyq_L;_5T^cYEl%`h7V ze+r3^LGJd`6t_NtuWUk7zIe*!sA(86n^+?O0h^wbfuGrVoEQmPUdO1DihL$<J_I8a%=g?z}Y#7#e`}qy4)=xCV+z< zbDI=WAr7&0=EE9OW3+Gh%8>vc#k^saPhmOUYPZBmG3Sm&vKzBQon+YZqzWR@-|7uV zqIPe74FJfy;q@hXJJwgFvu&ppB@9bal5@URBTZni%HHYhQ%qpV)bg0ep^vR$qJLxH zXK$x#AkXdLhO$PJk}o)f7NLeTQAYCy`b+l%_Y`^XyNomf$Z*tG$eQ z>Z)fl4C5au@fe1Gr|@NS4(iSj#S6XA@2BOSr~N9qFAY?eCs+uXNT*|sqDTI^rn6;< zk`NZw#`QE+slxTNvkA^iY^+wj>cnxcbz`eEf!^S5~q%lQ3FQE1CUYmY23LA|Vsw@)x1m!Cy&gH;P(~nt- zEJ`!Xv6S>#@W%2Eav|ITS-JuMU^32)hj~sYO}0M7U2kUZX67=I!WL_ewU{=GRCdcP zNa}|kht?SrzY3UxX==J>6jirg{pvkxR!XD2fx={Ox`9pvzFJDVvSbmOJvDvVv|C#( zIk~mUmlF|2-Ey2=_s;fGGa+v_5r`J=-3kROhr zz7BN3vUa0BUhb*Wl*DnpgoC0m^^#Ee@El@w1#$_xNS>zBIIsLZGOLVil-1n+REX>H zMjVaI)SW!!k}lJf&ON!4=%a~p-rgJBg%Ip&Ksk$+V?n8QeL^xM;ivn=<`q)nZf=3K zZWhPq$xGa$yi(XjuS)lf+bejm?r>vwEK4<%cdWbEPLjGJQW)XXrYyyV1%_R|dIEp0D4wYfg^SHhDIhMlW?yP}6moqv<%}#PZ=K)+z$s z6=GAwL*!Bi5t*}HyifwXdNe>4YTy6g*8FD@3)G*sUM!RfpdzdnzQfWWb#(A{rrvn#vW}r6RN58!pdTT{G3j z9{or5#xK)zjgI>xF$pU*&+qlY<-71=bsmR#zOS40k#Tg8UwU7^N^(fob)mjuCK4xS zwEQ&R`EH6St2iHj=j_w(jexM3L{WNE-H*WU$>JV0XSZWd*(sC1DX{#6{{={VL~m|d z;nR}p9bC029;Az2{&=8w5s89Y$w*uNL8k_%JF-?82d6Y(lNO*d(7%NX%iN^?3G$FO z_EpJ*TRTM2n$hDmjWC;sFKNrip|#k~J>dq)Je_Mqq&+@-*%CFf6T%;$K~7leCb?iSSDW**P(Z1L)q z{Y+oz%h>9Gh;!TqIT=S={qYK#SH^QiQc}ILF`3mmd>NpZ0Z|D-Scg)Z6EF$UVa0)5y~{_^`n8 zeS9|H@>Dh4HR#!bydgMNY_TZ&e5gNl zgk*Yg=;oexKvN5Fu_E3Wf2E$@OTT=DOff344{IqZTkf zl=icUvTXrRde;|0v?sNWujWcaufl)~7@eLCc%E22O3IFfkNm!m@FhmQ9g$|B->Mj{ z;bb{#byvP^;F<>=y6^Z2%W(Kw!uui2R|nBC4u&; zz+nO1n;`pxgwquUmpSJw9Gk~%JCxeG=+fkwMeBbFHE>kwDZg+zmwB?qSZ|E{vKDG4 z!9!DU&-Uh6fVikA4nF<}f&8F9Ia`Oe+b6^p{{b?5gp|%Sm@>ES4=8kKhVX^+ym*G{X z2YZ-l2kz>h)K$!`Cf`C9d@>S`s9>)qjH)bWe7Hbdv;tOL#}U|n;&Oxpq>TO2$&Fj( z=^xf|q_S$1m@nwWCb+w~u2I`Q!#U-N=6;#=q?>QpOP3{mseR}Lo+P-<&y z8^eYoTog<%bbEEpMRWE|iL6(Aet0XAp?K~bFVdK%W}R?veNXIJ;~>!~N{^p;L)(A9 z&%L5kZn3B%(_J=5Y&_k^XBjhzkOzXQ8T*xgX=Mp83YH{S9?ph9MXeeI_Q$5J$jiUm z?YiHeM}fYTlzK9su8@u^vpMmBI_fIglX5PD0$}Y`x)11D1|xK`UM+ zU+etpkL9DFl73JNlZ^5&gR;Mw$q46p+J}1<$tKk*TKCMGOgi|#W^XsTFla6pm?fHc zvqSb`x<(U$43>m^-uVHSgW9#1+i&L*K$yoh8>Pp=Z;>=460PIcLxwdTcpSOGjKRSO zt%_NE@|MW`?TrJCWek9=xf@5W?=CX#!T{ju-Se=j@|_&sb)X+?Eej@GMNRV@ zEFL=(@?{eNf8E0}C1*Wzsv>1?Lj5^5&=>G(i|QPp1&FgZhuNOkV(%`doIAX;V`_u0 z0%1Y$o~gs8EHHNdO7KZ%INsHPSHt52kDLE$0g>{j5bS0A(nGDz+R(!OV3~wyxe)$mSu3f^!;@G_I^Lc;OfnT$4$-4h}l(gW(c9vQlUdyN-GzFctT54ONAG5M6C25+JtdA%+-XqY>B_}c6<`S4ur3+F&B7)nL zRCp=0qKw4?2&c#@7=AIoA5P3C^83JjY%XH|WVqc65T5D`|EzVpGrpxco`hqnv6BCB|9m0ZxEgPjgM)__ zKB-dtzH{T{*)>W_fXaw!UcS_w5U0b^hwgsz!xr87 zP#t5shKH0|Ty9fTerxj_dF@Ncj#T2cDZ&`3_ID3CF$7HM*{O1oR0;I_+wHSzLZ+OJ zPBV!a!!W+?1B{ z1@6g{;#?82G`q{U7T{}g^o(O?bS&SM%`n3BFp*!T+!2e}$+xbD)l5L5Cs{KcFM)sA zUFNPM*lB>it#M4ts-FF7aKq`InED48GI@b?G9?lOj^_4RCx>*Z?n46*u5t}mlBR2) zxE@^;dF-pBmT%k&e@9e}SgX6IW%Q6l)yIYGJXC{L5WKZp?v;d$mG`wT7kl?(3J4%h z$C9dX?^d+$L(o5V;bUV`oO|iA9(Q3(BszQOudzFL(RC-im|VA+kfHIl;(DDx{y?V{dU{<(eM;K! zS%*4c_E5AK%OC4wC45I8h-XU;Wtkz(!q4CytH>8(WfHR&$g{%$waD775p~^?qNhCT zj_)z*B}l<<%GO`R$)g|C8sUOYmsy&|4LvGHbWXdS3 z1hif{5sK{7xtbxyo8+DIC3~iM1-DYZd;fkI!fx>jbk|h3c-q~xww|)$V)akuSe&Q! zQ=yC6nVG4?Cid8O66K#?(zDAKrN}?|T$|u#tesgZbCNA#y7S;DYu>ja(=z536xCzp z*S{(O11r{V@OkEhF}UHU*7mQ$>f7JH`|R$&k5#BVcT2xwOv}fXn_+ge;n@<#HkmTa zo(OL&6>?A(^(Zj0Nx|AQ1=)UBZIDWHejD6(VG7=9Z@hfhaLi4ex?2KR9`&zSHK;CG zvfm1qiq%|D_vPiBv}vzAfC2;h>N5=?C&LN%sw+B@F4#!jci!oj8^UEk^DBN{=ozyK zONzzNPRk`~lSR(b^^y%%C$tA+0Y1Ex^v6yIt!vuW`k3ns*oYrI2YZ*qDpemvVIiEnexjWF`M#h<NmNE<(N zcKuuO(wLPV5mUcBxN}fxcwu|Q%i&A07mcWwTDD;{~i2l@U zXe@N9whYLiy=QDI~q4J0X_G-@IUR!`cE zdsq?jP_%WDmHYA*p&k5~5q?WETFWnJ24~hiN3G+x7gvofQJAx9BE9Q``O~tWi}oq? z{AO9ofNwn(=fa+0VoEIH-zA#TI9|{$lvPXtA(jc zlmk#CnVC-G-QlsWj`l(0VTSh|1X7{ro5k_7YgM*{Rx`a+k2dO%N6kDhivOp`#e(Bc zo4(aoY3-F!29DY;>&SE4R`C;7&Y}WSjOst#uOs3s0@BG`b1SAwGHB_4aO*YK&Bfd& zyx<>w&*`Yr;7#R$?xYg!4%@PnFepzGp=9>JyA3aH_|C==6pUcdyrLo^{ZttF`i8D( zHdD1A`x@*;jC9bH5Hi5A5@DOxjH26+OL#JealDL`_^(0i@CfE`P5F4F zlqmSBTC ziZx6^>Gjntb;CJ2$V^}|U^%S!i{X4q3>OHo`(WF%{NX?kiv{16o(fbbKL2lz%`Sm( zclld7gV#rc&i57LJ@0z6dnChS%xJ@X;Hk|tvz6i=oCVV?+01jsIi*Xes1OC=n#756 zRznC1c(Um6fvit>(PzV3z+p4H9Kg*(p<2Vf6#DW{%c7ST8fgKroL!9M-!S3>_LmbVLg9!wAc*@1NY=f)QC?> zq*jJD#)5!ek~4=HrJ+j#-U2PliXC9|OyUN%nU>fE97b(1Ag&sMM#o9phBuQST(LBt zb}Q4K5;d9>6pqWcdb6s~Y*6J^zWEg2QzF;sFB@Q-AR;}~Iq#I=ZA&=ZKf8{sHxgK_ zV+ybZkF7epIOL$DGf-K9I#H|PV?F119(3l$nW7Y=v;DVz8c&G=mBeM!yoZR*QIqqb8ZZr&g1t z$73#Wac=^v>kZNSsGZKNz8c5Ujn~A*_m_((;#yPumFdg|(%}|bt*gPA&>}(sRUs?I zlG6tDo8T_|nvH2yuHuLCXOaOqB?;*;cA^v(UbwDq#W4-1?v=!lgT`;@b z=?>7{de{-=3u0K9=DIU^s$CaM7B?&~$QepdYOfLQ76dVgvihY_)5P4 zGHPRfc$Gp@`3V5v>JEL*cb}le;Qx*j$Yy#5QvBaykDurN3%d{hQ}lDzqiR-M{4*B- zYQKj66B7M={9nMiQHB;XEOj(=jPc`Oa>Jat|t_)CMqVrFKHF-D8qw@|Rd&c|El46{lOOws8Im)K0Cs zGS2pEj7Bx!y+3IY>8v^K$b86qmzqfSF^@A{T>I0LC{5GgY$$E$Q=u#&d5|D}sS(=f zqmIGdqugHG#HJ<3N`kmaueb${v*p3S`F{ozVf^HVk=pKOH6I#0Bag225?eHHW2hI( z)4nIwdrE9F;Qa@_{%>D6SAE45Q128oK(@TgHo(ztKlgdpyogUv6>op>@{_)J${%saQ$=3|QOjmsnK=HN0)Q z^Nd#-_bA-_&@If5MCZRaS+y&te*KPIU&oQ26rKA)H?7R}sN~IIRoV90>T4SHxxX3} zchCk(pW98|brAy%J!CJ~&n@exrFkDFx8bgWPMVybeG5UK(IlPBRPNEQ6gO_{fCXCg zRpt?xU)B$!wRN=N-|x8%de;^qpdY`hMmG*daQ)qb&oKCY@@jl4a(s{fY?fTtU*;}+ z{c0gYh((0^nTDam$vmAAQ>ZBI*lLb>kcxhf@BNsp9&WmUo|}w#cmb^*bbF^-cyLrA z0QFou8e%4dtpQ*P?2QEGHGQ3Wv$BYHxx2<%q($GsZUyM50NJ`312fVc>3vTPE2c@pHo8ynyZZkCqDGj_w9DtnNs`A96+Ncg z1q@+qRYYlQac7u;@_sf=v}R!>#$hQ+#7={Et~TflN7=(EenwTEC)~){ zSy>CGTk*=DnZ2h+4aH}WM4z7*5qJT#mm08rYL5VXX;O*%YV~7l4cnWzXmruo`l<-3 z)m6o1gwD7#@2bdX)Go!b=SZRb$;Dc(T zK;L(qm~AdkhtQ-giKWhSu{vpw%F#4}@s3U^7o2>MFzSrWAT~o?M#)(LRS>*78C~b& zP(z|L5%r9G8jY7Fl(mM8pVr&J)-O)<-K3mo)~of`;%=v?s_c)v_QQqX_7<(5md~wU z0Du|r@RI3y@OwgRf8ovZ=#)aV=9&H%c*6hTd;X^m`v1y-r-h~E*5_xgd?o+4#T5g* zK&yvH-s9h~ET)4rto@zd-{l(F5V|BjAyQe}Z-`x;+Lr3~E?7=IG1>&Y`)TZj3!}_M zZvp>Mk_8R#$`HMQCZ($z zt1U6UKtT3XORoux=d;5gyo7c0o0&5@B>MEiegj9)?1gCZ0*}aa`oAq+08Btz6t*PH!0Ymszb{?(#4x4hgdKUI zgWrw7Vz=@MXmAogKTpfQ$)?T5x74%3|6-3$2qUO|Vq%_O6=qr-lT$D6QK5&?Hpt~| z`|ELzumE4j*$`P*UJ*4Q+1-zb#{FFomD15sSbL^+tAQ~76RIT+yFIWd(uuNqGRS1- zy#Z*{GK+&}3r|J6{pP4rSHAH4xBU2Y%7fEDws~zC|LxGbVHac1S}}bwR=6al0^H4g zF}ugZ?Y33MH7>bM*O0Qc?vXdHiaf$unPhNQm z3VxNG`~Fo|qHu*X9-YIS08_G1-O~=zJUjrNeI-lUH%9%h zkb6U++-)x^$(c%;+&ROI=jr(EJ*KnJ&r^0i-W(|)me2Jbf9zonQ!}qzUG9RePb1_~ zR!t8-7>^ySUz$P7Mw}fnsv1XppSs?&?VP9*biIzmrt9==68KEWgi#n;dzBr-uK>Pb z9|c|6%&inf$~NY{FbE8;KSiV#jFfuo}qr0G}qg%#JWalQy~@dUQlRAT&NJ z-N{^wXKJLBH4ba%+R}8DtI-A!OR3urV&X1S(%X)YKR}jQAil131A|vW6pazDzcQvD zUtUPpgy5e+N|r*JNLp7ctv2<)z8Ac`vcJ!NwL)YOQ7}@z(G(5&w*Y}l26-p?MP0#Y3x=e|q1RwBT(zvlMNRJio}sMT z{I?Xv%=-T`Zd?9wzq`h8VDq`p`jmPGcRr!>o_^Z_nE#>w8y(9jzS(mq#q7|~;=mrF zYvV*Q(mD3%sRbF=@RCnIn%!>6cg{GDwfz0zKOesOj%chv$mEh_9#Zs5Vq&x@(igvDy3~tsVST?7V26TAL-FGK*7UOQ^O;=i+IuR=; zX1Te!6!i4rs+3>_&(x$O>ZD0hGeLtK>_u%YC}t=|<8(pW+UsV1Fd3*$!TdU>J#2=v&B*cXZNdZY0M^o+gQygF zENgVHA6Q<81=~)xntU-#eQc!$@UM1Fvc788uyWR4X-q^ERCuJ+S`6_c?!~$dRsDdPVIf_iigRJvvJG$@S9Lk=T^#^_Xt*7nh;jLim}#D4TUl&8 zwm3+QN1lNoTklF&HGR1KhDSnDp+n8TIDF|&z})s>HOwGBKHjEzD`|Rq8d7IkR8-_~ z<$X#q?-+hi^RwGuE`w(%kxw8aH8nLd^1UYL2DWw5ewEo2-tKW#NR{ufGk(%`9<5$i zW2Tmn&$;R<(wTMA`Pd;5L)zEZ2e!T3pC#dS&JS`llQI$#>OR{TAoDo<`o#S;Y=osA zt5Bj~#}AqG`ewq1d1scVGC#i;3e{tVm6n&6Q%Xb+5=6@}2@2|sf20};qdj)i)udKR zH8(il9&;=dygLoxnn7(giV_~Uzdfkx1Fsq9UMQ5(f-cGY#}8UhI`Ytlo<6DGQhX|I z0NEc#Stk@SzF)qd-B+F;!XUOJUDtF_B>7SQ0aP~2r;Mg({6mF;$?EX@2if9uCtHQQ zOu&1~#nMvxvpXpn8PENaBD2QIdMcG*@Ds3WkqsrrKOmq_Aj2L&xb^ocMXm8c$}*^) z4y09&)=+}^Po-l{bwCiD$xUmb6GhBzKT{}ckoq!MBn15k+Nn26%I}^)fqnFCul*jr zw7w^fpDI;8BW!afj3Z4IgR-748E@v^8;Xg8G4)in)5PgkQ7QT&Cc}xhGONsJ-F~T>>V4XcO!p#}BS%5&bycO=7Vk-0 zJhd5;RHXWDpcKYz@)M4oS(Eo)4G*fGG;t=Jj33% zkgkLxl!cKoAK$UXbXSH(K2xS~4oUah1kiog_G-l?>olGhC8hblf16NJQ0zic-ALK}^ktoDCdT^AXP}P5(OmP`b}LAP~#Syc-96kyrM8=Hl(~ zd{q`+(xg+)bl-dnTib^|#i2Ybx64_z88+Mj_|cN(m2SkkJ`b8&UKtgSB_rO5nw}Tc z+~3gOVaFn08FpEi+JizLFe}J2R;9#3BJ?FNR+f~}!uAd-TccMkO9XO_Jnzm1z+rRd za;^7=brYFhaE&9Tkjd3zFfb+lJW7HWKSM#&2AwLcV710aBF`9os%WzySvQwB`+dT> z=R|^9=k09Orl-hm`Q9=eaxdf7yRG-LeNRe*C*4~iW^)v*rYa4OqmsVfcfOT2adj?| z%fO;3(7PVzJy9S8n|Sn$h??gOC&NT}11%pE2=eF=B}JLJREx8(0ZKC#SaxZ=V^|f7 z_|nBDw)6dxPU!6^5r71)S-#5R?VQW4_Y(E-#aMLd1wKUY*&nV41Qq{?I*Dewu7~;D zZIC7ScHV8VC>6;Sn4`qfK`t;0xT&auvQmiOWZRfK7+l-t-=RO4q#1F;em^ zt3PG`{YpVlugt+CT)u3MBQ2d;t-g}?q=QLSo6s&LnuM3bco^ra^t@aIr*>ApGJUbt zhj2w&Rm=iuMx3l^<5?(Xpa2$cj=}cvvF&;WCQ>qgMV-l$9(%wKW2I+Dk}@;p?66ki z!GX+~P4vpfq0SnbArD*ppWz)Fhlhu_462h@ixnXx{9w+S_$Mo+X6 zHo``(w(i2K@*&r+`d#tlBj9r`zK11}7%)X@s4iK92~!n&rzlf=m)nT0+8Syl|Q9^W3zC zZf8p#<5JHOc#-ABA5@k}a_#dFdb^f_XQeiiFLY%u2&ds@I_0C;Pd=#Oi*J`P!OAJ$kRt9vwYzAz6kso^gSgn&|x5{1qqL1 zcD~&2FZxf0_Tt)gBZ||y^WM3%*8BcI&(y?ZnvDscCbvRHdY_DKOL+zbEC8ZPje34j z=(8bYK6iFTM)|)ltB;W;>OqKhoQ#YldeOWL>j(js<9xL;a4kugXQXNqFXn6Rj1 zdz+5eU$+e76eyG{Dl2c|q;RcZZeGso71@jNHlK7LE^1WyKlE`NKAF*rs@=x|OUZ6z zz0uGS@ijRanfE#XP2qY;A)yyc`7V0y?H_DX)}v?h0KmTr7nl{cn}e7Y&2qgymmj|x zyjGFC9mn^J7URu%2b>Spz_=>Ov8U7RHfM16$-VMe#K6_YOl8q(h?Gh@$P^os)|zSn zBM+mL1}G{i^$8%A78PrAeAHDc4C9a3?Hq!iK&y;lPeu~_!l*KNbnmd&LOCrWf4U*z zog)J1R9HU>s2Cit7 zXz|M;cVGH=41w#e5~+NNj$(e=Uj^;~7Bm=^r0&8~t(>#0;G8~#y_Hr@6#-?WsYaihR;ga6H-&h9SL`bI(%iYALWZsgJLgC8bh(WrrnaS~u&L;fR*Pc^mu1gl+$3ww zle!s@OgI4c^)C~3QiTx>g7vrL1+xFDO1;EqAq!>@haIhY>v<(l3$xE;3BrC76Pf%a z?`Ig1xRoQt>Ub)CUVpC0njsM?b__d^n~+GwE1R7_1(wexOU)?X)^Bem_4hcoRN{U3 z@b9n2M&P&mRgV+glWxLNC9^Uw1bh$CyK;7Qg+JyLXz{w#7c87jwx)m~wI4~qT0ETH zBEp#CUjQvq-+kw9ilv3Q_{tb*mIWtnnz0=?g(Bxs7{1qoA(-D1Y0|f#?Vda>tp9a; zZHOi-|NAd$vv)|x7kj_I8WvBUzv9VWR5(nL3w-h(IuY}9^0cPKO-h0ENugSy@I80q zgU$%Bw79t~53dMLg3tZsyrW*2R)WI)fqTkRoFRs<6SySdOLv9Wq;$x=DA{{$O{DU%(g4QA?z*;;M?FOO=P&tsng7z~ zIpT{VwCd+tmy7Y9m~EcFtm06nfDiY>M61|vX{(YI+gh4bx;%b#aUD%J@fI%m`{LY^ z?GdwnT;Kms)o$rmV5(PNR_{7}i`?BaNli?4UTqaqF9sAY{NS-?#;$bl6Y;0Vu0#gy}3aUL@7EDk^aL z)#h)W>aMRZo2YhvYAI^_G_z=SpXxf+yH0K4)23Zs{)p+Kdy-4zYt`cNvZO6yz8$Z3 z6g+gASYgz=xVU7m53mI+Jp01_EtYRvlwz;{S!k*`aY{(;f8I1!hAlNA%UDvjlc#E$2r@dCm&~NL~8O#1&p4_o^$?N56a?cocyehbRbkf;G&J;)m#Kt$g;Vv8%OE52oDaKRe zsqx>(f3ca+tt?9B|18Q|bA+1=H8tF+{4)yx+=N=haQ=~k}$ zsT{U=$+i9U3(a*|~pyeEfCGx$XhuTSmu;OFKSie17pF zBe4JK{}aWk_qJ$CiY&Yo1`OS~6>CafUCBJy#Ja_Qt)JNJ7dx+QIo>b7JS#w@E#im1 z&8}mJCO+Ru*B7zZ4Ix}2nUHT##>?|yw{WpEMbkSK-+L7#u- kn;rt6vt%IZ`m_G3Y32H7e(k_*vcQ9%T>V_uIVCg!05MXG)c^nh diff --git a/example/feature/file/itc-no-worker-label-2.png b/example/feature/file/itc-no-worker-label-2.png deleted file mode 100644 index 01e5e0a02fc87eb4de8406c7d7c92d6c990c42d5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 21698 zcmc$_1yq~gyDmtDQc8gq3jFZ4Sn&b{3KVEdad!z)+}(<%3dLQ66?cc=L5dUHf~7bF zf+s*A|qE*0`$^LrEWl?HT)-*o-;K2vYhyO@@c=V#t;kM4*?{DAIe-y(mzZJ*~0Pplp)Dk%IDZ{8Q3vZN?UN#Uf=%xCPlwJMD=Y47JaaZW-8_}}>`*vEw z=ZQGI1&ki~&1(1*^rAZ~rek-nlX}tigD>~#raJV+-`^5=+^|>I)kV=#>XZAGySC9bmSaR$o^gJB}3A1sn; z&h`tMChc(T1{Aba%9m&ao#dp$v3zC6IM*tc^#3q2ZVf?xXZC*)alTePO(kkF^YZIVXi?x zQnC>^apWSYy|NgvxpUU*5CuDoM$%zpwu7hq6y$bXDgt&79+PxP8(A6tAY>m@y$xbo zuOky~vKRqbR=)0Gtqew?7qtq5fXDL>#VpJMG25mMxZ|RYm69^(n3OaWw=APQw=cyW zA>}ijFYB~UBs7=dH%#fuV_b>;I0!$G_VvUtyC@qzsgENBX}Rv6`(Uq3$8?DF*&RF9 z*NYOa?zQQnQ1oCpI@EqKSPX5_P^`{z9>1O)xR^h)iWbe#vz=S$s2Pv^UbiYOo#|d# zHb#dRwT8>)VM3$bhKy^_T=n@d3GU6GwN!0grI9L3ks2)mn3?+_06ce)g2pT(_j8^f zEC>Lb*Wc{@nFQNjTKvS;Y02@xf_xw;aHbVrK6WEE;a!nyWXe@9(+FMmnu=aIyCS1y zMQ&Bf6}^3JO2r!ndU{7B)}v%fCmst6@6o%~UEZ&lm$sb$*&8sbb~C)lIoG%Qk>g-t z)fFpS;n7{4qf33g2Ni1w!0xi>?q;V@)J~bHW1!#5zo97l+i=^a9i4qf()QESq(2zt zI!^l?K?9E?q7^^wG{{q`FN%0BDc>?%TDj~3IWFG#BpcOJD#}$N$3@``Q@E2)o>~jb zSTAXBJNVVW%}kBsW}))Un=ALaB0<=m>$3Yw&0(SB_P*Ugeq5FFi=CGzP=UDx=H#D> z3>YJ&HG6nb3`om=J);7ew2eDw-jzIf`kcM-uFjdcr`u8y;N$;$!{M`8zf#a(K9 z&sq-N4`UAMVwOm7E{B8(rVx(qr!`)xd{@>oWiR+Sir1I?+HXX=9y+*xbJ6SfxnW>j z;>pStIp6AcaYw!%iCUiCx?y4I{EmD&8Ih+IU~TxGJYqSB7p(Ga9x^=chW!gVXeYR7D?O3}^^d$wu8$XoKVaCJuX8k7Fpfcw1ehYu>K zKb}e}0J1O6m=jo{2Sezmz??t02!zp;>Q(}*`x?wpOgp7#5)V$`%qIY1r{K#Yz^yM) z4DnQZWPlgl{a z9AdREEZsEiu&}@g;1X~ibG0*+c3Lcz^U$x9b`ox@SN`~u3acF8=}RPHK~?n*>`q>f z4SYYuJ)ChNC)TptUL#|+GU0rEhbHw2UNkvhhiDf!wu1R_vy$yi<9%(#>(tZkWu_`k z1DzbA^4^nN!I!e5jxX8^Q#U0F&qSozmQtt3hsH^1LR^>9i`Kou6m;mbEsLSC-l+AeHnzkYD(Yo z&Q+@@{tv(XktEa@Caph{Ofqzx6_CFM5={a|m<4BIdgXNa!t}jgLx-?|tHF-_^0=X@ zsbj7gx?Bq6x!VINtW)O9Q7Ka)iBVZ{L3!;L$O?aW1XTPDMh@bc>jU-MLF<3(c;se! zS~pEYd*=A+-R(ON3X49i3MSkD0506LTEIr@DkW79Jmg*jiNyW*;dcq?6#p1VT^}5Cd78+{ABkPQL%xLDnc`RX+q`@XZmie;>j~+^ zpP&^TS5!*Tds3c2wPr1tWJ5PE5C@sv05mMHnmY6;dpVyEBL=jjpu-pYt>P&-j8VcT zT}Bks=#Vb`gFUaE{XJ|!0H6YEdvyNWD#>u~q@q~FTzue%=TL7CJh0?kEL21#Xa`u- zEd6*q*Pe2C$n3mIq17{K7=yd=09!Y-T|c;zxxH`$tXj0bXX4|$@T9E>R_4L8clMnd z*;&vlhTgK#UE9j?@Grl}#%w~p{w@IV^HQ6v<@{g%qlYe&J8nS8MRCtChdO3~Rj0Pm zcR4>9hoQg4{3xL%zz8E=W;*GLk)N$~cb=5h{9$G*g*7~|Pw_omQ{MN|oK}Ycv(cNT zifw(YMtavKlIwZFrWg$UyTifl!gla%>D;C$_PL9Sm`u>QSj|dT6+8R@^T|sk8$hvV z;AnHhb`dd_w3{f7h`d)Xg-jK2t)P8a?V$H3>mVD8>6bRFS$3V9FdcygQL5L?j`*{! z@k{*`JcrhY8r5!HMub6Ya!_hJ+zjgMaad+avFD_WLn0HXw|394xU3|>je;X!U+u`* ziKkiXGUS?(gkw1v4v<(oxE>a2A!NTj$7*7{=Y|e%(4_Hgp1oA)6xMI3YXsi-btt`3 zR7xPa5ofP)IhbB_Q8!5}l8JAVO>lv1S1@j`TzZrO(Fz$8jzh!e;1b`Lj)CMHZDG6E zCCf0>2|Qo`a*dVHvsIzfvL{l#U4PTQmRFvnt#y_%f(D*$1>LdPx7X>v5$Lq{21CUD zbU=dgqMf^L+J3A9PxpuNwwEu@4uh7V7XC00=w>`KfjY?1)uro(5tmgaGKUp)!%N^Z z;#uv%yXBh;V~?xsZOK?K!}n`RoLl^x$|!7tVT`Hw{uGC6(%JX#zg-~e(nrJGJ;7AX zZ{8=vIlOx>&)d@Dq|Vj`dLmQlTTSh#sEml0O%7N3y;`|Cii=+&)<#WK!yivM4%He0 zn=tc5>n^HV{G%M`?54G)cU8f!1MJ3&idozz+Fc<_RrVF%3@QC_qu)h4Hu(dusRJ&F z=a$h&AVB(gN3&B^Rbu{N`_v6a!=i^jyF=3%hmuLDXOduP0I-~v4bw{E1W$j}Q9nfb z$iDBkRs>63&WQQmaPG|aon_VTXUbRc8C-ml4)Ek}MK?;fHM89mf@C1?@d6;ou+x;4 zm9=^F_u7HSbnmaj{eOx<{&qjpJoV)=PZUA4+o;Dht%!<@g@mbC$mxWSs# zTDW)+M`H2~9zPglB>!Q3Nr@@;lNsY7ASgcgy68-;ay{K*ldGrlW?hVVSjt!U%6Q^_ z{$@_mw8cGe<;;LoB;p@a>D=y!USc1aP`Lm-zLzXoDtZUbLBphad+*fsWl{RKIdp5# zB_dQ#nFUFXcsUxZ@b$t8YVv z)~EVRuW>DE$2P{gb<~HP823$%?3?XtNB+U|*A|F2_1_l4XtUO+wIgtLV7$pElcJ?C zN>7PhS{fCGhVtWvl#G(BFtzjsjY56O*TdyDBD~Q0b6A+vz8*uvSOm`Vpn+8CRqK(W z&lgE(Vf{o^6^74qwp{(D#%^=;J^X;^U>T^-YczLcl#rSra@=z-lkf6}IFFcNoei{( zd~0+LX}O!OKkLYBa;1TLU*u=hsTp6>TUKuNV^}1f7_LFoH|h=17@LM0pq8%<>qaiz z9|NBTtd!niPvQCGdc(PVHCReBpSI$%XzY-^u#bYB>Pe*8Zqs9Ig@=Dn3|m+`TCZo5 z45;Hw8m?PCB?e8eoRpNWu15m;R<_hNJyQ6=R+=4ctehT+rtpI3tUb5P?1U|@_*>Fx zEW3+p?I(iwUmG50j^f60f57^!o^{AP4ZPYhRbIC~z&7mnW%mF^RwPcv7k90Kf#&Wa z9p%Bs4LkHO3&U%ExBXVOmP5h(2?7|BfZT8I0}JPXnC;UH;3BS1!p6& zmNL{zFN>VYnPhH9@bBM_bTbZ;2H-bKFS~;uio3fFFP&2t_g8)c$d}3Itul9PJOXLo0AW>|yDlQUTkQ)%3q zZbxkNQ%>o~QGKK57lti@GpU>t>|iC<#D|E>If^ETR=P?;VZa^yft~@An9vX0?DSY9 zuX3?$Hrg9jE_|dldK&t{Big83ob}maUPR;G5+QRBAten+ZY80>ke=W(9b0~|#hmPh zh}3hZY|0Y5=ttU~575D84l#+dTDajf#P^3YDa&yx2yusO8E#Dy_EHn3I9m%&WwHI~@qLAB--vB8` z87+dvp;##nRefALK*lkw3nnK_kI5Lg*SV)V4E9xL+$-A%oxbYc!a^trY{G z9EjZ~Ab8z1qy-`7<4(8>Uria z6?CXHgcmZ-OM!O{N{mDt4N_0DO1y)wO`|1jj;ukS?bTkQMuKX)Tlu1cO2L5 zugHV%qvu>6ojPxSJ@=gk` z%9S0^C@mBw`@)h^_W12ng}lG(G#Zg3Cf}Zq8Ca+JfRZ!U&r9`n^HJ2{c_K7Jm*G) zJpQ&Xn(35O)0zGU^Y|NdRW~iaEI67e(Rpd=0v;wVkWX!D@TgG6=qbl(F_Ru9qwsR* zyR@}!&sr9Damw3idPAv$OrNB#zLsaKO^KJM`K42!F!|yWNB*4W=!=sgtbe3E;Nh}7 zcpRIQSERKh;f=j-DI=`pTtGBYz$NN?Kh6r(&K@Je!jdQ;Ok z4ev(IDDJ?)f1%6I=|xSp{rC$!;*4AIe<6rgb&#tzg4>xJP1jS5;KHdS3<0zu&!r zJY&bN!6$%*Q=RXYa)8Yb;LWF@?-0HWns;87Y|+rhs3>J-y;+TCk*3|_eA6ZE3#p03ia!LtG1`M?H1_W zuu%#q`?Z{UhGYR)@vPKznX+!Hb#ws(8!jIW(rwfsI%&8t+D%j%VkcxFr@5DYkx6*Lc_Q z`O8LFtfaX*G2@s+QAgd>L%Fx5-!7Z?MijhfeJyV&XbN=PLf4!{W8G4>7`! zrpuPN3fxC>ek@wz3imioLYlj#wbW?D8@25og1mLL+jt~NNS0DOIX&(_Lu^;MOSE}N zwSE?fNE07aF$_lRdT0chB*TTgD-WXc{L3%JCsQSB!Ag=IWMDt^LGdkI^@x|#4 z>*~jdg0J+R(H^$^HR9(wzoK0KSk;;)#goj$iqCsOYmzLH623Q62m>&inYBO4(hlG< znta_vGX2g=O*@q>F>Pb}brBD^(c-i`oSzaO)qCEmfp*B)SovO&|M8J&w>j5p;j)ez+yB*)AE`Rx4=r&CQGInXWmd;N>{=IJaW?4F zAZ)3COLK1_U$SyEOqSGb-Yg_jj%k&yUwsbTa?~qb$Ztx+Qf@5*MA-%QP_YR8ovj}T zN!hK2RwR^uMGx_{XP+&iX7-aVR2kXCNKd9+FqfYpJ zGaRpCn_V;8^b`KH)-s8$1r%FukwKXR^WZ4p*~>r#j8`^+IZhgy&eV2oc|jHG?s7JI zx#B0Dp=av8quUf$fsO4?`G672*P^0A{If=afC|Mzuw&0LXGfI6XaE#U+GVJan|JJ2 zD5G=EyL)5mZM}Y&ymc7Mu8aP&b-yYhBbLcGJAtyc561RB8rO6;GtCxNH>rI`Fs6F3 z=85jl+jX-`&pdoWP+H2t%f9Jb&7OEP`fdJg3sBwIIIe1OXIhREx9H$SBvI|>x{fSU z9oGh{Wrp|{-$h@E)WuugSy8{w;T{=WlBXZ@`jBm1%j}5tr`d zqGsyk5q{u9k?jE_TyfEe$p}@nqBsyvFqqQb>)xuV@nI`ZHGTCAq)%5udzCIl(PwSw z2QLS&0s1$I^lt)3w2?QvmzaTp{jhV;iblIuJc_zv(IO4{s67NF)b-YiLN!yk!&S@4 zRISnqM5}q)xg73}&4Dit>gI$c|Vo z-G1*+TR{}_`3h%d#y*kPa*DkSDx5X z6b2KrS*S$XW-O?c!j^PK7Sx>7xK1My1NHFNYF1=()vL9i8qfdgvg=EcA=`jlguyTS zFTcWmKzVW7yIX2%=M66tN4;NK+oK&>F(x!9-q+c?SPRQ5=mWczq0F-uznap0f7DNa zh=j1vm{a}(ra+<(yt7Qz^eQKU+Mb|u=dy+0C#KAeJe8qF*&lp6M5nh$`OzE0JhLM@ zl#$l?$*tw!Ouf` zLm}Rv6>F2QW#ltF%3=SmMycZW6@^WPcX_2`Eq?QU)ZR0|4CIU0@CYJ)JkTA}YwsOV zew~EXU0TuZ>nk`1(@2gWj61BA=VPU{`Fr&IGA+6fn-B~65Vl9sfmv9x80rg;Kpb0U zW~KG>rQEg9BJ}VSl#@6Os+wK@**dzeZy`#vIl*79TED$$M9IizOl*`+8?o*OzrqkB zP+1O=oGzRkR*6zH*q>?WnYJ2O7+Ke+BQ+Ts~F3HWs8IbW$RKR-m>T{7yFp3rb@MpLcmu7Q2;9G}Ov zZYTqA^r#gcy}1$@j^`Nz7BUbJ{KWIt2d1vm`q7&`@l1Z9y2bQ60+}-;) z->$!$J;&krjTpg0-r{6&Cls{>de4}rgXBwthZ}yIugGH#y z!}N8D7uD{V9#iSMMJ`017Vl2Bu{yP?LTiGx#=dmWyrAgMa*A2sCezH;uj@sFO#S%% zn3-u2>e$~cxFhRWB}XvvYqh$Ij_Q;NmpcTViWMRK|64jVjN%TKdlmpIE zlir*tlm!H%tQ-@JnQSy^(kI^1Pu?ugIkeemXbUkV^O(Cc2JGytTQC0uYk*|7%!)P**&@NgKJya0POK#Zh+06!|kUO>7V1|02_gJrhFFNkLItR|i#KRp}ca%Bp() z@)XwKDP9kPK_l;RUh3mL$2YZhs%1jABl;RSnBdrV5jnrL26Kmu!edb?DUFPoG8F{quE9puEQ zbrBJCvfN&DHWF=yg{Nm(_Wd47^2Yea+HG{Fjp+nuR;lDa_@|+M7Gt|WRf3}f49}%8 zKNmXvroCv!BIKju5;T~;PC5URuSQevhC=AMNCnKcx8B*lVOOX+*PEsUYqu)%HCdGO zRReX&(1@G!*X+9Qw`>86wkp|mifrs`GnwG9lg8T*n6@ z!M4GiVc%t9sun)Vm71PuI-W*@i*RHc;4d8(}CUi&H2I({%5*|1e%)K zEQr~nCbZdT+SLOC->q^9Y4P%;)hs)RKW&bnZTM!zU;n<|iC$Zydi(Wn21V7pT$YG? zw+MFgE>3;(~U|bBdmqPJ*cIMWD*BRd(*vVlZ@_wl6}hKGKz87X7= zhJH15e%A|TBKXV6qcIW_EG1bp=GQ%b0R62GkA2}CoD*K+t9(x-fpWBoG zz#kziYOtH@Li{H1A{T`x)_&uLE@>8`cg=G0f4c|;@Gd45H%@{DZfKD%cojcmh0sVW zeJGi3Oe^>yHPE^VQ;mZ3jUsC^C|Z?T{4V*C#dTitsS1}JnQ(7PFE(q?2kl`IeHFOl@|l%O06CSm-O;A zvOdo4ihZ-c_M~h)gYsHCwT+JTlk6Ak*(cjJq-a-t#`%n|?4G;tSCx-xn9af^Emh8b z$#8)=_K%Z-T2(F^S-tph@tIN{4kUZi583IW?_uO7AW*B0btP_UDYz8RZ{8W`-_1MO7vF0$LZY$d{-qoH7oFlQHU$W- zMsvSJZ#|ZPke52TDi&M;&-MTx8j^m@&$_&^ShnYA4A)ckF>KgNM7yuv0g)SJ{Gt)= z;d02fG0L#vO;$=}l|#WjH^LaWFKc{+=t|P<-m-bGhBwlacmY8$y~M+UuR~jShk7q| zE0HgA{)44VjmXpkA)%aIb8XssJR+H%{aRw^ zM77xi7@6hq`Mczl?%L$O%tRqaq)wQQa-4}|JkoAxDzK}%M!N*xNud5_z zdC$X3;_5hvi-UyI7};v<5thm0^~Z|;G!i6w82G$sH-U2W(;K5)GBX$c*WP*Qom60D zPM=1@9}J6G26PpLqIs*%3KD{4nkEggE{w>T623r6DUD69t-4l@SL19gH4TQLlIXa& zPa|1c;eHp`(x0RO(Nxp+HX++8qP_2^geA}~(|BvTZpu4vOm6w{JDKuoDx@`N=aD-% z@aZd}%4N7`5F$ZMD1-J70fMbNB`>$9g7VuLiDnW1Q^ZHU_vk09vVwvj&uO)+2uGbIp|?k*{rM`1ZEcdfPYK zoRJ+pLV*Hupm#-RxTk_X(PD!`dA46GWrOmlcndp!y-U4qp0`zFl>o=(7R#t3F}7+# zxS8bYm3Tpu=nI{h#u->kchRgJxy~&!aQ#chNf9JxY-eX>JajKByOAhzUC$sGpABzmi09$)i6kBJ1D8UqSnKUyOQcnv}$;u#fD@p54$mIc0mCw^` zzbaj5t7U~}7M}(<6xJh54oC{(DG-KNbFQ@(druM;(@7Jc8DONdZ`9%p@$yPpW#?|k_3_5t2$=FARZG)w($Ta1Y_hWG!e`_fUUApHlKb;QNL~!Q z9O!_tO6fs=78jh&dLGa-+qnEG+_RWl6(rY?M-C#>MG%UVVE2qUdzF+zIR$uK9jGrx z0Xfxv4Xoxv67mY-VuH(8f~tH7X;k9ebc-G`o@?$$5S>Csoy%xP z?w3_lLm#Gul%&!SYBLB=w#C9%u52=WkqWq(1YiAemp-Vb#6QGk8&=)#FfLW%JR%^O zhjnUrqBSEVvze3iXVzt7L7dp>1hFGuZ@WmW1${skJoLLV$du2!am{R<%tSWc27#CwfhiF4<37{&4pyAGmDGd2>Laik$s&y0`QIg{mi_B z`p;GJXZuOb3U&I{dCqeN`yws06VD4fn#MM;V3ZC*+Bd;|(@w9b6D7EKOsQqlgUF@R zx6pl};}6iN7C~A*2#;b2WYmzgu-ke5W8h)Z?>GM(2hqK|4XW@~(N<4Xas!7Ex-pgf ztQ8@=L*Vx2dziBqxV}RY1J_8yCAM8i!M=dj|=SYMalO zx!HKJvMDo;dX1IoRPd)QbBQm3Aw#iMtca|J`MI^YI*0eEuCC>vI(dC;gvmZnD@V}_ zcM}@4y^d|?T8@^zX!AT{&+{UWjyh;C&1m3OWd9xMgnyzx2(s%eR=*hvK2&g?zguU9 z*P#xW{F3d*6p_1eTobkG7R#3A{&9*mTpk@ljk`S+vgYZMLE&R%z4rk&5o}$4xy#hk zwVKH1MU-Z{c#6xIis}}UFoHi9ymB<1Vn`0$s>XGO&kxhl4`CE%$LEb!A8#QnX zlH{vI{%#8GJp)b5FOL&T+czC!bB4X}iC%}foN8BvT=gjqj$-{MXU3X%&r6e1!9wRM zs?|G}2jT=g$OH7#lB(cZI`lM?P>PhLb$VW2sSotx{Z9}9?tKS8aw$Nwd|btr=cbOZ zjxQs$jW%0tu6gP2l8X-nN=5axLSq-Q!pp4|#<_Cp?#>AJwgL7n&o?F@d*gDY54*lFe3f8+`cM$#gnv=WZZ;kvUOslPkhc(4C$^yUTW zhvuABoIfKfXW3vkwRNh_MO8(H1ZFY(wqM;m6=}g_HYJJWJqtTq=uahdugvpyl=M(+LZY!C6>l0!kN^RQCNvJqe_ zWm4&dI-Oj&kiqiZKrkuZ@bNmeLKTb3u`o4{g0s`ELC59!D7HZPyTTeJk~@FtV6`t$@bAfeprWEmd#Cez z%Uiwe>hw%8vdtXwMmq*iPY@J4{1&0oj}=}oTbwiMA!=k=66wni%0B&V zMjkw#b07_b7yeTG`=G{;pUzwkhuu{A!F~(W%9dRswuH$j`zPSF7v1q7An1)AUSJh4!`gj?L9R z)&-qhB%Pk+=t>-~u+O<%nxCB8;Nc~Lw==|%&Unx&#hE_gAJ7|R@=@h5QHY+xnn8Qh z{{EHcmHZQa-S}o1e>Ef55M`=tLH+82r%i1YN4m9ehpj?cmW{!ypn92U zIE{;_rS1?_3fW-nF@?-;EsKce4%Dk zWsWUh^0vtjS*#Xl3EDe_{;XYZ9uyOTR!>%&Ovf>{LKUcnG81ddPrWE5)pnh=Vx+^Y zDsP6v$>;XPz3Me?5^mA*^0jGJSP`e`<L|E ziu2j4U7Bg5Jiya(u}pL zZV7ma12sd@;tk_{*kg4Y{(IpcM`O27>scT5PUn#R%?M#p*G~W;S$lug!BHmAKa1+Z zc00~PbfWMTdg0{ON|co_Pv7r?qPv9l$qmLrh{aV&wNk|4%%VQ{aG6D=zxV@u zVUY|XEpP~`j*`{SpvF_0RO1?}jlKw)2@}fn^cpU<#}QaQ?Rn~#Q}2aeN=lD}k<69_ zSj>JZa#!ijANZz-pVy!7VLL(eMatT^G@r2|JbhYV)MK?Y^Nbg`)nu($7f$T-{W7_8 z{&yh9hp;G!2Du$g-_>o7P|8PP$A;`R@7MU;^rx_g@E8&k|36!su7d(Hw=5Ra9jx)y zD!w+;4w#Ma(StVg3$1SJ5k)hM0X4LeWIA`C_?6P|N)X70*xS<6PqgxKsiZp<+|7XB zH-WQ{WOI{({_c03N-axzRLhUu^rTElZWvAQ4u5IM{r>4Z%LH55lUwzscsTnK%yLjoXPCTF^$Q5Su zw<@Vvu%J(5uW^F|{jx&9Q`@^$wa<8pZ@kgfRv(~T#Tgd}Z`N1xSaB3oJ$!PCj!qV^ zGR6}mzIEz{OzuLVvZiHxrWfoqy6ftQvb=&$haf}Y{mZKese?~e^G!3*ek*uF%Soa1 zP3z^(p{WGwj173LYw>~9l4|=^1siK5xUUo(hTlM&yRO8)imV5Wjzf?YVm{rK z_>69r-?a_no;qWQLjd$}){2~JL>>3NBONbdLd936Z+M99H>7D}H5Fe;|22rZ?ElI! zY_Wd0o0ed)ptwKg04oW@#-BFUXNeY|^0V$td9x_5{c}Y~KX;}jl>EY4uQLD^T_=IcKU=hxz%P;$__zmqDi`Onmg|4(4Nb%e8EsHykNxQftoDLVh* z2oTiGyKeOoPd7f04OFht9zM`_1ByaE4Ufl+e1?e>JX|FD!CgIrq8TfS!ueTe`s8J*=+2OT#sW&;J|LrYM_HhkQ7Z|YaC5mBp>u-zc+`q~VPRwW^| zyB;%9`Bb=&nI#(LkZr|(klJq9tL=Xq*F#-ViC_lrv1g{IzGINS>0$C9N?hz8-_~8@1v2b)RJ;cNO4v8| zgK8RMyQ*Z*OAfX7#_QiRT-P{ou`9n#^JKHxq#tSSx1- z;)b}^eAJ^%JNr*H;N%#e_rRri8fq)Cf<@19*&D$tw6o#kYjNjADCbG{`rw6$t+;dK zpftU;Dk<5#9uHUxnVr6y%Iawsj*66YU%1o`8qR7rE*g8ZB zA)qB9X&Ub^_!OtTMNJNoMmDmSX8&BsVB&`X3ey>MXJqB%eRYtUs{pRHER#Fj9$K69 zGc|+G9tWm~Uuz7?L*=Q|TI=`>-7Ht)`cJQCqVJ9rZ{To%3HR#V@|pG3#d23s*tePR zmXD|)gHT#$CfX(UJ^z3!&=R^j2xUJ}wP!5nKt_x@O|w(bu&Utr}`=t_+{&0RH@7^~O+jAJZf=*25Pr85l( ze>-e2fM04M>4&sE79tKaS8@Y5_Le2otmd%vy4QlA>Z-8MVqu&>*#cJ6vC4N;Vi4$Q z3ZT&Ag|Mlv`I&KDQhyRhW1J^;Kqxagro7TQ(Ro2+@z_~|M~lpZICY^r7!yA$`c;k! zWia~Uijqt>HXKNHmtgh6^!l}c`cHKus@MCaD`^f<`RYdAo2AnApDu!Cih7!{S1Vxm zOf;i;*2aRQRB-#-+dnBU>&=A6BWu6XGvE(@zYUgsVkc)I@RODPw900Lv(Bz6vhB@g z6#yB>1UYExo^#|r?e(oV1(bfFJcq_Ik;5BNB{{kCh3XuR=m?DFP{?*srQK{}y>IWV z>&irV(gQ8lqx<_1x>Dq&Y6gpjjgRlVO-Fw)J^_t&pRxL`&$ewD1ykp*2j9!2pX515 zuyfxy6fRw_O;a%fk#{A1Z26}o^D*;n?v9k$C#XCez`W?L(J|16Q4tv-=^{Z}SGIhC z&aLb!&6_CaMb)JjwADoU=f^D!3VPo#@khB7ns!pC8|JB9bFk|WO^ptt?s0MrI2;cM z1%aiUW5|qFO6Elg0cX2yx{x#2I@ku22!-P5&$#YGhA#?`!kiUk-EvL&)GMae+i#&d z3ykAHz;SzeQ2)tjVLwSJs1x%oqhxxm^p)V|j!j`+dP~eA*xgKBWblExIGdiN88xLk zAP=b;pWaeB@(}8DeDpCvlg6OK2gC%?Yt0eQy4-uC$AR4&1eq+(^C$0Ro@W|PKh+E+ zXBPSPSHo*qCPDrcHxZPQ|JStU|83>-zhj=eIOM2Ft4|u9tH;|5v9Ff+JO=zl*iORV z&;Ot6r~k{a?mz8TE*Z0w6P}tPAc!$KnE{`V4MatA39dKx{Fs2W=5|%DFgnhJ3n~f^ zb$pY?;j6dcn3Mi*KPEIa_3Cx!ufF;NccxZHLQo@5IB5U%=hVgAE(W3{iEa&~qWMK3yAWriF7B_kjpaOd8`*N1JV zBV!!XA&0;VzDS)y+T@zq%5+L_(h)<0+wL`VK3*)_=HK$_k@T09cN;fVs@CE!LJJUe71q+uJ4Z!8N6|VFW4sX<%~@o#9>e^-|^CyKcV%#BNKiZDb%Y);0A;2Z)W{)6 zB(j!DB?**o=e{|EiTX+JB(61irnrsT%MO(AtSabH5=B3e5^-IeYt75XZGZ7I+en>$ z1P9`R#@1J_= z(@@{@>hv6tLj=WT#+wM0tTKR?OL*}9kzC|I?z;skTN@jZbi!Q1!VMl(@kuYdB)r2I zrGu{xZ8NRe!$0|=58=|*Lr>E7E5IxM>rtX-Y)!gldhyTcL|v{CWmO2yVI^ud5NnFU zxd&@#Ch+DG27|?4tOOhiJI!lQSkKhPDJSz>?iOV;efo3jQjz`QWSGCl)PF~+GMbT+0Yu)+r{ng zj%G@rvh+3*lfueohEoz-y8E{vPWj|TaC7p=V{2>q*V4|@3;Ax6*dmqTP z!%*FCGJg0XFGh+h^u1b|@yzB~8uRU{H_y+8IOpCrKNxHqO_GnUHh#Dn|02w~ow@lA z)1`jMGQDxx2gjr@fQ^bHV-0RU)59fw%>|;OID0 zidqe2Rx0+~O0yk`<}qpUf_DVjA>}*9Q@4696Wi!Xw!nmC3GIJ%=ES z59kJ3D1g7;J92-m7_a7D89;kkz zUZTD0<@!R#iN(;kQoAU-k}TIaKmVnK{)PivH}auG+J?#UQxIOOiun>N=bW!<7CU4 z{Uox5h|E|sDoZk_8XRkkk!36qN-}1~GWKQAWE&b=wwdQthZ#HB*J3aRBN@gp@_f9$ z-{(Jg{(#Tx{rTa(uIs+8`~AL4Vb_g#KW=B4H<%ckrv4lClctzrxSe^1kr+>=(Rx0= z#pS8GKDeR+-#te4sHi%>Y>n6BxA@pd&+^R2$QEj_vQ?c?!^4hHzjiBGq}m>_D--=x z#^MgEzse#G4XhZUY zTV*_*tCV5yfFx6qpATUT4(wI9yKH*eiZR@7Lt+n=ImkauP+KnQbir#Yk?l zy_;!6T9DhB-A!1=6G<)Q+%B>$Cdl3u`C9xA{4ZZYT>{SyK zZ-p!szeO>E z9jB;hZtT`leutYRL5GxT@RTWtWuw-sy^%%OU!G1R;?z1lqABRL!K7~EO(kj5^HjTC zb)|z&yqtYT@#!Sq9-}ygsg&}vxZ@Ogtj>qFE0uSUQ9Zl0Iq-CGDS*z31}F&i`?XJA zBmI5q3%7#TXAEU!Zx612oJZN_>4S5>oeRt(Lb2+#j7x+hoa8x|v|nYGV044m*LR|B zq5r(KXf+_!hlj9RHAf>G@(qzh*CvPuCSDim@&KSe8#}wvuyxA8{(8$gS>>R!rUcZ? z^R~O+Oxgl(-Xt_d58%yxMv2;G zZrDIzA1*!@?mwUW<-L6|fOPZf)}NihqmMYNwSCQ(pSyd(Cj%rU($ifLFae-!@Q*(y zfFooQZH;3j4a2)Luids=MoUBkn4vp1T?Q|Jp$%+d8>u5U>P{p&zhvkw;|~?yq$62b zS>YCR6l3bCsh#s!TWXjC1cJaDog_gaZ!9C*Ra;aWVP!&8y>Ot=-Gz%fEd195(`KUf z*04@+Mg4D~aNKdiq*Gw0BYxr2R{P~=43QIM(HhPq*xZ-I@Z{#g;kzU#yl+HQyh)lr znd!xLhYHFs4vo_8!oOorrwtt|i7NEj#<+$d4Ejj!T`;EyRX)GL7clYG`8SINaBvXjG`NM*6|%Avh)gxDuFEz! z7U!n6rG>2rp!yjQ(v^Xe)oqGXN`zdQgM=y%VG)rLVx$ZO)WBIf%5D6|t3JP#wR$?| zeZH6A*i?l5JOSrB%Y$xxfB({Ah);eDzqIR#}c=h!uD{fWZeT;i~%_YxKJ z(ce6|KNo@)yD^4?wIKA5-IeMUB2@E38m`9Xu}P9FHx`@Y!1R-OoSk`glQ})3L4a--RwA{GaFHSEOTngRls+# zCug(-oI2eSVioLYOH{S#gPUHM+yVXz^lRAgb;(Vu^dZ{^i2JuXVbyMJ>^4i*HF#sj zm-y47MmyhLLd+KMmDTh9f;Sj@89X^5-^qrau)6BvSV7P?6`k__C(aHv!otJPl_kot`i-%yNz1de_+szd)Z*7DaHT8+CWNrIn^c zDrYvD;%45zfk{i!AUpLIp8?@{$R|?EM@$gVLTs&jocx5%so0?jC~{5PllbL}nW3ui zI3y>%{Q5L#jxWxnIGZdZ(Db4!_?EiDGcwns|Hx2Z4V`&~-|`&JHBYAXCso2y`87Nb zBew9q!F$+J-3Ts#HK!eQkDbeYVAk?{4MD#^a}H3*P{T=JcHjHf?s2n4!b^os#H3e; zAymVS%G%z+uVztvwz^GSpx@@F%$UO!q7o8n#{}1?M(rIPD{P^;)f37r{Wkx#b2cj! zU>lGITi4=f&iAa@$ekWZ!x&x({~MvFlxMQR4zw%!q3S4c{zp*w?6hU?)7=CWf|D{C zzCKdRs6eNQ3cy#(P_LQ^`9J^SBq^L>FnF9XHHv<*NhT*P7W5dG-F8OztVXtfcNlxX zVqx~g**~cH6SVS~=-EilvcD2(v?p8p;fE*jzslxptDywvVh7{SZO(NbXHlXcVj=p> zMv8XWKt2A=N}a-_k~w5fJ&_#yN75%2O77Rrd!DNB&EQQI!`Gfj-OjnYdU6Pr0cK>sQFzw6ac94~`-=O+#fh57FI8%LJ_8!X25A zOL;Ol0gh(y6FFiFOWzQOmZuh>TToFAy8u^q@|jp=Ob$;zqo{@GiVd&xsKWgMXhgN1 z(wd+o^+gFtSCL}L*3eNy#vh?emf=(lWB^@enFYeF=~U|UJz}K3Ig=lLfOb#FcV7A} zVIfy@JfX1dw3TW7aD}6lmdpzF0*LxZkF9m6(zeY(<{4=4gNxx2uua$#q*39ic#(|G za%oF#txtk$=-8To@1NwYGC%|UN|0Fg3w}lg-Zf@H`HJYPLi_9XU+5RW(sHkS+TV&5 zIZ9r)m(DUbevsNsB|d7Q%gnd>$y1n~TJi;j<=@D9C)|G<&+M-#&F+Y@TeX}`%Em~= z2oWDtxj;_iHp$4w@Z|4+K}k+dR#R2YHAFTHeRgT^EmJhO;=dfqDjUI6AyIOD6Rmsw z14g3Sf&4eNsy^!kp9vix>cZc_D0Go$?G<<8hV#p74*L5j`zxY4wr<~foE#iw$7src z^B7zT_2#lH`XGv!OV{DzhGT7!i%cm>=lyF5jLGHLi@-(;?w9k%5~2HFd(J=i3HJ6A zMv7V}69psH+*-%pPQIDUYYtv9I^b;zPJ3Z=u1R>hzxI=>*72Wm3R%Qtx_M{m*OmLr zO!fBDni0!$CDgF=@N&Z<`TFN#=|`IGbZs>UE%j>t_*ZmM1*zHAot~L_^&nxp9&Obs ziS~)}y>mDIHvVXt*2*iVBB8kHP;lC%{lW?vI_l%|HJ2rBR)6WeSDw8dEr2p^1J3$6 z)VMcrPN)dkb4c6}p+O~*1$-8JxklW(EpCpg=}us~-@5@uC5nsb^}_4iAMLs#Rz}oT zv(ga+ZP%#5$U#6q1CgB4>C>(!*kQSUK8~FUy*8$>_j`DD_6@&sPfWD?bLZDXGo~pi z)=HO^a+s==q1#8&ZvXul6Eion5ukGAAyb}uD^A|UyAC3j-c%B?OlU%3H4xeu*O)^` ztoY{SA7=%_)#=+s1Vk^+f4@IhB_;o(u5BZ%X$R)v!932KF-nIxPm$b{$SA#0Smc|n zY8OWO4XtC0WPxTE=3{5_CjEoB#Wz0hnN}{4L#T3EfUbrXvkTz310{i^Q!y>9<}T~N zSl6HdZeAZmBYAbN>4(npLt>O)GH^q@WDJkmz!+gObR9@_xd^2y>{o=DGQoP=(e7>_ z>tPA(3awp#U-xe(N^fol5z6Tf?v3v1m=5rtzXSAMB_da}`8=IC>_E`kT=(@pYdbsK z_sG5NwmnvhzM+xPudPw&sP+XK5AexrV;_q7K5G^Vy^MWB@(%0GO5rqpEAoQ&aF%mg^JG*`W=^}etszLXC> zjlStDJpPkV5QT*lcBS>N3t0(ZP1VdFLSWhbDv!^~X#c4>TKo0%^GBH=5U^ODdsOA3_^}{5&%uWh z*q2UuJFf9raK?9f{vdbbzNdI5oRcnZ{3#@rC++M4Posy8YnM%Tim86TzOf)(0Rn@= zc9#k6wC0pjZjlSN)rArjxu>>k3lYYwWR|!F{BvB8&cLlpMY8P^ztTsIB6q2h%T3`N zmfF8X(Fea+O(mDKd`7GlnY1{hy@HZv-b8`HW>J?hbz;Jwt*B$e+ZkCh@;ClDKd|iM zwqvtrqQe`szJv!v3vN5yi(uBl7djOSA|QfR(-K*$9a7OlLFv1WfwqOfZUH$oUEu;@t@25Z`zEH u8oT}H;R%K9|92(oKlg_2ZaR+~9MiH(m${&E{BZNwZ(wsig3G diff --git a/example/feature/file/itc-no-worker-label.yaml b/example/feature/file/itc-no-worker-label.yaml deleted file mode 100644 index 1c83ed0..0000000 --- a/example/feature/file/itc-no-worker-label.yaml +++ /dev/null @@ -1,33 +0,0 @@ -protocolVersion: 2 -name: itc-no-worker-label -type: job -prerequisites: - - protocolVersion: 2 - name: keras_tensorflow_example - type: dockerimage - uri: openpai/pai.example.keras.tensorflow -taskRoles: - train: - instances: 1 - completion: - minFailedInstances: 1 - minSucceededInstances: 6 - dockerImage: keras_tensorflow_example - resourcePerInstance: - cpu: 4 - memoryMB: 8192 - gpu: 1 - commands: - - nvidia-smi -L - - printenv - - sleep 10000 -defaults: - virtualCluster: VC1 -extras: - gangAllocation: true - hivedScheduler: - jobPriorityClass: prod - taskRoles: - train: - skuType: M60 - submitFrom: submit-job-v2 diff --git a/example/feature/file/itc-oppo.yaml b/example/feature/file/itc-oppo.yaml index 2311837..d886aac 100644 --- a/example/feature/file/itc-oppo.yaml +++ b/example/feature/file/itc-oppo.yaml @@ -21,7 +21,7 @@ taskRoles: - rm /usr/local/cuda/lib64/stubs/libcuda.so.1 - python mnist_cnn.py defaults: - virtualCluster: VC1 + virtualCluster: vc1 extras: gangAllocation: false hivedScheduler: @@ -29,4 +29,3 @@ extras: taskRoles: train: skuType: K80 - submitFrom: submit-job-v2 diff --git a/example/feature/file/itc-pin.yaml b/example/feature/file/itc-pin.yaml index 22c3815..2e339d8 100644 --- a/example/feature/file/itc-pin.yaml +++ b/example/feature/file/itc-pin.yaml @@ -35,7 +35,7 @@ taskRoles: - python mnist_cnn.py defaults: - virtualCluster: VC1 + virtualCluster: vc1 extras: hivedScheduler: diff --git a/example/feature/file/itc-reconfig-1.yaml b/example/feature/file/itc-reconfig-1.yaml index a7a6fb6..3d3e99e 100644 --- a/example/feature/file/itc-reconfig-1.yaml +++ b/example/feature/file/itc-reconfig-1.yaml @@ -20,12 +20,12 @@ taskRoles: commands: - rm /usr/local/cuda/lib64/stubs/libcuda.so.1 - python mnist_cnn.py + - sleep 10m defaults: - virtualCluster: VC1 + virtualCluster: vc1 extras: hivedScheduler: jobPriorityClass: test taskRoles: train: skuType: M60 - submitFrom: submit-job-v2 diff --git a/example/feature/file/itc-reconfig-2.yaml b/example/feature/file/itc-reconfig-2.yaml index 07aba56..d14cc92 100644 --- a/example/feature/file/itc-reconfig-2.yaml +++ b/example/feature/file/itc-reconfig-2.yaml @@ -21,11 +21,10 @@ taskRoles: - rm /usr/local/cuda/lib64/stubs/libcuda.so.1 - python mnist_cnn.py defaults: - virtualCluster: VC1 + virtualCluster: vc1 extras: hivedScheduler: jobPriorityClass: test taskRoles: train: skuType: M60 - submitFrom: submit-job-v2 diff --git a/example/feature/file/itc-reconfig-3.yaml b/example/feature/file/itc-reconfig-3.yaml index 3348e39..53dcebb 100644 --- a/example/feature/file/itc-reconfig-3.yaml +++ b/example/feature/file/itc-reconfig-3.yaml @@ -20,6 +20,7 @@ taskRoles: commands: - rm /usr/local/cuda/lib64/stubs/libcuda.so.1 - python mnist_cnn.py + - sleep 10m defaults: virtualCluster: default extras: @@ -28,4 +29,3 @@ extras: taskRoles: train: skuType: K80 - submitFrom: submit-job-v2 diff --git a/example/feature/file/itc-reconfig-4.yaml b/example/feature/file/itc-reconfig-4.yaml index 88b7031..110f145 100644 --- a/example/feature/file/itc-reconfig-4.yaml +++ b/example/feature/file/itc-reconfig-4.yaml @@ -11,7 +11,7 @@ taskRoles: instances: 4 completion: minFailedInstances: 1 - minSucceededInstances: 2 + minSucceededInstances: 4 dockerImage: keras_tensorflow_example resourcePerInstance: cpu: 16 @@ -21,11 +21,10 @@ taskRoles: - rm /usr/local/cuda/lib64/stubs/libcuda.so.1 - python mnist_cnn.py defaults: - virtualCluster: VC1 + virtualCluster: vc1 extras: hivedScheduler: jobPriorityClass: test taskRoles: train: skuType: K80 - submitFrom: submit-job-v2 diff --git a/example/feature/file/itc-reconfig-5.yaml b/example/feature/file/itc-reconfig-5.yaml index bb46a67..7e6c3d3 100644 --- a/example/feature/file/itc-reconfig-5.yaml +++ b/example/feature/file/itc-reconfig-5.yaml @@ -11,7 +11,7 @@ taskRoles: instances: 3 completion: minFailedInstances: 1 - minSucceededInstances: 2 + minSucceededInstances: 3 dockerImage: keras_tensorflow_example resourcePerInstance: cpu: 16 @@ -21,11 +21,10 @@ taskRoles: - rm /usr/local/cuda/lib64/stubs/libcuda.so.1 - python mnist_cnn.py defaults: - virtualCluster: VC1 + virtualCluster: vc1 extras: hivedScheduler: jobPriorityClass: test taskRoles: train: skuType: K80 - submitFrom: submit-job-v2 diff --git a/example/feature/file/itc-safety-1.yaml b/example/feature/file/itc-safety-1.yaml index 0ef3a35..d954f16 100644 --- a/example/feature/file/itc-safety-1.yaml +++ b/example/feature/file/itc-safety-1.yaml @@ -22,7 +22,7 @@ taskRoles: - python mnist_cnn.py defaults: - virtualCluster: VC1 + virtualCluster: vc1 extras: hivedScheduler: diff --git a/example/feature/file/itc-safety-2.yaml b/example/feature/file/itc-safety-2.yaml index 773a19c..c3ea29f 100644 --- a/example/feature/file/itc-safety-2.yaml +++ b/example/feature/file/itc-safety-2.yaml @@ -22,7 +22,7 @@ taskRoles: - python mnist_cnn.py defaults: - virtualCluster: VC1 + virtualCluster: vc1 extras: hivedScheduler: diff --git a/pkg/algorithm/cell_allocation.go b/pkg/algorithm/cell_allocation.go index 6f56def..645f6bc 100644 --- a/pkg/algorithm/cell_allocation.go +++ b/pkg/algorithm/cell_allocation.go @@ -235,9 +235,9 @@ func getUsablePhysicalCells( return nil } // prioritize the cells with fewer opportunistic pods (to reduce preemption) - sort.SliceStable(candidates, func(i, j int) bool { - return candidates[i].GetUsedLeafCellNumAtPriorities()[opportunisticPriority] < - candidates[j].GetUsedLeafCellNumAtPriorities()[opportunisticPriority] + sort.SliceStable(usableCandidates, func(i, j int) bool { + return usableCandidates[i].GetUsedLeafCellNumAtPriorities()[opportunisticPriority] < + usableCandidates[j].GetUsedLeafCellNumAtPriorities()[opportunisticPriority] }) return usableCandidates } diff --git a/pkg/api/types.go b/pkg/api/types.go index 1bbe8b6..c604794 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -83,7 +83,7 @@ type PodSchedulingSpec struct { LeafCellNumber int32 `yaml:"leafCellNumber"` GangReleaseEnable bool `yaml:"gangReleaseEnable"` LazyPreemptionEnable bool `yaml:"lazyPreemptionEnable"` - IgnoreK8sSuggestedNodes bool `yaml:"ignoreK8sSuggestedNodes"` + IgnoreK8sSuggestedNodes bool `yaml:"ignoreK8sSuggestedNodes" default:"true"` AffinityGroup *AffinityGroupSpec `yaml:"affinityGroup"` } diff --git a/pkg/internal/utils.go b/pkg/internal/utils.go index ef57513..a480cde 100644 --- a/pkg/internal/utils.go +++ b/pkg/internal/utils.go @@ -232,7 +232,7 @@ func ExtractPodSchedulingSpec(pod *core.Pod) *si.PodSchedulingSpec { defer AsBadRequestPanic() errPfx := fmt.Sprintf("Pod annotation %v: ", si.AnnotationKeyPodSchedulingSpec) - podSchedulingSpec := si.PodSchedulingSpec{} + podSchedulingSpec := si.PodSchedulingSpec{IgnoreK8sSuggestedNodes: true} annotation := convertOldAnnotation(pod.Annotations[si.AnnotationKeyPodSchedulingSpec]) if annotation == "" {