azure amlt scripts

azure_scripts/chest.yaml

@@ -0,0 +1,139 @@
+description: ChestXray Dataset
+
+target:
+  service: philly
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  cluster: rr1
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements.txt
+    
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+ 
+
+# list of jobs to run, we run 2 jobs in this example
+jobs:
+  # name must be unique across the jobs
+# - name: erm_oracle
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains kaggle_trans  --test_domains kaggle_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 3 --os_env 1
+
+
+- name: erm
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 3 --os_env 1
+  
+- name: rand_match
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 10.0 --model_name densenet121 --n_runs 3 --os_env 1
+  
+- name: csd
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset chestxray --method_name csd --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 3 --rep_dim 1024 --os_env 1
+   
+- name: irm
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset chestxray --method_name irm --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 10.0 --penalty_s 5 --model_name densenet121 --n_runs 3 --os_env 1
+
+- name: irm-50
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset chestxray --method_name irm --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 50.0 --penalty_s 5 --model_name densenet121 --n_runs 3 --os_env 1
+
+- name: irm-100
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset chestxray --method_name irm --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 100.0 --penalty_s 5 --model_name densenet121 --n_runs 3 --os_env 1
+
+# - name: perf_match
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name hybrid --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 3 --penalty_aug 10.0 --os_env 1
+
+
+# - name: erm_oracle_nih
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains nih_trans  --test_domains nih_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 3 --os_env 1
+
+# - name: erm_nih
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains chex_trans kaggle_trans  --test_domains nih_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 3 --os_env 1
+  
+# - name: rand_match_nih
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains chex_trans kaggle_trans  --test_domains nih_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 10.0 --model_name densenet121 --n_runs 3 --os_env 1
+  
+# - name: csd_nih
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name csd --match_case 0.01 --train_domains chex_trans kaggle_trans  --test_domains nih_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 3 --rep_dim 1024 --os_env 1
+   
+# - name: irm_nih
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name irm --match_case 0.01 --train_domains chex_trans kaggle_trans  --test_domains nih_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 10.0 --penalty_s 5 --model_name densenet121 --n_runs 3 --os_env 1
+  
+ 
+# - name: erm_oracle_chex
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains chex_trans  --test_domains chex_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 3 --os_env 1
+
+# - name: erm_chex
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains nih_trans kaggle_trans  --test_domains chex_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 3 --os_env 1
+  
+# - name: rand_match_chex
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains nih_trans kaggle_trans  --test_domains chex_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 10.0 --model_name densenet121 --n_runs 3 --os_env 1
+  
+# - name: csd_chex
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name csd --match_case 0.01 --train_domains nih_trans kaggle_trans  --test_domains chex_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 3 --rep_dim 1024 --os_env 1
+   
+# - name: irm_chex
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name irm --match_case 0.01 --train_domains nih_trans kaggle_trans  --test_domains chex_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 10.0 --penalty_s 5 --model_name densenet121 --n_runs 3 --os_env 1
+
+

azure_scripts/chest_ctr.yaml

@@ -0,0 +1,43 @@
+description: ChestXray Dataset Constrastive Learning
+
+target:
+  service: philly
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  cluster: rr1
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements.txt
+    
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+    
+# list of jobs to run, we run 2 jobs in this example
+jobs:
+  # name must be unique across the jobs
+# - name: kaggle_test
+#   # one gpu
+#   sku: G1
+#   command:
+#   - python train.py --dataset chestxray --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 50 --batch_size 64 --pos_metric cos --train_domains nih_trans chex_trans --test_domains kaggle --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --os_env 1
+
+- name: chex_test
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset chestxray --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 50 --batch_size 64 --pos_metric cos --train_domains nih_trans kaggle_trans --test_domains chex_trans --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --os_env 1
+
+- name: nih_test
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset chestxray --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 50 --batch_size 64 --pos_metric cos --train_domains chex_trans kaggle_trans --test_domains nih_trans --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --os_env 1

azure_scripts/chest_ctr_spur.yaml

@@ -0,0 +1,43 @@
+description: ChestXray Dataset Constrastive Learning
+
+target:
+  service: philly
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  cluster: rr1
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements.txt
+    
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+    
+# list of jobs to run, we run 2 jobs in this example
+jobs:
+  # name must be unique across the jobs
+- name: kaggle_test
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset chestxray_spur --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 100 --batch_size 64 --pos_metric cos --train_domains nih_trans chex_trans --test_domains kaggle_trans --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --os_env 1
+
+# - name: chex_test
+#   # one gpu
+#   sku: G1
+#   command:
+#   - python train.py --dataset chestxray --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 50 --batch_size 64 --pos_metric cos --train_domains nih_trans kaggle_trans --test_domains chex_trans --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --os_env 1
+
+# - name: nih_test
+#   # one gpu
+#   sku: G1
+#   command:
+#   - python train.py --dataset chestxray --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 50 --batch_size 64 --pos_metric cos --train_domains chex_trans kaggle_trans --test_domains nih_trans --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --os_env 1

azure_scripts/chest_matchdg.yaml

@@ -0,0 +1,61 @@
+description: Hyperparam sweep on ChestXray Dataset
+
+target:
+  service: philly
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  cluster: rr1
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements.txt
+
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+
+search:
+  job_template:
+    # you may use {random_string:s} to avoid job name collisions
+    # {auto:3s} generates lr_0.00000_mom_0.5, .. etc
+    # {auto:2s} generates lr_0.00000_mo_0.5, .. etc
+    name: search_{experiment_name:s}_{auto:5s}
+    sku: G1
+    command:
+
+    - echo "--debug" && python train.py --dataset chestxray --method_name {method}  --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name densenet121 --model_name densenet121 --n_runs 2 --train_domains nih_trans chex_trans --test_domains kaggle_trans --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40 --os_env 1  --lr {lr} --batch_size {batch_size} --weight_decay {weight_decay} --penalty_ws {penalty} --penalty_aug {penalty_aug} --model_name densenet121
+    
+  type: grid
+  max_trials: 100
+  params:
+    - name: penalty
+      spec: discrete
+#       values: [10.0, 50.0]
+      values: [1.0]
+    - name: penalty_aug
+      spec: discrete
+#       values: [10.0]
+      values: [50.0]
+    - name: lr
+      spec: discrete
+      values: [0.001]
+#       values: [0.001, 0.0005]
+    - name: method
+      spec: discrete
+      values: [hybrid]
+#       values: [matchdg_erm, hybrid]
+    - name: batch_size
+      spec: discrete
+#       values: [16, 32, 64]
+      values: [16]
+    - name: weight_decay
+      spec: discrete
+      values: [0.0005]
+#       values: [0.0005, 0.001]

azure_scripts/chest_matchdg_spur.yaml

@@ -0,0 +1,62 @@
+description: Hyperparam sweep on ChestXray Dataset
+
+target:
+  service: philly
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  cluster: rr1
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements.txt
+
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+
+search:
+  job_template:
+    # you may use {random_string:s} to avoid job name collisions
+    # {auto:3s} generates lr_0.00000_mom_0.5, .. etc
+    # {auto:2s} generates lr_0.00000_mo_0.5, .. etc
+    name: search_{experiment_name:s}_{auto:5s}
+    sku: G1
+    command:
+
+    - echo "--debug" && python train.py --dataset chestxray_spur --method_name {method}  --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name densenet121 --model_name densenet121 --n_runs 2 --train_domains nih_trans chex_trans --test_domains kaggle_trans --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40 --os_env 1  --lr {lr} --batch_size {batch_size} --weight_decay {weight_decay} --penalty_ws {penalty} --penalty_aug {penalty_aug} --model_name densenet121
+    
+  type: grid
+  max_trials: 100
+  params:
+    - name: penalty
+      spec: discrete
+#       values: [10.0, 50.0]
+#       values: [1.0]
+      values: [50.0]
+    - name: penalty_aug
+      spec: discrete
+      values: [1.0]
+#       values: [100.0]
+    - name: lr
+      spec: discrete
+      values: [0.001]
+#       values: [0.001, 0.0005]
+    - name: method
+      spec: discrete
+      values: [matchdg_erm]
+#       values: [matchdg_erm, hybrid]
+    - name: batch_size
+      spec: discrete
+#       values: [16, 32, 64]
+      values: [16]
+    - name: weight_decay
+      spec: discrete
+      values: [0.0005]
+#       values: [0.0005, 0.001]

azure_scripts/chest_spur.yaml

@@ -0,0 +1,139 @@
+description: ChestXray Dataset
+
+target:
+  service: philly
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  cluster: rr1
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements.txt
+    
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+ 
+
+# list of jobs to run, we run 2 jobs in this example
+jobs:
+  # name must be unique across the jobs
+# - name: erm_oracle
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains kaggle_trans  --test_domains kaggle_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 3 --os_env 1
+
+
+- name: erm
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset chestxray_spur --method_name erm_match --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 3 --os_env 1
+  
+- name: rand_match
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset chestxray_spur --method_name erm_match --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 50.0 --model_name densenet121 --n_runs 3 --os_env 1
+  
+- name: csd
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset chestxray_spur --method_name csd --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 3 --rep_dim 1024 --os_env 1
+   
+- name: irm
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset chestxray_spur --method_name irm --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 50.0 --penalty_s 5 --model_name densenet121 --n_runs 3 --os_env 1
+
+# - name: irm-50
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name irm --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 50.0 --penalty_s 5 --model_name densenet121 --n_runs 3 --os_env 1
+
+# - name: irm-100
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name irm --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 100.0 --penalty_s 5 --model_name densenet121 --n_runs 3 --os_env 1
+
+# - name: perf_match
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name hybrid --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 3 --penalty_aug 10.0 --os_env 1
+
+
+# - name: erm_oracle_nih
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains nih_trans  --test_domains nih_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 3 --os_env 1
+
+# - name: erm_nih
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains chex_trans kaggle_trans  --test_domains nih_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 3 --os_env 1
+  
+# - name: rand_match_nih
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains chex_trans kaggle_trans  --test_domains nih_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 10.0 --model_name densenet121 --n_runs 3 --os_env 1
+  
+# - name: csd_nih
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name csd --match_case 0.01 --train_domains chex_trans kaggle_trans  --test_domains nih_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 3 --rep_dim 1024 --os_env 1
+   
+# - name: irm_nih
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name irm --match_case 0.01 --train_domains chex_trans kaggle_trans  --test_domains nih_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 10.0 --penalty_s 5 --model_name densenet121 --n_runs 3 --os_env 1
+  
+ 
+# - name: erm_oracle_chex
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains chex_trans  --test_domains chex_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 3 --os_env 1
+
+# - name: erm_chex
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains nih_trans kaggle_trans  --test_domains chex_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 3 --os_env 1
+  
+# - name: rand_match_chex
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains nih_trans kaggle_trans  --test_domains chex_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 10.0 --model_name densenet121 --n_runs 3 --os_env 1
+  
+# - name: csd_chex
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name csd --match_case 0.01 --train_domains nih_trans kaggle_trans  --test_domains chex_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 3 --rep_dim 1024 --os_env 1
+   
+# - name: irm_chex
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset chestxray --method_name irm --match_case 0.01 --train_domains nih_trans kaggle_trans  --test_domains chex_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 10.0 --penalty_s 5 --model_name densenet121 --n_runs 3 --os_env 1
+
+

azure_scripts/fmnist.yaml

@@ -0,0 +1,89 @@
+description: Fashion MNIST Dataset
+
+target:
+  service: philly
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  cluster: rr1
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements.txt
+    
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+ 
+
+# list of jobs to run, we run 2 jobs in this example
+jobs:
+  # name must be unique across the jobs
+
+- name: erm
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset fashion_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.0 --epochs 60  --os_env 1
+
+
+- name: random_match
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset fashion_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.1 --epochs 60  --os_env 1
+
+
+# - name: approx_25
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset fashion_mnist --method_name erm_match --match_case 0.25 --penalty_ws 0.1 --epochs 25  --os_env 1
+
+
+# - name: approx_50
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset fashion_mnist --method_name erm_match --match_case 0.50 --penalty_ws 0.1 --epochs 25  --os_env 1
+
+
+# - name: approx_75
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset fashion_mnist --method_name erm_match --match_case 0.75 --penalty_ws 0.1 --epochs 25  --os_env 1
+
+
+- name: perfect
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset fashion_mnist --method_name erm_match --match_case 1.0 --penalty_ws 0.1 --epochs 60  --os_env 1
+
+
+- name: csd
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset fashion_mnist --method_name csd --match_case 0.01 --penalty_ws 0.0 --rep_dim 512 --epochs 60  --os_env 1
+
+
+- name: irm
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset fashion_mnist --method_name irm --match_case 0.01 --penalty_irm 0.05 --penalty_s -1   --epochs 60  --os_env 1
+
+  
+- name: matchdg
+  # one gpu
+  sku: G1
+  command: 
+  - echo "--debug" && python train.py --dataset fashion_mnist --method_name matchdg_erm --match_case -1 --penalty_ws 0.1 --epochs 60 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet18 --n_runs 3  --os_env 1

azure_scripts/irm_fashion.yaml

@@ -0,0 +1,41 @@
+description: Hyperparam sweep on IRM MNIST
+
+target:
+  service: philly
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  cluster: rr1
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements.txt
+
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+
+search:
+  job_template:
+    # you may use {random_string:s} to avoid job name collisions
+    # {auto:3s} generates lr_0.00000_mom_0.5, .. etc
+    # {auto:2s} generates lr_0.00000_mo_0.5, .. etc
+    name: search_{experiment_name:s}_{auto:5s}
+    sku: G1
+    command:
+    - python train.py --dataset fashion_mnist --method_name irm --match_case 0.01 --lr 0.01 --penalty_irm {penalty} --penalty_s {threshold} --epochs 60 --os_env 1
+  type: grid
+  max_trials: 60
+  params:
+    - name: penalty
+      spec: discrete
+      values: [0.05, 0.1, 0.5, 1.0, 5.0]
+    - name: threshold
+      spec: discrete
+      values: [-1, 5, 15, 30, 45 ]

azure_scripts/irm_mnist.yaml

@@ -0,0 +1,41 @@
+description: Hyperparam sweep on IRM MNIST
+
+target:
+  service: philly
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  cluster: rr1
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements.txt
+
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+
+search:
+  job_template:
+    # you may use {random_string:s} to avoid job name collisions
+    # {auto:3s} generates lr_0.00000_mom_0.5, .. etc
+    # {auto:2s} generates lr_0.00000_mo_0.5, .. etc
+    name: search_{experiment_name:s}_{auto:5s}
+    sku: G1
+    command:
+    - python train.py --dataset rot_mnist --method_name irm --match_case 0.01 --lr 0.01 --penalty_irm {penalty} --penalty_s {threshold} --epochs 60  --os_env 1
+  type: grid
+  max_trials: 60
+  params:
+    - name: penalty
+      spec: discrete
+      values: [0.05, 0.1, 0.5, 1.0, 5.0]
+    - name: threshold
+      spec: discrete
+      values: [-1, 5, 15, 30, 45 ]

azure_scripts/mnist.yaml

@@ -0,0 +1,90 @@
+description: Fashion MNIST Dataset
+
+target:
+  service: amlk8s
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  name: itpeusp100cl
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements_new.txt
+    
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+ 
+
+# list of jobs to run, we run 2 jobs in this example
+jobs:
+  # name must be unique across the jobs
+
+# - name: erm
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset rot_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.0 --epochs 25  --os_env 1
+
+
+# - name: random_match
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset rot_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.1 --epochs 25  --os_env 1
+
+
+# - name: approx_25
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset rot_mnist --method_name erm_match --match_case 0.25 --penalty_ws 0.1 --epochs 25  --os_env 1
+
+
+# - name: approx_50
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset rot_mnist --method_name erm_match --match_case 0.50 --penalty_ws 0.1 --epochs 25  --os_env 1
+
+
+# - name: approx_75
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset rot_mnist --method_name erm_match --match_case 0.75 --penalty_ws 0.1 --epochs 25  --os_env 1
+
+
+# - name: perfect
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset rot_mnist --method_name erm_match --match_case 1.0 --penalty_ws 0.1 --epochs 25  --os_env 1
+
+
+# - name: csd
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset rot_mnist --method_name csd --match_case 0.01 --penalty_ws 0.0 --rep_dim 512 --epochs 25  --os_env 1
+
+
+- name: irm
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" &&  python data/data_gen.py rot_mnist lenet
+#   - echo "--debug" && python train.py --dataset rot_mnist --method_name irm --match_case 0.01 --penalty_irm 1.0 --penalty_s 5   --epochs 25  --os_env 1
+
+  
+# - name: matchdg
+#   # one gpu
+#   sku: G1
+#   command: 
+#   - echo "--debug" && python train.py --dataset rot_mnist --method_name matchdg_erm --match_case -1 --penalty_ws 0.1 --epochs 25 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet18 --n_runs 3  --os_env 1

azure_scripts/mnist_ctr.yaml

@@ -0,0 +1,61 @@
+description: MNIST Dataset Constrastive Learning
+
+target:
+  service: philly
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  cluster: rr1
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements.txt
+    
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+    
+# list of jobs to run, we run 2 jobs in this example
+jobs:
+  # name must be unique across the jobs
+- name: fmnist_ctr_standard
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset fashion_mnist --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 60 --batch_size 64 --pos_metric cos --os_env 1
+  
+- name: fmnist_ctr_perfect
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset fashion_mnist --method_name matchdg_ctr --match_case 1.0 --match_flag 1 --epochs 60 --batch_size 64 --pos_metric cos --os_env 1
+  
+- name: fmnist_ctr_non_iterative
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset fashion_mnist --method_name matchdg_ctr --match_case 0.01 --match_flag 0 --epochs 60 --batch_size 256 --pos_metric cos --os_env 1
+  
+- name: rmnist_ctr_standard
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset rot_mnist --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 100 --batch_size 256 --pos_metric cos --train_domains 30 45 --os_env 1
+  
+- name: rmnist_ctr_perfect
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset rot_mnist --method_name matchdg_ctr --match_case 1.0 --match_flag 1 --epochs 100 --batch_size 256 --pos_metric cos --os_env 1
+  
+- name: rmnist_ctr_non_iterative
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset rot_mnist --method_name matchdg_ctr --match_case 0.01 --match_flag 0 --epochs 100 --batch_size 256 --pos_metric cos --os_env 1

azure_scripts/mnist_ctr_spur.yaml

@@ -0,0 +1,39 @@
+description: MNIST Dataset Constrastive Learning
+
+target:
+  service: philly
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  cluster: rr1
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements.txt
+    
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+    
+# list of jobs to run, we run 2 jobs in this example
+jobs:
+  # name must be unique across the jobs
+# - name: fmnist_ctr_standard
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset fashion_mnist --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 60 --batch_size 64 --pos_metric cos --os_env 1
+
+  
+- name: rmnist_ctr_standard
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset rot_mnist_spur --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 100 --batch_size 256 --pos_metric cos --img_c 3 --os_env 1
+

azure_scripts/mnist_spur.yaml

@@ -0,0 +1,89 @@
+description: Fashion MNIST Dataset
+
+target:
+  service: philly
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  cluster: rr1
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements.txt
+    
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+ 
+
+# list of jobs to run, we run 2 jobs in this example
+jobs:
+  # name must be unique across the jobs
+
+# - name: erm
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset rot_mnist_spur --method_name erm_match --match_case 0.01 --penalty_ws 0.0 --epochs 25 --img_c 3  --os_env 1
+
+
+# - name: random_match
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset rot_mnist_spur --method_name erm_match --match_case 0.01 --penalty_ws 10.0 --epochs 25 --img_c 3  --os_env 1
+
+
+# - name: approx_25
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset rot_mnist --method_name erm_match --match_case 0.25 --penalty_ws 0.1 --epochs 25  --os_env 1
+
+
+# - name: approx_50
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset rot_mnist --method_name erm_match --match_case 0.50 --penalty_ws 0.1 --epochs 25  --os_env 1
+
+
+# - name: approx_75
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset rot_mnist --method_name erm_match --match_case 0.75 --penalty_ws 0.1 --epochs 25  --os_env 1
+
+
+# - name: perfect
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset rot_mnist_spur --method_name erm_match --match_case 1.0 --penalty_ws 10.0 --epochs 25  --img_c 3 --os_env 1
+
+
+# - name: csd
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset rot_mnist_spur --method_name csd --match_case 0.01 --penalty_ws 0.0 --rep_dim 512 --epochs 25 --img_c 3  --os_env 1
+
+
+# - name: irm
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset rot_mnist_spur --method_name irm --match_case 0.01 --penalty_irm 50.0 --penalty_s 5   --epochs 25 --img_c 3  --os_env 1
+
+  
+- name: matchdg
+  # one gpu
+  sku: G1
+  command: 
+  - echo "--debug" && python train.py --dataset rot_mnist_spur --method_name matchdg_erm --match_case -1 --penalty_ws 0.1 --epochs 25 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet18 --n_runs 3 --img_c 3 --os_env 1

azure_scripts/pacs.yaml

@@ -0,0 +1,73 @@
+description: PACS Dataset
+
+target:
+  service: philly
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  cluster: rr1
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements.txt
+    
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+    
+# list of jobs to run, we run 2 jobs in this example
+jobs:
+  # name must be unique across the jobs
+- name: photo
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset pacs --method_name erm_match --match_case 0.01 --penalty_ws 0.0 --train_domains art_painting cartoon sketch --test_domains photo --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --lr 0.005 --epochs 50 --os_env 1
+  
+- name: art_painting
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset pacs --method_name erm_match --match_case 0.01 --penalty_ws 0.0 --train_domains photo cartoon sketch --test_domains art_painting --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --lr 0.005 --epochs 50 --os_env 1
+  
+- name: cartoon
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset pacs --method_name erm_match --match_case 0.01 --penalty_ws 0.0 --train_domains photo art_painting sketch --test_domains cartoon --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --lr 0.005 --epochs 50 --os_env 1
+
+- name: sketch
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset pacs --method_name erm_match --match_case 0.01 --penalty_ws 0.0 --train_domains photo art_painting cartoon --test_domains sketch --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --lr 0.005 --epochs 50 --os_env 1
+
+- name: photo_random_match
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset pacs --method_name erm_match --match_case 0.01 --penalty_ws 0.5 --train_domains art_painting cartoon sketch --test_domains photo --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --lr 0.005 --epochs 50 --os_env 1
+  
+- name: art_painting_random_match
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset pacs --method_name erm_match --match_case 0.01 --penalty_ws 0.5 --train_domains photo cartoon sketch --test_domains art_painting --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --lr 0.005 --epochs 50 --os_env 1
+  
+- name: cartoon_random_match
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset pacs --method_name erm_match --match_case 0.01 --penalty_ws 0.5 --train_domains photo art_painting sketch --test_domains cartoon --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --lr 0.005 --epochs 50 --os_env 1
+
+- name: sketch_random_match
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset pacs --method_name erm_match --match_case 0.01 --penalty_ws 0.5 --train_domains photo art_painting cartoon --test_domains sketch --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --lr 0.005 --epochs 50 --os_env 1

azure_scripts/pacs_art_painting.yaml

@@ -0,0 +1,71 @@
+description: Hyperparam sweep on PACS
+
+target:
+  service: amlk8s
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  name: itpeusp100cl
+#   name:  itpeusp40cl
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements_new.txt
+
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+
+search:
+  job_template:
+    # you may use {random_string:s} to avoid job name collisions
+    # {auto:3s} generates lr_0.00000_mom_0.5, .. etc
+    # {auto:2s} generates lr_0.00000_mo_0.5, .. etc
+    name: search_{experiment_name:s}_{auto:5s}
+    sku: G1
+    command:
+    
+    # ERM, RandMatch
+#     - echo "--debug" && python train.py --dataset pacs --method_name {method} --match_case 0.0 --n_runs 3 --train_domains photo cartoon sketch --test_domains art_painting --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr {lr} --batch_size {batch_size} --weight_decay {weight_decay} --penalty_ws {penalty} --penalty_aug {penalty_aug} --model_name {model} 
+    
+    
+    # MDG, Hybrid
+    - echo "--debug" && python train.py --dataset pacs --method_name {method}  --match_case -1 --ctr_match_case 0.0 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains photo cartoon sketch --test_domains art_painting --out_classes 7 --perfect_match 0  --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr {lr} --batch_size {batch_size} --weight_decay {weight_decay} --penalty_ws {penalty} --penalty_aug {penalty_aug} --model_name {model} 
+
+    
+  type: grid
+  max_trials: 100
+  params:
+    - name: penalty
+      spec: discrete
+#       values: [0.1, 0.5, 1.0, 5.0]
+      values: [0.01, 1.0]
+    - name: penalty_aug
+      spec: discrete
+#       values: [0.1, 1.0, 5.0, 10.0]
+      values: [0.1, 1.0]
+    - name: lr
+      spec: discrete
+      values: [0.001]
+#       values: [0.01, 0.001, 0.0005]
+    - name: model
+      spec: discrete
+      values: [alexnet]
+#       values: [alexnet, resnet18, resnet50]
+    - name: method
+      spec: discrete
+#       values: [erm_match, matchdg_erm, hybrid]
+      values: [hybrid]
+    - name: batch_size
+      spec: discrete
+      values: [16]
+    - name: weight_decay
+      spec: discrete
+      values: [0.0005]
+      

azure_scripts/pacs_cartoon.yaml

@@ -0,0 +1,71 @@
+description: Hyperparam sweep on PACS
+
+target:
+  service: amlk8s
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  name: itpeusp100cl
+#   name:  itpeusp40cl
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements_new.txt
+
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+
+search:
+  job_template:
+    # you may use {random_string:s} to avoid job name collisions
+    # {auto:3s} generates lr_0.00000_mom_0.5, .. etc
+    # {auto:2s} generates lr_0.00000_mo_0.5, .. etc
+    name: search_{experiment_name:s}_{auto:5s}
+    sku: G1
+    command:
+    
+    # ERM, RandMatch
+#     - echo "--debug" && python train.py --dataset pacs --method_name {method} --match_case 0.0 --n_runs 3 --train_domains photo art_painting sketch --test_domains cartoon --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr {lr} --batch_size {batch_size} --weight_decay {weight_decay} --penalty_ws {penalty} --penalty_aug {penalty_aug} --model_name {model} 
+    
+    
+    # MDG, Hybrid
+    - echo "--debug" && python train.py --dataset pacs --method_name {method}  --match_case -1 --ctr_match_case 0.0 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains photo art_painting sketch --test_domains cartoon --out_classes 7 --perfect_match 0  --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr {lr} --batch_size {batch_size} --weight_decay {weight_decay} --penalty_ws {penalty} --penalty_aug {penalty_aug} --model_name {model} 
+
+    
+  type: grid
+  max_trials: 100
+  params:
+    - name: penalty
+      spec: discrete
+#       values: [0.1, 0.5, 1.0, 5.0]
+      values: [0.01, 1.0]
+    - name: penalty_aug
+      spec: discrete
+#       values: [0.1, 1.0, 5.0, 10.0]
+      values: [0.1, 1.0]
+    - name: lr
+      spec: discrete
+      values: [0.001]
+#       values: [0.01, 0.001, 0.0005]
+    - name: model
+      spec: discrete
+      values: [alexnet]
+#       values: [alexnet, resnet18, resnet50]
+    - name: method
+      spec: discrete
+#       values: [erm_match, matchdg_erm, hybrid]
+      values: [hybrid]
+    - name: batch_size
+      spec: discrete
+      values: [16]
+    - name: weight_decay
+      spec: discrete
+      values: [0.0005]
+      

azure_scripts/pacs_ctr.yaml

@@ -0,0 +1,74 @@
+description: PACS Dataset
+
+target:
+  service: philly
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  cluster: rr1
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements.txt
+    
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+    
+# list of jobs to run, we run 2 jobs in this example
+jobs:
+  # name must be unique across the jobs
+- name: photo_ctr_r18
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 50 --batch_size 256 --pos_metric cos --train_domains art_painting cartoon sketch --test_domains photo --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --os_env 1
+  
+# - name: art_painting_ctr_r18
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset pacs --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 50 --batch_size 256 --pos_metric cos --train_domains photo cartoon sketch --test_domains art_painting --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --os_env 1
+  
+- name: cartoon_ctr_r18
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 50 --batch_size 256 --pos_metric cos --train_domains photo art_painting sketch --test_domains cartoon --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --os_env 1
+
+- name: sketch_ctr_r18
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 50 --batch_size 256 --pos_metric cos --train_domains photo art_painting cartoon --test_domains sketch --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --os_env 1
+  
+  
+- name: photo_ctr_r50
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 50 --batch_size 64 --pos_metric cos --train_domains art_painting cartoon sketch --test_domains photo --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet50 --os_env 1
+  
+- name: art_painting_ctr_r50
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 50 --batch_size 64 --pos_metric cos --train_domains photo cartoon sketch --test_domains art_painting --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet50 --os_env 1
+  
+- name: cartoon_ctr_r50
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 50 --batch_size 64 --pos_metric cos --train_domains photo art_painting sketch --test_domains cartoon --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet50 --os_env 1
+
+- name: sketch_ctr_r50
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 50 --batch_size 64 --pos_metric cos --train_domains photo art_painting cartoon --test_domains sketch --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet50 --os_env 1

azure_scripts/pacs_erm.yaml

@@ -0,0 +1,49 @@
+description: PACS ERM Dataset
+
+target:
+  service: amlk8s
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  name: itpeusp100cl
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements_new.txt
+    
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+    
+# list of jobs to run, we run 2 jobs in this example
+jobs:
+  # name must be unique across the jobs
+# - name: photo_r18
+#   # one gpu
+#   sku: G1
+#   command:
+#   - python train.py --dataset pacs --method_name erm_match --match_case 0.01 --penalty_ws 0.0 --train_domains art_painting cartoon sketch --test_domains photo --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --lr 0.0005 --epochs 50 --model_name resnet18 --weight_decay 0.001 --os_env 1
+  
+# - name: art_painting_r18
+#   # one gpu
+#   sku: G1
+#   command:
+#   - python train.py --dataset pacs --method_name erm_match --match_case 0.01 --penalty_ws 0.0 --train_domains photo cartoon sketch --test_domains art_painting --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --lr 0.01 --epochs 50 --model_name resnet18 --weight_decay 0.001 --os_env 1
+  
+# - name: cartoon_r18
+#   # one gpu
+#   sku: G1
+#   command:
+#   - python train.py --dataset pacs --method_name erm_match --match_case 0.01 --penalty_ws 0.0 --train_domains photo art_painting sketch --test_domains cartoon --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --lr 0.001 --epochs 50 --model_name resnet18 --weight_decay 0.001 --os_env 1
+
+- name: sketch_r18
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset pacs --method_name erm_match --match_case 0.01 --penalty_ws 0.0 --train_domains photo art_painting cartoon --test_domains sketch --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --lr 0.01 --epochs 50 --model_name resnet18 --weight_decay 0.001 --os_env 1

azure_scripts/pacs_hybrid.yaml

@@ -0,0 +1,87 @@
+description: PACS MatchDG Dataset
+
+target:
+  service: philly
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  cluster: rr1
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements.txt
+    
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+    
+# list of jobs to run, we run 2 jobs in this example
+jobs:
+  # name must be unique across the jobs
+# - name: photo_r18
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset pacs --method_name hybrid --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet18 --n_runs 3 --train_domains art_painting cartoon sketch --test_domains photo --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.0005  --batch_size 16 --weight_decay 0.001 --penalty_ws 0.1 --penalty_aug 0.1 --model_name resnet18
+  
+# - name: art_painting_r18
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset pacs --method_name hybrid --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet18 --n_runs 3 --train_domains photo cartoon sketch --test_domains art_painting --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.001 --batch_size 16 --weight_decay 0.001 --penalty_ws 0.01 --penalty_aug 0.1 --model_name resnet18
+  
+# - name: cartoon_r18
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset pacs --method_name hybrid --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet18 --n_runs 3 --train_domains photo art_painting sketch --test_domains cartoon --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.001 --batch_size 16 --weight_decay 0.001 --penalty_ws 0.01 --penalty_aug 0.1 --model_name resnet18
+
+- name: sketch_r18_0.01
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name hybrid --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet18 --n_runs 3 --train_domains photo art_painting cartoon --test_domains sketch --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.001 --batch_size 16 --weight_decay 0.001 --penalty_ws 0.01 --penalty_aug 0.1 --model_name resnet18
+
+
+- name: sketch_r18_0.1
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name hybrid --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet18 --n_runs 3 --train_domains photo art_painting cartoon --test_domains sketch --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.001 --batch_size 16 --weight_decay 0.001 --penalty_ws 0.1 --penalty_aug 0.1 --model_name resnet18
+
+
+- name: sketch_r18_0.5
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name hybrid --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet18 --n_runs 3 --train_domains photo art_painting cartoon --test_domains sketch --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.001 --batch_size 16 --weight_decay 0.001 --penalty_ws 0.5 --penalty_aug 0.1 --model_name resnet18
+
+# - name: photo_r50
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset pacs --method_name hybrid --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains art_painting cartoon sketch --test_domains photo --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.0005  --batch_size 16 --weight_decay 0.001 --penalty_ws 0.1 --penalty_aug 0.1 --model_name resnet50
+  
+# - name: art_painting_r50
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset pacs --method_name hybrid --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains photo cartoon sketch --test_domains art_painting --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.001 --batch_size 16 --weight_decay 0.001 --penalty_ws 0.01 --penalty_aug 0.1 --model_name resnet50
+  
+# - name: cartoon_r50
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset pacs --method_name hybrid --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains photo art_painting sketch --test_domains cartoon --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.0005 --batch_size 16 --weight_decay 0.001 --penalty_ws 0.01 --penalty_aug 0.1 --model_name resnet50
+
+# - name: sketch_r50
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset pacs --method_name hybrid --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains photo art_painting cartoon --test_domains sketch --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.001 --batch_size 16 --weight_decay 0.001 --penalty_ws 0.01 --penalty_aug 0.1 --model_name resnet50

azure_scripts/pacs_matchdg.yaml

@@ -0,0 +1,73 @@
+description: PACS MatchDG Dataset
+
+target:
+  service: philly
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  cluster: rr1
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements.txt
+    
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+    
+# list of jobs to run, we run 2 jobs in this example
+jobs:
+  # name must be unique across the jobs
+# - name: photo_r18
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset pacs --method_name matchdg_erm --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains art_painting cartoon sketch --test_domains photo --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.0005  --batch_size 16 --weight_decay 0.001 --penalty_ws 0.1 --penalty_aug 0.1 --model_name resnet18
+  
+# - name: art_painting_r18
+#   # one gpu
+#   sku: G1
+#   command:
+#   - echo "--debug" && python train.py --dataset pacs --method_name matchdg_erm --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains photo cartoon sketch --test_domains art_painting --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.001 --batch_size 16 --weight_decay 0.001 --penalty_ws 0.5 --penalty_aug 0.1 --model_name resnet18
+  
+- name: cartoon_r18
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name matchdg_erm --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains photo art_painting sketch --test_domains cartoon --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.001 --batch_size 16 --weight_decay 0.001 --penalty_ws 1.0 --penalty_aug 0.1 --model_name resnet18
+
+- name: sketch_r18
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name matchdg_erm --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains photo art_painting cartoon --test_domains sketch --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.01 --batch_size 16 --weight_decay 0.001 --penalty_ws 0.5 --penalty_aug 0.1 --model_name resnet18
+
+- name: photo_r50
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name matchdg_erm --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains art_painting cartoon sketch --test_domains photo --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.0005  --batch_size 16 --weight_decay 0.001 --penalty_ws 0.1 --penalty_aug 0.1 --model_name resnet50
+  
+- name: art_painting_r50
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name matchdg_erm --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains photo cartoon sketch --test_domains art_painting --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.001 --batch_size 16 --weight_decay 0.001 --penalty_ws 0.1 --penalty_aug 0.1 --model_name resnet50
+  
+- name: cartoon_r50
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name matchdg_erm --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains photo art_painting sketch --test_domains cartoon --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.0005 --batch_size 16 --weight_decay 0.001 --penalty_ws 1.0 --penalty_aug 0.1 --model_name resnet50
+
+- name: sketch_r50
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name matchdg_erm --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains photo art_painting cartoon --test_domains sketch --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.001 --batch_size 16 --weight_decay 0.001 --penalty_ws 0.5 --penalty_aug 0.1 --model_name resnet50

azure_scripts/pacs_perfect.yaml

@@ -0,0 +1,73 @@
+description: PACS MatchDG Dataset
+
+target:
+  service: philly
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  cluster: rr1
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements.txt
+    
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+    
+# list of jobs to run, we run 2 jobs in this example
+jobs:
+  # name must be unique across the jobs
+- name: photo_r18
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name hybrid --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains art_painting cartoon sketch --test_domains photo --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.0005  --batch_size 16 --weight_decay 0.001 --penalty_ws 0.0 --penalty_aug 0.1 --model_name resnet18
+  
+- name: art_painting_r18
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name hybrid --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains photo cartoon sketch --test_domains art_painting --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.001 --batch_size 16 --weight_decay 0.001 --penalty_ws 0.0 --penalty_aug 0.1 --model_name resnet18
+  
+- name: cartoon_r18
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name hybrid --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains photo art_painting sketch --test_domains cartoon --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.001 --batch_size 16 --weight_decay 0.001 --penalty_ws 0.0 --penalty_aug 0.1 --model_name resnet18
+
+- name: sketch_r18
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name hybrid --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains photo art_painting cartoon --test_domains sketch --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.01 --batch_size 16 --weight_decay 0.001 --penalty_ws 0.0 --penalty_aug 0.1 --model_name resnet18
+
+- name: photo_r50
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name hybrid --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains art_painting cartoon sketch --test_domains photo --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.0005  --batch_size 16 --weight_decay 0.001 --penalty_ws 0.0 --penalty_aug 0.1 --model_name resnet50
+  
+- name: art_painting_r50
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name hybrid --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains photo cartoon sketch --test_domains art_painting --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.001 --batch_size 16 --weight_decay 0.001 --penalty_ws 0.0 --penalty_aug 0.1 --model_name resnet50
+  
+- name: cartoon_r50
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name hybrid --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains photo art_painting sketch --test_domains cartoon --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.0005 --batch_size 16 --weight_decay 0.001 --penalty_ws 0.0 --penalty_aug 0.1 --model_name resnet50
+
+- name: sketch_r50
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" && python train.py --dataset pacs --method_name hybrid --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains photo art_painting cartoon --test_domains sketch --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr 0.001 --batch_size 16 --weight_decay 0.001 --penalty_ws 0.0 --penalty_aug 0.1 --model_name resnet50

azure_scripts/pacs_photo.yaml

@@ -0,0 +1,72 @@
+description: Hyperparam sweep on PACS
+
+target:
+  service: amlk8s
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  name: itpeusp100cl
+#   name: itplabrl1cl1
+#   name:  itpeusp40cl
+  
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements_new.txt
+
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+
+search:
+  job_template:
+    # you may use {random_string:s} to avoid job name collisions
+    # {auto:3s} generates lr_0.00000_mom_0.5, .. etc
+    # {auto:2s} generates lr_0.00000_mo_0.5, .. etc
+    name: search_{experiment_name:s}_{auto:5s}
+    sku: G1
+    command:
+    
+    # ERM, RandMatch
+#     - echo "--debug" && python train.py --dataset pacs --method_name {method} --match_case 0.0 --n_runs 3 --train_domains art_painting cartoon sketch --test_domains photo --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr {lr} --batch_size {batch_size} --weight_decay {weight_decay} --penalty_ws {penalty} --penalty_aug {penalty_aug} --model_name {model} 
+    
+    
+    # MDG, Hybrid
+    - echo "--debug" && python train.py --dataset pacs --method_name {method}  --match_case -1 --ctr_match_case 0.0 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains art_painting cartoon sketch --test_domains photo --out_classes 7 --perfect_match 0  --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr {lr} --batch_size {batch_size} --weight_decay {weight_decay} --penalty_ws {penalty} --penalty_aug {penalty_aug} --model_name {model} 
+
+    
+  type: grid
+  max_trials: 100
+  params:
+    - name: penalty
+      spec: discrete
+#       values: [0.1, 0.5, 1.0, 5.0]
+      values: [0.1]
+    - name: penalty_aug
+      spec: discrete
+#       values: [0.1, 1.0, 5.0, 10.0]
+      values: [0.1, 1.0]
+    - name: lr
+      spec: discrete
+      values: [0.0005]
+#       values: [0.01, 0.001, 0.0005]
+    - name: model
+      spec: discrete
+      values: [alexnet]
+#       values: [alexnet, resnet18, resnet50]
+    - name: method
+      spec: discrete
+#       values: [erm_match, matchdg_erm, hybrid]
+      values: [hybrid]
+    - name: batch_size
+      spec: discrete
+      values: [16]
+    - name: weight_decay
+      spec: discrete
+      values: [0.0005]
+      

azure_scripts/pacs_random.yaml

@@ -0,0 +1,73 @@
+description: PACS Random Match Dataset
+
+target:
+  service: philly
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  cluster: rr1
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements.txt
+    
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+    
+# list of jobs to run, we run 2 jobs in this example
+jobs:
+  # name must be unique across the jobs
+- name: photo_r18
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset pacs --method_name erm_match --match_case 0.01 --penalty_ws 0.1 --train_domains art_painting cartoon sketch --test_domains photo --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --lr 0.001 --epochs 50 --model_name resnet18 --weight_decay 0.001 --os_env 1
+  
+- name: art_painting_r18
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset pacs --method_name erm_match --match_case 0.01 --penalty_ws 0.5 --train_domains photo cartoon sketch --test_domains art_painting --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --lr 0.01 --epochs 50 --model_name resnet18 --os_env 1
+  
+- name: cartoon_r18
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset pacs --method_name erm_match --match_case 0.01 --penalty_ws 0.1 --train_domains photo art_painting sketch --test_domains cartoon --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --lr 0.01 --epochs 50 --model_name resnet18 --os_env 1
+
+- name: sketch_r18
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset pacs --method_name erm_match --match_case 0.01 --penalty_ws 0.1 --train_domains photo art_painting cartoon --test_domains sketch --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --lr 0.01 --epochs 50 --model_name resnet18 --os_env 1
+
+- name: photo_r50
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset pacs --method_name erm_match --match_case 0.01 --penalty_ws 0.1 --train_domains art_painting cartoon sketch --test_domains photo --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --lr 0.0005 --epochs 50 --model_name resnet50 --weight_decay 0.001 --os_env 1
+  
+- name: art_painting_r50
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset pacs --method_name erm_match --match_case 0.01 --penalty_ws 0.5 --train_domains photo cartoon sketch --test_domains art_painting --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --lr 0.001 --epochs 50 --model_name resnet50 --os_env 1
+  
+- name: cartoon_r50
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset pacs --method_name erm_match --match_case 0.01 --penalty_ws 0.5 --train_domains photo art_painting sketch --test_domains cartoon --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --lr 0.0005 --epochs 50 --model_name resnet50 --os_env 1
+
+- name: sketch_r50
+  # one gpu
+  sku: G1
+  command:
+  - python train.py --dataset pacs --method_name erm_match --match_case 0.01 --penalty_ws 0.1 --train_domains photo art_painting cartoon --test_domains sketch --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --lr 0.01 --epochs 50 --model_name resnet50 --os_env 1

azure_scripts/pacs_sketch.yaml

@@ -0,0 +1,70 @@
+description: Hyperparam sweep on PACS
+
+target:
+  service: amlk8s
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  name: itpeusp100cl
+#   name:  itpeusp40cl
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements_new.txt
+
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+
+search:
+  job_template:
+    # you may use {random_string:s} to avoid job name collisions
+    # {auto:3s} generates lr_0.00000_mom_0.5, .. etc
+    # {auto:2s} generates lr_0.00000_mo_0.5, .. etc
+    name: search_{experiment_name:s}_{auto:5s}
+    sku: G1
+    command:
+    
+    # ERM, RandMatch
+#     - echo "--debug" && python train.py --dataset pacs --method_name {method} --match_case 0.0 --n_runs 3 --train_domains photo art_painting cartoon --test_domains sketch --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr {lr} --batch_size {batch_size} --weight_decay {weight_decay} --penalty_ws {penalty} --penalty_aug {penalty_aug} --model_name {model}     
+    
+    # MDG, Hybrid
+    - echo "--debug" && python train.py --dataset pacs --method_name {method}  --match_case -1 --ctr_match_case 0.0 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet50 --n_runs 3 --train_domains photo art_painting cartoon  --test_domains sketch --out_classes 7 --perfect_match 0  --img_c 3 --pre_trained 1 --epochs 50 --os_env 1  --lr {lr} --batch_size {batch_size} --weight_decay {weight_decay} --penalty_ws {penalty} --penalty_aug {penalty_aug} --model_name {model} 
+
+    
+  type: grid
+  max_trials: 100
+  params:
+    - name: penalty
+      spec: discrete
+#       values: [0.1, 0.5, 1.0, 5.0]
+      values: [0.01, 0.1]
+    - name: penalty_aug
+      spec: discrete
+#       values: [0.1, 1.0, 5.0, 10.0]
+      values: [0.1, 1.0]
+    - name: lr
+      spec: discrete
+      values: [0.001]
+#       values: [0.01, 0.001, 0.0005]
+    - name: model
+      spec: discrete
+      values: [alexnet]
+#       values: [alexnet, resnet18, resnet50]
+    - name: method
+      spec: discrete
+#       values: [erm_match, matchdg_erm, hybrid]
+      values: [hybrid]
+    - name: batch_size
+      spec: discrete
+      values: [16]
+    - name: weight_decay
+      spec: discrete
+      values: [0.0005]
+      

azure_scripts/setup_data_mnist.yaml

@@ -0,0 +1,46 @@
+description: MNIST Dat Setup
+
+target:
+  service: amlk8s
+  # which virtual cluster you belong to (msrlabs, etc.). Everyone has access to "msrlabs".
+  vc: resrchvc 
+  # physical cluster to use (cam, gcr, rr1, rr2) or Azure clusters (eu1, eu2, etc.)
+  name: itpeusp100cl
+
+environment:
+  image: pytorch/pytorch:1.5-cuda10.1-cudnn7-devel
+  setup:
+    - pip install --user -r requirements_new.txt
+    
+code:
+  # local directory of the code. this will be uploaded to the server.
+  # $CONFIG_DIR is expanded to the directory of this config file
+  local_dir: $CONFIG_DIR
+
+data:
+    local_dir: $CONFIG_DIR/data/datasets/
+    remote_dir: data/datasets/
+ 
+
+# list of jobs to run, we run 2 jobs in this example
+jobs:
+  # name must be unique across the jobs
+
+- name: rot_mnist_resnet18
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" &&  python data/data_gen.py rot_mnist resnet18
+
+- name: fashion_mnist_resnet18
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" &&  python data/data_gen.py fashion_mnist resnet18
+
+- name: rot_mnist_lenet
+  # one gpu
+  sku: G1
+  command:
+  - echo "--debug" &&  python data/data_gen.py rot_mnist lenet
+

misc_scripts/acc_pacs.sh

@@ -0,0 +1,29 @@
+python  test.py --test_metric acc --mia_sample_size 1000 --batch_size 64 --dataset pacs  --method_name erm_match --match_case 0.01 --penalty_ws 0 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --train_domains art_painting cartoon sketch --test_domains photo > results/pacs/acc/photo_erm.txt
+
+python  test.py --test_metric acc --mia_sample_size 1000 --batch_size 64 --dataset pacs  --method_name erm_match --match_case 0.01 --penalty_ws 0 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --train_domains photo cartoon sketch --test_domains art_painting > results/pacs/acc/painting_erm.txt
+
+python  test.py --test_metric acc --mia_sample_size 1000 --batch_size 64 --dataset pacs  --method_name erm_match --match_case 0.01 --penalty_ws 0 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --train_domains photo art_painting sketch --test_domains cartoon > results/pacs/acc/cartoon_erm.txt
+
+python  test.py --test_metric acc --mia_sample_size 1000 --batch_size 64 --dataset pacs  --method_name erm_match --match_case 0.01 --penalty_ws 0 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --train_domains photo art_painting cartoon --test_domains sketch > results/pacs/acc/sketch_erm.txt
+
+
+
+
+python  test.py --test_metric acc --mia_sample_size 1000 --batch_size 64 --dataset pacs  --method_name erm_match --match_case 0.01 --penalty_ws 0.1 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --train_domains art_painting cartoon sketch --test_domains photo > results/pacs/acc/photo_random.txt
+
+python  test.py --test_metric acc --mia_sample_size 1000 --batch_size 64 --dataset pacs  --method_name erm_match --match_case 0.01 --penalty_ws 0.5 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --train_domains photo cartoon sketch --test_domains art_painting > results/pacs/acc/painting_random.txt
+
+python  test.py --test_metric acc --mia_sample_size 1000 --batch_size 64 --dataset pacs  --method_name erm_match --match_case 0.01 --penalty_ws 0.1 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --train_domains photo art_painting sketch --test_domains cartoon > results/pacs/acc/cartoon_random.txt
+
+python  test.py --test_metric acc --mia_sample_size 1000 --batch_size 64 --dataset pacs  --method_name erm_match --match_case 0.01 --penalty_ws 0.1 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --train_domains photo art_painting cartoon --test_domains sketch > results/pacs/acc/sketch_random.txt
+
+
+
+
+python test.py --test_metric acc --mia_sample_size 1000 --batch_size 64 --dataset pacs --method_name hybrid --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet18 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --penalty_ws 0.1 --match_case -1 --train_domains art_painting cartoon sketch --test_domains photo >  results/pacs/acc/photo_hybrid.txt
+
+python test.py --test_metric acc --mia_sample_size 1000 --batch_size 64 --dataset pacs --method_name hybrid --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet18 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --penalty_ws 0.01 --match_case -1 --train_domains photo cartoon sketch --test_domains art_painting > results/pacs/acc/painting_hybrid.txt
+
+python test.py --test_metric acc --mia_sample_size 1000 --batch_size 64 --dataset pacs --method_name hybrid --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet18 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --penalty_ws 0.01 --match_case -1 --train_domains photo art_painting sketch --test_domains cartoon > results/pacs/acc/cartoon_hybrid.txt
+
+python test.py --test_metric acc --mia_sample_size 1000 --batch_size 64 --dataset pacs --method_name hybrid --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet18 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --penalty_ws 0.5 --match_case -1 --train_domains photo art_painting cartoon --test_domains sketch > results/pacs/acc/sketch_hybrid.txt

misc_scripts/adv_attack.sh

@@ -0,0 +1,33 @@
+# python test.py --dataset rot_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.0 --test_metric adv_attack --penalty_diff_ctr 0 --adv_eps 5
+# python test.py --dataset rot_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.1 --test_metric adv_attack --penalty_diff_ctr 0 --adv_eps 5
+# python test.py --dataset rot_mnist --method_name erm_match --match_case 1.0 --penalty_ws 0.1 --test_metric adv_attack --penalty_diff_ctr 0 --adv_eps 5
+
+# python test.py --dataset rot_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.0 --test_metric adv_attack --penalty_diff_ctr 0 --adv_eps 25
+# python test.py --dataset rot_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.1 --test_metric adv_attack --penalty_diff_ctr 0 --adv_eps 25
+# python test.py --dataset rot_mnist --method_name erm_match --match_case 1.0 --penalty_ws 0.1 --test_metric adv_attack --penalty_diff_ctr 0 --adv_eps 25
+
+# python test.py --dataset rot_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.0 --test_metric adv_attack --penalty_diff_ctr 0 --adv_eps 50
+# python test.py --dataset rot_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.1 --test_metric adv_attack --penalty_diff_ctr 0 --adv_eps 50
+# python test.py --dataset rot_mnist --method_name erm_match --match_case 1.0 --penalty_ws 0.1 --test_metric adv_attack --penalty_diff_ctr 0 --adv_eps 50
+
+# python test.py --dataset rot_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.0 --test_metric adv_attack --penalty_diff_ctr 0 --adv_eps 75
+# python test.py --dataset rot_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.1 --test_metric adv_attack --penalty_diff_ctr 0 --adv_eps 75
+# python test.py --dataset rot_mnist --method_name erm_match --match_case 1.0 --penalty_ws 0.1 --test_metric adv_attack --penalty_diff_ctr 0 --adv_eps 75
+
+# python test.py --dataset rot_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.0 --test_metric adv_attack --penalty_diff_ctr 0 --adv_eps 100
+# python test.py --dataset rot_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.1 --test_metric adv_attack --penalty_diff_ctr 0 --adv_eps 100
+# python test.py --dataset rot_mnist --method_name erm_match --match_case 1.0 --penalty_ws 0.1 --test_metric adv_attack --penalty_diff_ctr 0 --adv_eps 100
+
+python3 test.py --test_metric adv_attack --dataset fashion_mnist --method_name erm_match --match_case 1.0 --penalty_ws 0.1 --adv_eps 0.3 > perfect.txt
+
+python3 test.py --test_metric adv_attack --dataset fashion_mnist --method_name erm_match --match_case 0.75 --penalty_ws 0.1 --adv_eps 0.3 > case_75.txt
+
+python3 test.py --test_metric adv_attack --dataset fashion_mnist --method_name erm_match --match_case 0.5 --penalty_ws 0.1 --adv_eps 0.3 > case_50.txt
+
+python3 test.py --test_metric adv_attack --dataset fashion_mnist --method_name erm_match --match_case 0.25 --penalty_ws 0.1 --adv_eps 0.3 > case_25.txt
+
+python3 test.py --test_metric adv_attack --dataset fashion_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.1 --adv_eps 0.3 > random.txt
+
+python3 test.py --test_metric adv_attack --dataset fashion_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.0 --adv_eps 0.3 > erm.txt
+
+python  test.py --test_metric adv_attack --dataset fashion_mnist --method_name matchdg_erm --penalty_ws 0.1 --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --adv_eps 0.3 --n_runs 2 > match_dg.txt

misc_scripts/chex.sh

@@ -0,0 +1,72 @@
+# Chex Evaluation
+python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains nih_trans kaggle_trans  --test_domains chex  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 2 > erm_chex.txt
+
+python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains nih_trans kaggle_trans  --test_domains chex  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 50.0 --model_name densenet121 --n_runs 2 > rand_chex.txt
+
+python train.py --dataset chestxray --method_name csd --match_case 0.01 --train_domains nih_trans kaggle_trans  --test_domains chex  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 2 --rep_dim 1024 > csd_chex.txt
+
+python train.py --dataset chestxray --method_name irm --match_case 0.01 --train_domains nih_trans kaggle_trans  --test_domains chex  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 50.0 --penalty_s 5 --model_name densenet121 --n_runs 2 > irm_chex.txt
+
+
+# NIH Evaluation
+python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains chex_trans kaggle_trans  --test_domains nih  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 2 > erm_nih.txt
+
+python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains chex_trans kaggle_trans  --test_domains nih  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 50.0 --model_name densenet121 --n_runs 2 > rand_nih.txt
+
+python train.py --dataset chestxray --method_name csd --match_case 0.01 --train_domains chex_trans kaggle_trans  --test_domains nih  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 2 --rep_dim 1024 > csd_nih.txt
+
+python train.py --dataset chestxray --method_name irm --match_case 0.01 --train_domains chex_trans kaggle_trans  --test_domains nih  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 50.0 --penalty_s 5 --model_name densenet121 --n_runs 2 > irm_nih.txt
+
+
+# Kaggle Evaluation
+python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 2 > erm_kaggle.txt
+
+python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 50.0 --model_name densenet121 --n_runs 2 > rand_kaggle.txt
+
+python train.py --dataset chestxray --method_name csd --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 2 --rep_dim 1024 > csd_kaggle.txt
+
+python train.py --dataset chestxray --method_name irm --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 50.0 --penalty_s 5 --model_name densenet121 --n_runs 2 > irm_kaggle.txt
+
+
+# python train.py --dataset chestxray --method_name hybrid --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle_trans  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 3 --penalty_aug 10.0 > perf_chex_trans.txt
+
+
+
+
+
+
+# python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 32 --penalty_ws 0.0 --model_name densenet121 --n_runs 1 > erm_chex.txt
+
+# python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 32 --penalty_ws 1.0 --model_name densenet121 --n_runs 1 > rand_chex.txt
+
+# python train.py --dataset chestxray --method_name csd --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 32 --penalty_ws 0.0 --model_name densenet121 --n_runs 1 --rep_dim 1024 > csd_chex.txt
+
+# python train.py --dataset chestxray --method_name irm --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 32 --penalty_ws 1.0 --penalty_s 5 --model_name densenet121 --n_runs 1 > irm_chex.txt
+
+# python train.py --dataset chestxray --method_name hybrid --match_case 0.01 --train_domains nih_trans chex_trans  --test_domains kaggle  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 16 --penalty_ws 0.0 --model_name densenet121 --n_runs 1 --penalty_aug 10.0 > perf_chex.txt
+
+# #Kaggle Evaluation
+# python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains nih chex  --test_domains kaggle  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --penalty_ws 0.0 --model_name densenet121 > erm_.txt
+
+# python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains nih chex  --test_domains kaggle  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --penalty_ws 1.0 --model_name densenet121 > rand_chex.txt
+
+# python train.py --dataset chestxray --method_name csd --match_case 0.01 --train_domains nih chex  --test_domains kaggle  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --penalty_ws 0.0 --model_name densenet121 > csd_chex.txt
+
+# python train.py --dataset chestxray --method_name irm --match_case 0.01 --train_domains nih chex  --test_domains kaggle  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --penalty_ws 1.0 --penalty_s 5 --model_name densenet121 > irm_chex.txt
+
+
+# #NIH Evaluation
+# python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains kaggle chex  --test_domains nih  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --penalty_ws 0.0 --model_name densenet121 > erm_chex.txt
+
+# python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains kaggle chex  --test_domains nih  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --penalty_ws 1.0 --model_name densenet121 > rand_chex.txt
+
+# python train.py --dataset chestxray --method_name csd --match_case 0.01 --train_domains kaggle chex  --test_domains nih  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --penalty_ws 0.0 --model_name densenet121 > csd_chex.txt
+
+# python train.py --dataset chestxray --method_name irm --match_case 0.01 --train_domains kaggle chex  --test_domains nih  --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --penalty_ws 1.0 --penalty_s 5 --model_name densenet121 > irm_chex.txt
+
+#Alone
+# python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains nih --test_domains nih --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 32 --penalty_ws 0.0 --model_name densenet121 --n_runs 1 > nih_alone.txt
+
+# python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains chex --test_domains chex --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 32 --penalty_ws 0.0 --model_name densenet121 --n_runs 1 > chex_alone.txt
+
+# python train.py --dataset chestxray --method_name erm_match --match_case 0.01 --train_domains kaggle --test_domains kaggle --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 40  --lr 0.001 --batch_size 32 --penalty_ws 0.0 --model_name densenet121 --n_runs 1 > kaggle_alone.txt

misc_scripts/domainbed.sh

@@ -0,0 +1,11 @@
+python train.py --dataset rot_mnist --method_name erm_match --match_case 0.01 --penalty_ws 1.0 --epochs 50 --model_name domain_bed_mnist --img_h 28 --img_w 28 --train_domains 15 30 45 60 75 --test_domains 0 > rand_1.txt
+
+python train.py --dataset rot_mnist --method_name erm_match --match_case 0.30 --penalty_ws 1.0 --epochs 50 --model_name domain_bed_mnist --img_h 28 --img_w 28 --train_domains 15 30 45 60 75 --test_domains 0 > mdg_1.txt
+
+python train.py --dataset rot_mnist --method_name erm_match --match_case 1.0 --penalty_ws 1.0 --epochs 50 --model_name domain_bed_mnist --img_h 28 --img_w 28 --train_domains 15 30 45 60 75 --test_domains 0 > perf_1.txt
+
+python train.py --dataset rot_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.1 --epochs 50 --model_name domain_bed_mnist --img_h 28 --img_w 28 --train_domains 15 30 45 60 75 --test_domains 0 > rand_0.1.txt
+
+python train.py --dataset rot_mnist --method_name erm_match --match_case 0.30 --penalty_ws 0.1 --epochs 50 --model_name domain_bed_mnist --img_h 28 --img_w 28 --train_domains 15 30 45 60 75 --test_domains 0 > mdg_0.1.txt
+
+python train.py --dataset rot_mnist --method_name erm_match --match_case 1.0 --penalty_ws 0.1 --epochs 50 --model_name domain_bed_mnist --img_h 28 --img_w 28 --train_domains 15 30 45 60 75 --test_domains 0 > perf_0.1.txt

misc_scripts/mnist_abl.sh

@@ -0,0 +1,16 @@
+##Rotated MNIST
+
+#Perfect MatchDG
+python train.py --dataset rot_mnist --method_name matchdg_ctr --match_case 1.0 --match_flag 1 --epochs 50 --batch_size 64 --pos_metric cos
+
+#Non Interative MatchDG
+python train.py --dataset rot_mnist --method_name matchdg_ctr --match_case 0.0 --match_flag 0 --epochs 50 --batch_size 64 --pos_metric cos
+
+
+##Fashion MNIST
+
+#Perfect MatchDG
+python train.py --dataset fashion_mnist --method_name matchdg_ctr --match_case 1.0 --match_flag 1 --epochs 50 --batch_size 64 --pos_metric cos
+
+#Non Interative MatchDG
+python train.py --dataset fashion_mnist --method_name matchdg_ctr --match_case 0.0 --match_flag 0 --epochs 50 --batch_size 64 --pos_metric cos

misc_scripts/pacs_ctr.sh

@@ -0,0 +1,15 @@
+#Photo
+python train.py --dataset pacs --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 15 --batch_size 128 --pos_metric cos --train_domains art_painting cartoon sketch --test_domains photo --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name $1
+
+
+#Art Painting
+python train.py --dataset pacs --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 15 --batch_size 128 --pos_metric cos --train_domains photo cartoon sketch --test_domains art_painting --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name $1
+
+
+#Cartoon
+python train.py --dataset pacs --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 15 --batch_size 128 --pos_metric cos --train_domains photo art_painting sketch --test_domains cartoon --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name $1
+
+
+#Sketch
+python train.py --dataset pacs --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 15 --batch_size 128 --pos_metric cos --train_domains photo art_painting cartoon --test_domains sketch --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name $1
+

misc_scripts/privacy_chest.sh

@@ -0,0 +1,104 @@
+#Kaggle
+python3 cxray_run.py kaggle acc train
+python3 cxray_run.py kaggle acc test
+python3 cxray_run.py kaggle privacy_loss_attack
+python3 cxray_run.py kaggle privacy_entropy
+python3 cxray_run.py kaggle match_score test
+
+#ChexPert
+python3 cxray_run.py chex acc train
+python3 cxray_run.py chex acc test
+python3 cxray_run.py chex privacy_loss_attack
+python3 cxray_run.py chex privacy_entropy
+python3 cxray_run.py chex match_score test
+
+#NIH
+python3 cxray_run.py nih acc train
+python3 cxray_run.py nih acc test
+python3 cxray_run.py nih privacy_loss_attack
+python3 cxray_run.py nih privacy_entropy
+python3 cxray_run.py nih match_score test
+
+# #Accuracy Train
+
+# python test.py --test_metric acc --batch_size 64 --dataset chestxray --acc_data_case train --match_func_aug_case 1 --method_name erm_match --match_case 0.01 --penalty_ws 0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/acc_train/erm.txt   
+
+# python test.py --test_metric acc --batch_size 64 --dataset chestxray --acc_data_case train --match_func_aug_case 1 --method_name erm_match --match_case 0.01 --penalty_ws 10.0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/acc_train/random.txt
+
+# python test.py --test_metric acc --batch_size 64 --dataset chestxray --acc_data_case train --match_func_aug_case 1 --method_name csd --match_case 0.01 --penalty_ws 0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/acc_train/csd.txt
+ 
+# python test.py --test_metric acc --batch_size 64 --dataset chestxray --acc_data_case train --match_func_aug_case 1 --method_name irm --match_case 0.01 --penalty_s 5 --penalty_ws 10.0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/acc_train/irm.txt
+
+# python test.py --test_metric acc --batch_size 64 --dataset chestxray --acc_data_case train --match_func_aug_case 1 --method_name matchdg_erm --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name densenet121 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --penalty_ws 50.0 --match_case -1 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/acc_train/matchdg.txt
+
+# # #Accuracy Test
+
+# # python test.py --test_metric acc --batch_size 64 --dataset chestxray --acc_data_case test --match_func_aug_case 1 --method_name erm_match --match_case 0.01 --penalty_ws 0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/acc_test/erm.txt   
+
+# # python test.py --test_metric acc --batch_size 64 --dataset chestxray --acc_data_case test --match_func_aug_case 1 --method_name erm_match --match_case 0.01 --penalty_ws 10.0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/acc_test/random.txt
+
+# # python test.py --test_metric acc --batch_size 64 --dataset chestxray --acc_data_case test --match_func_aug_case 1 --method_name csd --match_case 0.01 --penalty_ws 0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/acc_test/csd.txt
+ 
+# # python test.py --test_metric acc --batch_size 64 --dataset chestxray --acc_data_case test --match_func_aug_case 1 --method_name irm --match_case 0.01 --penalty_s 5 --penalty_ws 10.0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/acc_test/irm.txt
+
+# # python test.py --test_metric acc --batch_size 64 --dataset chestxray --acc_data_case test --match_func_aug_case 1 --method_name matchdg_erm --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name densenet121 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --penalty_ws 50.0 --match_case -1 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/acc_test/matchdg.txt
+
+# # #Mean Rank
+
+# # python test.py --test_metric match_score --batch_size 64 --dataset chestxray --match_func_data_case test --match_func_aug_case 1 --method_name erm_match --match_case 0.01 --penalty_ws 0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/match_score/erm.txt   
+
+# # python test.py --test_metric match_score --batch_size 64 --dataset chestxray --match_func_data_case test --match_func_aug_case 1 --method_name erm_match --match_case 0.01 --penalty_ws 10.0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle 
+
+# # python test.py --test_metric match_score --batch_size 64 --dataset chestxray --match_func_data_case test --match_func_aug_case 1 --method_name csd --match_case 0.01 --penalty_ws 0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle
+ 
+# # python test.py --test_metric match_score --batch_size 64 --dataset chestxray --match_func_data_case test --match_func_aug_case 1 --method_name irm --match_case 0.01 --penalty_s 5 --penalty_ws 10.0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle
+
+# # python test.py --test_metric match_score --batch_size 64 --dataset chestxray --match_func_data_case test --match_func_aug_case 1 --method_name matchdg_erm --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name densenet121 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --penalty_ws 50.0 --match_case -1 --train_domains nih_trans chex_trans --test_domains kaggle
+
+# # #MIA Attack
+
+# # python  test.py --test_metric mia --mia_sample_size 1000 --mia_logit 1 --batch_size 64 --dataset chestxray --method_name erm_match --match_case 0.01 --penalty_ws 0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/privacy/erm.txt 
+
+# # python  test.py --test_metric mia --mia_sample_size 1000 --batch_size 64 --dataset chestxray --method_name erm_match --match_case 0.01 --penalty_ws 10 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/privacy/random.txt
+
+# # python test.py --test_metric mia --mia_sample_size 1000 --batch_size 64 --dataset chestxray --method_name csd --match_case 0.01 --penalty_ws 0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/privacy/csd.txt
+
+# # python test.py --test_metric mia --mia_sample_size 1000 --batch_size 64 --dataset chestxray --method_name irm --match_case 0.01 --penalty_s 5 --penalty_ws 10.0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/privacy/irm.txt
+
+# # python test.py --test_metric mia --mia_sample_size 1000 --batch_size 64 --dataset chestxray --method_name matchdg_erm --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name densenet121 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --penalty_ws 50.0 --match_case -1 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/privacy/matchdg.txt
+
+# # #Entorpy Attack
+
+# # python  test.py --test_metric privacy_entropy --mia_sample_size 1000 --batch_size 64 --dataset chestxray --method_name erm_match --match_case 0.01 --penalty_ws 0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/privacy_entropy/erm.txt
+
+# # python  test.py --test_metric privacy_entropy --mia_sample_size 1000 --batch_size 64 --dataset chestxray --method_name erm_match --match_case 0.01 --penalty_ws 10 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/privacy_entropy/random.txt
+
+# # python test.py --test_metric privacy_entropy --mia_sample_size 1000 --batch_size 64 --dataset chestxray --method_name csd --match_case 0.01 --penalty_ws 0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle  > results/chestxray/privacy_entropy/csd.txt
+
+# # python test.py --test_metric privacy_entropy --mia_sample_size 1000 --batch_size 64 --dataset chestxray --method_name irm --match_case 0.01 --penalty_s 5 --penalty_ws 10.0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/privacy_entropy/irm.txt
+
+# # python test.py --test_metric privacy_entropy --mia_sample_size 1000 --batch_size 64 --dataset chestxray --method_name matchdg_erm --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name densenet121 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --penalty_ws 50.0 --match_case -1 --train_domains nih_trans chex_trans --test_domains kaggle  > results/chestxray/privacy_entropy/matchdg.txt
+
+# # #Loss Attack
+
+# # python  test.py --test_metric privacy_loss_attack --mia_sample_size 1000 --batch_size 64 --dataset chestxray --method_name erm_match --match_case 0.01 --penalty_ws 0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/privacy_loss/erm.txt
+
+# # python  test.py --test_metric privacy_loss_attack --mia_sample_size 1000 --batch_size 64 --dataset chestxray --method_name erm_match --match_case 0.01 --penalty_ws 10 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/privacy_loss/random.txt
+
+# # python test.py --test_metric privacy_loss_attack --mia_sample_size 1000 --batch_size 64 --dataset chestxray --method_name csd --match_case 0.01 --penalty_ws 0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle  > results/chestxray/privacy_loss/csd.txt
+
+# # python test.py --test_metric privacy_loss_attack --mia_sample_size 1000 --batch_size 64 --dataset chestxray --method_name irm --match_case 0.01 --penalty_s 5 --penalty_ws 10.0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/privacy_loss/irm.txt
+
+# # python test.py --test_metric privacy_loss_attack --mia_sample_size 1000 --batch_size 64 --dataset chestxray --method_name matchdg_erm --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name densenet121 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --penalty_ws 50.0 --match_case -1 --train_domains nih_trans chex_trans --test_domains kaggle  > results/chestxray/privacy_loss/matchdg.txt
+
+#Attribute Attack
+
+# python test.py --test_metric attribute_attack --mia_logit 1 --batch_size 64 --attribute_domain 1 --dataset chestxray --method_name erm_match --match_case 0.01 --penalty_ws 0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/attribute_attack/erm.txt
+
+# python test.py --test_metric attribute_attack --mia_logit 1 --batch_size 64 --attribute_domain 1 --dataset chestxray --method_name erm_match --match_case 0.01 --penalty_ws 10.0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/attribute_attack/random.txt
+
+# python test.py --test_metric attribute_attack --mia_logit 1 --batch_size 64 --attribute_domain 1 --dataset chestxray --method_name csd --match_case 0.01 --penalty_ws 0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/attribute_attack/csd.txt
+
+# python test.py --test_metric attribute_attack --mia_logit 1 --batch_size 64 --attribute_domain 1 --dataset chestxray --method_name irm --match_case 0.01 --penalty_s 5 --penalty_ws 10.0 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/attribute_attack/irm.txt 
+
+# python test.py --test_metric attribute_attack --mia_logit 1 --batch_size 64 --attribute_domain 1 --dataset chestxray --method_name matchdg_erm --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name densenet121 --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --penalty_ws 50.0 --match_case -1 --train_domains nih_trans chex_trans --test_domains kaggle > results/chestxray/attribute_attack/matchdg.txt

misc_scripts/privacy_mnist.sh

@@ -0,0 +1,39 @@
+#Rot MNIST
+# python3 mnist_run.py rot_mnist train_all acc train
+# python3 mnist_run.py rot_mnist train_all acc test
+# python3 mnist_run.py rot_mnist train_all privacy_loss_attack
+# python3 mnist_run.py rot_mnist train_all privacy_entropy
+# python3 mnist_run.py rot_mnist train_all match_score test
+
+# python3 mnist_run.py rot_mnist train_abl_3 acc train
+# python3 mnist_run.py rot_mnist train_abl_3 acc test
+# python3 mnist_run.py rot_mnist train_abl_3 privacy_loss_attack
+# python3 mnist_run.py rot_mnist train_abl_3 privacy_entropy
+# python3 mnist_run.py rot_mnist train_abl_3 match_score test
+
+# python3 mnist_run.py rot_mnist train_abl_2 acc train
+# python3 mnist_run.py rot_mnist train_abl_2 acc test
+# python3 mnist_run.py rot_mnist train_abl_2 privacy_loss_attack
+# python3 mnist_run.py rot_mnist train_abl_2 privacy_entropy
+# python3 mnist_run.py rot_mnist train_abl_2 match_score test
+
+
+#Fashion MNIST
+# python3 mnist_run.py fashion_mnist train_all acc train
+# python3 mnist_run.py fashion_mnist train_all acc test
+python3 mnist_run.py fashion_mnist train_all privacy_loss_attack
+# python3 mnist_run.py fashion_mnist train_all privacy_entropy
+# python3 mnist_run.py fashion_mnist train_all match_score test
+
+# python3 mnist_run.py fashion_mnist train_abl_3 acc train
+# python3 mnist_run.py fashion_mnist train_abl_3 acc test
+python3 mnist_run.py fashion_mnist train_abl_3 privacy_loss_attack
+# python3 mnist_run.py fashion_mnist train_abl_3 privacy_entropy
+# python3 mnist_run.py fashion_mnist train_abl_3 match_score test
+
+# python3 mnist_run.py fashion_mnist train_abl_2 acc train
+# python3 mnist_run.py fashion_mnist train_abl_2 acc test
+python3 mnist_run.py fashion_mnist train_abl_2 privacy_loss_attack
+# python3 mnist_run.py fashion_mnist train_abl_2 privacy_entropy
+# python3 mnist_run.py fashion_mnist train_abl_2 match_score test
+

misc_scripts/privacy_pacs.sh

@@ -0,0 +1,24 @@
+# python  test.py --test_metric privacy_entropy --mia_sample_size 1000 --batch_size 64 --dataset pacs  --method_name erm_match --match_case 0.01 --penalty_ws 0 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --train_domains art_painting cartoon sketch --test_domains photo > results/pacs/entropy/photo_erm.txt
+
+# python  test.py --test_metric privacy_entropy --mia_sample_size 1000 --batch_size 64 --dataset pacs  --method_name erm_match --match_case 0.01 --penalty_ws 0 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --train_domains photo cartoon sketch --test_domains art_painting > results/pacs/entropy/painting_erm.txt
+
+# python  test.py --test_metric privacy_entropy --mia_sample_size 1000 --batch_size 64 --dataset pacs  --method_name erm_match --match_case 0.01 --penalty_ws 0 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --train_domains photo art_painting sketch --test_domains cartoon > results/pacs/entropy/cartoon_erm.txt
+
+# python  test.py --test_metric privacy_entropy --mia_sample_size 1000 --batch_size 64 --dataset pacs  --method_name erm_match --match_case 0.01 --penalty_ws 0 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --train_domains photo art_painting cartoon --test_domains sketch > results/pacs/entropy/sketch_erm.txt
+
+
+# python  test.py --test_metric privacy_entropy --mia_sample_size 1000 --batch_size 64 --dataset pacs  --method_name erm_match --match_case 0.01 --penalty_ws 0.1 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --train_domains art_painting cartoon sketch --test_domains photo > results/pacs/entropy/photo_random.txt
+
+# python  test.py --test_metric privacy_entropy --mia_sample_size 1000 --batch_size 64 --dataset pacs  --method_name erm_match --match_case 0.01 --penalty_ws 0.5 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --train_domains photo cartoon sketch --test_domains art_painting > results/pacs/entropy/painting_random.txt
+
+# python  test.py --test_metric privacy_entropy --mia_sample_size 1000 --batch_size 64 --dataset pacs  --method_name erm_match --match_case 0.01 --penalty_ws 0.1 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --train_domains photo art_painting sketch --test_domains cartoon > results/pacs/entropy/cartoon_random.txt
+
+# python  test.py --test_metric privacy_entropy --mia_sample_size 1000 --batch_size 64 --dataset pacs  --method_name erm_match --match_case 0.01 --penalty_ws 0.1 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --train_domains photo art_painting cartoon --test_domains sketch > results/pacs/entropy/sketch_random.txt
+
+python test.py --test_metric privacy_entropy --mia_sample_size 1000 --batch_size 64 --dataset pacs --method_name hybrid --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet18 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --penalty_ws 0.1 --match_case -1 --train_domains art_painting cartoon sketch --test_domains photo >  results/pacs/entropy/photo_hybrid.txt
+
+python test.py --test_metric privacy_entropy --mia_sample_size 1000 --batch_size 64 --dataset pacs --method_name hybrid --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet18 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --penalty_ws 0.01 --match_case -1 --train_domains photo cartoon sketch --test_domains art_painting > results/pacs/entropy/painting_hybrid.txt
+
+python test.py --test_metric privacy_entropy --mia_sample_size 1000 --batch_size 64 --dataset pacs --method_name hybrid --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet18 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --penalty_ws 0.01 --match_case -1 --train_domains photo art_painting sketch --test_domains cartoon > results/pacs/entropy/cartoon_hybrid.txt
+
+python test.py --test_metric privacy_entropy --mia_sample_size 1000 --batch_size 64 --dataset pacs --method_name hybrid --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet18 --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name resnet18 --penalty_ws 0.5 --match_case -1 --train_domains photo art_painting cartoon --test_domains sketch > results/pacs/entropy/sketch_hybrid.txt

misc_scripts/reproduce_paper.sh

@@ -0,0 +1,39 @@
+#RotMNIST
+python train.py --dataset rot_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.0 --epochs 25
+python train.py --dataset rot_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.1 --epochs 25
+python train.py --dataset rot_mnist --method_name erm_match --match_case 0.25 --penalty_ws 0.1 --epochs 25
+python train.py --dataset rot_mnist --method_name erm_match --match_case 0.50 --penalty_ws 0.1 --epochs 25
+python train.py --dataset rot_mnist --method_name erm_match --match_case 1.0 --penalty_ws 0.1 --epochs 25
+python train.py --dataset rot_mnist --method_name erm_match --match_case 0.75 --penalty_ws 0.1 --epochs 25
+python train.py --dataset rot_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.0 --epochs 25
+
+python train.py --dataset rot_mnist --method_name irm --match_case 0.01 --penalty_irm 1.0 --penalty_s 5 -- epochs 25
+python train.py --dataset rot_mnist --method_name csd --match_case 0.01 --penalty_ws 0.0 -- epochs 25
+
+# python train.py --dataset rot_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.0 --epochs 25 --train_domains 30 45
+# python train.py --dataset rot_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.1 --epochs 25 --train_domains 30 45
+# python train.py --dataset rot_mnist --method_name erm_match --match_case 0.25 --penalty_ws 0.1 --epochs 25 --train_domains 30 45
+# python train.py --dataset rot_mnist --method_name erm_match --match_case 0.50 --penalty_ws 0.1 --epochs 25 --train_domains 30 45
+# python train.py --dataset rot_mnist --method_name erm_match --match_case 0.75 --penalty_ws 0.1 --epochs 25 --train_domains 30 45
+# python train.py --dataset rot_mnist --method_name erm_match --match_case 1.0 --penalty_ws 0.1 --epochs 25 --train_domains 30 45
+# python train.py --dataset rot_mnist --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 30 --batch_size 64 --pos_metric cos  
+# python train.py --dataset rot_mnist --method_name matchdg_erm --penalty_ws 0.1 --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5  --n_runs 3 --epochs 25
+# python train.py --dataset rot_mnist --method_name csd --match_case 0.01 --penalty_ws 0.0 -- epochs 25 --train_domains 30 45
+# python train.py --dataset rot_mnist --method_name irm --match_case 0.01 --penalty_irm 1.0 --penalty_s 5 -- epochs 25 --train_domains 30 45
+# python train.py --dataset rot_mnist --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 30 --batch_size 64 --pos_metric cos  
+# python train.py --dataset rot_mnist --method_name matchdg_erm --penalty_ws 0.1 --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5  --n_runs 3 --epochs 25
+
+#FashionMNIST
+# python train.py --dataset fashion_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.0 --epochs 25
+# python train.py --dataset fashion_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.1 --epochs 25
+# python train.py --dataset fashion_mnist --method_name erm_match --match_case 0.25 --penalty_ws 0.1 --epochs 25
+# python train.py --dataset fashion_mnist --method_name erm_match --match_case 0.50 --penalty_ws 0.1 --epochs 25
+# python train.py --dataset fashion_mnist --method_name erm_match --match_case 0.75 --penalty_ws 0.1 --epochs 25
+# python train.py --dataset fashion_mnist --method_name erm_match --match_case 1.0 --penalty_ws 0.1 --epochs 25
+# python train.py --dataset fashion_mnist --method_name csd --match_case 0.01 --penalty_ws 0.0 -- epochs 25
+
+
+#ChestXRay
+
+# python train.py --dataset chestxray --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 3 --batch_size 32 --pos_metric cos --train_domains nih_trans chex_trans --test_domains kaggle_trans --out_classes 2 --perfect_match 0 --img_c 3 --pre_trained 1 --model_name densenet121 --n_runs 2
+

misc_scripts/run_domainbed.sh

@@ -0,0 +1,16 @@
+#ERM MatchDG
+python train.py --dataset rot_mnist --method_name erm_match --match_case 0.5 --penalty_ws 0.25 --model domain_bed_mnist --img_w 28 --img_h 28 --epochs 25 --train_domains 15 30 45 60 75 --test_domain 0
+
+python train.py --dataset rot_mnist --method_name erm_match --match_case 0.5 --penalty_ws 0.25 --model domain_bed_mnist --img_w 28 --img_h 28 --epochs 25 --train_domains 0 15 30 45 60 --test_domain 75
+
+#ERM RandMatch
+python train.py --dataset rot_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.25 --model domain_bed_mnist --img_w 28 --img_h 28 --epochs 25 --train_domains 15 30 45 60 75 --test_domain 0
+
+python train.py --dataset rot_mnist --method_name erm_match --match_case 0.01 --penalty_ws 0.25 --model domain_bed_mnist --img_w 28 --img_h 28 --epochs 25 --train_domains 0 15 30 45 60 --test_domain 75
+
+#ERM PerfMatch
+python train.py --dataset rot_mnist --method_name erm_match --match_case 1.0 --penalty_ws 0.25 --model domain_bed_mnist --img_w 28 --img_h 28 --epochs 25 --train_domains 15 30 45 60 75 --test_domain 0
+
+python train.py --dataset rot_mnist --method_name erm_match --match_case 1.0 --penalty_ws 0.25 --model domain_bed_mnist --img_w 28 --img_h 28 --epochs 25 --train_domains 0 15 30 45 60 --test_domain 75
+
+

misc_scripts/slab-run.sh

@@ -0,0 +1,112 @@
+# python train.py --dataset slab --model_name slab --method_name perf_match --batch_size 64 --lr 0.1 --penalty_ws 0.0 --epochs 30 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.50 --n_runs 3 > slab_temp/erm.txt
+
+# python  test.py --test_metric acc --dataset slab --model_name slab --method_name perf_match --penalty_ws 0.0 --out_classes 2 --train_domains 0.0 0.10 --n_runs 3  --acc_data_case train
+
+# python  test.py --test_metric slab_feat_eval --dataset slab --model_name slab --method_name perf_match --penalty_ws 0.0 --out_classes 2 --train_domains 0.0 0.10 --n_runs 3  --match_func_data_case train
+
+# python test_slab.py --method_name perf_match --penalty_ws 0.0 --n_runs 3 --train_domains 0.0 0.10 --test_domain 0.3
+
+# # python  test.py --test_metric mia --dataset slab --model_name slab --method_name perf_match --penalty_ws 0.0 --mia_logit 1 --mia_sample_size 400 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.50 --n_runs 3  > slab_temp/erm-mi.txt
+
+# # python  test.py --test_metric privacy_entropy --dataset slab --model_name slab --method_name perf_match --penalty_ws 0.0 --mia_sample_size 400 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.50 --n_runs 3  > slab_temp/erm-entropy.txt
+
+# # python  test.py --test_metric privacy_loss_attack --dataset slab --model_name slab --method_name perf_match --penalty_ws 0.0 --mia_sample_size 400 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.50 --n_runs 3  > slab_temp/erm-loss.txt
+
+
+
+# python train.py --dataset slab --model_name slab --method_name rand_match --batch_size 64 --lr 0.1 --penalty_ws 1.0 --epochs 30 --out_classes 2 --train_domains 0.0 0.10 --test_domains 1.0 --n_runs 3  > slab_temp/rand.txt
+
+# python  test.py --test_metric acc --dataset slab --model_name slab --method_name rand_match --penalty_ws 1.0 --out_classes 2 --train_domains 0.0 0.10 --n_runs 3  --acc_data_case train
+
+# python test_slab.py --method_name rand_match --penalty_ws 1.0 --n_runs 3 --train_domains 0.0 0.10 --test_domain 0.3
+
+# # python  test.py --test_metric mia --dataset slab --model_name slab --method_name rand_match --penalty_ws 10.0 --mia_logit 1 --mia_sample_size 400 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.50 --n_runs 3  > slab_temp/rand-mi.txt
+
+# # python  test.py --test_metric privacy_entropy --dataset slab --model_name slab --method_name rand_match --penalty_ws 10.0 --mia_sample_size 400 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.50 --n_runs 3  > slab_temp/rand-entropy.txt
+
+# # python  test.py --test_metric privacy_loss_attack --dataset slab --model_name slab --method_name rand_match --penalty_ws 10.0 --mia_sample_size 400 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.50 --n_runs 3  > slab_temp/rand-loss.txt
+
+
+
+# python train.py --dataset slab --model_name slab --method_name perf_match --batch_size 64 --lr 0.1 --penalty_ws 1.0 --epochs 30 --out_classes 2 --train_domains 0.0 0.10 --test_domains 1.0 --n_runs 3  > slab_temp/perf.txt
+
+# python  test.py --test_metric acc --dataset slab --model_name slab --method_name perf_match --penalty_ws 1.0 --out_classes 2 --train_domains 0.0 0.10 --n_runs 3  --acc_data_case train
+
+# python test_slab.py --method_name perf_match --penalty_ws 1.0 --n_runs 3 --train_domains 0.0 0.10 --test_domain 0.3
+
+# # python  test.py --test_metric mia --dataset slab --model_name slab --method_name perf_match --penalty_ws 10.0 --mia_logit 1 --mia_sample_size 400 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.50 --n_runs 3  > slab_temp/perf-mi.txt
+
+# # python  test.py --test_metric privacy_entropy --dataset slab --model_name slab --method_name perf_match --penalty_ws 10.0 --mia_sample_size 400 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.50 --n_runs 3  > slab_temp/perf-entropy.txt
+
+# # python  test.py --test_metric privacy_loss_attack --dataset slab --model_name slab --method_name perf_match --penalty_ws 10.0 --mia_sample_size 400 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.50 --n_runs 3
+
+
+
+# #IRM
+# python train.py --dataset slab --model_name slab --method_name irm_slab --batch_size 64 --lr 0.1 --penalty_irm 10.0 --penalty_s 2 --epochs 30 --out_classes 2 --train_domains 0.0 0.10 --test_domains 1.0 --n_runs 3 > slab_temp/irm.txt
+
+
+# #CSD
+# python train.py --dataset slab --model_name slab --method_name csd_slab --batch_size 64 --lr 0.1 --penalty_ws 0.0 --epochs 30 --out_classes 2 --train_domains 0.0 0.10 --test_domains 1.0 --n_runs 3 --rep_dim 100  > slab_temp/csd.txt
+
+
+
+
+
+
+# ## Slab Spur
+# python train.py --dataset slab_spur --model_name slab --method_name perf_match --batch_size 64 --lr 0.1 --penalty_ws 0.0 --epochs 30 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.90 --n_runs 3 --slab_data_dim 3
+
+# python train.py --dataset slab_spur --model_name slab --method_name perf_match --batch_size 64 --lr 0.1 --penalty_ws 50.0 --epochs 30 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.90 --n_runs 3 --slab_data_dim 3
+
+# python train.py --dataset slab_spur --model_name slab --method_name rand_match --batch_size 64 --lr 0.1 --penalty_ws 50.0 --epochs 30 --out_classes 2 --train_domains 0.0 0.10 --test_domains 1.0 --n_runs 3  --slab_data_dim 3
+
+# python train.py --dataset slab_spur --model_name slab --method_name irm_slab --batch_size 64 --lr 0.1 --penalty_irm 10.0 --penalty_s 2 --epochs 30 --out_classes 2 --train_domains 0.0 0.10 --test_domains 1.0 --n_runs 3  --slab_data_dim 3 
+
+# python train.py --dataset slab_spur --model_name slab --method_name csd_slab --batch_size 64 --lr 0.1 --penalty_ws 0.0 --epochs 30 --out_classes 2 --train_domains 0.0 0.10 --test_domains 1.0 --n_runs 3 --rep_dim 100  --slab_data_dim 3 
+
+
+# # Train Case
+# python  test.py --test_metric acc --dataset slab_spur --model_name slab --method_name perf_match --penalty_ws 50.0 --out_classes 2 --train_domains 0.0 0.10 --n_runs 3  --acc_data_case train --slab_data_dim 3
+
+# # #Attribute Attack
+# # python test.py --test_metric attribute_attack --mia_logit 1 --attribute_domain 0 --batch_size 64 --dataset slab_spur --model_name slab --method_name perf_match --penalty_ws 0.0 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.90 --n_runs 3 --slab_data_dim 3 
+
+# python test.py --test_metric attribute_attack --mia_logit 1 --attribute_domain 0 --batch_size 64 --dataset slab_spur --model_name slab --method_name perf_match --penalty_ws 0.0 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.90 --n_runs 3 --slab_data_dim 3
+
+# python test.py --test_metric attribute_attack --mia_logit 1 --attribute_domain 0 --batch_size 64 --dataset slab_spur --model_name slab --method_name perf_match --penalty_ws 50.0 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.90 --n_runs 3 --slab_data_dim 3
+
+
+# python3 slab-run.py train
+# python3 slab-run.py test
+# python3 slab-run.py train_plot
+
+python test_slab.py --method_name perf_match --penalty_ws 0.0 --n_runs 3 --train_domains 0.0 0.10 --test_domain 0.90 --slab_noise 0.05 > slab_noise/erm_auc.txt
+
+python test_slab.py --method_name perf_match --penalty_ws 1.0 --n_runs 3 --train_domains 0.0 0.10 --test_domain 0.90 --slab_noise 0.05 > slab_noise/perf_auc.txt
+
+python  test.py --test_metric privacy_entropy --dataset slab --model_name slab --method_name perf_match --penalty_ws 0.0 --mia_sample_size 400 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.90 --n_runs 3 --slab_noise 0.05 > slab_noise/erm_entropy.txt
+
+python  test.py --test_metric privacy_entropy --dataset slab --model_name slab --method_name perf_match --penalty_ws 1.0 --mia_sample_size 400 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.90 --n_runs 3 --slab_noise 0.05 > slab_noise/perf_entropy.txt
+
+python  test.py --test_metric privacy_loss_attack --dataset slab --model_name slab --method_name perf_match --penalty_ws 0.0 --mia_sample_size 400 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.90 --n_runs 3 --slab_noise 0.05 > slab_noise/erm_loss.txt
+
+python  test.py --test_metric privacy_loss_attack --dataset slab --model_name slab --method_name perf_match --penalty_ws 1.0 --mia_sample_size 400 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.90 --n_runs 3 --slab_noise 0.05 > slab_noise/perf_loss.txt
+
+# python  test.py --test_metric mia --dataset slab --model_name slab --method_name perf_match --penalty_ws 0.0 --mia_logit 1 --mia_sample_size 400 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.15 --n_runs 3 --slab_noise 0.05 > slab_noise/erm_mi.txt
+
+# python  test.py --test_metric mia --dataset slab --model_name slab --method_name perf_match --penalty_ws 1.0 --mia_logit 1 --mia_sample_size 400 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.15 --n_runs 3 --slab_noise 0.05 > slab_noise/perf_mi.txt
+
+
+
+python train.py --dataset slab --model_name slab --method_name perf_match --batch_size 64 --lr 0.1 --penalty_ws 0.0 --epochs 30 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.90 --n_runs 3 --slab_noise 0.20    
+
+python train.py --dataset slab --model_name slab --method_name mask_linear --batch_size 64 --lr 0.1 --penalty_ws 0.0 --epochs 30 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.90 --n_runs 3 --slab_noise 0.20
+
+python test_slab.py --method_name perf_match --penalty_ws 0.0 --n_runs 3 --train_domains 0.0 0.10 --test_domain 0.30 --slab_noise 0.20
+
+python test_slab.py --method_name mask_linear --penalty_ws 0.0 --n_runs 3 --train_domains 0.0 0.10 --test_domain 0.3 --slab_noise 0.20
+
+python  test.py --test_metric mia --dataset slab --model_name slab --method_name perf_match --penalty_ws 0.0 --mia_logit 1 --mia_sample_size 400 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.30 --n_runs 3 --slab_noise 0.20
+
+ python  test.py --test_metric mia --dataset slab --model_name slab --method_name mask_linear --penalty_ws 0.0 --mia_logit 1 --mia_sample_size 400 --out_classes 2 --train_domains 0.0 0.10 --test_domains 0.30 --n_runs 3 --slab_noise 0.20

misc_scripts/t3.sh

@@ -0,0 +1,3 @@
+python3 cxray_run.py kaggle train 
+python3 cxray_run.py chex train 
+python3 cxray_run.py nih train

misc_scripts/temp.sh

@@ -0,0 +1,39 @@
+# # python3 metric_eval.py fashion_mnist mia
+# # python3 metric_eval.py rot_mnist mia
+
+# # python train.py --dataset pacs --method_name hybrid --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --n_runs 3 --train_domains photo cartoon sketch --test_domains art_painting --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --lr 0.001 --batch_size 16 --weight_decay 0.001 --penalty_ws 0.1 --penalty_aug 0.1 --model_name resnet18 --ctr_model_name resnet18
+
+# #python train.py --dataset pacs --method_name hybrid --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --n_runs 3 --train_domains photo art_painting cartoon --test_domains sketch --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --lr 0.01 --batch_size 16 --weight_decay 0.001 --penalty_ws 0.1 --penalty_aug 0.1 --model_name resnet18 --ctr_model_name resnet18
+
+# #python train.py --dataset pacs --method_name hybrid --match_case -1 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --n_runs 3 --train_domains photo cartoon sketch --test_domains art_painting --out_classes 7 --perfect_match 0 --img_c 3 --pre_trained 1 --epochs 50 --lr 0.001 --batch_size 16 --weight_decay 0.001 --penalty_ws 0.1 --penalty_aug 0.1 --model_name resnet18 --ctr_model_name resnet18
+
+
+# #python3 metric_eval.py fashion_mnist acc test
+# #python3 metric_eval.py fashion_mnist match_score test
+# #python3 metric_eval.py fashion_mnist mia
+# python3 metric_eval.py fashion_mnist privacy_entropy
+# #python3 metric_eval.py fashion_mnist attribute_attack
+# #python3 metric_eval.py fashion_mnist privacy_loss_attack
+
+
+# python train.py --dataset rot_mnist --method_name erm_match --match_case 0.0 --penalty_ws 0.0 --epochs 25
+# python train.py --dataset rot_mnist --method_name erm_match --match_case 0.0 --penalty_ws 0.1 --epochs 25
+# python train.py --dataset rot_mnist --method_name erm_match --match_case 0.25 --penalty_ws 0.1 --epochs 25
+# python train.py --dataset rot_mnist --method_name erm_match --match_case 0.50 --penalty_ws 0.1 --epochs 25
+# python train.py --dataset rot_mnist --method_name erm_match --match_case 0.75 --penalty_ws 0.1 --epochs 25
+# python train.py --dataset rot_mnist --method_name erm_match --match_case 1.0 --penalty_ws 0.1 --epochs 25
+
+# python train.py --dataset fashion_mnist --method_name erm_match --match_case 0.0 --penalty_ws 0.0 --epochs 25
+# python train.py --dataset fashion_mnist --method_name erm_match --match_case 0.0 --penalty_ws 0.1 --epochs 25
+# python train.py --dataset fashion_mnist --method_name erm_match --match_case 0.25 --penalty_ws 0.1 --epochs 25
+# python train.py --dataset fashion_mnist --method_name erm_match --match_case 0.50 --penalty_ws 0.1 --epochs 25
+# python train.py --dataset fashion_mnist --method_name erm_match --match_case 0.75 --penalty_ws 0.1 --epochs 25
+# python train.py --dataset fashion_mnist --method_name erm_match --match_case 1.0 --penalty_ws 0.1 --epochs 25
+
+#python train.py --dataset rot_mnist --method_name irm --match_case 0.0 --penalty_irm 1.0 --penalty_s 5 --epochs 25
+
+#python train.py --dataset rot_mnist --method_name csd --match_case 0.0 --penalty_ws 0.0 --epochs 25 --rep_dim 512
+
+python train.py --dataset rot_mnist --method_name matchdg_ctr --match_case 0.0 --match_flag 1 --epochs 50 --batch_size 64 --pos_metric cos
+
+python train.py --dataset rot_mnist --method_name matchdg_erm --match_case -1 --penalty_ws 0.1 --epochs 25 --ctr_match_case 0.0 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet18

misc_scripts/temp2.sh

@@ -0,0 +1,19 @@
+# #MatchDG CTR Phase: RotMNIST
+# python train.py --dataset rot_mnist --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 30 --batch_size 64 --pos_metric cos
+
+# python train.py --dataset rot_mnist --method_name matchdg_ctr --match_case 1.0 --match_flag 1 --epochs 30 --batch_size 64 --pos_metric cos
+
+# python train.py --dataset rot_mnist --method_name matchdg_ctr --match_case 0.01 --match_flag 0 --epochs 30 --batch_size 64 --pos_metric cos
+
+# #MatchDG CTR Phase: FashionMNIST
+# python train.py --dataset fashion_mnist --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 30 --batch_size 64 --pos_metric cos
+
+# python train.py --dataset fashion_mnist --method_name matchdg_ctr --match_case 1.0 --match_flag 1 --epochs 30 --batch_size 64 --pos_metric cos
+
+# python train.py --dataset fashion_mnist --method_name matchdg_ctr --match_case 0.01 --match_flag 0 --epochs 30 --batch_size 64 --pos_metric cos
+
+
+
+python train.py --dataset rot_mnist --method_name matchdg_ctr --match_case 0.01 --match_flag 1 --epochs 100 --batch_size 128 --pos_metric cos
+
+python train.py --dataset rot_mnist --method_name matchdg_erm --match_case -1 --penalty_ws 0.1 --epochs 25 --ctr_match_case 0.01 --ctr_match_flag 1 --ctr_match_interrupt 5 --ctr_model_name resnet18