From 563d0a744f1591826a555b1dc1381ee3359d3dec Mon Sep 17 00:00:00 2001 From: Debadeepta Dey Date: Wed, 1 Sep 2021 20:28:19 -0700 Subject: [PATCH] Updated experimental results. --- .vscode/launch.json | 6 +- confs/algos/proxynas_natsbench_space.yaml | 2 +- run_all_ft_analysis.bat | 55 +++++++++++++++++++ .../analysis_freeze_natsbench_space_new.py | 18 +++++- .../analysis_random_search_darts_reg.py | 2 +- .../reports/fear_plots/cross_exp_conf.yaml | 47 ++++++++++++++-- .../fear_plots/cross_random_search.yaml | 1 + 7 files changed, 121 insertions(+), 10 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index eff7358f..2c636ff9 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -236,7 +236,7 @@ "request": "launch", "program": "${cwd}/scripts/main.py", "console": "integratedTerminal", - "args": ["--full", "--algos", "proxynas_natsbench_space", "--datasets", "cifar100"], + "args": ["--full", "--algos", "proxynas_natsbench_space", "--datasets", "cifar10"], "justMyCode": true }, { @@ -718,7 +718,7 @@ "request": "launch", "program": "${cwd}/scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py", "console": "integratedTerminal", - "args": ["--results-dir", "F:\\archaiphilly\\phillytools\\ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2_nofreeze", + "args": ["--results-dir", "F:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_scu", "--out-dir", "F:\\archai_experiment_reports"] }, { @@ -772,7 +772,7 @@ "request": "launch", "program": "${cwd}/scripts/reports/fear_plots/cross_exp_plots.py", "console": "integratedTerminal", - "args": ["--dataset", "natsbench_sss_cifar10", + "args": ["--dataset", "natsbench_imagenet16-120", "--conf-location", "scripts/reports/fear_plots/cross_exp_conf.yaml"] }, { diff --git a/confs/algos/proxynas_natsbench_space.yaml b/confs/algos/proxynas_natsbench_space.yaml index 5a6751f5..66d9114c 100644 --- a/confs/algos/proxynas_natsbench_space.yaml +++ b/confs/algos/proxynas_natsbench_space.yaml @@ -23,7 +23,7 @@ nas: trainer: plotsdir: '' use_val: False - top1_acc_threshold: 0.15 # after some accuracy we will shift into training only the last 'n' layers + top1_acc_threshold: 0.2 # after some accuracy we will shift into training only the last 'n' layers apex: _copy: '/common/apex' aux_weight: '_copy: /nas/eval/model_desc/aux_weight' diff --git a/run_all_ft_analysis.bat b/run_all_ft_analysis.bat index 54ee6e39..593f86d9 100644 --- a/run_all_ft_analysis.bat +++ b/run_all_ft_analysis.bat @@ -62,6 +62,26 @@ @REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_fb512_ftlr0.1_fte5_ct256_ftt0.6_scu --out-dir D:\\archai_experiment_reports @REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_fb512_ftlr0.1_fte10_ct256_ftt0.6_scu --out-dir D:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_classifier --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_c16 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_c15 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_c14 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_c12 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_c11 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_c10 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_c9 --out-dir F:\\archai_experiment_reports + +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte5_ct256_ftt0.55_c13 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte5_ct256_ftt0.50_c13 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte5_ct256_ftt0.45_c13 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte5_ct256_ftt0.40_c13 --out-dir F:\\archai_experiment_reports + +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte10_ct256_ftt0.6_c13 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte15_ct256_ftt0.6_c13 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte20_ct256_ftt0.6_c13 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte25_ct256_ftt0.6_c13 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte30_ct256_ftt0.6_c13 --out-dir F:\\archai_experiment_reports + @REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e01 --out-dir D:\\archai_experiment_reports @REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e02 --out-dir D:\\archai_experiment_reports @REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e04 --out-dir D:\\archai_experiment_reports @@ -118,6 +138,26 @@ @REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_c100_fb512_ftlr0.1_fte5_ct256_ftt0.3_scu --out-dir D:\\archai_experiment_reports @REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_c100_fb512_ftlr0.1_fte10_ct256_ftt0.3_scu --out-dir D:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_c100_fb512_ftlr0.1_fte5_ct256_ftt0.3_classifier --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_c100_fb512_ftlr0.1_fte5_ct256_ftt0.3_c16 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_c100_fb512_ftlr0.1_fte5_ct256_ftt0.3_c15 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_c100_fb512_ftlr0.1_fte5_ct256_ftt0.3_c14 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_c100_fb512_ftlr0.1_fte5_ct256_ftt0.3_c12 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_c100_fb512_ftlr0.1_fte5_ct256_ftt0.3_c11 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_c100_fb512_ftlr0.1_fte5_ct256_ftt0.3_c10 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_c100_fb512_ftlr0.1_fte5_ct256_ftt0.3_c9 --out-dir F:\\archai_experiment_reports + +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_c100_fb512_ftlr0.1_fte5_ct256_ftt0.25_c13 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_c100_fb512_ftlr0.1_fte5_ct256_ftt0.20_c13 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_c100_fb512_ftlr0.1_fte5_ct256_ftt0.15_c13 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_c100_fb512_ftlr0.1_fte5_ct256_ftt0.10_c13 --out-dir F:\\archai_experiment_reports + +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_c100_fb512_ftlr0.1_fte10_ct256_ftt0.3_c13 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_c100_fb512_ftlr0.1_fte15_ct256_ftt0.3_c13 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_c100_fb512_ftlr0.1_fte20_ct256_ftt0.3_c13 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_c100_fb512_ftlr0.1_fte25_ct256_ftt0.3_c13 --out-dir F:\\archai_experiment_reports +@REM python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_c100_fb512_ftlr0.1_fte30_ct256_ftt0.3_c13 --out-dir F:\\archai_experiment_reports + @REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_c100_reg_b256_e10 --out-dir D:\\archai_experiment_reports @REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_c100_reg_b256_e20 --out-dir D:\\archai_experiment_reports @REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_c100_reg_b256_e30 --out-dir D:\\archai_experiment_reports @@ -265,6 +305,21 @@ python scripts/reports/analysis_freeze_natsbench_space.py --results-dir D:\\arch @REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_i16_reg_b512_e20_scu --out-dir D:\\archai_experiment_reports @REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_i16_reg_b512_e30_scu --out-dir D:\\archai_experiment_reports +python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2_classifier --out-dir F:\\archai_experiment_reports +python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2_c16 --out-dir F:\\archai_experiment_reports +python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2_c12 --out-dir F:\\archai_experiment_reports +python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2_c11 --out-dir F:\\archai_experiment_reports +python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2_c10 --out-dir F:\\archai_experiment_reports +python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2_c9 --out-dir F:\\archai_experiment_reports + +python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.25_c13 --out-dir F:\\archai_experiment_reports +python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.15_c13 --out-dir F:\\archai_experiment_reports +python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.10_c13 --out-dir F:\\archai_experiment_reports + +python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_i6_fb512_ftlr0.1_fte15_ct256_ftt0.2_c13 --out-dir F:\\archai_experiment_reports +python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_i6_fb512_ftlr0.1_fte20_ct256_ftt0.2_c13 --out-dir F:\\archai_experiment_reports +python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_i6_fb512_ftlr0.1_fte25_ct256_ftt0.2_c13 --out-dir F:\\archai_experiment_reports +python scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py --results-dir F:\\archaiphilly\\phillytools\\ft_i6_fb512_ftlr0.1_fte30_ct256_ftt0.2_c13 --out-dir F:\\archai_experiment_reports @REM Flower 102 diff --git a/scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py b/scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py index cbf08123..04e4fa2b 100644 --- a/scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py +++ b/scripts/reports/fear_analysis/analysis_freeze_natsbench_space_new.py @@ -80,7 +80,7 @@ def main(): # a = parse_a_job(job_dir) # parallel parsing of yaml logs - num_workers = 8 + num_workers = 6 with Pool(num_workers) as p: a = p.map(parse_a_job, job_dirs) @@ -283,6 +283,22 @@ def main(): fig.show() + # scatter plot between time to threshold accuracy and fear evaluation + all_freeze_evals_last_scaled = [x*100.0 for x in all_freeze_evals_last] + fig = px.scatter(x=all_cond_time_last, y=all_freeze_evals_last_scaled, labels={'x': 'Time to reach threshold train accuracy (s)', 'y': 'FEAR Accuracy'}) + fig.update_layout(font=dict( + size=48, + )) + + savename = os.path.join(out_dir, 'cond_time_vs_fear_acc.html') + fig.write_html(savename) + + savename_pdf = os.path.join(out_dir, 'cond_time_vs_fear_acc.pdf') + fig.write_image(savename_pdf, engine="kaleido", width=1500, height=1500, scale=1) + + fig.show() + + # histogram of training accuracies fig = px.histogram(all_reg_evals, labels={'x': 'Test Accuracy', 'y': 'Counts'}) savename = os.path.join(out_dir, 'distribution_of_reg_evals.html') diff --git a/scripts/reports/fear_analysis/analysis_random_search_darts_reg.py b/scripts/reports/fear_analysis/analysis_random_search_darts_reg.py index 49e869b1..43f687c2 100644 --- a/scripts/reports/fear_analysis/analysis_random_search_darts_reg.py +++ b/scripts/reports/fear_analysis/analysis_random_search_darts_reg.py @@ -68,7 +68,7 @@ def main(): # a = parse_a_job(job_dir) # parallel parsing of yaml logs - num_workers = 8 + num_workers = 6 with Pool(num_workers) as p: a = p.map(parse_a_job, job_dirs) diff --git a/scripts/reports/fear_plots/cross_exp_conf.yaml b/scripts/reports/fear_plots/cross_exp_conf.yaml index 225b2884..8c0b6657 100644 --- a/scripts/reports/fear_plots/cross_exp_conf.yaml +++ b/scripts/reports/fear_plots/cross_exp_conf.yaml @@ -46,7 +46,6 @@ natsbench_cifar10: # ft_fb256_ftlr1.5_fte10_ct256_ftt0.6: 'fear stage 2: batch 256, lr 1.5,
epochs 10, stage 1: batch 256, thresh 0.6' # ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6: 'fear stage 2: batch 1024, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.6' # ft_fb1024_ftlr0.1_fte10_ct256_ftt0.6: 'fear stage 2: batch 1024, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.6' - ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_scu: 'fear stage 2: batch 1024, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.6' #ft_fb1024_ftlr0.1_fte10_ct256_ftt0.6_scu: 'fear stage 2: batch 1024, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.6' # ft_fb512_ftlr0.1_fte5_ct256_ftt0.6: 'fear stage 2: batch 512, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.6' # ft_fb512_ftlr0.1_fte10_ct256_ftt0.6: 'fear stage 2: batch 512, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.6' @@ -60,7 +59,28 @@ natsbench_cifar10: # ft_fb512_ftlr0.1_fte10_ct256_ftt0.6_c9: 'fear stage 2: batch 512, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.6, frozen till: cell 9' # ft_fb256_ftlr0.1_fte5_ct256_ftt0.6_c9: 'fear stage 2: batch 256, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.6, frozen till: cell 9' # ft_fb256_ftlr0.1_fte10_ct256_ftt0.6_c9: 'fear stage 2: batch 256, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.6, frozen till: cell 9' - ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_nofreeze: 'ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_nofreeze' + # ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_nofreeze: 'fear stage 2: nofreeze, batch 1024, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.6' + # ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_classifier: 'fear stage 2: last layer, batch 1024, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.6' + # ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_c16: 'fear stage 2: c16 onwards, batch 1024, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.6' + # ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_c15: 'fear stage 2: c15 onwards, batch 1024, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.6' + ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_scu: 'fear stage 2: c13 onwards, batch 1024, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.6' + # ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_c14: 'fear stage 2: c14 onwards, batch 1024, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.6' + # ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_c12: 'fear stage 2: c12 onwards, batch 1024, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.6' + # ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_c11: 'fear stage 2: c11 onwards, batch 1024, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.6' + # ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_c10: 'fear stage 2: c10 onwards, batch 1024, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.6' + # ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_c9: 'fear stage 2: c9 onwards, batch 1024, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.6' + # ft_fb1024_ftlr0.1_fte5_ct256_ftt0.55_c13: 'fear stage 2: c13 onwards, batch 1024, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.55' + # ft_fb1024_ftlr0.1_fte5_ct256_ftt0.50_c13: 'fear stage 2: c13 onwards, batch 1024, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.50' + # ft_fb1024_ftlr0.1_fte5_ct256_ftt0.45_c13: 'fear stage 2: c13 onwards, batch 1024, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.45' + # ft_fb1024_ftlr0.1_fte5_ct256_ftt0.40_c13: 'fear stage 2: c13 onwards, batch 1024, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.40' + ft_fb1024_ftlr0.1_fte10_ct256_ftt0.6_c13: 'fear stage 2: c13 onwards, batch 1024, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.6' + ft_fb1024_ftlr0.1_fte15_ct256_ftt0.6_c13: 'fear stage 2: c13 onwards, batch 1024, lr 0.1,
epochs 15, stage 1: batch 256, thresh 0.6' + ft_fb1024_ftlr0.1_fte20_ct256_ftt0.6_c13: 'fear stage 2: c13 onwards, batch 1024, lr 0.1,
epochs 20, stage 1: batch 256, thresh 0.6' + ft_fb1024_ftlr0.1_fte25_ct256_ftt0.6_c13: 'fear stage 2: c13 onwards, batch 1024, lr 0.1,
epochs 25, stage 1: batch 256, thresh 0.6' + ft_fb1024_ftlr0.1_fte30_ct256_ftt0.6_c13: 'fear stage 2: c13 onwards, batch 1024, lr 0.1,
epochs 30, stage 1: batch 256, thresh 0.6' + + + zero_cost: @@ -168,7 +188,6 @@ natsbench_imagenet16-120: # ft_i6_fb512_ftlr0.1_fte5_ct256_ftt0.2: 'fear stage 2: batch 512, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.2' # ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2: 'fear stage 2: batch 512, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.2' # ft_i6_fb512_ftlr0.1_fte5_ct256_ftt0.2_scu: 'fear stage 2: batch 512, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.2 scu' - ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2_scu: 'fear stage 2: batch 512, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.2' # ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2_c14: 'fear stage 2: batch 512, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.2 c14' # ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2_c15: 'fear stage 2: batch 512, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.2 c15' # ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2_c16: 'fear stage 2: batch 512, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.2 c16' @@ -209,7 +228,27 @@ natsbench_imagenet16-120: # ft_i6_fb2048_ftlr0.1_fte10_ct256_ftt0.1_scu: 'fear stage 2: batch 2048, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.1 val scu' # ft_i6_fb1024_ftlr0.1_fte5_ct256_ftt0.1_scu: 'fear stage 2: batch 1024, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.1 val scu' # ft_i6_fb1024_ftlr0.1_fte10_ct256_ftt0.1_scu: 'fear stage 2: batch 1024, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.1 val' - ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2_nofreeze: 'ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2_nofreeze' + # ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2_nofreeze: 'fear stage 2: nofreeze, batch 512, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.2' + # ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2_classifier: 'fear stage 2: last layer, batch 512, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.2' + # ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2_c16: 'fear stage 2: c16 onwards, batch 512, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.2' + # ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2_c15: 'fear stage 2: c15 onwards, batch 512, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.2' + # ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2_c14: 'fear stage 2: c14 onwards, batch 512, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.2' + ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2_scu: 'fear stage 2: c13 onwards, batch 512, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.2' + # ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2_c12: 'fear stage 2: c12 onwards, batch 512, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.2' + # ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2_c11: 'fear stage 2: c11 onwards, batch 512, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.2' + # ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2_c10: 'fear stage 2: c10 onwards, batch 512, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.2' + # ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.2_c9: 'fear stage 2: c9 onwards, batch 512, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.2' + # ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.25_c13: 'fear stage 2: c13 onwards, batch 512, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.25' + # ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.15_c13: 'fear stage 2: c13 onwards, batch 512, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.15' + # ft_i6_fb512_ftlr0.1_fte10_ct256_ftt0.10_c13: 'fear stage 2: c13 onwards, batch 512, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.10' + ft_i6_fb512_ftlr0.1_fte15_ct256_ftt0.2_c13: 'fear stage 2: c13 onwards, batch 512, lr 0.1,
epochs 15, stage 1: batch 256, thresh 0.2' + ft_i6_fb512_ftlr0.1_fte20_ct256_ftt0.2_c13: 'fear stage 2: c13 onwards, batch 512, lr 0.1,
epochs 20, stage 1: batch 256, thresh 0.2' + ft_i6_fb512_ftlr0.1_fte25_ct256_ftt0.2_c13: 'fear stage 2: c13 onwards, batch 512, lr 0.1,
epochs 25, stage 1: batch 256, thresh 0.2' + ft_i6_fb512_ftlr0.1_fte30_ct256_ftt0.2_c13: 'fear stage 2: c13 onwards, batch 512, lr 0.1,
epochs 30, stage 1: batch 256, thresh 0.2' + + + + zero_cost: zc_imagenet16-120: 'zero cost score at initialization' diff --git a/scripts/reports/fear_plots/cross_random_search.yaml b/scripts/reports/fear_plots/cross_random_search.yaml index 0870f19e..25b2eb97 100644 --- a/scripts/reports/fear_plots/cross_random_search.yaml +++ b/scripts/reports/fear_plots/cross_random_search.yaml @@ -10,6 +10,7 @@ darts_cifar10: rs_darts_reg_e5: 'rs_darts_reg_e5' rs_darts_reg_e10: 'rs_darts_reg_e10' rs_darts_reg_e15: 'rs_darts_reg_e15' + rs_darts_reg_e50: 'rs_darts_reg_e50' regular: rs_darts_far_ftt0.6_fte10_ratio_1.2: 'rs_darts_far_ftt0.6_fte10_ratio_1.2'