AML Notebooks: filtering dataframes and displaying them.

This commit is contained in:
Luis França 2023-03-31 09:42:07 -03:00
Родитель 324397f586
Коммит ddbb796ca9
3 изменённых файлов: 296 добавлений и 659 удалений

Просмотреть файл

@ -28,27 +28,6 @@ def get_search_csv(output_path: Union[str, Path], iteration_num: Optional[int] =
return df
def get_csv_as_stylized_html(df: pd.DataFrame) -> str:
"""Returns a stylized html table from a pandas dataframe with a scrollbar
Args:
df (pd.DataFrame): Pandas dataframe to convert to html table
Returns:
str: Stylized html table
"""
styled_table = df.style.set_properties(**{'background-color': 'lightblue',
'color': 'black',
'border-color': 'white',
'font-size': '12pt'
})
html_table = styled_table.to_html()
html_with_scrollbar = f'<div style="height: 300px; overflow-y: scroll;">{html_table}</div>'
return html_with_scrollbar
def get_arch_abs_path(archid: str, downloaded_folder: Union[str, Path], iteration_num: Optional[int] = -1) -> Path:
"""Returns the absolute path to the architecture file

Различия файлов скрыты, потому что одна или несколько строк слишком длинны

Просмотреть файл

@ -338,202 +338,127 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div style=\"height: 300px; overflow-y: scroll;\"><style type=\"text/css\">\n",
"#T_8a23d_row0_col0, #T_8a23d_row0_col1, #T_8a23d_row0_col2, #T_8a23d_row0_col3, #T_8a23d_row0_col4, #T_8a23d_row1_col0, #T_8a23d_row1_col1, #T_8a23d_row1_col2, #T_8a23d_row1_col3, #T_8a23d_row1_col4, #T_8a23d_row2_col0, #T_8a23d_row2_col1, #T_8a23d_row2_col2, #T_8a23d_row2_col3, #T_8a23d_row2_col4, #T_8a23d_row3_col0, #T_8a23d_row3_col1, #T_8a23d_row3_col2, #T_8a23d_row3_col3, #T_8a23d_row3_col4, #T_8a23d_row4_col0, #T_8a23d_row4_col1, #T_8a23d_row4_col2, #T_8a23d_row4_col3, #T_8a23d_row4_col4, #T_8a23d_row5_col0, #T_8a23d_row5_col1, #T_8a23d_row5_col2, #T_8a23d_row5_col3, #T_8a23d_row5_col4, #T_8a23d_row6_col0, #T_8a23d_row6_col1, #T_8a23d_row6_col2, #T_8a23d_row6_col3, #T_8a23d_row6_col4, #T_8a23d_row7_col0, #T_8a23d_row7_col1, #T_8a23d_row7_col2, #T_8a23d_row7_col3, #T_8a23d_row7_col4, #T_8a23d_row8_col0, #T_8a23d_row8_col1, #T_8a23d_row8_col2, #T_8a23d_row8_col3, #T_8a23d_row8_col4, #T_8a23d_row9_col0, #T_8a23d_row9_col1, #T_8a23d_row9_col2, #T_8a23d_row9_col3, #T_8a23d_row9_col4, #T_8a23d_row10_col0, #T_8a23d_row10_col1, #T_8a23d_row10_col2, #T_8a23d_row10_col3, #T_8a23d_row10_col4, #T_8a23d_row11_col0, #T_8a23d_row11_col1, #T_8a23d_row11_col2, #T_8a23d_row11_col3, #T_8a23d_row11_col4, #T_8a23d_row12_col0, #T_8a23d_row12_col1, #T_8a23d_row12_col2, #T_8a23d_row12_col3, #T_8a23d_row12_col4, #T_8a23d_row13_col0, #T_8a23d_row13_col1, #T_8a23d_row13_col2, #T_8a23d_row13_col3, #T_8a23d_row13_col4, #T_8a23d_row14_col0, #T_8a23d_row14_col1, #T_8a23d_row14_col2, #T_8a23d_row14_col3, #T_8a23d_row14_col4, #T_8a23d_row15_col0, #T_8a23d_row15_col1, #T_8a23d_row15_col2, #T_8a23d_row15_col3, #T_8a23d_row15_col4, #T_8a23d_row16_col0, #T_8a23d_row16_col1, #T_8a23d_row16_col2, #T_8a23d_row16_col3, #T_8a23d_row16_col4, #T_8a23d_row17_col0, #T_8a23d_row17_col1, #T_8a23d_row17_col2, #T_8a23d_row17_col3, #T_8a23d_row17_col4, #T_8a23d_row18_col0, #T_8a23d_row18_col1, #T_8a23d_row18_col2, #T_8a23d_row18_col3, #T_8a23d_row18_col4 {\n",
" background-color: lightblue;\n",
" color: black;\n",
" border-color: white;\n",
" font-size: 12pt;\n",
"}\n",
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table id=\"T_8a23d\">\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr>\n",
" <th class=\"blank level0\" >&nbsp;</th>\n",
" <th id=\"T_8a23d_level0_col0\" class=\"col_heading level0 col0\" >archid</th>\n",
" <th id=\"T_8a23d_level0_col1\" class=\"col_heading level0 col1\" >non_embedding_params</th>\n",
" <th id=\"T_8a23d_level0_col2\" class=\"col_heading level0 col2\" >onnx_latency</th>\n",
" <th id=\"T_8a23d_level0_col3\" class=\"col_heading level0 col3\" >onnx_memory</th>\n",
" <th id=\"T_8a23d_level0_col4\" class=\"col_heading level0 col4\" >is_pareto</th>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>archid</th>\n",
" <th>non_embedding_params</th>\n",
" <th>onnx_latency</th>\n",
" <th>onnx_memory</th>\n",
" <th>is_pareto</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th id=\"T_8a23d_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
" <td id=\"T_8a23d_row0_col0\" class=\"data row0 col0\" >gpt2_cea35b3f3fd242d2af609b8d2a3936cc814a7f41</td>\n",
" <td id=\"T_8a23d_row0_col1\" class=\"data row0 col1\" >15144192.000000</td>\n",
" <td id=\"T_8a23d_row0_col2\" class=\"data row0 col2\" >0.761083</td>\n",
" <td id=\"T_8a23d_row0_col3\" class=\"data row0 col3\" >246.531515</td>\n",
" <td id=\"T_8a23d_row0_col4\" class=\"data row0 col4\" >True</td>\n",
" <th>0</th>\n",
" <td>gpt2_cea35b3f3fd242d2af609b8d2a3936cc814a7f41</td>\n",
" <td>15144192.0</td>\n",
" <td>0.761083</td>\n",
" <td>246.531515</td>\n",
" <td>True</td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_8a23d_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
" <td id=\"T_8a23d_row1_col0\" class=\"data row1 col0\" >gpt2_df106863e1a0c9c140036b05661aa88e92f07701</td>\n",
" <td id=\"T_8a23d_row1_col1\" class=\"data row1 col1\" >7769472.000000</td>\n",
" <td id=\"T_8a23d_row1_col2\" class=\"data row1 col2\" >0.575391</td>\n",
" <td id=\"T_8a23d_row1_col3\" class=\"data row1 col3\" >143.315580</td>\n",
" <td id=\"T_8a23d_row1_col4\" class=\"data row1 col4\" >True</td>\n",
" <th>1</th>\n",
" <td>gpt2_df106863e1a0c9c140036b05661aa88e92f07701</td>\n",
" <td>7769472.0</td>\n",
" <td>0.575391</td>\n",
" <td>143.315580</td>\n",
" <td>True</td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_8a23d_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
" <td id=\"T_8a23d_row2_col0\" class=\"data row2 col0\" >gpt2_0f371e7b893319c0d20944d60143fad14e2695d0</td>\n",
" <td id=\"T_8a23d_row2_col1\" class=\"data row2 col1\" >14724096.000000</td>\n",
" <td id=\"T_8a23d_row2_col2\" class=\"data row2 col2\" >0.850879</td>\n",
" <td id=\"T_8a23d_row2_col3\" class=\"data row2 col3\" >157.321033</td>\n",
" <td id=\"T_8a23d_row2_col4\" class=\"data row2 col4\" >True</td>\n",
" <th>2</th>\n",
" <td>gpt2_0f371e7b893319c0d20944d60143fad14e2695d0</td>\n",
" <td>14724096.0</td>\n",
" <td>0.850879</td>\n",
" <td>157.321033</td>\n",
" <td>True</td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_8a23d_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
" <td id=\"T_8a23d_row3_col0\" class=\"data row3 col0\" >gpt2_889f69f6f6193194b0b78cf645bc44324928785e</td>\n",
" <td id=\"T_8a23d_row3_col1\" class=\"data row3 col1\" >31372992.000000</td>\n",
" <td id=\"T_8a23d_row3_col2\" class=\"data row3 col2\" >1.538318</td>\n",
" <td id=\"T_8a23d_row3_col3\" class=\"data row3 col3\" >295.865806</td>\n",
" <td id=\"T_8a23d_row3_col4\" class=\"data row3 col4\" >True</td>\n",
" <th>9</th>\n",
" <td>gpt2_85a39fce7fd60bf8df99d17bb91298a450e2b0b1</td>\n",
" <td>12424512.0</td>\n",
" <td>0.676473</td>\n",
" <td>148.552288</td>\n",
" <td>True</td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_8a23d_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
" <td id=\"T_8a23d_row4_col0\" class=\"data row4 col0\" >gpt2_e848f3b384f04fdd238e08a79fdd2fa74d8c5194</td>\n",
" <td id=\"T_8a23d_row4_col1\" class=\"data row4 col1\" >27327616.000000</td>\n",
" <td id=\"T_8a23d_row4_col2\" class=\"data row4 col2\" >1.434555</td>\n",
" <td id=\"T_8a23d_row4_col3\" class=\"data row4 col3\" >267.924564</td>\n",
" <td id=\"T_8a23d_row4_col4\" class=\"data row4 col4\" >False</td>\n",
" <th>10</th>\n",
" <td>gpt2_3748aa9c59880395ceabfba51d411bfa3ca198e8</td>\n",
" <td>5917568.0</td>\n",
" <td>0.376884</td>\n",
" <td>148.768559</td>\n",
" <td>True</td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_8a23d_level0_row5\" class=\"row_heading level0 row5\" >5</th>\n",
" <td id=\"T_8a23d_row5_col0\" class=\"data row5 col0\" >gpt2_ecd66243c69a1a3273be0599297e7f7ca4943938</td>\n",
" <td id=\"T_8a23d_row5_col1\" class=\"data row5 col1\" >20050048.000000</td>\n",
" <td id=\"T_8a23d_row5_col2\" class=\"data row5 col2\" >1.082883</td>\n",
" <td id=\"T_8a23d_row5_col3\" class=\"data row5 col3\" >215.160275</td>\n",
" <td id=\"T_8a23d_row5_col4\" class=\"data row5 col4\" >True</td>\n",
" <th>15</th>\n",
" <td>gpt2_c710d31b0c06ad032125e28d4303a05676902937</td>\n",
" <td>11966848.0</td>\n",
" <td>0.714716</td>\n",
" <td>134.300035</td>\n",
" <td>True</td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_8a23d_level0_row6\" class=\"row_heading level0 row6\" >6</th>\n",
" <td id=\"T_8a23d_row6_col0\" class=\"data row6 col0\" >gpt2_e446e91026f819b689775cba65accbaafd81c4ef</td>\n",
" <td id=\"T_8a23d_row6_col1\" class=\"data row6 col1\" >35854592.000000</td>\n",
" <td id=\"T_8a23d_row6_col2\" class=\"data row6 col2\" >1.613641</td>\n",
" <td id=\"T_8a23d_row6_col3\" class=\"data row6 col3\" >312.946812</td>\n",
" <td id=\"T_8a23d_row6_col4\" class=\"data row6 col4\" >True</td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_8a23d_level0_row7\" class=\"row_heading level0 row7\" >7</th>\n",
" <td id=\"T_8a23d_row7_col0\" class=\"data row7 col0\" >gpt2_6606054e0222dccc9ab29ff8254706f6e190fb5b</td>\n",
" <td id=\"T_8a23d_row7_col1\" class=\"data row7 col1\" >27327616.000000</td>\n",
" <td id=\"T_8a23d_row7_col2\" class=\"data row7 col2\" >1.417154</td>\n",
" <td id=\"T_8a23d_row7_col3\" class=\"data row7 col3\" >267.924564</td>\n",
" <td id=\"T_8a23d_row7_col4\" class=\"data row7 col4\" >True</td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_8a23d_level0_row8\" class=\"row_heading level0 row8\" >8</th>\n",
" <td id=\"T_8a23d_row8_col0\" class=\"data row8 col0\" >gpt2_2ff7eebbc32f2bb8554e2241b2f1eed495d7b2a4</td>\n",
" <td id=\"T_8a23d_row8_col1\" class=\"data row8 col1\" >12431680.000000</td>\n",
" <td id=\"T_8a23d_row8_col2\" class=\"data row8 col2\" >1.038724</td>\n",
" <td id=\"T_8a23d_row8_col3\" class=\"data row8 col3\" >148.587824</td>\n",
" <td id=\"T_8a23d_row8_col4\" class=\"data row8 col4\" >True</td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_8a23d_level0_row9\" class=\"row_heading level0 row9\" >9</th>\n",
" <td id=\"T_8a23d_row9_col0\" class=\"data row9 col0\" >gpt2_85a39fce7fd60bf8df99d17bb91298a450e2b0b1</td>\n",
" <td id=\"T_8a23d_row9_col1\" class=\"data row9 col1\" >12424512.000000</td>\n",
" <td id=\"T_8a23d_row9_col2\" class=\"data row9 col2\" >0.676473</td>\n",
" <td id=\"T_8a23d_row9_col3\" class=\"data row9 col3\" >148.552288</td>\n",
" <td id=\"T_8a23d_row9_col4\" class=\"data row9 col4\" >True</td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_8a23d_level0_row10\" class=\"row_heading level0 row10\" >10</th>\n",
" <td id=\"T_8a23d_row10_col0\" class=\"data row10 col0\" >gpt2_3748aa9c59880395ceabfba51d411bfa3ca198e8</td>\n",
" <td id=\"T_8a23d_row10_col1\" class=\"data row10 col1\" >5917568.000000</td>\n",
" <td id=\"T_8a23d_row10_col2\" class=\"data row10 col2\" >0.376884</td>\n",
" <td id=\"T_8a23d_row10_col3\" class=\"data row10 col3\" >148.768559</td>\n",
" <td id=\"T_8a23d_row10_col4\" class=\"data row10 col4\" >True</td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_8a23d_level0_row11\" class=\"row_heading level0 row11\" >11</th>\n",
" <td id=\"T_8a23d_row11_col0\" class=\"data row11 col0\" >gpt2_4170664dbb38d8c19458ae9c518d6ff62d134fe5</td>\n",
" <td id=\"T_8a23d_row11_col1\" class=\"data row11 col1\" >15144192.000000</td>\n",
" <td id=\"T_8a23d_row11_col2\" class=\"data row11 col2\" >0.825369</td>\n",
" <td id=\"T_8a23d_row11_col3\" class=\"data row11 col3\" >246.531515</td>\n",
" <td id=\"T_8a23d_row11_col4\" class=\"data row11 col4\" >False</td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_8a23d_level0_row12\" class=\"row_heading level0 row12\" >12</th>\n",
" <td id=\"T_8a23d_row12_col0\" class=\"data row12 col0\" >gpt2_4b37e60e4c0017177c7722fe3b4b08b4a6969ccb</td>\n",
" <td id=\"T_8a23d_row12_col1\" class=\"data row12 col1\" >29808768.000000</td>\n",
" <td id=\"T_8a23d_row12_col2\" class=\"data row12 col2\" >1.627816</td>\n",
" <td id=\"T_8a23d_row12_col3\" class=\"data row12 col3\" >252.355514</td>\n",
" <td id=\"T_8a23d_row12_col4\" class=\"data row12 col4\" >True</td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_8a23d_level0_row13\" class=\"row_heading level0 row13\" >13</th>\n",
" <td id=\"T_8a23d_row13_col0\" class=\"data row13 col0\" >gpt2_d042734419c3f3b4e04e2db9ca184f5db02ee2ba</td>\n",
" <td id=\"T_8a23d_row13_col1\" class=\"data row13 col1\" >40381312.000000</td>\n",
" <td id=\"T_8a23d_row13_col2\" class=\"data row13 col2\" >1.889455</td>\n",
" <td id=\"T_8a23d_row13_col3\" class=\"data row13 col3\" >342.717808</td>\n",
" <td id=\"T_8a23d_row13_col4\" class=\"data row13 col4\" >True</td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_8a23d_level0_row14\" class=\"row_heading level0 row14\" >14</th>\n",
" <td id=\"T_8a23d_row14_col0\" class=\"data row14 col0\" >gpt2_b0111ea37ec42adb5040daa60db0e0b11d164017</td>\n",
" <td id=\"T_8a23d_row14_col1\" class=\"data row14 col1\" >17080384.000000</td>\n",
" <td id=\"T_8a23d_row14_col2\" class=\"data row14 col2\" >0.905016</td>\n",
" <td id=\"T_8a23d_row14_col3\" class=\"data row14 col3\" >228.869057</td>\n",
" <td id=\"T_8a23d_row14_col4\" class=\"data row14 col4\" >True</td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_8a23d_level0_row15\" class=\"row_heading level0 row15\" >15</th>\n",
" <td id=\"T_8a23d_row15_col0\" class=\"data row15 col0\" >gpt2_c710d31b0c06ad032125e28d4303a05676902937</td>\n",
" <td id=\"T_8a23d_row15_col1\" class=\"data row15 col1\" >11966848.000000</td>\n",
" <td id=\"T_8a23d_row15_col2\" class=\"data row15 col2\" >0.714716</td>\n",
" <td id=\"T_8a23d_row15_col3\" class=\"data row15 col3\" >134.300035</td>\n",
" <td id=\"T_8a23d_row15_col4\" class=\"data row15 col4\" >True</td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_8a23d_level0_row16\" class=\"row_heading level0 row16\" >16</th>\n",
" <td id=\"T_8a23d_row16_col0\" class=\"data row16 col0\" >gpt2_3c57938472ca145510df49098166e3259e1b6509</td>\n",
" <td id=\"T_8a23d_row16_col1\" class=\"data row16 col1\" >23652864.000000</td>\n",
" <td id=\"T_8a23d_row16_col2\" class=\"data row16 col2\" >1.557708</td>\n",
" <td id=\"T_8a23d_row16_col3\" class=\"data row16 col3\" >241.403567</td>\n",
" <td id=\"T_8a23d_row16_col4\" class=\"data row16 col4\" >True</td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_8a23d_level0_row17\" class=\"row_heading level0 row17\" >17</th>\n",
" <td id=\"T_8a23d_row17_col0\" class=\"data row17 col0\" >gpt2_0ffcee3a558c89448aea22ff34c8dba77208072a</td>\n",
" <td id=\"T_8a23d_row17_col1\" class=\"data row17 col1\" >7769472.000000</td>\n",
" <td id=\"T_8a23d_row17_col2\" class=\"data row17 col2\" >0.592398</td>\n",
" <td id=\"T_8a23d_row17_col3\" class=\"data row17 col3\" >143.315580</td>\n",
" <td id=\"T_8a23d_row17_col4\" class=\"data row17 col4\" >False</td>\n",
" </tr>\n",
" <tr>\n",
" <th id=\"T_8a23d_level0_row18\" class=\"row_heading level0 row18\" >18</th>\n",
" <td id=\"T_8a23d_row18_col0\" class=\"data row18 col0\" >gpt2_1d573928a72694068c1af9608548862d8519c533</td>\n",
" <td id=\"T_8a23d_row18_col1\" class=\"data row18 col1\" >14199296.000000</td>\n",
" <td id=\"T_8a23d_row18_col2\" class=\"data row18 col2\" >0.851394</td>\n",
" <td id=\"T_8a23d_row18_col3\" class=\"data row18 col3\" >155.320788</td>\n",
" <td id=\"T_8a23d_row18_col4\" class=\"data row18 col4\" >True</td>\n",
" <th>18</th>\n",
" <td>gpt2_1d573928a72694068c1af9608548862d8519c533</td>\n",
" <td>14199296.0</td>\n",
" <td>0.851394</td>\n",
" <td>155.320788</td>\n",
" <td>True</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
" archid non_embedding_params \\\n",
"0 gpt2_cea35b3f3fd242d2af609b8d2a3936cc814a7f41 15144192.0 \n",
"1 gpt2_df106863e1a0c9c140036b05661aa88e92f07701 7769472.0 \n",
"2 gpt2_0f371e7b893319c0d20944d60143fad14e2695d0 14724096.0 \n",
"9 gpt2_85a39fce7fd60bf8df99d17bb91298a450e2b0b1 12424512.0 \n",
"10 gpt2_3748aa9c59880395ceabfba51d411bfa3ca198e8 5917568.0 \n",
"15 gpt2_c710d31b0c06ad032125e28d4303a05676902937 11966848.0 \n",
"18 gpt2_1d573928a72694068c1af9608548862d8519c533 14199296.0 \n",
"\n",
" onnx_latency onnx_memory is_pareto \n",
"0 0.761083 246.531515 True \n",
"1 0.575391 143.315580 True \n",
"2 0.850879 157.321033 True \n",
"9 0.676473 148.552288 True \n",
"10 0.376884 148.768559 True \n",
"15 0.714716 134.300035 True \n",
"18 0.851394 155.320788 True "
]
},
"execution_count": 5,
"metadata": {},
"output_type": "display_data"
"output_type": "execute_result"
}
],
"source": [
"df = nb_helper.get_search_csv(downloaded_folder)\n",
"df = df[['archid', 'non_embedding_params', 'onnx_latency', 'onnx_memory', 'is_pareto']]\n",
"\n",
"csv_as_html = nb_helper.get_csv_as_stylized_html(df)\n",
"display(HTML(csv_as_html))"
"df[(df['onnx_latency'] < 0.9) & (df['is_pareto'] == True)]"
]
},
{