зеркало из https://github.com/microsoft/FLAML.git
notebook
This commit is contained in:
Anonymous-submission-repo
Родитель
4e37826417
Коммит
ff995d08d8
|
|
@ -0,0 +1,285 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Tune neural networks with leicographic preference across objectives\n",
|
||||
"This example is to tune neural networks model with two objectives \"error_rate\", \"flops\" on FashionMnist dataset. \n",
|
||||
"\n",
|
||||
"**Requirements.** This notebook requires:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# %pip install torch torchvision flaml[blendsearch,ray] thop"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"import thop\n",
|
||||
"import torch.nn as nn\n",
|
||||
"from flaml import tune\n",
|
||||
"import torch.nn.functional as F\n",
|
||||
"import torchvision\n",
|
||||
"import numpy as np\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"DEVICE = torch.device(\"cpu\")\n",
|
||||
"BATCHSIZE = 128\n",
|
||||
"N_TRAIN_EXAMPLES = BATCHSIZE * 30\n",
|
||||
"N_VALID_EXAMPLES = BATCHSIZE * 10\n",
|
||||
"data_dir = os.path.abspath(\"data\")\n",
|
||||
"\n",
|
||||
"train_dataset = torchvision.datasets.FashionMNIST(\n",
|
||||
" data_dir,\n",
|
||||
" train=True,\n",
|
||||
" download=True,\n",
|
||||
" transform=torchvision.transforms.ToTensor(),\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"train_loader = torch.utils.data.DataLoader(\n",
|
||||
" torch.utils.data.Subset(train_dataset, list(range(N_TRAIN_EXAMPLES))),\n",
|
||||
" batch_size=BATCHSIZE,\n",
|
||||
" shuffle=True,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"val_dataset = torchvision.datasets.FashionMNIST(\n",
|
||||
" data_dir, train=False, transform=torchvision.transforms.ToTensor()\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"val_loader = torch.utils.data.DataLoader(\n",
|
||||
" torch.utils.data.Subset(val_dataset, list(range(N_VALID_EXAMPLES))),\n",
|
||||
" batch_size=BATCHSIZE,\n",
|
||||
" shuffle=True,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Specify the model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def define_model(configuration):\n",
|
||||
" n_layers = configuration[\"n_layers\"]\n",
|
||||
" layers = []\n",
|
||||
" in_features = 28 * 28\n",
|
||||
" for i in range(n_layers):\n",
|
||||
" out_features = configuration[\"n_units_l{}\".format(i)]\n",
|
||||
" layers.append(nn.Linear(in_features, out_features))\n",
|
||||
" layers.append(nn.ReLU())\n",
|
||||
" p = configuration[\"dropout_{}\".format(i)]\n",
|
||||
" layers.append(nn.Dropout(p))\n",
|
||||
" in_features = out_features\n",
|
||||
" layers.append(nn.Linear(in_features, 10))\n",
|
||||
" layers.append(nn.LogSoftmax(dim=1))\n",
|
||||
" return nn.Sequential(*layers)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Train"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def train_model(model, optimizer, train_loader):\n",
|
||||
" model.train()\n",
|
||||
" for batch_idx, (data, target) in enumerate(train_loader):\n",
|
||||
" data, target = data.view(-1, 28 * 28).to(DEVICE), target.to(DEVICE)\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
" F.nll_loss(model(data), target).backward()\n",
|
||||
" optimizer.step()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Metrics "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def eval_model(model, valid_loader):\n",
|
||||
" model.eval()\n",
|
||||
" correct = 0\n",
|
||||
" with torch.no_grad():\n",
|
||||
" for batch_idx, (data, target) in enumerate(valid_loader):\n",
|
||||
" data, target = data.view(-1, 28 * 28).to(DEVICE), target.to(DEVICE)\n",
|
||||
" pred = model(data).argmax(dim=1, keepdim=True)\n",
|
||||
" correct += pred.eq(target.view_as(pred)).sum().item()\n",
|
||||
"\n",
|
||||
" accuracy = correct / N_VALID_EXAMPLES\n",
|
||||
" flops, params = thop.profile(\n",
|
||||
" model, inputs=(torch.randn(1, 28 * 28).to(DEVICE),), verbose=False\n",
|
||||
" )\n",
|
||||
" return np.log2(flops), 1 - accuracy, params"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Evaluate function"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def evaluate_function(configuration):\n",
|
||||
" model = define_model(configuration).to(DEVICE)\n",
|
||||
" optimizer = torch.optim.Adam(model.parameters(), configuration[\"lr\"])\n",
|
||||
" n_epoch = configuration[\"n_epoch\"]\n",
|
||||
" for epoch in range(n_epoch):\n",
|
||||
" train_model(model, optimizer, train_loader)\n",
|
||||
" flops, error_rate, params = eval_model(model, val_loader)\n",
|
||||
" return {\"error_rate\": error_rate, \"flops\": flops, \"params\": params}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Lexicographic information across objectives"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"lexico_objectives = {}\n",
|
||||
"lexico_objectives[\"metrics\"] = [\"error_rate\", \"flops\"]\n",
|
||||
"lexico_objectives[\"tolerances\"] = {\"error_rate\": 0.02, \"flops\": 0.0}\n",
|
||||
"lexico_objectives[\"targets\"] = {\"error_rate\": 0.0, \"flops\": 0.0}\n",
|
||||
"lexico_objectives[\"modes\"] = [\"min\", \"min\"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Search space"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"search_space = {\n",
|
||||
" \"n_layers\": tune.randint(lower=1, upper=3),\n",
|
||||
" \"n_units_l0\": tune.randint(lower=4, upper=128),\n",
|
||||
" \"n_units_l1\": tune.randint(lower=4, upper=128),\n",
|
||||
" \"n_units_l2\": tune.randint(lower=4, upper=128),\n",
|
||||
" \"dropout_0\": tune.uniform(lower=0.2, upper=0.5),\n",
|
||||
" \"dropout_1\": tune.uniform(lower=0.2, upper=0.5),\n",
|
||||
" \"dropout_2\": tune.uniform(lower=0.2, upper=0.5),\n",
|
||||
" \"lr\": tune.loguniform(lower=1e-5, upper=1e-1),\n",
|
||||
" \"n_epoch\": tune.randint(lower=1, upper=20),\n",
|
||||
"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Launch the tuning"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"low_cost_partial_config = {\n",
|
||||
" \"n_layers\": 1,\n",
|
||||
" \"n_units_l0\": 4,\n",
|
||||
" \"n_units_l1\": 4,\n",
|
||||
" \"n_units_l2\": 4,\n",
|
||||
" \"n_epoch\": 1,\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"analysis = tune.run(\n",
|
||||
" evaluate_function,\n",
|
||||
" num_samples=100000000,\n",
|
||||
" time_budget_s=100,\n",
|
||||
" config=search_space,\n",
|
||||
" use_ray=False,\n",
|
||||
" lexico_objectives=lexico_objectives,\n",
|
||||
" low_cost_partial_config=low_cost_partial_config,\n",
|
||||
")\n",
|
||||
"result = analysis.best_result\n",
|
||||
"print(result)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.9.14 64-bit",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.14"
|
||||
},
|
||||
"orig_nbformat": 4,
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "949777d72b0d2535278d3dc13498b2535136f6dfe0678499012e853ee9abcab1"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
Загрузка…
Ссылка в новой задаче