зеркало из https://github.com/microsoft/EdgeML.git
Merge branch 'master' into harsha/pytorch
This commit is contained in:
Harsha Vardhan Simhadri
Коммит
bf71343ba8
|
|
@ -41,7 +41,7 @@ namespace EdgeML
|
|||
|
||||
//
|
||||
// ProtoNNModel includes hyperparameters, parameters, and some state on initialization
|
||||
//
|
||||
//
|
||||
class ProtoNNModel
|
||||
{
|
||||
|
||||
|
|
@ -71,6 +71,7 @@ namespace EdgeML
|
|||
|
||||
void setHyperParamsFromArgs(const int argc, const char** argv);
|
||||
void finalizeHyperParams();
|
||||
void exitWithHelp();
|
||||
|
||||
//
|
||||
// Create a string with hyperParam settings
|
||||
|
|
@ -115,7 +116,7 @@ namespace EdgeML
|
|||
class ProtoNNTrainer
|
||||
{
|
||||
////////////////////////////////////////////////////////
|
||||
// DO NOT REORDER model and data.
|
||||
// DO NOT REORDER model and data.
|
||||
// They should be in this order for constructors to work
|
||||
ProtoNNModel model;
|
||||
Data data;
|
||||
|
|
@ -236,7 +237,7 @@ namespace EdgeML
|
|||
void RBF();
|
||||
|
||||
void setFromArgs(const int argc, const char** argv);
|
||||
|
||||
|
||||
void createOutputDirs();
|
||||
|
||||
public:
|
||||
|
|
@ -253,7 +254,7 @@ namespace EdgeML
|
|||
|
||||
~ProtoNNPredictor();
|
||||
|
||||
// Not thread safe
|
||||
// Not thread safe
|
||||
FP_TYPE testDenseDataPoint(
|
||||
const FP_TYPE *const values,
|
||||
const labelCount_t *const labels,
|
||||
|
|
@ -277,13 +278,13 @@ namespace EdgeML
|
|||
dataCount_t batchSize);
|
||||
|
||||
ResultStruct testBatchWise();
|
||||
|
||||
|
||||
ResultStruct testPointWise();
|
||||
|
||||
ResultStruct test();
|
||||
|
||||
void saveTopKScores(std::string filename="", int topk=5);
|
||||
|
||||
|
||||
void normalize();
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ ProtoNNModel::ProtoNNHyperParams::ProtoNNHyperParams()
|
|||
|
||||
ntrain = 0;
|
||||
nvalidation = 0;
|
||||
|
||||
|
||||
iters = 20;
|
||||
epochs = 20;
|
||||
batchSize = 1024;
|
||||
|
|
@ -87,7 +87,7 @@ void ProtoNNModel::ProtoNNHyperParams::finalizeHyperParams()
|
|||
assert(epochs >= 1 && "number of epochs should be >= 1");
|
||||
// Following asserts removed to faciliate support for TLC
|
||||
// which does not know how many datapoints are going to be fed before-hand!
|
||||
// assert(ntrain >= 1);
|
||||
// assert(ntrain >= 1);
|
||||
// assert(nvalidation >= 0);
|
||||
// assert(m <= ntrain);
|
||||
if (d > D) {
|
||||
|
|
@ -112,7 +112,7 @@ void ProtoNNModel::ProtoNNHyperParams::setHyperParamsFromArgs(const int argc, co
|
|||
{
|
||||
for (int i = 1; i < argc; ++i) {
|
||||
if (i % 2 == 1)
|
||||
assert(argv[i][0] == '-'); //odd arguments must be specifiers, not values
|
||||
assert(argv[i][0] == '-'); //odd arguments must be specifiers, not values
|
||||
else {
|
||||
|
||||
switch (argv[i - 1][1]) {
|
||||
|
|
@ -143,7 +143,7 @@ void ProtoNNModel::ProtoNNHyperParams::setHyperParamsFromArgs(const int argc, co
|
|||
if (argv[i][0] == '0') problemType = binary;
|
||||
else if (argv[i][0] == '1') problemType = multiclass;
|
||||
else if (argv[i][0] == '2') problemType = multilabel;
|
||||
else assert(false); //Problem type unknown
|
||||
else exitWithHelp();
|
||||
break;
|
||||
case 'W':
|
||||
lambdaW = (FP_TYPE)strtod(argv[i], NULL);
|
||||
|
|
@ -193,7 +193,7 @@ void ProtoNNModel::ProtoNNHyperParams::setHyperParamsFromArgs(const int argc, co
|
|||
|
||||
default:
|
||||
LOG_INFO("Command line argument not recognized; saw character: " + std::string(1, argv[i - 1][1]));
|
||||
assert(false);
|
||||
exitWithHelp();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -201,3 +201,32 @@ void ProtoNNModel::ProtoNNHyperParams::setHyperParamsFromArgs(const int argc, co
|
|||
|
||||
finalizeHyperParams();
|
||||
}
|
||||
|
||||
|
||||
void ProtoNNModel::ProtoNNHyperParams::exitWithHelp()
|
||||
{
|
||||
LOG_INFO("Options:");
|
||||
|
||||
LOG_INFO("-P : [Required] Option to load a predefined model, Visit docs for format. [Default: 0]");
|
||||
LOG_INFO("-R : [Required] A random number seed which can be used to re-generate previously obtained experimental results. [Default: 42]");
|
||||
LOG_INFO("-r : [Required] Number of training points.");
|
||||
LOG_INFO("-v : [Required] Number of validation/test points.");
|
||||
LOG_INFO("-D : [Required] The original dimension of the data.");
|
||||
LOG_INFO("-l : [Required] Number of Classes");
|
||||
LOG_INFO("-C : [Required] Problem Format. Specify one from 0 (binary), 1 (multiclass), 2 (multilabel)");
|
||||
LOG_INFO("-d : [Required] Projection dimension (the dimension into which the data is projected). [Default: 15]");
|
||||
LOG_INFO("-m : [m or k Required] Number of Prototypes. [Default: 20]");
|
||||
LOG_INFO("-k : [m or k Required] Number of Prototypes Per Class.\n");
|
||||
|
||||
LOG_INFO("-g : [Optional] GammaNumerator, also alters RBF kernel parameter 𝛾 =(2.5⋅𝐺𝑎𝑚𝑚𝑎𝑁𝑢𝑚𝑒𝑟𝑎𝑡𝑜𝑟)/(𝑚𝑒𝑑𝑖𝑎𝑛(||𝐵𝑗,𝑊−𝑋𝑖||22)). [Default: 1.0] ");
|
||||
LOG_INFO("-W : [Optional] Projection sparsity ( 𝜆𝑊 ). [Default: 1.0] ");
|
||||
LOG_INFO("-Z : [Optional] Label Sparsity. [Default: 1.0]");
|
||||
LOG_INFO("-B : [Optional] Prototype sparsity. [Default: 1.0]\n");
|
||||
|
||||
|
||||
LOG_INFO("-T : [Optional] Total number of optimization iterations. [Default: 20]");
|
||||
LOG_INFO("-E : [Optional] Number of epochs (complete see-through's) of the data for each iteration, and each parameter. [Default: 20]");
|
||||
LOG_INFO("-N : [Optional] Normalization. Default: 0 (No Normalization), 1 (Min-Max Normalization), 2 (L2-Normalization)\n");
|
||||
|
||||
exit(1);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -37,15 +37,7 @@ class EMI_Trainer:
|
|||
it by adding regularization or other terms.
|
||||
3. After the modification has been performed, the user needs to
|
||||
call the `createOpCollections()` method so that the newly edited
|
||||
operations can be added to Tensorflow collections. This helps in
|
||||
|
||||
HELP_WANTED: Automode is more of a hack than a systematic way of
|
||||
supporting multiple loss functions/ optimizers. One way of
|
||||
accomplishing this would be to make __createTrainOp and __createLossOp
|
||||
methods protected or public, and having users override these.
|
||||
Alternatively, we can change the structure to incorporate the
|
||||
_createExtendedGraph and _restoreExtendedGraph operations used in
|
||||
EMI-LSTM and so forth.
|
||||
operations can be added to Tensorflow collections.
|
||||
'''
|
||||
self.numTimeSteps = numTimeSteps
|
||||
self.numOutput = numOutput
|
||||
|
|
@ -94,7 +86,7 @@ class EMI_Trainer:
|
|||
target: The target labels in one hot-encoding. Expects [-1,
|
||||
numSubinstance, numClass]
|
||||
'''
|
||||
if self.graphCreated is True:
|
||||
if self.graphCreated:
|
||||
# TODO: These statements are redundant after self.validInit call
|
||||
# A simple check to self.__validInit should suffice. Test this.
|
||||
assert self.lossOp is not None
|
||||
|
|
@ -120,7 +112,7 @@ class EMI_Trainer:
|
|||
A__ = tf.tile(A_, [1, 1, self.numTimeSteps, 1])
|
||||
return A__
|
||||
|
||||
def __createLossOp(self, predicted, target):
|
||||
def createLossOp(self, predicted, target):
|
||||
assert self.__validInit is True, 'Initialization failure'
|
||||
with tf.name_scope(self.scope):
|
||||
# Loss indicator tensor
|
||||
|
|
@ -157,7 +149,7 @@ class EMI_Trainer:
|
|||
lossOp = tf.nn.l2_loss(diff, name='l2-loss')
|
||||
return lossOp
|
||||
|
||||
def __createTrainOp(self):
|
||||
def createTrainOp(self):
|
||||
with tf.name_scope(self.scope):
|
||||
tst = tf.train.AdamOptimizer(self.stepSize).minimize(self.lossOp)
|
||||
return tst
|
||||
|
|
@ -178,10 +170,9 @@ class EMI_Trainer:
|
|||
self.equalTilda = tf.cast(equal, tf.float32, name='equal-tilda')
|
||||
self.accTilda = tf.reduce_mean(self.equalTilda, name='acc-tilda')
|
||||
|
||||
self.lossOp = self.__createLossOp(predicted, target)
|
||||
self.trainOp = self.__createTrainOp()
|
||||
if self.automode:
|
||||
self.createOpCollections()
|
||||
self.lossOp = self.createLossOp(predicted, target)
|
||||
self.trainOp = self.createTrainOp()
|
||||
self.createOpCollections()
|
||||
self.graphCreated = True
|
||||
|
||||
def _restoreGraph(self, predicted, target):
|
||||
|
|
@ -359,7 +350,7 @@ class EMI_Driver:
|
|||
|
||||
graph: The computation graph needed to be used for the current session.
|
||||
reuse: If True, global_variables_initializer will not be invoked and
|
||||
the graph will retain the current tensor states/values.
|
||||
the graph will retain the current tensor states/values.
|
||||
feedDict: Not used
|
||||
'''
|
||||
sess = self.__sess
|
||||
|
|
@ -955,7 +946,7 @@ class EMI_Driver:
|
|||
# longestContinuousClass[i] is the class label having
|
||||
# longest substring in bag i
|
||||
longestContinuousClass = np.argmax(scoreList, axis=1) + 1
|
||||
# longestContinuousClassLength[i] is length of
|
||||
# longestContinuousClassLength[i] is length of
|
||||
# longest class substring in bag i
|
||||
longestContinuousClassLength = np.max(scoreList, axis=1)
|
||||
assert longestContinuousClass.ndim == 1
|
||||
|
|
|
|||
|
|
@ -26,8 +26,6 @@
|
|||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%load_ext autoreload\n",
|
||||
"%autoreload 2\n",
|
||||
"from __future__ import print_function\n",
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
|
|
@ -40,10 +38,10 @@
|
|||
"tf.set_random_seed(42)\n",
|
||||
"\n",
|
||||
"# MI-RNN and EMI-RNN imports\n",
|
||||
"from edgeml.tf.graph.rnn import EMI_DataPipeline\n",
|
||||
"from edgeml.tf.graph.rnn import EMI_BasicLSTM\n",
|
||||
"from edgeml.tf.trainer.emirnnTrainer import EMI_Trainer, EMI_Driver\n",
|
||||
"import edgeml.tf.utils"
|
||||
"from edgeml.graph.rnn import EMI_DataPipeline\n",
|
||||
"from edgeml.graph.rnn import EMI_BasicLSTM\n",
|
||||
"from edgeml.trainer.emirnnTrainer import EMI_Trainer, EMI_Driver\n",
|
||||
"import edgeml.utils"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -122,10 +120,10 @@
|
|||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"x_train shape is: (6339, 6, 48, 9)\n",
|
||||
"y_train shape is: (6339, 6, 6)\n",
|
||||
"x_test shape is: (1013, 6, 48, 9)\n",
|
||||
"y_test shape is: (1013, 6, 6)\n"
|
||||
"x_train shape is: (6294, 6, 48, 9)\n",
|
||||
"y_train shape is: (6294, 6, 6)\n",
|
||||
"x_test shape is: (1058, 6, 48, 9)\n",
|
||||
"y_test shape is: (1058, 6, 6)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -240,17 +238,7 @@
|
|||
"start_time": "2018-08-22T16:02:22.218949Z"
|
||||
}
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"WARNING:tensorflow:From ../../edgeml/graph/rnn.py:706: BasicLSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
|
||||
"Instructions for updating:\n",
|
||||
"This class is deprecated, please use tf.nn.rnn_cell.LSTMCell, which supports all the feature this cell currently has. Please replace the existing code with tf.nn.rnn_cell.LSTMCell(name='basic_lstm_cell').\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"tf.reset_default_graph()\n",
|
||||
"g1 = tf.Graph() \n",
|
||||
|
|
@ -291,36 +279,36 @@
|
|||
"Update policy: top-k\n",
|
||||
"Training with MI-RNN loss for 3 rounds\n",
|
||||
"Round: 0\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.00375 Acc 0.94271 | Val acc 0.97236 | Model saved to /tmp/model-lstm, global_step 1000\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.00342 Acc 0.96354 | Val acc 0.96644 | Model saved to /tmp/model-lstm, global_step 1001\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.00327 Acc 0.93750 | Val acc 0.96841 | Model saved to /tmp/model-lstm, global_step 1002\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.00308 Acc 0.94271 | Val acc 0.96841 | Model saved to /tmp/model-lstm, global_step 1003\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1000\n",
|
||||
"Epoch 1 Batch 193 ( 390) Loss 0.00467 Acc 0.90104 | Val acc 0.90454 | Model saved to /tmp/model-lstm, global_step 1000\n",
|
||||
"Epoch 1 Batch 193 ( 390) Loss 0.00262 Acc 0.93750 | Val acc 0.91777 | Model saved to /tmp/model-lstm, global_step 1001\n",
|
||||
"Epoch 1 Batch 193 ( 390) Loss 0.00285 Acc 0.90625 | Val acc 0.91871 | Model saved to /tmp/model-lstm, global_step 1002\n",
|
||||
"Epoch 1 Batch 193 ( 390) Loss 0.00263 Acc 0.91146 | Val acc 0.92344 | Model saved to /tmp/model-lstm, global_step 1003\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1003\n",
|
||||
"Round: 1\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.00342 Acc 0.96354 | Val acc 0.96644 | Model saved to /tmp/model-lstm, global_step 1004\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.00326 Acc 0.94271 | Val acc 0.96841 | Model saved to /tmp/model-lstm, global_step 1005\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.00318 Acc 0.93750 | Val acc 0.96742 | Model saved to /tmp/model-lstm, global_step 1006\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.00301 Acc 0.92708 | Val acc 0.96841 | Model saved to /tmp/model-lstm, global_step 1007\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1005\n",
|
||||
"Epoch 1 Batch 193 ( 390) Loss 0.00260 Acc 0.91146 | Val acc 0.92155 | Model saved to /tmp/model-lstm, global_step 1004\n",
|
||||
"Epoch 1 Batch 193 ( 390) Loss 0.00268 Acc 0.91146 | Val acc 0.92628 | Model saved to /tmp/model-lstm, global_step 1005\n",
|
||||
"Epoch 1 Batch 193 ( 390) Loss 0.00241 Acc 0.92188 | Val acc 0.92911 | Model saved to /tmp/model-lstm, global_step 1006\n",
|
||||
"Epoch 1 Batch 193 ( 390) Loss 0.00245 Acc 0.91667 | Val acc 0.91493 | Model saved to /tmp/model-lstm, global_step 1007\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1006\n",
|
||||
"Round: 2\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.00321 Acc 0.94271 | Val acc 0.96644 | Model saved to /tmp/model-lstm, global_step 1008\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.00324 Acc 0.92708 | Val acc 0.96742 | Model saved to /tmp/model-lstm, global_step 1009\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.00328 Acc 0.90625 | Val acc 0.96841 | Model saved to /tmp/model-lstm, global_step 1010\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.00290 Acc 0.92188 | Val acc 0.96841 | Model saved to /tmp/model-lstm, global_step 1011\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1010\n",
|
||||
"Epoch 1 Batch 193 ( 390) Loss 0.00245 Acc 0.91667 | Val acc 0.91493 | Model saved to /tmp/model-lstm, global_step 1008\n",
|
||||
"Epoch 1 Batch 193 ( 390) Loss 0.00247 Acc 0.93750 | Val acc 0.91210 | Model saved to /tmp/model-lstm, global_step 1009\n",
|
||||
"Epoch 1 Batch 193 ( 390) Loss 0.00238 Acc 0.93750 | Val acc 0.91115 | Model saved to /tmp/model-lstm, global_step 1010\n",
|
||||
"Epoch 1 Batch 193 ( 390) Loss 0.00247 Acc 0.91667 | Val acc 0.90737 | Model saved to /tmp/model-lstm, global_step 1011\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1008\n",
|
||||
"Round: 3\n",
|
||||
"Switching to EMI-Loss function\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.25792 Acc 0.91146 | Val acc 0.96742 | Model saved to /tmp/model-lstm, global_step 1012\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.23910 Acc 0.93229 | Val acc 0.96644 | Model saved to /tmp/model-lstm, global_step 1013\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.21696 Acc 0.97396 | Val acc 0.96841 | Model saved to /tmp/model-lstm, global_step 1014\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.21577 Acc 0.95833 | Val acc 0.96841 | Model saved to /tmp/model-lstm, global_step 1015\n",
|
||||
"Epoch 1 Batch 193 ( 390) Loss 0.19644 Acc 0.92188 | Val acc 0.91304 | Model saved to /tmp/model-lstm, global_step 1012\n",
|
||||
"Epoch 1 Batch 193 ( 390) Loss 0.19590 Acc 0.92188 | Val acc 0.91304 | Model saved to /tmp/model-lstm, global_step 1013\n",
|
||||
"Epoch 1 Batch 193 ( 390) Loss 0.18886 Acc 0.91667 | Val acc 0.92250 | Model saved to /tmp/model-lstm, global_step 1014\n",
|
||||
"Epoch 1 Batch 193 ( 390) Loss 0.17789 Acc 0.92708 | Val acc 0.91210 | Model saved to /tmp/model-lstm, global_step 1015\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1014\n",
|
||||
"Round: 4\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.21659 Acc 0.94271 | Val acc 0.96644 | Model saved to /tmp/model-lstm, global_step 1016\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.20983 Acc 0.96354 | Val acc 0.96841 | Model saved to /tmp/model-lstm, global_step 1017\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.19877 Acc 0.96354 | Val acc 0.96841 | Model saved to /tmp/model-lstm, global_step 1018\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.20167 Acc 0.94271 | Val acc 0.96841 | Model saved to /tmp/model-lstm, global_step 1019\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1017\n"
|
||||
"Epoch 1 Batch 193 ( 390) Loss 0.17789 Acc 0.92708 | Val acc 0.91210 | Model saved to /tmp/model-lstm, global_step 1016\n",
|
||||
"Epoch 1 Batch 193 ( 390) Loss 0.17308 Acc 0.93750 | Val acc 0.90737 | Model saved to /tmp/model-lstm, global_step 1017\n",
|
||||
"Epoch 1 Batch 193 ( 390) Loss 0.16609 Acc 0.93750 | Val acc 0.91682 | Model saved to /tmp/model-lstm, global_step 1018\n",
|
||||
"Epoch 1 Batch 193 ( 390) Loss 0.16253 Acc 0.93750 | Val acc 0.91115 | Model saved to /tmp/model-lstm, global_step 1019\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1018\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -406,10 +394,10 @@
|
|||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Accuracy at k = 2: 0.913471\n",
|
||||
"Accuracy at k = 2: 0.894130\n",
|
||||
"Savings due to MI-RNN : 0.625000\n",
|
||||
"Savings due to Early prediction: 0.577056\n",
|
||||
"Total Savings: 0.841396\n"
|
||||
"Savings due to Early prediction: 0.696645\n",
|
||||
"Total Savings: 0.886242\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -443,27 +431,27 @@
|
|||
"output_type": "stream",
|
||||
"text": [
|
||||
" len acc macro-fsc macro-pre macro-rec micro-fsc micro-pre \\\n",
|
||||
"0 1 0.892433 0.891689 0.896050 0.895365 0.892433 0.892433 \n",
|
||||
"1 2 0.913471 0.913896 0.914488 0.916145 0.913471 0.913471 \n",
|
||||
"2 3 0.913471 0.913863 0.914164 0.915745 0.913471 0.913471 \n",
|
||||
"3 4 0.900916 0.901157 0.904399 0.901916 0.900916 0.900916 \n",
|
||||
"4 5 0.882932 0.883396 0.894413 0.882276 0.882932 0.882932 \n",
|
||||
"5 6 0.867323 0.868491 0.888064 0.865587 0.867323 0.867323 \n",
|
||||
"0 1 0.869019 0.868122 0.878439 0.871618 0.869019 0.869019 \n",
|
||||
"1 2 0.894130 0.894577 0.898733 0.896412 0.894130 0.894130 \n",
|
||||
"2 3 0.893451 0.893584 0.897599 0.894880 0.893451 0.893451 \n",
|
||||
"3 4 0.873770 0.873280 0.882517 0.873642 0.873770 0.873770 \n",
|
||||
"4 5 0.853410 0.853243 0.870547 0.851575 0.853410 0.853410 \n",
|
||||
"5 6 0.836105 0.836891 0.863634 0.833071 0.836105 0.836105 \n",
|
||||
"\n",
|
||||
" micro-rec \n",
|
||||
"0 0.892433 \n",
|
||||
"1 0.913471 \n",
|
||||
"2 0.913471 \n",
|
||||
"3 0.900916 \n",
|
||||
"4 0.882932 \n",
|
||||
"5 0.867323 \n",
|
||||
"Max accuracy 0.913471 at subsequencelength 2\n",
|
||||
"Max micro-f 0.913471 at subsequencelength 2\n",
|
||||
"Micro-precision 0.913471 at subsequencelength 2\n",
|
||||
"Micro-recall 0.913471 at subsequencelength 2\n",
|
||||
"Max macro-f 0.913896 at subsequencelength 2\n",
|
||||
"macro-precision 0.914488 at subsequencelength 2\n",
|
||||
"macro-recall 0.916145 at subsequencelength 2\n"
|
||||
"0 0.869019 \n",
|
||||
"1 0.894130 \n",
|
||||
"2 0.893451 \n",
|
||||
"3 0.873770 \n",
|
||||
"4 0.853410 \n",
|
||||
"5 0.836105 \n",
|
||||
"Max accuracy 0.894130 at subsequencelength 2\n",
|
||||
"Max micro-f 0.894130 at subsequencelength 2\n",
|
||||
"Micro-precision 0.894130 at subsequencelength 2\n",
|
||||
"Micro-recall 0.894130 at subsequencelength 2\n",
|
||||
"Max macro-f 0.894577 at subsequencelength 2\n",
|
||||
"macro-precision 0.898733 at subsequencelength 2\n",
|
||||
"macro-recall 0.896412 at subsequencelength 2\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -495,16 +483,16 @@
|
|||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1000\n",
|
||||
"Round: 0, Validation accuracy: 0.9724, Test Accuracy (k = 2): 0.893790, Total Savings: 0.725069\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1005\n",
|
||||
"Round: 1, Validation accuracy: 0.9684, Test Accuracy (k = 2): 0.906345, Total Savings: 0.773529\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1010\n",
|
||||
"Round: 2, Validation accuracy: 0.9684, Test Accuracy (k = 2): 0.904988, Total Savings: 0.793007\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1003\n",
|
||||
"Round: 0, Validation accuracy: 0.9234, Test Accuracy (k = 2): 0.899559, Total Savings: 0.790902\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1006\n",
|
||||
"Round: 1, Validation accuracy: 0.9291, Test Accuracy (k = 2): 0.896844, Total Savings: 0.814705\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1008\n",
|
||||
"Round: 2, Validation accuracy: 0.9149, Test Accuracy (k = 2): 0.894469, Total Savings: 0.821671\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1014\n",
|
||||
"Round: 3, Validation accuracy: 0.9684, Test Accuracy (k = 2): 0.916186, Total Savings: 0.833013\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1017\n",
|
||||
"Round: 4, Validation accuracy: 0.9684, Test Accuracy (k = 2): 0.913471, Total Savings: 0.841396\n"
|
||||
"Round: 3, Validation accuracy: 0.9225, Test Accuracy (k = 2): 0.894130, Total Savings: 0.876447\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1018\n",
|
||||
"Round: 4, Validation accuracy: 0.9168, Test Accuracy (k = 2): 0.894130, Total Savings: 0.886242\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -549,7 +537,7 @@
|
|||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.3"
|
||||
"version": "3.6.8"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
|
|
|
|||
|
|
@ -34,7 +34,14 @@ data into the required format.
|
|||
|
||||
Along with the example notebook, a command line execution script for ProtoNN is
|
||||
provided in `protoNN_example.py`. After the USPS data has been setup, this
|
||||
script can be used with the following command:
|
||||
script can be used to run ProtoNN on the processed data.
|
||||
|
||||
Create a temporary matrices for the model matrices.
|
||||
```
|
||||
mkdir ./model
|
||||
```
|
||||
|
||||
The run the ProtoNN script;
|
||||
|
||||
```
|
||||
python protoNN_example.py \
|
||||
|
|
@ -45,7 +52,7 @@ python protoNN_example.py \
|
|||
--learning-rate 0.1 \
|
||||
--epochs 200 \
|
||||
--val-step 10 \
|
||||
--output-dir ./
|
||||
--output-dir ./model
|
||||
```
|
||||
|
||||
You can expect a test set accuracy of about 92.5%.
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
antlr4-python3-runtime==4.7
|
||||
jupyter==1.0.0
|
||||
numpy==1.14.5
|
||||
numpy==1.16.4
|
||||
pandas==0.23.4
|
||||
scikit-learn==0.19.2
|
||||
scipy==1.1.0
|
||||
tensorflow==1.12.2
|
||||
requests
|
||||
scikit-learn==0.21.2
|
||||
scipy==1.3.0
|
||||
tensorflow==1.13.1
|
||||
requests
|
||||
|
|
|
|||
|
|
@ -0,0 +1,606 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Using EMI-RNN on the HAR Dataset with custom Loss and Optimizer\n",
|
||||
"\n",
|
||||
"This is a very simple example of how the existing EMI-RNN implementation can be used on the HAR dataset with a custom loss function and optimizer. We illustrate how to train a model that predicts on 48 step sequence in place of the 128 length baselines while attempting to predict early.\n",
|
||||
"\n",
|
||||
"In the preprint of our work, we use the terms *bag* and *instance* to refer to the LSTM input sequence of original length and the shorter ones we want to learn to predict on, respectively. In the code though, *bag* is replaced with *instance* and *instance* is replaced with *sub-instance*. We will use the term *instance* and *sub-instance* interchangeably.\n",
|
||||
"\n",
|
||||
"The network used here is a simple LSTM + Linear classifier network. \n",
|
||||
"\n",
|
||||
"The UCI [Human Activity Recognition](https://archive.ics.uci.edu/ml/datasets/human+activity+recognition+using+smartphones) dataset."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-08-22T16:02:21.994456Z",
|
||||
"start_time": "2018-08-22T16:02:21.080183Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from __future__ import print_function\n",
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"import tensorflow as tf\n",
|
||||
"import numpy as np\n",
|
||||
"# Making sure edgeml is part of python path\n",
|
||||
"os.environ['CUDA_VISIBLE_DEVICES'] ='0'\n",
|
||||
"\n",
|
||||
"np.random.seed(42)\n",
|
||||
"tf.set_random_seed(42)\n",
|
||||
"\n",
|
||||
"# MI-RNN and EMI-RNN imports\n",
|
||||
"from edgeml.graph.rnn import EMI_DataPipeline\n",
|
||||
"from edgeml.graph.rnn import EMI_BasicLSTM\n",
|
||||
"from edgeml.trainer.emirnnTrainer import EMI_Trainer, EMI_Driver\n",
|
||||
"import edgeml.utils"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Let us set up some network parameters for the computation graph."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-08-22T16:02:22.011139Z",
|
||||
"start_time": "2018-08-22T16:02:21.996670Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Network parameters for our LSTM + FC Layer\n",
|
||||
"NUM_HIDDEN = 32\n",
|
||||
"NUM_TIMESTEPS = 48\n",
|
||||
"ORIGINAL_NUM_TIMESTEPS = 128\n",
|
||||
"NUM_FEATS = 9\n",
|
||||
"FORGET_BIAS = 1.0\n",
|
||||
"NUM_OUTPUT = 6\n",
|
||||
"USE_DROPOUT = False\n",
|
||||
"KEEP_PROB = 0.75\n",
|
||||
"\n",
|
||||
"# For dataset API\n",
|
||||
"PREFETCH_NUM = 5\n",
|
||||
"BATCH_SIZE = 32\n",
|
||||
"\n",
|
||||
"# Number of epochs in *one iteration*\n",
|
||||
"NUM_EPOCHS = 2\n",
|
||||
"# Number of iterations in *one round*. After each iteration,\n",
|
||||
"# the model is dumped to disk. At the end of the current\n",
|
||||
"# round, the best model among all the dumped models in the\n",
|
||||
"# current round is picked up..\n",
|
||||
"NUM_ITER = 4\n",
|
||||
"# A round consists of multiple training iterations and a belief\n",
|
||||
"# update step using the best model from all of these iterations\n",
|
||||
"NUM_ROUNDS = 5\n",
|
||||
"LEARNING_RATE=0.001\n",
|
||||
"# Fraction of rounds to use with EMI loss function rather than\n",
|
||||
"# MI loss function. It is usually better to let the model stabilize\n",
|
||||
"# with the MI loss function before enforcing early prediciton\n",
|
||||
"# requirement with EMI loss. Setting to 0 runs purely MI-RNN.\n",
|
||||
"FRAC_EMI=0.5\n",
|
||||
"\n",
|
||||
"# Beta term for Regularization\n",
|
||||
"BETA = 0.01\n",
|
||||
"\n",
|
||||
"# A staging direcory to store models\n",
|
||||
"MODEL_PREFIX = '/tmp/model-lstm'"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Loading Data\n",
|
||||
"\n",
|
||||
"Please make sure the data is preprocessed to a format that is compatible with EMI-RNN. `tf/examples/EMI-RNN/fetch_har.py` can be used to download and setup the HAR dataset.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-08-22T16:02:22.127637Z",
|
||||
"start_time": "2018-08-22T16:02:22.012833Z"
|
||||
}
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"x_train shape is: (6340, 6, 48, 9)\n",
|
||||
"y_train shape is: (6340, 6, 6)\n",
|
||||
"x_test shape is: (1012, 6, 48, 9)\n",
|
||||
"y_test shape is: (1012, 6, 6)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Loading the data\n",
|
||||
"x_train, y_train = np.load('./HAR/48_16/x_train.npy'), np.load('./HAR/48_16/y_train.npy')\n",
|
||||
"x_test, y_test = np.load('./HAR/48_16/x_test.npy'), np.load('./HAR/48_16/y_test.npy')\n",
|
||||
"x_val, y_val = np.load('./HAR/48_16/x_val.npy'), np.load('./HAR/48_16/y_val.npy')\n",
|
||||
"\n",
|
||||
"# BAG_TEST, BAG_TRAIN, BAG_VAL represent bag_level labels. These are used for the label update\n",
|
||||
"# step of EMI/MI RNN\n",
|
||||
"BAG_TEST = np.argmax(y_test[:, 0, :], axis=1)\n",
|
||||
"BAG_TRAIN = np.argmax(y_train[:, 0, :], axis=1)\n",
|
||||
"BAG_VAL = np.argmax(y_val[:, 0, :], axis=1)\n",
|
||||
"NUM_SUBINSTANCE = x_train.shape[1]\n",
|
||||
"print(\"x_train shape is:\", x_train.shape)\n",
|
||||
"print(\"y_train shape is:\", y_train.shape)\n",
|
||||
"print(\"x_test shape is:\", x_val.shape)\n",
|
||||
"print(\"y_test shape is:\", y_val.shape)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Computation Graph\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"The *EMI-RNN* computation graph is constructed out of the following three mutually disjoint parts:\n",
|
||||
"\n",
|
||||
"1. `EMI_DataPipeline`: An efficient data input pipeline that using the Tensorflow Dataset API. This module ingests data compatible with EMI-RNN and provides two iterators for a batch of input data, $x$ and label $y$. \n",
|
||||
"2. `EMI_RNN`: The 'abstract' `EMI-RNN` class defines the methods and attributes required for the forward computation graph. An implementation based on LSTM - `EMI_LSTM` is used in this document, though the user is free to implement his own computation graphs compatible with `EMI-RNN`. This module expects two Dataset API iterators for $x$-batch and $y$-batch as inputs and constructs the forward computation graph based on them. Every implementation of this class defines an `output` operation - the output of the forward computation graph.\n",
|
||||
"3. `EMI_Trainer`: An instance of `EMI_Trainer` class which defines the loss functions and the training routine. This expects an `output` operator from an `EMI-RNN` implementation and attaches loss functions and training routines to it. \n",
|
||||
"\n",
|
||||
"To build the computation graph, we create an instance of all the above and then connect them together.\n",
|
||||
"\n",
|
||||
"Note that, the `EMI_BasicLSTM` class is an implementation that uses an LSTM cell and pushes the LSTM output at each step to a secondary classifier for classification. This secondary classifier is not implemented as part of `EMI_BasicLSTM` and is left to the user to define by overriding the `createExtendedGraph` method, and the `restoreExtendedgraph` method.\n",
|
||||
"\n",
|
||||
"For the purpose of this example, we will be using a simple linear layer as a secondary classifier."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-08-22T16:02:22.159977Z",
|
||||
"start_time": "2018-08-22T16:02:22.129530Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Define the linear secondary classifier\n",
|
||||
"def createExtendedGraph(self, baseOutput, *args, **kwargs):\n",
|
||||
" W1 = tf.Variable(np.random.normal(size=[NUM_HIDDEN, NUM_OUTPUT]).astype('float32'), name='W1')\n",
|
||||
" B1 = tf.Variable(np.random.normal(size=[NUM_OUTPUT]).astype('float32'), name='B1')\n",
|
||||
" y_cap = tf.add(tf.tensordot(baseOutput, W1, axes=1), B1, name='y_cap_tata')\n",
|
||||
" self.output = y_cap\n",
|
||||
" self.graphCreated = True\n",
|
||||
"\n",
|
||||
"def restoreExtendedGraph(self, graph, *args, **kwargs):\n",
|
||||
" y_cap = graph.get_tensor_by_name('y_cap_tata:0')\n",
|
||||
" self.output = y_cap\n",
|
||||
" self.graphCreated = True\n",
|
||||
" \n",
|
||||
"def feedDictFunc(self, keep_prob=None, inference=False, **kwargs):\n",
|
||||
" if inference is False:\n",
|
||||
" feedDict = {self._emiGraph.keep_prob: keep_prob}\n",
|
||||
" else:\n",
|
||||
" feedDict = {self._emiGraph.keep_prob: 1.0}\n",
|
||||
" return feedDict\n",
|
||||
" \n",
|
||||
"EMI_BasicLSTM._createExtendedGraph = createExtendedGraph\n",
|
||||
"EMI_BasicLSTM._restoreExtendedGraph = restoreExtendedGraph\n",
|
||||
"\n",
|
||||
"if USE_DROPOUT is True:\n",
|
||||
" EMI_Driver.feedDictFunc = feedDictFunc\n",
|
||||
"\n",
|
||||
"# Build the Computation Graph\n",
|
||||
"inputPipeline = EMI_DataPipeline(NUM_SUBINSTANCE, NUM_TIMESTEPS, NUM_FEATS, NUM_OUTPUT)\n",
|
||||
"emiLSTM = EMI_BasicLSTM(NUM_SUBINSTANCE, NUM_HIDDEN, NUM_TIMESTEPS, NUM_FEATS,\n",
|
||||
" forgetBias=FORGET_BIAS, useDropout=USE_DROPOUT)\n",
|
||||
"emiTrainer = EMI_Trainer(NUM_TIMESTEPS, NUM_OUTPUT,\n",
|
||||
" stepSize=LEARNING_RATE, automode=False)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Note that, to use your custom loss and optimizer, overload `createLossOp` and `createTrainOp` function and return the newly created operators. Use `automode=False` while creating the EMI_Trainer call. For the purpose of this example, we will use Cross Entropy loss with L2 regularizer and Adagrad Optimizer."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-08-22T16:02:22.217131Z",
|
||||
"start_time": "2018-08-22T16:02:22.161816Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Define Custom Loss Function\n",
|
||||
"def createLossOp(self, predicted, target):\n",
|
||||
" with tf.name_scope(self.scope):\n",
|
||||
" li = np.zeros([NUM_TIMESTEPS, NUM_OUTPUT])\n",
|
||||
" li[-1, :] = 1\n",
|
||||
" liTensor = tf.Variable(li.astype('float32'),\n",
|
||||
" name='loss-indicator',\n",
|
||||
" trainable=False)\n",
|
||||
" name='loss-indicator-placeholder'\n",
|
||||
" liPlaceholder = tf.placeholder(tf.float32,\n",
|
||||
" name=name)\n",
|
||||
" liAssignOp = tf.assign(liTensor, liPlaceholder,\n",
|
||||
" name='loss-indicator-assign-op')\n",
|
||||
" self.lossIndicatorTensor = liTensor\n",
|
||||
" self.lossIndicatorPlaceholder = liPlaceholder\n",
|
||||
" self.lossIndicatorAssignOp = liAssignOp\n",
|
||||
" # predicted of dim [-1, numSubinstance, numTimeSteps, numOutput]\n",
|
||||
" dims = [-1, NUM_TIMESTEPS, NUM_OUTPUT]\n",
|
||||
" logits__ = tf.reshape(predicted, dims)\n",
|
||||
" labels__ = tf.reshape(target, dims)\n",
|
||||
" diff = (logits__ - labels__)\n",
|
||||
" diff = tf.multiply(self.lossIndicatorTensor, diff)\n",
|
||||
" # take loss only for the timesteps indicated by lossIndicator for softmax\n",
|
||||
" logits__ = tf.multiply(self.lossIndicatorTensor, logits__)\n",
|
||||
" labels__ = tf.multiply(self.lossIndicatorTensor, labels__)\n",
|
||||
" logits__ = tf.reshape(logits__, [-1, self.numOutput])\n",
|
||||
" labels__ = tf.reshape(labels__, [-1, self.numOutput])\n",
|
||||
" softmax1 = tf.nn.softmax_cross_entropy_with_logits_v2(labels=labels__,\n",
|
||||
" logits=logits__)\n",
|
||||
" lossOp = tf.reduce_mean(softmax1)\n",
|
||||
" regularizer = tf.nn.l2_loss(self.lossIndicatorTensor)\n",
|
||||
" loss = tf.reduce_mean(lossOp + BETA * regularizer)\n",
|
||||
" return loss\n",
|
||||
"\n",
|
||||
"#Define Custom Optimizer\n",
|
||||
"def createTrainOp(self):\n",
|
||||
" with tf.name_scope(self.scope):\n",
|
||||
" tst = tf.train.AdagradOptimizer(self.stepSize).minimize(self.lossOp)\n",
|
||||
" return tst\n",
|
||||
"\n",
|
||||
"# Add to Tensorflow Collections\n",
|
||||
"def createOpCollections(self):\n",
|
||||
" tf.add_to_collection('EMI-train-op', self.trainOp)\n",
|
||||
" tf.add_to_collection('EMI-loss-op', self.lossOp)\n",
|
||||
"\n",
|
||||
"# Override functions in EMI Trainer\n",
|
||||
"EMI_Trainer.createLossOp = createLossOp\n",
|
||||
"EMI_Trainer.createTrainOp = createTrainOp\n",
|
||||
"EMI_Trainer.createOpCollections = createOpCollections"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Now that we have all the elementary parts of the computation graph setup, we connect them together to form the forward graph."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-08-22T16:02:27.440097Z",
|
||||
"start_time": "2018-08-22T16:02:22.218949Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"tf.reset_default_graph()\n",
|
||||
"g1 = tf.Graph() \n",
|
||||
"with g1.as_default():\n",
|
||||
" # Obtain the iterators to each batch of the data\n",
|
||||
" x_batch, y_batch = inputPipeline()\n",
|
||||
" # Create the forward computation graph based on the iterators\n",
|
||||
" y_cap = emiLSTM(x_batch)\n",
|
||||
" # Create loss graphs and training routines\n",
|
||||
" emiTrainer(y_cap, y_batch)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# EMI Driver\n",
|
||||
"\n",
|
||||
"The `EMI_Driver` implements the `EMI_RNN` algorithm. For more information on how the driver works, please refer to `tf/docs/EMI-RNN.md`.\n",
|
||||
"\n",
|
||||
"Note that, during the training period, the accuracy printed is instance level accuracy with the current label information as target. Bag level accuracy, with which we are actually concerned, is calculated after the training ends."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-08-22T16:06:20.676437Z",
|
||||
"start_time": "2018-08-22T16:02:27.442355Z"
|
||||
}
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Update policy: top-k\n",
|
||||
"Training with MI-RNN loss for 3 rounds\n",
|
||||
"Round: 0\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.07134 Acc 0.10938 | Val acc 0.20553 | Model saved to /tmp/model-lstm, global_step 1000\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.06691 Acc 0.17188 | Val acc 0.24605 | Model saved to /tmp/model-lstm, global_step 1001\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.06411 Acc 0.41146 | Val acc 0.38636 | Model saved to /tmp/model-lstm, global_step 1002\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.06228 Acc 0.41667 | Val acc 0.42589 | Model saved to /tmp/model-lstm, global_step 1003\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1003\n",
|
||||
"Round: 1\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.06100 Acc 0.42708 | Val acc 0.48123 | Model saved to /tmp/model-lstm, global_step 1004\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.06003 Acc 0.49479 | Val acc 0.52767 | Model saved to /tmp/model-lstm, global_step 1005\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.05925 Acc 0.52083 | Val acc 0.56225 | Model saved to /tmp/model-lstm, global_step 1006\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.05860 Acc 0.52604 | Val acc 0.60079 | Model saved to /tmp/model-lstm, global_step 1007\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1007\n",
|
||||
"Round: 2\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.05804 Acc 0.52083 | Val acc 0.61759 | Model saved to /tmp/model-lstm, global_step 1008\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.05754 Acc 0.52083 | Val acc 0.62846 | Model saved to /tmp/model-lstm, global_step 1009\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.05708 Acc 0.51562 | Val acc 0.63834 | Model saved to /tmp/model-lstm, global_step 1010\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 0.05667 Acc 0.53646 | Val acc 0.65613 | Model saved to /tmp/model-lstm, global_step 1011\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1011\n",
|
||||
"Round: 3\n",
|
||||
"Switching to EMI-Loss function\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 2.53950 Acc 0.59896 | Val acc 0.70356 | Model saved to /tmp/model-lstm, global_step 1012\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 2.43370 Acc 0.63542 | Val acc 0.73419 | Model saved to /tmp/model-lstm, global_step 1013\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 2.36060 Acc 0.65104 | Val acc 0.78953 | Model saved to /tmp/model-lstm, global_step 1014\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 2.30109 Acc 0.66667 | Val acc 0.82411 | Model saved to /tmp/model-lstm, global_step 1015\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1015\n",
|
||||
"Round: 4\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 2.24863 Acc 0.68750 | Val acc 0.84684 | Model saved to /tmp/model-lstm, global_step 1016\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 2.20104 Acc 0.71875 | Val acc 0.86858 | Model saved to /tmp/model-lstm, global_step 1017\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 2.15813 Acc 0.73958 | Val acc 0.87945 | Model saved to /tmp/model-lstm, global_step 1018\n",
|
||||
"Epoch 1 Batch 191 ( 390) Loss 2.12039 Acc 0.75000 | Val acc 0.88538 | Model saved to /tmp/model-lstm, global_step 1019\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1019\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"with g1.as_default():\n",
|
||||
" emiDriver = EMI_Driver(inputPipeline, emiLSTM, emiTrainer)\n",
|
||||
"\n",
|
||||
"emiDriver.initializeSession(g1)\n",
|
||||
"# y_updated,modelStats\n",
|
||||
"y_updated, modelStats = emiDriver.run(numClasses=NUM_OUTPUT, x_train=x_train,\n",
|
||||
" y_train=y_train, bag_train=BAG_TRAIN,\n",
|
||||
" x_val=x_val, y_val=y_val, bag_val=BAG_VAL,\n",
|
||||
" numIter=NUM_ITER, keep_prob=KEEP_PROB,\n",
|
||||
" numRounds=NUM_ROUNDS, batchSize=BATCH_SIZE,\n",
|
||||
" numEpochs=NUM_EPOCHS, modelPrefix=MODEL_PREFIX,\n",
|
||||
" fracEMI=FRAC_EMI, updatePolicy='top-k', k=1)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Evaluating the trained model\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"## Accuracy\n",
|
||||
"\n",
|
||||
"Since the trained model predicts on a smaller 48-step input while our test data has labels for 128 step inputs (i.e. bag level labels), evaluating the accuracy of the trained model is not straight forward. We perform the evaluation as follows:\n",
|
||||
"\n",
|
||||
"1. Divide the test data also into sub-instances; similar to what was done for the train data.\n",
|
||||
"2. Obtain sub-instance level predictions for each bag in the test data.\n",
|
||||
"3. Obtain bag level predictions from sub-instance level predictions. For this, we use our estimate of the length of the signature to estimate the expected number of sub-instances that would be non negative - $k$ illustrated in the figure. If a bag has $k$ consecutive sub-instances with the same label, that becomes the label of the bag. All other bags are labeled negative.\n",
|
||||
"4. Compare the predicted bag level labels with the known bag level labels in test data.\n",
|
||||
"\n",
|
||||
"## Early Savings\n",
|
||||
"\n",
|
||||
"Early prediction is accomplished by defining an early prediction policy method. This method receives the prediction at each step of the learned LSTM for a sub-instance as input and is expected to return a predicted class and the 0-indexed step at which it made this prediction. This is illustrated below in code. "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-08-22T16:06:20.702818Z",
|
||||
"start_time": "2018-08-22T16:06:20.679077Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Early Prediction Policy: We make an early prediction based on the predicted classes\n",
|
||||
"# probability. If the predicted class probability > minProb at some step, we make\n",
|
||||
"# a prediction at that step.\n",
|
||||
"def earlyPolicy_minProb(instanceOut, minProb, **kwargs):\n",
|
||||
" assert instanceOut.ndim == 2\n",
|
||||
" classes = np.argmax(instanceOut, axis=1)\n",
|
||||
" prob = np.max(instanceOut, axis=1)\n",
|
||||
" index = np.where(prob >= minProb)[0]\n",
|
||||
" if len(index) == 0:\n",
|
||||
" assert (len(instanceOut) - 1) == (len(classes) - 1)\n",
|
||||
" return classes[-1], len(instanceOut) - 1\n",
|
||||
" index = index[0]\n",
|
||||
" return classes[index], index\n",
|
||||
"\n",
|
||||
"def getEarlySaving(predictionStep, numTimeSteps, returnTotal=False):\n",
|
||||
" predictionStep = predictionStep + 1\n",
|
||||
" predictionStep = np.reshape(predictionStep, -1)\n",
|
||||
" totalSteps = np.sum(predictionStep)\n",
|
||||
" maxSteps = len(predictionStep) * numTimeSteps\n",
|
||||
" savings = 1.0 - (totalSteps / maxSteps)\n",
|
||||
" if returnTotal:\n",
|
||||
" return savings, totalSteps\n",
|
||||
" return savings"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Accuracy at k = 2: 0.793010\n",
|
||||
"Savings due to MI-RNN : 0.625000\n",
|
||||
"Savings due to Early prediction: 0.086214\n",
|
||||
"Total Savings: 0.657330\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"k = 2\n",
|
||||
"predictions, predictionStep = emiDriver.getInstancePredictions(x_test, y_test, earlyPolicy_minProb,\n",
|
||||
" minProb=0.99, keep_prob=1.0)\n",
|
||||
"bagPredictions = emiDriver.getBagPredictions(predictions, minSubsequenceLen=k, numClass=NUM_OUTPUT)\n",
|
||||
"print('Accuracy at k = %d: %f' % (k, np.mean((bagPredictions == BAG_TEST).astype(int))))\n",
|
||||
"mi_savings = (1 - NUM_TIMESTEPS / ORIGINAL_NUM_TIMESTEPS)\n",
|
||||
"emi_savings = getEarlySaving(predictionStep, NUM_TIMESTEPS)\n",
|
||||
"total_savings = mi_savings + (1 - mi_savings) * emi_savings\n",
|
||||
"print('Savings due to MI-RNN : %f' % mi_savings)\n",
|
||||
"print('Savings due to Early prediction: %f' % emi_savings)\n",
|
||||
"print('Total Savings: %f' % (total_savings))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-08-22T16:06:21.736513Z",
|
||||
"start_time": "2018-08-22T16:06:21.279557Z"
|
||||
},
|
||||
"scrolled": false
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" len acc macro-fsc macro-pre macro-rec micro-fsc micro-pre \\\n",
|
||||
"0 1 0.732270 0.674675 0.782901 0.733758 0.732270 0.732270 \n",
|
||||
"1 2 0.793010 0.788136 0.790928 0.791973 0.793010 0.793010 \n",
|
||||
"2 3 0.780794 0.778230 0.811512 0.773139 0.780794 0.780794 \n",
|
||||
"3 4 0.738378 0.726696 0.816650 0.724994 0.738378 0.738378 \n",
|
||||
"4 5 0.698677 0.672229 0.815220 0.680764 0.698677 0.698677 \n",
|
||||
"5 6 0.675263 0.635061 0.828203 0.654302 0.675263 0.675263 \n",
|
||||
"\n",
|
||||
" micro-rec \n",
|
||||
"0 0.732270 \n",
|
||||
"1 0.793010 \n",
|
||||
"2 0.780794 \n",
|
||||
"3 0.738378 \n",
|
||||
"4 0.698677 \n",
|
||||
"5 0.675263 \n",
|
||||
"Max accuracy 0.793010 at subsequencelength 2\n",
|
||||
"Max micro-f 0.793010 at subsequencelength 2\n",
|
||||
"Micro-precision 0.793010 at subsequencelength 2\n",
|
||||
"Micro-recall 0.793010 at subsequencelength 2\n",
|
||||
"Max macro-f 0.788136 at subsequencelength 2\n",
|
||||
"macro-precision 0.790928 at subsequencelength 2\n",
|
||||
"macro-recall 0.791973 at subsequencelength 2\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# A slightly more detailed analysis method is provided. \n",
|
||||
"df = emiDriver.analyseModel(predictions, BAG_TEST, NUM_SUBINSTANCE, NUM_OUTPUT)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Picking the best model\n",
|
||||
"\n",
|
||||
"The `EMI_Driver.run()` method, upon finishing, returns a list containing information about the best models after each EMI-RNN round. This can be used to identify the best model (based on validation accuracy) at the end of each round - illustrated below."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-08-22T16:06:49.331237Z",
|
||||
"start_time": "2018-08-22T16:06:21.738312Z"
|
||||
}
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1003\n",
|
||||
"Round: 0, Validation accuracy: 0.4259, Test Accuracy (k = 2): 0.392942, Total Savings: 0.625000\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1007\n",
|
||||
"Round: 1, Validation accuracy: 0.6008, Test Accuracy (k = 2): 0.523923, Total Savings: 0.625000\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1011\n",
|
||||
"Round: 2, Validation accuracy: 0.6561, Test Accuracy (k = 2): 0.532406, Total Savings: 0.625000\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1015\n",
|
||||
"Round: 3, Validation accuracy: 0.8241, Test Accuracy (k = 2): 0.688157, Total Savings: 0.632898\n",
|
||||
"INFO:tensorflow:Restoring parameters from /tmp/model-lstm-1019\n",
|
||||
"Round: 4, Validation accuracy: 0.8854, Test Accuracy (k = 2): 0.793010, Total Savings: 0.657330\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"devnull = open(os.devnull, 'r')\n",
|
||||
"for val in modelStats:\n",
|
||||
" round_, acc, modelPrefix, globalStep = val\n",
|
||||
" emiDriver.loadSavedGraphToNewSession(modelPrefix, globalStep, redirFile=devnull)\n",
|
||||
" predictions, predictionStep = emiDriver.getInstancePredictions(x_test, y_test, earlyPolicy_minProb,\n",
|
||||
" minProb=0.99, keep_prob=1.0)\n",
|
||||
"\n",
|
||||
" bagPredictions = emiDriver.getBagPredictions(predictions, minSubsequenceLen=k, numClass=NUM_OUTPUT)\n",
|
||||
" print(\"Round: %2d, Validation accuracy: %.4f\" % (round_, acc), end='')\n",
|
||||
" print(', Test Accuracy (k = %d): %f, ' % (k, np.mean((bagPredictions == BAG_TEST).astype(int))), end='')\n",
|
||||
" mi_savings = (1 - NUM_TIMESTEPS / ORIGINAL_NUM_TIMESTEPS)\n",
|
||||
" emi_savings = getEarlySaving(predictionStep, NUM_TIMESTEPS)\n",
|
||||
" total_savings = mi_savings + (1 - mi_savings) * emi_savings\n",
|
||||
" print(\"Total Savings: %f\" % total_savings)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.8"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
Загрузка…
Ссылка в новой задаче