1283 строки
44 KiB
Plaintext
1283 строки
44 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"<i>Copyright (c) Microsoft Corporation. All rights reserved.</i>\n",
|
|
"\n",
|
|
"<i>Licensed under the MIT License.</i>"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Wide and Deep Model for Movie Recommendation\n",
|
|
"\n",
|
|
"<br>\n",
|
|
"\n",
|
|
"A linear model with a wide set of crossed-column (co-occurrence) features can memorize the feature interactions, while deep neural networks (DNN) can generalize the feature patterns through low-dimensional dense embeddings learned for the sparse features. [**Wide-and-deep**](https://arxiv.org/abs/1606.07792) learning jointly trains wide linear model and deep neural networks to combine the benefits of memorization and generalization for recommender systems.\n",
|
|
"\n",
|
|
"This notebook shows how to build and test the wide-and-deep model using [TensorFlow high-level Estimator API (v1.12)](https://www.tensorflow.org/api_docs/python/tf/estimator/DNNLinearCombinedRegressor). With the [movie recommendation dataset](https://grouplens.org/datasets/movielens/), we quickly demonstrate following topics:\n",
|
|
"1. How to prepare data\n",
|
|
"2. Build the model\n",
|
|
"3. Use log-hook to estimate performance while training\n",
|
|
"4. Test the model and export\n",
|
|
"\n",
|
|
"> Note: The output cells in this notebook are from the result of run on Azure DSVM (Data Science Virtual Machine) with *Standard NC6* virtual machine."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Tensorflow Version: 1.12.0\n",
|
|
"['/device:CPU:0', '/device:XLA_CPU:0', '/device:XLA_GPU:0', '/device:GPU:0']\n",
|
|
"Num CPUs: 6\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import sys\n",
|
|
"sys.path.append(\"notebooks\")\n",
|
|
"\n",
|
|
"import os\n",
|
|
"import shutil\n",
|
|
"import itertools\n",
|
|
"\n",
|
|
"import papermill as pm\n",
|
|
"import pandas as pd\n",
|
|
"import numpy as np\n",
|
|
"import sklearn.preprocessing\n",
|
|
"\n",
|
|
"import tensorflow as tf\n",
|
|
"from tensorflow.python.client import device_lib\n",
|
|
"\n",
|
|
"import reco_utils.recommender.wide_deep.wide_deep_utils as wide_deep\n",
|
|
"from reco_utils.common import tf_utils\n",
|
|
"from reco_utils.dataset import movielens\n",
|
|
"from reco_utils.dataset.pandas_df_utils import user_item_pairs\n",
|
|
"from reco_utils.dataset.python_splitters import python_random_split\n",
|
|
"import reco_utils.evaluation.python_evaluation\n",
|
|
"\n",
|
|
"print(\"Tensorflow Version:\", tf.VERSION)\n",
|
|
"\n",
|
|
"devices = device_lib.list_local_devices()\n",
|
|
"print([x.name for x in devices])\n",
|
|
"\n",
|
|
"num_cpus = os.cpu_count()\n",
|
|
"print(\"Num CPUs:\", num_cpus)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {
|
|
"tags": [
|
|
"parameters"
|
|
]
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"\"\"\"Parameters. This cell is being used to pass parameters from other scripts via papermill\"\"\"\n",
|
|
"\n",
|
|
"# Recommend top k items\n",
|
|
"TOP_K = 10\n",
|
|
"# Select Movielens data size: 100k, 1m, 10m, or 20m\n",
|
|
"MOVIELENS_DATA_SIZE = '100k'\n",
|
|
"# Metrics to use for evaluation. reco_utils.evaluation.python_evaluation function names\n",
|
|
"RANKING_METRICS = ['map_at_k', 'ndcg_at_k', 'precision_at_k', 'recall_at_k']\n",
|
|
"RATING_METRICS = ['rmse', 'mae', 'rsquared', 'exp_var']\n",
|
|
"EVALUATE_WHILE_TRAINING = True # If true, use session hook to evaluate model while training\n",
|
|
"# Data column names\n",
|
|
"USER_COL = 'UserId'\n",
|
|
"ITEM_COL = 'MovieId'\n",
|
|
"RATING_COL = 'Rating'\n",
|
|
"ITEM_FEAT_COL = 'Genres'\n",
|
|
"\n",
|
|
"# Prepared train and test set pickle file paths. If None, load.\n",
|
|
"DATA_DIR = None\n",
|
|
"TRAIN_PICKLE_PATH = None\n",
|
|
"TEST_PICKLE_PATH = None\n",
|
|
"EXPORT_DIR_BASE = './outputs/model'\n",
|
|
"\n",
|
|
"\"\"\"Hyperparameters\"\"\"\n",
|
|
"MODEL_TYPE = 'wide_deep'\n",
|
|
"EPOCHS = 50\n",
|
|
"BATCH_SIZE = 64\n",
|
|
"# Wide (linear) model hyperparameters\n",
|
|
"LINEAR_OPTIMIZER = 'Ftrl'\n",
|
|
"LINEAR_OPTIMIZER_LR = 0.0029 # Learning rate\n",
|
|
"LINEAR_L1_REG = 0.0 # L1 Regularization rate for FtrlOptimizer\n",
|
|
"LINEAR_MOMENTUM = 0.9 # Momentum for MomentumOptimizer or RMSPropOptimizer\n",
|
|
"# DNN model hyperparameters\n",
|
|
"DNN_OPTIMIZER = 'Adagrad'\n",
|
|
"DNN_OPTIMIZER_LR = 0.1\n",
|
|
"DNN_L1_REG = 0.0 # L1 Regularization rate for FtrlOptimizer\n",
|
|
"DNN_MOMENTUM = 0.9 # Momentum for MomentumOptimizer or RMSPropOptimizer\n",
|
|
"DNN_HIDDEN_LAYER_1 = 0 # Set 0 to not use this layer\n",
|
|
"DNN_HIDDEN_LAYER_2 = 128 # Set 0 to not use this layer\n",
|
|
"DNN_HIDDEN_LAYER_3 = 256 # Set 0 to not use this layer\n",
|
|
"DNN_HIDDEN_LAYER_4 = 32 # With this setting, DNN hidden units will be = [512, 256, 128, 128]\n",
|
|
"DNN_USER_DIM = 4\n",
|
|
"DNN_ITEM_DIM = 4\n",
|
|
"DNN_DROPOUT = 0.4\n",
|
|
"DNN_BATCH_NORM = 1 # 1 to use batch normalization, 0 if not.\n",
|
|
"\n",
|
|
"MODEL_DIR = 'model_checkpoints'"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"data_loaded = False\n",
|
|
"# If local paths of train and test sets have given, use them\n",
|
|
"if TRAIN_PICKLE_PATH is not None and TEST_PICKLE_PATH is not None:\n",
|
|
" if DATA_DIR is not None:\n",
|
|
" train_pickle_path = os.path.join(DATA_DIR, TRAIN_PICKLE_PATH)\n",
|
|
" test_pickle_path = os.path.join(DATA_DIR, TEST_PICKLE_PATH)\n",
|
|
" train = pd.read_pickle(path=train_pickle_path)\n",
|
|
" test = pd.read_pickle(path=test_pickle_path)\n",
|
|
" data = pd.concat([train, test])\n",
|
|
" data_loaded = True"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"### 1. Prepare Data\n",
|
|
"\n",
|
|
"#### 1.1 Movie Rating and Genres Data\n",
|
|
"First, download [MovieLens](https://grouplens.org/datasets/movielens/) data. Movies in the data set are tagged as one or more genres where there are total 19 genres including '*unknown*'. We load *movie genres* to use them as item features."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>UserId</th>\n",
|
|
" <th>MovieId</th>\n",
|
|
" <th>Rating</th>\n",
|
|
" <th>Genres_string</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>196</td>\n",
|
|
" <td>242</td>\n",
|
|
" <td>3.0</td>\n",
|
|
" <td>Comedy</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>63</td>\n",
|
|
" <td>242</td>\n",
|
|
" <td>3.0</td>\n",
|
|
" <td>Comedy</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>226</td>\n",
|
|
" <td>242</td>\n",
|
|
" <td>5.0</td>\n",
|
|
" <td>Comedy</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>154</td>\n",
|
|
" <td>242</td>\n",
|
|
" <td>3.0</td>\n",
|
|
" <td>Comedy</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td>306</td>\n",
|
|
" <td>242</td>\n",
|
|
" <td>5.0</td>\n",
|
|
" <td>Comedy</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" UserId MovieId Rating Genres_string\n",
|
|
"0 196 242 3.0 Comedy\n",
|
|
"1 63 242 3.0 Comedy\n",
|
|
"2 226 242 5.0 Comedy\n",
|
|
"3 154 242 3.0 Comedy\n",
|
|
"4 306 242 5.0 Comedy"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"if not data_loaded:\n",
|
|
" # The genres of each movie are returned as '|' separated string, e.g. \"Animation|Children's|Comedy\".\n",
|
|
" data = movielens.load_pandas_df(\n",
|
|
" size=MOVIELENS_DATA_SIZE,\n",
|
|
" header=[USER_COL, ITEM_COL, RATING_COL],\n",
|
|
" genres_col='Genres_string' # load genres as a temporal column 'Genres_string'\n",
|
|
" )\n",
|
|
" display(data.head())"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"#### 1.2 Encode Item Features (Genres)\n",
|
|
"To use genres from our model, we multi-hot-encode them with scikit-learn's [MultiLabelBinarizer](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.MultiLabelBinarizer.html).\n",
|
|
"\n",
|
|
"For example, *Movie id=2355* has three genres, *Animation|Children's|Comedy*, which are being converted into an integer array of the indicator value for each genre like `[0, 0, 1, 1, 1, 0, 0, 0, ...]`. In the later step, we convert this into a float array and feed into the model."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Genres: ['Action' 'Adventure' 'Animation' \"Children's\" 'Comedy' 'Crime'\n",
|
|
" 'Documentary' 'Drama' 'Fantasy' 'Film-Noir' 'Horror' 'Musical' 'Mystery'\n",
|
|
" 'Romance' 'Sci-Fi' 'Thriller' 'War' 'Western' 'unknown']\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>MovieId</th>\n",
|
|
" <th>Genres_string</th>\n",
|
|
" <th>Genres</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>242</td>\n",
|
|
" <td>Comedy</td>\n",
|
|
" <td>[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>117</th>\n",
|
|
" <td>302</td>\n",
|
|
" <td>Crime|Film-Noir|Mystery|Thriller</td>\n",
|
|
" <td>[0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, ...</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>414</th>\n",
|
|
" <td>377</td>\n",
|
|
" <td>Children's|Comedy</td>\n",
|
|
" <td>[0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>427</th>\n",
|
|
" <td>51</td>\n",
|
|
" <td>Drama|Romance|War|Western</td>\n",
|
|
" <td>[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, ...</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>508</th>\n",
|
|
" <td>346</td>\n",
|
|
" <td>Crime|Drama</td>\n",
|
|
" <td>[0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" MovieId Genres_string \\\n",
|
|
"0 242 Comedy \n",
|
|
"117 302 Crime|Film-Noir|Mystery|Thriller \n",
|
|
"414 377 Children's|Comedy \n",
|
|
"427 51 Drama|Romance|War|Western \n",
|
|
"508 346 Crime|Drama \n",
|
|
"\n",
|
|
" Genres \n",
|
|
"0 [0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ... \n",
|
|
"117 [0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, ... \n",
|
|
"414 [0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ... \n",
|
|
"427 [0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, ... \n",
|
|
"508 [0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, ... "
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"if not data_loaded:\n",
|
|
" # Encode 'genres' into int array (multi-hot representation) to use as item features\n",
|
|
" genres_encoder = sklearn.preprocessing.MultiLabelBinarizer()\n",
|
|
" data[ITEM_FEAT_COL] = genres_encoder.fit_transform(\n",
|
|
" data['Genres_string'].apply(lambda s: s.split(\"|\"))\n",
|
|
" ).tolist()\n",
|
|
" print(\"Genres:\", genres_encoder.classes_)\n",
|
|
" display(data.drop_duplicates(ITEM_COL)[[ITEM_COL, 'Genres_string', ITEM_FEAT_COL]].head())"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"#### 1.3 Train and Test Split"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Train = 75000, test = 25000\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"if not data_loaded:\n",
|
|
" train, test = python_random_split(\n",
|
|
" data.drop('Genres_string', axis=1), # We don't need Genres original string column\n",
|
|
" ratio=0.75,\n",
|
|
" seed=42 \n",
|
|
" )\n",
|
|
" data_loaded = True\n",
|
|
"\n",
|
|
"print(\"Train = {}, test = {}\".format(len(train), len(test)))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"metadata": {
|
|
"scrolled": false
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Num items = 1682, num users = 943\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# Unique items\n",
|
|
"if ITEM_FEAT_COL is None:\n",
|
|
" items = data.drop_duplicates(ITEM_COL)[[ITEM_COL]].reset_index(drop=True)\n",
|
|
" item_feat_shape = None\n",
|
|
"else:\n",
|
|
" items = data.drop_duplicates(ITEM_COL)[[ITEM_COL, ITEM_FEAT_COL]].reset_index(drop=True)\n",
|
|
" item_feat_shape = len(items[ITEM_FEAT_COL][0])\n",
|
|
"# Unique users\n",
|
|
"users = data.drop_duplicates(USER_COL)[[USER_COL]].reset_index(drop=True)\n",
|
|
"\n",
|
|
"print(\"Num items = {}, num users = {}\".format(len(items), len(users)))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"### 2. Build Model\n",
|
|
"\n",
|
|
"Wide-and-deep model consists of a linear model and DNN. We use the following hyperparameters and feature sets for the model:\n",
|
|
"\n",
|
|
"<br> | <div align=\"center\">Wide (linear) model</div> | <div align=\"center\">Deep neural networks</div>\n",
|
|
"---|---|---\n",
|
|
"Feature set | <ul><li>User-item co-occurrence features<br>to capture how their co-occurrence<br>correlates with the target rating</li></ul> | <ul><li>Deep, lower-dimensional embedding vectors<br>for every user and item</li><li>Item feature vector</li></ul>\n",
|
|
"Hyperparameters | <ul><li>FTRL optimizer</li><li>Learning rate = 0.0029</li><li>L1 regularization = 0.0</li></ul> | <ul><li>Adagrad optimizer</li><li>Learning rate = 0.1</li><li>Hidden units = [128, 256, 32]</li><li>Dropout rate = 0.4</li><li>Use batch normalization (Batch size = 64)</li><li>User embedding vector size = 4</li><li>Item embedding vector size = 4</li></ul>\n",
|
|
"\n",
|
|
"<br>\n",
|
|
"\n",
|
|
"* [FTRL optimizer](https://www.eecs.tufts.edu/~dsculley/papers/ad-click-prediction.pdf)\n",
|
|
"* [Adagrad optimizer](http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf)\n",
|
|
"\n",
|
|
"The hyperparameters are found on *MovieLens 100k* **train set** (split by using the same `seed` we used in this notebook). We used **Azure Machine Learning service**([AzureML](https://azure.microsoft.com/en-us/services/machine-learning-service/)) for the Hyperparameter tuning. Please find details from [aml_hyperparameter_tuning](../04_model_select_and_optimize/hypertune_aml_wide_and_deep_quickstart.ipynb)."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"DNN hidden units = [128, 256, 32]\n",
|
|
"Embedding 943 users to 4-dim vector\n",
|
|
"Embedding 1682 items to 4-dim vector\n",
|
|
"\n",
|
|
"{'l1_regularization_strength': 0.0} {}\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"train_steps = EPOCHS * len(train) // BATCH_SIZE\n",
|
|
"\n",
|
|
"# Clean-up previous model dir if already exists. Otherwise, it will try to train on top of the existing one.\n",
|
|
"shutil.rmtree(MODEL_DIR, ignore_errors=True)\n",
|
|
"\n",
|
|
"DNN_HIDDEN_UNITS = []\n",
|
|
"if DNN_HIDDEN_LAYER_1 > 0:\n",
|
|
" DNN_HIDDEN_UNITS.append(DNN_HIDDEN_LAYER_1)\n",
|
|
"if DNN_HIDDEN_LAYER_2 > 0:\n",
|
|
" DNN_HIDDEN_UNITS.append(DNN_HIDDEN_LAYER_2)\n",
|
|
"if DNN_HIDDEN_LAYER_3 > 0:\n",
|
|
" DNN_HIDDEN_UNITS.append(DNN_HIDDEN_LAYER_3)\n",
|
|
"if DNN_HIDDEN_LAYER_4 > 0:\n",
|
|
" DNN_HIDDEN_UNITS.append(DNN_HIDDEN_LAYER_4)\n",
|
|
"\n",
|
|
"if MODEL_TYPE is 'deep' or MODEL_TYPE is 'wide_deep':\n",
|
|
" print(\"DNN hidden units =\", DNN_HIDDEN_UNITS)\n",
|
|
" print(\"Embedding {} users to {}-dim vector\".format(len(users), DNN_USER_DIM))\n",
|
|
" print(\"Embedding {} items to {}-dim vector\\n\".format(len(items), DNN_ITEM_DIM))\n",
|
|
"\n",
|
|
"save_checkpoints_steps = max(1, train_steps // 5)\n",
|
|
" \n",
|
|
"# Model type is tf.estimator.DNNLinearCombinedRegressor, known as 'wide-and-deep'\n",
|
|
"wide_columns, deep_columns = wide_deep.build_feature_columns(\n",
|
|
" users=users[USER_COL].values,\n",
|
|
" items=items[ITEM_COL].values,\n",
|
|
" user_col=USER_COL,\n",
|
|
" item_col=ITEM_COL,\n",
|
|
" item_feat_col=ITEM_FEAT_COL,\n",
|
|
" user_dim=DNN_USER_DIM,\n",
|
|
" item_dim=DNN_ITEM_DIM,\n",
|
|
" item_feat_shape=item_feat_shape,\n",
|
|
" model_type=MODEL_TYPE,\n",
|
|
")\n",
|
|
"\n",
|
|
"# Optimizer specific parameters\n",
|
|
"linear_params = {}\n",
|
|
"if LINEAR_OPTIMIZER == 'Ftrl':\n",
|
|
" linear_params['l1_regularization_strength'] = LINEAR_L1_REG\n",
|
|
"elif LINEAR_OPTIMIZER == 'Momentum' or LINEAR_OPTIMIZER == 'RMSProp':\n",
|
|
" linear_params['momentum'] = LINEAR_MOMENTUM\n",
|
|
"\n",
|
|
"dnn_params = {}\n",
|
|
"if DNN_OPTIMIZER == 'Ftrl':\n",
|
|
" dnn_params['l1_regularization_strength'] = DNN_L1_REG\n",
|
|
"elif DNN_OPTIMIZER == 'Momentum' or DNN_OPTIMIZER == 'RMSProp':\n",
|
|
" dnn_params['momentum'] = DNN_MOMENTUM\n",
|
|
"\n",
|
|
"print(linear_params, dnn_params)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 9,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"INFO:tensorflow:Using config: {'_model_dir': 'model_checkpoints', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 11718, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
|
|
"graph_options {\n",
|
|
" rewrite_options {\n",
|
|
" meta_optimizer_iterations: ONE\n",
|
|
" }\n",
|
|
"}\n",
|
|
", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 2929, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f7880268160>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
|
|
"\n",
|
|
"Feature specs:\n",
|
|
"_CrossedColumn(keys=(_VocabularyListCategoricalColumn(key='UserId', vocabulary_list=(196, 63, 226, 1 ...\n",
|
|
"_EmbeddingColumn(categorical_column=_VocabularyListCategoricalColumn(key='UserId', vocabulary_list=( ...\n",
|
|
"_EmbeddingColumn(categorical_column=_VocabularyListCategoricalColumn(key='MovieId', vocabulary_list= ...\n",
|
|
"_NumericColumn(key='Genres', shape=(19,), default_value=None, dtype=tf.float32, normalizer_fn=None) ...\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"model = wide_deep.build_model(\n",
|
|
" model_dir=MODEL_DIR,\n",
|
|
" wide_columns=wide_columns,\n",
|
|
" deep_columns=deep_columns,\n",
|
|
" linear_optimizer=tf_utils.build_optimizer(LINEAR_OPTIMIZER, LINEAR_OPTIMIZER_LR, **linear_params),\n",
|
|
" dnn_optimizer=tf_utils.build_optimizer(DNN_OPTIMIZER, DNN_OPTIMIZER_LR, **dnn_params),\n",
|
|
" dnn_hidden_units=DNN_HIDDEN_UNITS,\n",
|
|
" dnn_dropout=DNN_DROPOUT,\n",
|
|
" dnn_batch_norm=(DNN_BATCH_NORM==1),\n",
|
|
" log_every_n_iter=max(1, train_steps//20), # log 20 times\n",
|
|
" save_checkpoints_steps=save_checkpoints_steps\n",
|
|
")\n",
|
|
"\n",
|
|
"# Wide columns are the features for wide model, and deep columns are for DNN\n",
|
|
"print(\"\\nFeature specs:\")\n",
|
|
"for c in wide_columns + deep_columns:\n",
|
|
" print(str(c)[:100], \"...\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"### 3. Train and Evaluate Model\n",
|
|
"\n",
|
|
"Now we are all set to train the model. Here, we show how to utilize session hooks to track model performance while training. Our custom hook `tf_utils.evaluation_log_hook` estimates the model performance on the given data based on the specified evaluation functions. Note we pass test set to evaluate the model on rating metrics while we use <span id=\"ranking-pool\">ranking-pool (all the user-item pairs)</span> for ranking metrics.\n",
|
|
"\n",
|
|
"> Note: The TensorFlow Estimator's default loss calculates Mean Squared Error. Square root of the loss is the same as [RMSE](https://en.wikipedia.org/wiki/Root-mean-square_deviation)."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"cols = {\n",
|
|
" 'col_user': USER_COL,\n",
|
|
" 'col_item': ITEM_COL,\n",
|
|
" 'col_rating': RATING_COL,\n",
|
|
" 'col_prediction': 'prediction'\n",
|
|
"}\n",
|
|
"\n",
|
|
"# Prepare ranking evaluation set, i.e. get the cross join of all user-item pairs\n",
|
|
"ranking_pool = user_item_pairs(\n",
|
|
" user_df=users,\n",
|
|
" item_df=items,\n",
|
|
" user_col=USER_COL,\n",
|
|
" item_col=ITEM_COL,\n",
|
|
" user_item_filter_df=train, # Remove seen items\n",
|
|
" shuffle=True\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"metadata": {
|
|
"scrolled": false
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Training steps = 58593, Batch size = 64 (num epochs = 50)\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"\"\"\" Training hooks to track training performance (evaluate on 'train' data) \n",
|
|
"\"\"\"\n",
|
|
"hooks = []\n",
|
|
"evaluation_logger = None\n",
|
|
"if EVALUATE_WHILE_TRAINING:\n",
|
|
" class EvaluationLogger(tf_utils.Logger):\n",
|
|
" def __init__(self):\n",
|
|
" self.eval_log = {}\n",
|
|
"\n",
|
|
" def log(self, metric, value):\n",
|
|
" if metric not in self.eval_log:\n",
|
|
" self.eval_log[metric] = []\n",
|
|
" self.eval_log[metric].append(value)\n",
|
|
" print(\"eval_{} = {}\".format(metric, value))\n",
|
|
"\n",
|
|
" def get_log(self):\n",
|
|
" return self.eval_log\n",
|
|
"\n",
|
|
" evaluation_logger = EvaluationLogger()\n",
|
|
"\n",
|
|
" if len(RANKING_METRICS) > 0:\n",
|
|
" hooks.append(\n",
|
|
" tf_utils.evaluation_log_hook(\n",
|
|
" model,\n",
|
|
" logger=evaluation_logger,\n",
|
|
" true_df=test,\n",
|
|
" y_col=RATING_COL,\n",
|
|
" eval_df=ranking_pool,\n",
|
|
" every_n_iter=save_checkpoints_steps,\n",
|
|
" model_dir=MODEL_DIR,\n",
|
|
" eval_fns=[getattr(reco_utils.evaluation.python_evaluation, m) for m in RANKING_METRICS],\n",
|
|
" **{**cols, 'k': TOP_K}\n",
|
|
" )\n",
|
|
" )\n",
|
|
" if len(RATING_METRICS) > 0:\n",
|
|
" hooks.append(\n",
|
|
" tf_utils.evaluation_log_hook(\n",
|
|
" model,\n",
|
|
" logger=evaluation_logger,\n",
|
|
" true_df=test,\n",
|
|
" y_col=RATING_COL,\n",
|
|
" eval_df=test.drop(RATING_COL, axis=1),\n",
|
|
" every_n_iter=save_checkpoints_steps,\n",
|
|
" model_dir=MODEL_DIR,\n",
|
|
" eval_fns=[getattr(reco_utils.evaluation.python_evaluation, m) for m in RATING_METRICS],\n",
|
|
" **cols\n",
|
|
" )\n",
|
|
" )\n",
|
|
"\n",
|
|
"print(\"Training steps = {}, Batch size = {} (num epochs = {})\".format(train_steps, BATCH_SIZE, EPOCHS))\n",
|
|
"\n",
|
|
"train_fn = tf_utils.pandas_input_fn(\n",
|
|
" df=train,\n",
|
|
" y_col=RATING_COL,\n",
|
|
" batch_size=BATCH_SIZE,\n",
|
|
" num_epochs=None, # None == run forever. We use steps=TRAIN_STEPS instead.\n",
|
|
" shuffle=True,\n",
|
|
" num_threads=num_cpus-1\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 12,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"WARNING:tensorflow:From /home/jumin/miniconda3/envs/reco_gpu/lib/python3.6/site-packages/tensorflow/python/estimator/inputs/queues/feeding_queue_runner.py:62: QueueRunner.__init__ (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.\n",
|
|
"Instructions for updating:\n",
|
|
"To construct input pipelines, use the `tf.data` module.\n",
|
|
"WARNING:tensorflow:From /home/jumin/miniconda3/envs/reco_gpu/lib/python3.6/site-packages/tensorflow/python/estimator/inputs/queues/feeding_functions.py:500: add_queue_runner (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.\n",
|
|
"Instructions for updating:\n",
|
|
"To construct input pipelines, use the `tf.data` module.\n",
|
|
"INFO:tensorflow:Calling model_fn.\n",
|
|
"INFO:tensorflow:Done calling model_fn.\n",
|
|
"INFO:tensorflow:Create CheckpointSaverHook.\n",
|
|
"INFO:tensorflow:Graph was finalized.\n",
|
|
"INFO:tensorflow:Running local_init_op.\n",
|
|
"INFO:tensorflow:Done running local_init_op.\n",
|
|
"WARNING:tensorflow:From /home/jumin/miniconda3/envs/reco_gpu/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py:804: start_queue_runners (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.\n",
|
|
"Instructions for updating:\n",
|
|
"To construct input pipelines, use the `tf.data` module.\n",
|
|
"INFO:tensorflow:Saving checkpoints for 0 into model_checkpoints/model.ckpt.\n",
|
|
"INFO:tensorflow:loss = 932.2452, step = 0\n",
|
|
"INFO:tensorflow:global_step/sec: 150.817\n",
|
|
"INFO:tensorflow:loss = 42.562725, step = 2929 (19.421 sec)\n",
|
|
"INFO:tensorflow:global_step/sec: 153.639\n",
|
|
"INFO:tensorflow:loss = 61.396675, step = 5858 (19.064 sec)\n",
|
|
"INFO:tensorflow:global_step/sec: 155.273\n",
|
|
"INFO:tensorflow:loss = 49.38915, step = 8787 (18.864 sec)\n",
|
|
"INFO:tensorflow:global_step/sec: 155.743\n",
|
|
"INFO:tensorflow:loss = 67.17851, step = 11716 (18.807 sec)\n",
|
|
"INFO:tensorflow:Saving checkpoints for 11718 into model_checkpoints/model.ckpt.\n",
|
|
"eval_map_at_k = 0.007616734294189487\n",
|
|
"eval_ndcg_at_k = 0.06743113778797069\n",
|
|
"eval_precision_at_k = 0.07324840764331211\n",
|
|
"eval_recall_at_k = 0.02374049652834253\n",
|
|
"eval_rmse = 0.9421604900657937\n",
|
|
"eval_mae = 0.7441799917883426\n",
|
|
"eval_rsquared = 0.30093904698626484\n",
|
|
"eval_exp_var = 0.30167929819298833\n",
|
|
"INFO:tensorflow:global_step/sec: 50.8499\n",
|
|
"INFO:tensorflow:loss = 51.37668, step = 14645 (57.601 sec)\n",
|
|
"INFO:tensorflow:global_step/sec: 154.955\n",
|
|
"INFO:tensorflow:loss = 46.42402, step = 17574 (18.902 sec)\n",
|
|
"INFO:tensorflow:global_step/sec: 154.111\n",
|
|
"INFO:tensorflow:loss = 45.922108, step = 20503 (19.006 sec)\n",
|
|
"INFO:tensorflow:global_step/sec: 155.611\n",
|
|
"INFO:tensorflow:loss = 44.25303, step = 23432 (18.823 sec)\n",
|
|
"INFO:tensorflow:Saving checkpoints for 23436 into model_checkpoints/model.ckpt.\n",
|
|
"eval_map_at_k = 0.005282381352239611\n",
|
|
"eval_ndcg_at_k = 0.05232036421287402\n",
|
|
"eval_precision_at_k = 0.059447983014861996\n",
|
|
"eval_recall_at_k = 0.019680933276829217\n",
|
|
"eval_rmse = 0.939755619704993\n",
|
|
"eval_mae = 0.7380850439405441\n",
|
|
"eval_rsquared = 0.3045032070417637\n",
|
|
"eval_exp_var = 0.3045892461871469\n",
|
|
"INFO:tensorflow:global_step/sec: 51.5327\n",
|
|
"INFO:tensorflow:loss = 46.728806, step = 26361 (56.838 sec)\n",
|
|
"INFO:tensorflow:global_step/sec: 154.394\n",
|
|
"INFO:tensorflow:loss = 68.13846, step = 29290 (18.971 sec)\n",
|
|
"INFO:tensorflow:global_step/sec: 154.743\n",
|
|
"INFO:tensorflow:loss = 48.842392, step = 32219 (18.928 sec)\n",
|
|
"INFO:tensorflow:global_step/sec: 154.304\n",
|
|
"INFO:tensorflow:loss = 55.659912, step = 35148 (18.982 sec)\n",
|
|
"INFO:tensorflow:Saving checkpoints for 35154 into model_checkpoints/model.ckpt.\n",
|
|
"eval_map_at_k = 0.00397994926269106\n",
|
|
"eval_ndcg_at_k = 0.04061161661873355\n",
|
|
"eval_precision_at_k = 0.04787685774946921\n",
|
|
"eval_recall_at_k = 0.016326311177037908\n",
|
|
"eval_rmse = 0.9397414775204982\n",
|
|
"eval_mae = 0.7356724128818511\n",
|
|
"eval_rsquared = 0.30452413965391645\n",
|
|
"eval_exp_var = 0.30486639672614235\n",
|
|
"INFO:tensorflow:global_step/sec: 51.703\n",
|
|
"INFO:tensorflow:loss = 64.73981, step = 38077 (56.650 sec)\n",
|
|
"INFO:tensorflow:global_step/sec: 154.366\n",
|
|
"INFO:tensorflow:loss = 50.060017, step = 41006 (18.974 sec)\n",
|
|
"INFO:tensorflow:global_step/sec: 156.552\n",
|
|
"INFO:tensorflow:loss = 49.136665, step = 43935 (18.712 sec)\n",
|
|
"INFO:tensorflow:global_step/sec: 155.29\n",
|
|
"INFO:tensorflow:loss = 33.41443, step = 46864 (18.859 sec)\n",
|
|
"INFO:tensorflow:Saving checkpoints for 46872 into model_checkpoints/model.ckpt.\n",
|
|
"eval_map_at_k = 0.0034277165107839884\n",
|
|
"eval_ndcg_at_k = 0.033525255041823354\n",
|
|
"eval_precision_at_k = 0.0381104033970276\n",
|
|
"eval_recall_at_k = 0.013090803781510604\n",
|
|
"eval_rmse = 0.9396357563690039\n",
|
|
"eval_mae = 0.7332056822863221\n",
|
|
"eval_rsquared = 0.30468061326857676\n",
|
|
"eval_exp_var = 0.3052303508899866\n",
|
|
"INFO:tensorflow:global_step/sec: 51.3759\n",
|
|
"INFO:tensorflow:loss = 48.65799, step = 49793 (57.011 sec)\n",
|
|
"INFO:tensorflow:global_step/sec: 154.842\n",
|
|
"INFO:tensorflow:loss = 44.42491, step = 52722 (18.916 sec)\n",
|
|
"INFO:tensorflow:global_step/sec: 154.678\n",
|
|
"INFO:tensorflow:loss = 57.47725, step = 55651 (18.936 sec)\n",
|
|
"INFO:tensorflow:global_step/sec: 154.598\n",
|
|
"INFO:tensorflow:loss = 34.94922, step = 58580 (18.946 sec)\n",
|
|
"INFO:tensorflow:Saving checkpoints for 58590 into model_checkpoints/model.ckpt.\n",
|
|
"eval_map_at_k = 0.0024880331040521833\n",
|
|
"eval_ndcg_at_k = 0.025429799187102142\n",
|
|
"eval_precision_at_k = 0.029617834394904466\n",
|
|
"eval_recall_at_k = 0.010823063887818299\n",
|
|
"eval_rmse = 0.94190933971966\n",
|
|
"eval_mae = 0.7343136594682187\n",
|
|
"eval_rsquared = 0.301311692626853\n",
|
|
"eval_exp_var = 0.3023860484774723\n",
|
|
"INFO:tensorflow:Saving checkpoints for 58593 into model_checkpoints/model.ckpt.\n",
|
|
"INFO:tensorflow:Loss for final step: 40.66799.\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/papermill.record+json": {
|
|
"eval_map_at_k": [
|
|
0.007616734294189487,
|
|
0.005282381352239611,
|
|
0.00397994926269106,
|
|
0.0034277165107839884,
|
|
0.0024880331040521833
|
|
]
|
|
}
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/papermill.record+json": {
|
|
"eval_ndcg_at_k": [
|
|
0.06743113778797069,
|
|
0.05232036421287402,
|
|
0.04061161661873355,
|
|
0.033525255041823354,
|
|
0.025429799187102142
|
|
]
|
|
}
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/papermill.record+json": {
|
|
"eval_precision_at_k": [
|
|
0.07324840764331211,
|
|
0.059447983014861996,
|
|
0.04787685774946921,
|
|
0.0381104033970276,
|
|
0.029617834394904466
|
|
]
|
|
}
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/papermill.record+json": {
|
|
"eval_recall_at_k": [
|
|
0.02374049652834253,
|
|
0.019680933276829217,
|
|
0.016326311177037908,
|
|
0.013090803781510604,
|
|
0.010823063887818299
|
|
]
|
|
}
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/papermill.record+json": {
|
|
"eval_rmse": [
|
|
0.9421604900657937,
|
|
0.939755619704993,
|
|
0.9397414775204982,
|
|
0.9396357563690039,
|
|
0.94190933971966
|
|
]
|
|
}
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/papermill.record+json": {
|
|
"eval_mae": [
|
|
0.7441799917883426,
|
|
0.7380850439405441,
|
|
0.7356724128818511,
|
|
0.7332056822863221,
|
|
0.7343136594682187
|
|
]
|
|
}
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/papermill.record+json": {
|
|
"eval_rsquared": [
|
|
0.30093904698626484,
|
|
0.3045032070417637,
|
|
0.30452413965391645,
|
|
0.30468061326857676,
|
|
0.301311692626853
|
|
]
|
|
}
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"application/papermill.record+json": {
|
|
"eval_exp_var": [
|
|
0.30167929819298833,
|
|
0.3045892461871469,
|
|
0.30486639672614235,
|
|
0.3052303508899866,
|
|
0.3023860484774723
|
|
]
|
|
}
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"tf.logging.set_verbosity(tf.logging.INFO)\n",
|
|
"\n",
|
|
"try:\n",
|
|
" model.train(\n",
|
|
" input_fn=train_fn,\n",
|
|
" hooks=hooks,\n",
|
|
" steps=train_steps\n",
|
|
" )\n",
|
|
"except tf.train.NanLossDuringTrainingError:\n",
|
|
" raise ValueError(\n",
|
|
" \"Training stopped with NanLossDuringTrainingError. Try other optimizers, smaller batch size and smaller learning rate.\"\n",
|
|
" )\n",
|
|
" \n",
|
|
"if EVALUATE_WHILE_TRAINING:\n",
|
|
" for m, v in evaluation_logger.get_log().items():\n",
|
|
" pm.record(\"eval_{}\".format(m), v)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"#### 3.2 TensorBoard\n",
|
|
"\n",
|
|
"Once the train is done, you can browse the details of the training results as well as the metrics we logged from [TensorBoard](https://www.tensorflow.org/guide/summaries_and_tensorboard).\n",
|
|
"\n",
|
|
"[]()|[]()|[]()\n",
|
|
":---:|:---:|:---:\n",
|
|
"<img src=\"https://recodatasets.blob.core.windows.net/images/tensorboard_0.png?sanitize=true\"> | <img src=\"https://recodatasets.blob.core.windows.net/images/tensorboard_1.png?sanitize=true\"> | <img src=\"https://recodatasets.blob.core.windows.net/images/tensorboard_2.png?sanitize=true\">\n",
|
|
"\n",
|
|
"To open the TensorBoard, open a terminal from the same directory of this notebook, run `tensorboard --logdir=model_checkpoints`, and open http://localhost:6006 from a browser.\n",
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"### 4. Test and Export Model\n",
|
|
"\n",
|
|
"#### 4.1 Item rating prediction"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 13,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"INFO:tensorflow:Calling model_fn.\n",
|
|
"INFO:tensorflow:Done calling model_fn.\n",
|
|
"INFO:tensorflow:Graph was finalized.\n",
|
|
"INFO:tensorflow:Restoring parameters from model_checkpoints/model.ckpt-58593\n",
|
|
"INFO:tensorflow:Running local_init_op.\n",
|
|
"INFO:tensorflow:Done running local_init_op.\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/papermill.record+json": {
|
|
"rmse": 0.9426291944019234
|
|
}
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"rmse = 0.9426291944019234\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/papermill.record+json": {
|
|
"mae": 0.7339886412034928
|
|
}
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"mae = 0.7339886412034928\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/papermill.record+json": {
|
|
"rsquared": 0.30024333876363185
|
|
}
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"rsquared = 0.30024333876363185\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/papermill.record+json": {
|
|
"exp_var": 0.30290971689748925
|
|
}
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"exp_var = 0.30290971689748925\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"if len(RATING_METRICS) > 0:\n",
|
|
" predictions = list(model.predict(input_fn=tf_utils.pandas_input_fn(df=test)))\n",
|
|
" prediction_df = test.drop(RATING_COL, axis=1)\n",
|
|
" prediction_df['prediction'] = [p['predictions'][0] for p in predictions]\n",
|
|
" prediction_df['prediction'].describe()\n",
|
|
" \n",
|
|
" for m in RATING_METRICS:\n",
|
|
" fn = getattr(reco_utils.evaluation.python_evaluation, m)\n",
|
|
" result = fn(test, prediction_df, **cols)\n",
|
|
" pm.record(m, result)\n",
|
|
" print(m, \"=\", result)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"#### 4.2 Recommend k items\n",
|
|
"For top-k recommendation evaluation, we use the ranking pool (all the user-item pairs) we prepared at the [training step](#ranking-pool). The difference is we remove users' seen items from the pool in this step which is more natural to the movie recommendation scenario."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 14,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"INFO:tensorflow:Calling model_fn.\n",
|
|
"INFO:tensorflow:Done calling model_fn.\n",
|
|
"INFO:tensorflow:Graph was finalized.\n",
|
|
"INFO:tensorflow:Restoring parameters from model_checkpoints/model.ckpt-58593\n",
|
|
"INFO:tensorflow:Running local_init_op.\n",
|
|
"INFO:tensorflow:Done running local_init_op.\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/papermill.record+json": {
|
|
"map_at_k": 0.0024952697678336188
|
|
}
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"map_at_k = 0.0024952697678336188\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/papermill.record+json": {
|
|
"ndcg_at_k": 0.025467919551724758
|
|
}
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"ndcg_at_k = 0.025467919551724758\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/papermill.record+json": {
|
|
"precision_at_k": 0.029617834394904466
|
|
}
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"precision_at_k = 0.029617834394904466\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"application/papermill.record+json": {
|
|
"recall_at_k": 0.01080877083868074
|
|
}
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"recall_at_k = 0.01080877083868074\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"if len(RANKING_METRICS) > 0:\n",
|
|
" predictions = list(model.predict(input_fn=tf_utils.pandas_input_fn(df=ranking_pool)))\n",
|
|
" prediction_df = ranking_pool.copy()\n",
|
|
" prediction_df['prediction'] = [p['predictions'][0] for p in predictions]\n",
|
|
"\n",
|
|
" for m in RANKING_METRICS:\n",
|
|
" fn = getattr(reco_utils.evaluation.python_evaluation, m)\n",
|
|
" result = fn(test, prediction_df, **{**cols, 'k': TOP_K})\n",
|
|
" pm.record(m, result)\n",
|
|
" print(m, \"=\", result)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"#### 4.3 Export Model\n",
|
|
"Finally, we export the model so that we can load later for re-training, evaluation, and prediction.\n",
|
|
"Examples of how to load, re-train, and evaluate the saved model can be found from [aml_hyperparameter_tuning](../04_model_select_and_optimize/hypertune_aml_wide_and_deep_quickstart.ipynb) notebook."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 15,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"os.makedirs(EXPORT_DIR_BASE, exist_ok=True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 16,
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"application/papermill.record+json": {
|
|
"saved_model_dir": "b'./outputs/model/1550346652'"
|
|
}
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Model exported to b'./outputs/model/1550346652'\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"tf.logging.set_verbosity(tf.logging.ERROR)\n",
|
|
"\n",
|
|
"train_rcvr_fn = tf.contrib.estimator.build_supervised_input_receiver_fn_from_input_fn(\n",
|
|
" train_fn\n",
|
|
")\n",
|
|
"eval_rcvr_fn = tf.contrib.estimator.build_supervised_input_receiver_fn_from_input_fn(\n",
|
|
" tf_utils.pandas_input_fn(df=test, y_col=RATING_COL)\n",
|
|
")\n",
|
|
"serve_rcvr_fn = tf.estimator.export.build_parsing_serving_input_receiver_fn(\n",
|
|
" tf.feature_column.make_parse_example_spec(wide_columns+deep_columns)\n",
|
|
")\n",
|
|
"rcvr_fn_map = {\n",
|
|
" tf.estimator.ModeKeys.TRAIN: train_rcvr_fn,\n",
|
|
" tf.estimator.ModeKeys.EVAL: eval_rcvr_fn,\n",
|
|
" tf.estimator.ModeKeys.PREDICT: serve_rcvr_fn\n",
|
|
"}\n",
|
|
"\n",
|
|
"export_dir = tf.contrib.estimator.export_all_saved_models(\n",
|
|
" model,\n",
|
|
" export_dir_base=EXPORT_DIR_BASE,\n",
|
|
" input_receiver_fn_map=rcvr_fn_map\n",
|
|
")\n",
|
|
"pm.record('saved_model_dir', str(export_dir))\n",
|
|
"print(\"Model exported to\", str(export_dir))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"#### Cleanup"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 17,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"\"\"\"\n",
|
|
"Do not directly delete EXPORT_DIR_BASE directory since hypertune_aml_wide_and_deep_quickstart\n",
|
|
"notebook uses this notebook to train and export model.\n",
|
|
"Instead, use the same name for both MODEL_DIR and EXPORT_DIR_BASE to test so that can cleaned up\n",
|
|
"\"\"\"\n",
|
|
"shutil.rmtree(MODEL_DIR, ignore_errors=True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.6.8"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|