microsoft
/
az-deep-realtime-score
зеркало из https://github.com/microsoft/az-deep-realtime-score.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

AKS deployment - debugging code -- work in progress

This commit is contained in:
jayamathew 2018-09-10 11:43:43 -04:00
Родитель b67cb4523e
Коммит ce2205652b
1 изменённых файлов: 769 добавлений и 0 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

769
Keras_Tensorflow/AML_02_AKS_deployment.ipynb Normal file
Просмотреть файл

@ -0,0 +1,769 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Refer to \n",
"https://github.com/Azure/ViennaDocs/blob/master/PrivatePreview/notebooks/01.train-within-notebook.ipynb\n",
"https://github.com/Azure/ViennaDocs/blob/master/PrivatePreview/notebooks/11.production-deploy-to-aks.ipynb"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"from azureml.core import Workspace\n",
"from azureml.core.compute import AksCompute, ComputeTarget\n",
"from azureml.core.webservice import Webservice, AksWebservice\n",
"from azureml.core.image import Image\n",
"from azureml.core.model import Model"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0.1.4\n"
]
}
],
"source": [
"import azureml.core\n",
"print(azureml.core.VERSION)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Found the config file in: /home/jayaubuntudsvm/Desktop/AKSDeploymentTutorial/Keras_Tensorflow/aml_config/config.json\n",
"jayavienna\n",
"jayavienna\n",
"eastus2\n",
"edf507a2-6235-46c5-b560-fd463ba2e771\n"
]
}
],
"source": [
"from azureml.core.workspace import Workspace\n",
"\n",
"ws = Workspace.from_config()\n",
"print(ws.name, ws.resource_group, ws.location, ws.subscription_id, sep = '\\n')"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Using TensorFlow backend.\n"
]
}
],
"source": [
"#Creating the model pickle file\n",
"import tensorflow as tf\n",
"from resnet152 import ResNet152\n",
"from keras.preprocessing import image\n",
"from keras.applications.imagenet_utils import preprocess_input, decode_predictions\n",
"\n",
"model = ResNet152(weights='imagenet')"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<keras.engine.training.Model at 0x7f82aa527da0>"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"model_json = model.to_json()\n",
"with open(\"model_resnet.json\", \"w\") as json_file:\n",
" json_file.write(model_json)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"model.save_weights(\"model_resnet_weights.h5\")"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"#from keras.models import model_from_json\n",
"#model = model_from_json(model_json)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"model.load_weights('model_resnet_weights.h5')"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<keras.engine.training.Model at 0x7f82aa527da0>"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Registering model resnet_model\n",
"resnet_model resnet 152 model 5\n"
]
}
],
"source": [
"#Register the model\n",
"from azureml.core.model import Model\n",
"model = Model.register(model_path = \"model_resnet_weights.h5\", # this points to a local file\n",
" model_name = \"resnet_model\", # this is the name the model is registered as\n",
" tags = [\"dl\", \"resnet\"],\n",
" description = \"resnet 152 model\",\n",
" workspace = ws)\n",
"\n",
"print(model.name, model.description, model.version)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"#define init() function\n",
"def init():\n",
" import tensorflow as tf\n",
" from resnet152 import ResNet152\n",
" from keras.preprocessing import image\n",
" from keras.applications.imagenet_utils import preprocess_input, decode_predictions\n",
"\n",
" import numpy as np\n",
" import timeit as t\n",
" import base64\n",
" import json\n",
" from PIL import Image, ImageOps\n",
" from io import BytesIO\n",
" import logging\n",
"\n",
" global model\n",
" model = ResNet152(weights='imagenet')\n",
" print('Model loaded')"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Model loaded\n"
]
}
],
"source": [
"init()"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"#define run() function \n",
"def run(img_path):\n",
" \n",
" import tensorflow as tf\n",
" from resnet152 import ResNet152\n",
" from keras.preprocessing import image\n",
" from keras.applications.imagenet_utils import preprocess_input, decode_predictions\n",
"\n",
" import numpy as np\n",
" import timeit as t\n",
" import base64\n",
" import json\n",
" from PIL import Image, ImageOps\n",
" from io import BytesIO\n",
" import logging \n",
" \n",
" model = ResNet152(weights='imagenet')\n",
" print('Model loaded')\n",
" \n",
" img = image.load_img(img_path, target_size=(224, 224))\n",
" img = image.img_to_array(img)\n",
" img = np.expand_dims(img, axis=0)\n",
" img = preprocess_input(img)\n",
" \n",
" preds = model.predict(img)\n",
" print('Predicted:', decode_predictions(preds, top=3))"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'220px-Lynx_lynx_poing.jpg'"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import wget\n",
"wget.download('https://upload.wikimedia.org/wikipedia/commons/thumb/6/68/Lynx_lynx_poing.jpg/220px-Lynx_lynx_poing.jpg')"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"img_path = '220px-Lynx_lynx_poing.jpg'"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Model loaded\n",
"Predicted: [[('n02127052', 'lynx', 0.9959085), ('n02128385', 'leopard', 0.0011504436), ('n02123159', 'tiger_cat', 0.0009417995)]]\n"
]
}
],
"source": [
"run(img_path)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Overwriting score.py\n"
]
}
],
"source": [
"%%writefile score.py\n",
"def init():\n",
" import tensorflow as tf\n",
" from resnet152 import ResNet152\n",
" from keras.preprocessing import image\n",
" from keras.applications.imagenet_utils import preprocess_input, decode_predictions\n",
"\n",
" import numpy as np\n",
" import timeit as t\n",
" import base64\n",
" import json\n",
" from PIL import Image, ImageOps\n",
" from io import BytesIO\n",
" import logging\n",
"\n",
" global model\n",
" model = ResNet152(weights='imagenet')\n",
" print('Model loaded')\n",
" \n",
"def run(img_path):\n",
" \n",
" import tensorflow as tf\n",
" from resnet152 import ResNet152\n",
" from keras.preprocessing import image\n",
" from keras.applications.imagenet_utils import preprocess_input, decode_predictions\n",
"\n",
" import numpy as np\n",
" import timeit as t\n",
" import base64\n",
" import json\n",
" from PIL import Image, ImageOps\n",
" from io import BytesIO\n",
" import logging \n",
" \n",
" model = ResNet152(weights='imagenet')\n",
" print('Model loaded')\n",
" \n",
" img = image.load_img(img_path, target_size=(224, 224))\n",
" img = image.img_to_array(img)\n",
" img = np.expand_dims(img, axis=0)\n",
" img = preprocess_input(img)\n",
" \n",
" preds = model.predict(img)\n",
" print('Predicted:', decode_predictions(preds, top=3)) "
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [],
"source": [
"%run score.py"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"def init():\n",
" import tensorflow as tf\n",
" from resnet152 import ResNet152\n",
" from keras.preprocessing import image\n",
" from keras.applications.imagenet_utils import preprocess_input, decode_predi\n",
"ctions\n",
"\n",
" import numpy as np\n",
" import timeit as t\n",
" import base64\n",
" import json\n",
" from PIL import Image, ImageOps\n",
" from io import BytesIO\n",
" import logging\n",
"\n",
" global model\n",
" model = ResNet152(weights='imagenet')\n",
" print('Model loaded')\n",
" \n",
"def run(img_path):\n",
" \n",
" import tensorflow as tf\n",
" from resnet152 import ResNet152\n",
"\u001b[Km--More--(46%)\u001b[m"
]
}
],
"source": [
"!more score.py"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"# Conda environment specification. The dependencies defined in this file will\n",
"# be automatically provisioned for runs with userManagedDependencies=False.\n",
"\n",
"# Details about the Conda environment file format:\n",
"# https://conda.io/docs/user-guide/tasks/manage-environments.html#create-env-fil\n",
"e-manually\n",
"\n",
"name: project_environment\n",
"dependencies:\n",
" # The python interpreter version.\n",
" # Currently Azure ML only supports 3.5.2 and later.\n",
"- python=3.6\n",
"\n",
"- pip:\n",
" # Required packages for AzureML execution, history, and data preparation.\n",
" - --index-url https://azuremlsdktestpypi.azureedge.net/sdk-release/Preview/E75\n",
"01C02541B433786111FE8E140CAA1\n",
" - --extra-index-url https://pypi.python.org/simple\n",
" - azureml-defaults\n",
" - papermill==0.14.1\n",
" - python-dotenv==0.9.0\n",
" - Pillow==5.2.0\n",
" - wget==3.2\n",
"\u001b[Km--More--(86%)\u001b[m"
]
}
],
"source": [
"!more conda_dependencies.yml"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Creating image\n",
"Running................................................\n",
"SucceededImage creation operation finished for image myimage1:3, operation \"Succeeded\"\n"
]
}
],
"source": [
"from azureml.core.image import ContainerImage\n",
"\n",
"image_config = ContainerImage.image_configuration(execution_script = \"score.py\",\n",
" runtime = \"python\",\n",
" conda_file = \"conda_dependencies.yml\",\n",
" description = \"Image for AKS Deployment Tutorial\",\n",
" tags = [\"AKS\",\"AML\"]\n",
" )\n",
"\n",
"image = ContainerImage.create(name = \"myimage1\",\n",
" # this is the model object\n",
" models = [], \n",
" image_config = image_config,\n",
" workspace = ws)\n",
"\n",
"image.wait_for_creation(show_output = True)"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [],
"source": [
"#Provision AKS cluster\n",
"# Use the default configuration (can also provide parameters to customize)\n",
"prov_config = AksCompute.provisioning_configuration()\n",
"\n",
"aks_name = 'jaya-aks-1' \n",
"# Create the cluster\n",
"aks_target = ComputeTarget.create(workspace = ws, \n",
" name = aks_name, \n",
" provisioning_configuration = prov_config)\n"
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {},
"outputs": [],
"source": [
"#!az aks delete --resource-group jayavienna --name jaya-aks --yes"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Creating.....................................................................................................................................................................................................................................................................................................................\n",
"SucceededAKS provisioning operation finished, operation \"Succeeded\"\n",
"Succeeded\n",
"None\n",
"CPU times: user 4.42 s, sys: 502 ms, total: 4.92 s\n",
"Wall time: 27min 7s\n"
]
}
],
"source": [
"%%time\n",
"aks_target.wait_for_provisioning(show_output = True)\n",
"print(aks_target.provisioning_state)\n",
"print(aks_target.provisioning_errors)"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [],
"source": [
"#Deploy web service to AKS\n",
"#Set the web service configuration (using default here)\n",
"aks_config = AksWebservice.deploy_configuration()"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Creating service\n",
"Running.............................................\n",
"FailedAKS service creation operation finished, operation \"Failed\"\n",
"Service creation failed with\n",
"StatusCode: 400\n",
"Message: Kubernetes Deployment failed\n",
"{'desiredReplicas': 1, 'updatedReplicas': 0, 'availableReplicas': 0}\n",
"CPU times: user 835 ms, sys: 42 ms, total: 877 ms\n",
"Wall time: 4min 16s\n"
]
}
],
"source": [
"%%time\n",
"aks_service_name ='jaya-aks-service-2'\n",
"\n",
"aks_service = Webservice.deploy_from_image(workspace = ws, \n",
" name = aks_service_name,\n",
" image = image,\n",
" deployment_config = aks_config,\n",
" deployment_target = aks_target)\n",
"aks_service.wait_for_deployment(show_output = True)\n",
"print(aks_service.state)"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Found the config file in: /home/jayaubuntudsvm/Desktop/AKSDeploymentTutorial/Keras_Tensorflow/aml_config/config.json\n",
"jaya-aks-service-2\n",
"jaya-aks-service-1\n",
"2018-09-06 20:02:57,782 CRIT Supervisor running as root (no user in config file)\n",
"2018-09-06 20:02:57,784 INFO supervisord started with pid 1\n",
"2018-09-06 20:02:58,786 INFO spawned: 'rsyslog' with pid 9\n",
"2018-09-06 20:02:58,788 INFO spawned: 'program_exit' with pid 10\n",
"2018-09-06 20:02:58,789 INFO spawned: 'nginx' with pid 11\n",
"2018-09-06 20:02:58,790 INFO spawned: 'iot' with pid 12\n",
"2018-09-06 20:02:58,791 INFO spawned: 'gunicorn' with pid 13\n",
"2018-09-06 20:02:58,802 INFO success: iot entered RUNNING state, process has stayed up for > than 0 seconds (startsecs)\n",
"EdgeHubConnectionString and IOTEDGE_IOTHUBHOSTNAME are not set. Exiting...\n",
"2018-09-06 20:02:58,869 INFO exited: iot (exit status 1; expected)\n",
"2018-09-06 20:02:59,870 INFO success: rsyslog entered RUNNING state, process has stayed up for > than 1 seconds (startsecs)\n",
"2018-09-06 20:02:59,870 INFO success: program_exit entered RUNNING state, process has stayed up for > than 1 seconds (startsecs)\n",
"2018-09-06 20:03:03,875 INFO success: nginx entered RUNNING state, process has stayed up for > than 5 seconds (startsecs)\n",
"{\"logger\": \"gunicorn.error\", \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\", \"level\": \"INFO\", \"msg\": \"Starting gunicorn %s\", \"message\": \"Starting gunicorn 19.6.0\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"path\": \"/home/mmlspark/lib/conda/lib/python3.5/site-packages/gunicorn/glogging.py\", \"stack_info\": null, \"timestamp\": \"2018-09-06T20:03:04.247413Z\"}\n",
"{\"logger\": \"gunicorn.error\", \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\", \"level\": \"INFO\", \"msg\": \"Listening at: %s (%s)\", \"message\": \"Listening at: http://127.0.0.1:9090 (13)\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"path\": \"/home/mmlspark/lib/conda/lib/python3.5/site-packages/gunicorn/glogging.py\", \"stack_info\": null, \"timestamp\": \"2018-09-06T20:03:04.248233Z\"}\n",
"{\"logger\": \"gunicorn.error\", \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\", \"level\": \"INFO\", \"msg\": \"Using worker: %s\", \"message\": \"Using worker: sync\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"path\": \"/home/mmlspark/lib/conda/lib/python3.5/site-packages/gunicorn/glogging.py\", \"stack_info\": null, \"timestamp\": \"2018-09-06T20:03:04.248351Z\"}\n",
"{\"logger\": \"gunicorn.error\", \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\", \"level\": \"INFO\", \"message\": \"worker timeout is set to 300\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"path\": \"/home/mmlspark/lib/conda/lib/python3.5/site-packages/gunicorn/glogging.py\", \"stack_info\": null, \"timestamp\": \"2018-09-06T20:03:04.248946Z\"}\n",
"{\"logger\": \"gunicorn.error\", \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\", \"level\": \"INFO\", \"msg\": \"Booting worker with pid: %s\", \"message\": \"Booting worker with pid: 29\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"path\": \"/home/mmlspark/lib/conda/lib/python3.5/site-packages/gunicorn/glogging.py\", \"stack_info\": null, \"timestamp\": \"2018-09-06T20:03:04.249864Z\"}\n",
"Initializing logger\n",
"{\"logger\": \"root\", \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\", \"level\": \"INFO\", \"message\": \"{\\\"message\\\": \\\"Starting up app insights client\\\", \\\"apiName\\\": \\\"\\\", \\\"requestId\\\": \\\"00000000-0000-0000-0000-000000000000\\\"}\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"path\": \"/var/azureml-app/aml_logger.py\", \"stack_info\": null, \"timestamp\": \"2018-09-06T20:03:04.757389Z\"}\n",
"{\"logger\": \"root\", \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\", \"level\": \"INFO\", \"message\": \"{\\\"message\\\": \\\"Starting up request id generator\\\", \\\"apiName\\\": \\\"\\\", \\\"requestId\\\": \\\"00000000-0000-0000-0000-000000000000\\\"}\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"path\": \"/var/azureml-app/aml_logger.py\", \"stack_info\": null, \"timestamp\": \"2018-09-06T20:03:04.757627Z\"}\n",
"{\"logger\": \"root\", \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\", \"level\": \"INFO\", \"message\": \"{\\\"message\\\": \\\"Starting up app insight hooks\\\", \\\"apiName\\\": \\\"\\\", \\\"requestId\\\": \\\"00000000-0000-0000-0000-000000000000\\\"}\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"path\": \"/var/azureml-app/aml_logger.py\", \"stack_info\": null, \"timestamp\": \"2018-09-06T20:03:04.757757Z\"}\n",
"{\"logger\": \"root\", \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\", \"level\": \"INFO\", \"message\": \"{\\\"message\\\": \\\"Invoking user's init function\\\", \\\"apiName\\\": \\\"\\\", \\\"requestId\\\": \\\"00000000-0000-0000-0000-000000000000\\\"}\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"path\": \"/var/azureml-app/aml_logger.py\", \"stack_info\": null, \"timestamp\": \"2018-09-06T20:03:04.757867Z\"}\n",
"{\"logger\": \"root\", \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\", \"level\": \"ERROR\", \"message\": \"{\\\"message\\\": \\\"User's init function failed\\\", \\\"apiName\\\": \\\"\\\", \\\"requestId\\\": \\\"00000000-0000-0000-0000-000000000000\\\"}\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"path\": \"/var/azureml-app/aml_logger.py\", \"stack_info\": null, \"timestamp\": \"2018-09-06T20:03:06.402341Z\"}\n",
"{\"logger\": \"root\", \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\", \"level\": \"ERROR\", \"message\": \"{\\\"message\\\": \\\"Encountered Exception Traceback (most recent call last):\\\\n File \\\\\\\"/var/azureml-app/aml_blueprint.py\\\\\\\", line 109, in register\\\\n main.init()\\\\n File \\\\\\\"/var/azureml-app/main.py\\\\\\\", line 75, in init\\\\n driver_module.init()\\\\n File \\\\\\\"score.py\\\\\\\", line 3, in init\\\\n from resnet152 import ResNet152\\\\nImportError: No module named 'resnet152'\\\\n\\\", \\\"apiName\\\": \\\"\\\", \\\"requestId\\\": \\\"00000000-0000-0000-0000-000000000000\\\"}\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"path\": \"/var/azureml-app/aml_logger.py\", \"stack_info\": null, \"timestamp\": \"2018-09-06T20:03:06.402633Z\"}\n",
"{\"logger\": \"gunicorn.error\", \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\", \"level\": \"INFO\", \"msg\": \"Worker exiting (pid: %s)\", \"message\": \"Worker exiting (pid: 29)\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"path\": \"/home/mmlspark/lib/conda/lib/python3.5/site-packages/gunicorn/glogging.py\", \"stack_info\": null, \"timestamp\": \"2018-09-06T20:03:06.402787Z\"}\n",
"{\"logger\": \"gunicorn.error\", \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\", \"level\": \"INFO\", \"msg\": \"Shutting down: %s\", \"message\": \"Shutting down: Master\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"path\": \"/home/mmlspark/lib/conda/lib/python3.5/site-packages/gunicorn/glogging.py\", \"stack_info\": null, \"timestamp\": \"2018-09-06T20:03:06.589150Z\"}\n",
"{\"logger\": \"gunicorn.error\", \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\", \"level\": \"INFO\", \"msg\": \"Reason: %s\", \"message\": \"Reason: Worker failed to boot.\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"path\": \"/home/mmlspark/lib/conda/lib/python3.5/site-packages/gunicorn/glogging.py\", \"stack_info\": null, \"timestamp\": \"2018-09-06T20:03:06.589449Z\"}\n",
"2018-09-06 20:03:06,617 INFO exited: gunicorn (exit status 3; not expected)\n",
"2018-09-06 20:03:07,619 INFO spawned: 'gunicorn' with pid 35\n",
"{\"logger\": \"gunicorn.error\", \"message\": \"Starting gunicorn 19.6.0\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"level\": \"INFO\", \"msg\": \"Starting gunicorn %s\", \"timestamp\": \"2018-09-06T20:03:13.066570Z\", \"path\": \"/home/mmlspark/lib/conda/lib/python3.5/site-packages/gunicorn/glogging.py\", \"stack_info\": null, \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\"}\n",
"{\"logger\": \"gunicorn.error\", \"message\": \"Listening at: http://127.0.0.1:9090 (35)\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"level\": \"INFO\", \"msg\": \"Listening at: %s (%s)\", \"timestamp\": \"2018-09-06T20:03:13.067406Z\", \"path\": \"/home/mmlspark/lib/conda/lib/python3.5/site-packages/gunicorn/glogging.py\", \"stack_info\": null, \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\"}\n",
"{\"logger\": \"gunicorn.error\", \"message\": \"Using worker: sync\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"level\": \"INFO\", \"msg\": \"Using worker: %s\", \"timestamp\": \"2018-09-06T20:03:13.067547Z\", \"path\": \"/home/mmlspark/lib/conda/lib/python3.5/site-packages/gunicorn/glogging.py\", \"stack_info\": null, \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\"}\n",
"{\"logger\": \"gunicorn.error\", \"message\": \"worker timeout is set to 300\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"level\": \"INFO\", \"timestamp\": \"2018-09-06T20:03:13.068255Z\", \"path\": \"/home/mmlspark/lib/conda/lib/python3.5/site-packages/gunicorn/glogging.py\", \"stack_info\": null, \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\"}\n",
"{\"logger\": \"gunicorn.error\", \"message\": \"Booting worker with pid: 39\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"level\": \"INFO\", \"msg\": \"Booting worker with pid: %s\", \"timestamp\": \"2018-09-06T20:03:13.069285Z\", \"path\": \"/home/mmlspark/lib/conda/lib/python3.5/site-packages/gunicorn/glogging.py\", \"stack_info\": null, \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\"}\n",
"Initializing logger\n",
"{\"logger\": \"root\", \"message\": \"{\\\"requestId\\\": \\\"00000000-0000-0000-0000-000000000000\\\", \\\"message\\\": \\\"Starting up app insights client\\\", \\\"apiName\\\": \\\"\\\"}\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"level\": \"INFO\", \"timestamp\": \"2018-09-06T20:03:13.480881Z\", \"path\": \"/var/azureml-app/aml_logger.py\", \"stack_info\": null, \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\"}\n",
"{\"logger\": \"root\", \"message\": \"{\\\"requestId\\\": \\\"00000000-0000-0000-0000-000000000000\\\", \\\"message\\\": \\\"Starting up request id generator\\\", \\\"apiName\\\": \\\"\\\"}\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"level\": \"INFO\", \"timestamp\": \"2018-09-06T20:03:13.481104Z\", \"path\": \"/var/azureml-app/aml_logger.py\", \"stack_info\": null, \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\"}\n",
"{\"logger\": \"root\", \"message\": \"{\\\"requestId\\\": \\\"00000000-0000-0000-0000-000000000000\\\", \\\"message\\\": \\\"Starting up app insight hooks\\\", \\\"apiName\\\": \\\"\\\"}\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"level\": \"INFO\", \"timestamp\": \"2018-09-06T20:03:13.481212Z\", \"path\": \"/var/azureml-app/aml_logger.py\", \"stack_info\": null, \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\"}\n",
"{\"logger\": \"root\", \"message\": \"{\\\"requestId\\\": \\\"00000000-0000-0000-0000-000000000000\\\", \\\"message\\\": \\\"Invoking user's init function\\\", \\\"apiName\\\": \\\"\\\"}\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"level\": \"INFO\", \"timestamp\": \"2018-09-06T20:03:13.481322Z\", \"path\": \"/var/azureml-app/aml_logger.py\", \"stack_info\": null, \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\"}\n",
"{\"logger\": \"root\", \"message\": \"{\\\"requestId\\\": \\\"00000000-0000-0000-0000-000000000000\\\", \\\"message\\\": \\\"User's init function failed\\\", \\\"apiName\\\": \\\"\\\"}\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"level\": \"ERROR\", \"timestamp\": \"2018-09-06T20:03:14.924091Z\", \"path\": \"/var/azureml-app/aml_logger.py\", \"stack_info\": null, \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\"}\n",
"{\"logger\": \"root\", \"message\": \"{\\\"requestId\\\": \\\"00000000-0000-0000-0000-000000000000\\\", \\\"message\\\": \\\"Encountered Exception Traceback (most recent call last):\\\\n File \\\\\\\"/var/azureml-app/aml_blueprint.py\\\\\\\", line 109, in register\\\\n main.init()\\\\n File \\\\\\\"/var/azureml-app/main.py\\\\\\\", line 75, in init\\\\n driver_module.init()\\\\n File \\\\\\\"score.py\\\\\\\", line 3, in init\\\\n from resnet152 import ResNet152\\\\nImportError: No module named 'resnet152'\\\\n\\\", \\\"apiName\\\": \\\"\\\"}\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"level\": \"ERROR\", \"timestamp\": \"2018-09-06T20:03:14.924467Z\", \"path\": \"/var/azureml-app/aml_logger.py\", \"stack_info\": null, \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\"}\n",
"{\"logger\": \"gunicorn.error\", \"message\": \"Worker exiting (pid: 39)\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"level\": \"INFO\", \"msg\": \"Worker exiting (pid: %s)\", \"timestamp\": \"2018-09-06T20:03:14.924632Z\", \"path\": \"/home/mmlspark/lib/conda/lib/python3.5/site-packages/gunicorn/glogging.py\", \"stack_info\": null, \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\"}\n",
"{\"logger\": \"gunicorn.error\", \"message\": \"Shutting down: Master\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"level\": \"INFO\", \"msg\": \"Shutting down: %s\", \"timestamp\": \"2018-09-06T20:03:15.119963Z\", \"path\": \"/home/mmlspark/lib/conda/lib/python3.5/site-packages/gunicorn/glogging.py\", \"stack_info\": null, \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\"}\n",
"{\"logger\": \"gunicorn.error\", \"message\": \"Reason: Worker failed to boot.\", \"host\": \"jaya-aks-service-2-7f96c94d75-bc4tq\", \"level\": \"INFO\", \"msg\": \"Reason: %s\", \"timestamp\": \"2018-09-06T20:03:15.120300Z\", \"path\": \"/home/mmlspark/lib/conda/lib/python3.5/site-packages/gunicorn/glogging.py\", \"stack_info\": null, \"tags\": \"%(module)s, %(asctime)s, %(levelname)s, %(message)s\"}\n",
"2018-09-06 20:03:15,149 INFO exited: gunicorn (exit status 3; not expected)\n",
"2018-09-06 20:03:16,150 INFO gave up: gunicorn entered FATAL state, too many start retries too quickly\n",
"2018-09-06 20:03:17,152 WARN program_exit: bad result line: 'Killing supervisor with this event: ver:3.0 server:supervisor serial:0 pool:program_exit poolserial:0 eventname:PROCESS_STATE_FATAL len:58'\n",
"2018-09-06 20:03:17,152 WARN program_exit: has entered the UNKNOWN state and will no longer receive events, this usually indicates the process violated the eventlistener protocol\n",
"2018-09-06 20:03:17,152 WARN received SIGQUIT indicating exit request\n",
"2018-09-06 20:03:17,152 INFO waiting for nginx, rsyslog, program_exit to die\n",
"2018-09-06 20:03:18,157 INFO stopped: nginx (exit status 0)\n",
"2018-09-06 20:03:18,157 INFO stopped: program_exit (terminated by SIGTERM)\n",
"2018-09-06 20:03:18,159 INFO stopped: rsyslog (exit status 0)\n",
"\n"
]
}
],
"source": [
"#Debug\n",
"\n",
"# load workspace from default config file\n",
"ws = Workspace.from_config()\n",
"\n",
"# list all web services in the workspace\n",
"for s in ws.webservices():\n",
" print(s.name)\n",
"\n",
"# instantiate the web service object from workspace and service name\n",
"svc = Webservice(workspace = ws, name = 'jaya-aks-service-2')\n",
"\n",
"# print out the Docker log\n",
"print(svc.get_logs())"
]
},
{
"cell_type": "code",
"execution_count": 42,
"metadata": {},
"outputs": [],
"source": [
"#clean up resources\n",
"aks_target = AksCompute(name='my-aks-',workspace=ws)\n",
"aks_target.delete()"
]
},
{
"cell_type": "code",
"execution_count": 43,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"jaya-aks-service-2\n",
"jaya-aks-service-1\n"
]
}
],
"source": [
"for s in ws.webservices():\n",
" print(s.name)"
]
},
{
"cell_type": "code",
"execution_count": 48,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Found the config file in: /home/jayaubuntudsvm/Desktop/AKSDeploymentTutorial/Keras_Tensorflow/aml_config/config.json\n",
"jaya-aks-1\n"
]
}
],
"source": [
"from azureml.core import Workspace\n",
"from azureml.core.compute import AksCompute, ComputeTarget\n",
"\n",
"ws = Workspace.from_config()\n",
"\n",
"for c in ws.compute_targets():\n",
" print(c.name)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python [conda env:myenv]",
"language": "python",
"name": "conda-env-myenv-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.6"
}
},
"nbformat": 4,
"nbformat_minor": 2
}