diff --git a/.ci/azure-pipeline-azureml-notebook-test.yml b/.ci/azure-pipeline-azureml-notebook-test.yml index 79c415a..a9eee3f 100644 --- a/.ci/azure-pipeline-azureml-notebook-test.yml +++ b/.ci/azure-pipeline-azureml-notebook-test.yml @@ -13,10 +13,6 @@ variables: value : 'reports/test-unit.xml' trigger: none -pr: -- staging -- master - jobs: - job: AzureMLNotebookTest timeoutInMinutes: 300 @@ -61,4 +57,4 @@ jobs: inputs: testResultsFiles: '**/test-*.xml' failTaskOnFailedTests: true - condition: succeededOrFailed() \ No newline at end of file + condition: succeededOrFailed() diff --git a/.ci/repo_metrics_pipeline.yml b/.ci/repo_metrics_pipeline.yml index 40a0c81..9a3b660 100644 --- a/.ci/repo_metrics_pipeline.yml +++ b/.ci/repo_metrics_pipeline.yml @@ -1,3 +1,23 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# More info on scheduling: https://docs.microsoft.com/en-us/azure/devops/pipelines/build/triggers?view=azure-devops&tabs=yaml#scheduled-triggers +# Implementing the scheduler from the dashboard +# Uncomment in case it wants to be done from using the yml +# schedules: +# - cron: "56 22 * * *" +# displayName: Daily track of metrics +# branches: +# include: +# - master +# always: true + + +# no PR builds +pr: none + +# no CI trigger +trigger: none jobs: - job: Repometrics @@ -5,7 +25,6 @@ jobs: vmImage: 'ubuntu-16.04' steps: - - task: UsePythonVersion@0 inputs: versionSpec: '3.6' @@ -13,13 +32,13 @@ jobs: - script: | cp tools/repo_metrics/config_template.py tools/repo_metrics/config.py - sed -i ''s/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/$(github_token)/g'' tools/repo_metrics/config.py - sed -i ''s/XXXXXXXXXXXXXXXXXXXXXXXXX/$(cosmosdb_connectionstring)/g'' tools/repo_metrics/config.py + sed -i 's##$(github_token)#' tools/repo_metrics/config.py + sed -i "s##`echo '$(cosmosdb_connectionstring)' | sed 's@&@\\\\&@g'`#" tools/repo_metrics/config.py displayName: Configure CosmosDB Connection - script: | - python -m pip install python-dateutil>=2.80 pymongo>=3.8.0 gitpython>2.1.11 requests>=2.21.0 - python tools/repo_metrics/track_metrics.py --github_repo "https://github.com/microsoft/ComputerVision" --save_to_database + python -m pip install 'python-dateutil>=2.8.0' 'pymongo>=3.8.0' 'gitpython>2.1.11' 'requests>=2.21.0' + python tools/repo_metrics/track_metrics.py --github_repo 'https://github.com/microsoft/ComputerVision' --save_to_database displayName: Python script to record stats diff --git a/.ci/templates/unit-test-steps.yml b/.ci/templates/unit-test-steps.yml index a8d1769..da25e58 100644 --- a/.ci/templates/unit-test-steps.yml +++ b/.ci/templates/unit-test-steps.yml @@ -5,6 +5,10 @@ steps: echo "##vso[task.prependpath]/data/anaconda/bin" displayName: Add Conda to PATH +- bash: | + rm -rf /data/anaconda/envs/cv + displayName: 'Remove conda env in case it was not created correctly' + - bash: | conda env create -f environment.yml source activate cv diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bc4172f..7d0fac4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,12 +20,12 @@ When you submit a pull request, a CLA-bot will automatically determine whether y ## Steps to Contributing Here are the basic steps to get started with your first contribution. Please reach out with any questions. -1. Use [open issues](https://github.com/Microsoft/Recommenders/issues) to discuss the proposed changes. Create an issue describing changes if necessary to collect feedback. Also, please use provided labels to tag issues so everyone can easily sort issues of interest. +1. Use [open issues](https://github.com/Microsoft/ComputerVision/issues) to discuss the proposed changes. Create an issue describing changes if necessary to collect feedback. Also, please use provided labels to tag issues so everyone can easily sort issues of interest. 1. [Fork the repo](https://help.github.com/articles/fork-a-repo/) so you can make and test local changes. 1. Create a new branch for the issue. We suggest prefixing the branch with your username and then a descriptive title: (e.g. gramhagen/update_contributing_docs) 1. Create a test that replicates the issue. 1. Make code changes. -1. Ensure unit tests pass and code style / formatting is consistent (see [wiki](https://github.com/Microsoft/Recommenders/wiki/Coding-Guidelines#python-and-docstrings-style) for more details). +1. Ensure unit tests pass and code style / formatting is consistent, and follows the [Zen of Python](https://github.com/Microsoft/Recommenders/wiki/Coding-Guidelines#the-zen-of-python). 1. We use [pre-commit](https://pre-commit.com/) package to run our pre-commit hooks. We use black formatter and flake8 linting on each commit. In order to set up pre-commit on your machine, follow the steps here, please note that you only need to run these steps the first time you use pre-commit for this project. * Update your conda environment, pre-commit is part of the yaml file or just do @@ -49,7 +49,6 @@ Here are the basic steps to get started with your first contribution. Please rea Note: We use the staging branch to land all new features, so please remember to create the Pull Request against staging. -Once the features included in a milestone are complete we will merge staging into master and make a release. See the wiki for more detail about our [merge strategy](https://github.com/Microsoft/Recommenders/wiki/Strategy-to-merge-the-code-to-master-branch). ## Working with Notebooks @@ -77,8 +76,6 @@ nbdiff notebook_1.ipynb notebook_2.ipynb We strive to maintain high quality code to make the utilities in the repository easy to understand, use, and extend. We also work hard to maintain a friendly and constructive environment. We've found that having clear expectations on the development process and consistent style helps to ensure everyone can contribute and collaborate effectively. -Please review the [coding guidelines](https://github.com/Microsoft/Recommenders/wiki/Coding-Guidelines) wiki page to see more details about the expectations for development approach and style. - We follow the Google docstring guidlines outlined on this [styleguide](https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings) page. For example: ```python def bite(n:int, animal:animal_object) -> bool: @@ -103,7 +100,7 @@ This project has adopted the [Microsoft Open Source Code of Conduct](https://ope For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. -Apart from the official Code of Conduct developed by Microsoft, in the Recommenders team we adopt the following behaviors, to ensure a great working environment: +Apart from the official Code of Conduct developed by Microsoft, we adopt the following behaviors, to ensure a great working environment: #### Do not point fingers Let’s be constructive. For example: "This method is missing docstrings" instead of "YOU forgot to put docstrings". diff --git a/README.md b/README.md index 22ca245..f775c55 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,10 @@ The current main priority is to support image classification. Additionally, we a ## Getting Started -To get started on your local machine: +To get started: -1. Install Anaconda with Python >= 3.6. [Miniconda](https://conda.io/miniconda.html) is a quick way to get started. +1. (Optional) Create an Azure Data Science Virtual Machine with e.g. a V100 GPU ([instructions](https://docs.microsoft.com/en-us/azure/machine-learning/data-science-virtual-machine/provision-deep-learning-dsvm), [price table](https://azure.microsoft.com/en-us/pricing/details/virtual-machines/windows/)). +1. Install Anaconda with Python >= 3.6. [Miniconda](https://conda.io/miniconda.html). This step can be skipped if working on a Data Science Virtual Machine. 1. Clone the repository ``` git clone https://github.com/Microsoft/ComputerVision diff --git a/classification/notebooks/11_exploring_hyperparameters.ipynb b/classification/notebooks/11_exploring_hyperparameters.ipynb index 97478b5..bbd07d9 100644 --- a/classification/notebooks/11_exploring_hyperparameters.ipynb +++ b/classification/notebooks/11_exploring_hyperparameters.ipynb @@ -22,6 +22,7 @@ "source": [ "In this notebook, we'll cover how to test different hyperparameters for a particular dataset and how to benchmark different parameters across a group of datasets.\n", "\n", + "For an example of how to scale up with remote GPU clusters on Azure Machine Learning, please view [24_exploring_hyperparameters_on_azureml.ipynb](../24_exploring_hyperparameters_on_azureml).\n", "## Table of Contents\n", "\n", "* [Testing hyperparameters](#hyperparam)\n", @@ -52,7 +53,7 @@ "metadata": {}, "source": [ "Ensure edits to libraries are loaded and plotting is shown in the notebook." - ] + ] }, { "cell_type": "code", diff --git a/classification/notebooks/24_exploring_hyperparameters_on_azureml.ipynb b/classification/notebooks/24_exploring_hyperparameters_on_azureml.ipynb index 05f2807..a887cde 100644 --- a/classification/notebooks/24_exploring_hyperparameters_on_azureml.ipynb +++ b/classification/notebooks/24_exploring_hyperparameters_on_azureml.ipynb @@ -20,16 +20,16 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "In this notebook, we'll cover how to test different hyperparameters for a particular dataset and how to benchmark different parameters across a group of datasets using AzureML" + "In this notebook, we'll cover how to test different hyperparameters for a particular dataset and how to benchmark different parameters across a group of datasets using AzureML. We assume familiarity with the basic concepts and parameters, which are discussed in the [01_training_introduction.ipynb](01_training_introduction.ipynb), [02_multilabel_classification.ipynb](02_multilabel_classification.ipynb) and [03_training_accuracy_vs_speed.ipynb](03_training_accuracy_vs_speed.ipynb) notebooks. " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Similar to [11_exploring_hyperparameters.ipynb](https://github.com/microsoft/ComputerVision/blob/master/classification/notebooks/11_exploring_hyperparameters.ipynb), we will learn more about how different learning rates and different image sizes affect our model's accuracy when restricted to 10 epochs, and we want to build an AzureML experiment to test out these hyperparameters. \n", + "Similar to [11_exploring_hyperparameters.ipynb](https://github.com/microsoft/ComputerVision/blob/master/classification/notebooks/11_exploring_hyperparameters.ipynb), we will learn more about how different learning rates and different image sizes affect our model's accuracy when restricted to 16 epochs, and we want to build an AzureML experiment to test out these hyperparameters. \n", "\n", - "We will be using a ResNet50 model to classify a set of images into 4 categories - 'can', 'carton', 'milk_bottle', 'water_bottle'. We will then conduct hyper-parameter tuning to find the best set of parameters for this model. For this,\n", + "We will be using a ResNet18 model to classify a set of images into 4 categories: 'can', 'carton', 'milk_bottle', 'water_bottle'. We will then conduct hyper-parameter tuning to find the best set of parameters for this model. For this,\n", "we present an overall process of utilizing AzureML, specifically [Hyperdrive](https://docs.microsoft.com/en-us/python/api/azureml-train-core/azureml.train.hyperdrive?view=azure-ml-py) component to run this tuning in parallel (and not successively).We demonstrate the following key steps: \n", "* Configure AzureML Workspace\n", "* Create Remote Compute Target (GPU cluster)\n", @@ -43,15 +43,7 @@ "cell_type": "code", "execution_count": 1, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "SDK version: 1.0.48\n" - ] - } - ], + "outputs": [], "source": [ "import os\n", "import sys\n", @@ -65,15 +57,14 @@ "from azureml.core.compute import ComputeTarget, AmlCompute\n", "from azureml.core.compute_target import ComputeTargetException\n", "import azureml.data\n", - "from azureml.train.hyperdrive import RandomParameterSampling, BanditPolicy, HyperDriveConfig, PrimaryMetricGoal, choice\n", "from azureml.train.estimator import Estimator\n", - "\n", + "from azureml.train.hyperdrive import (\n", + " RandomParameterSampling, BanditPolicy, HyperDriveConfig, PrimaryMetricGoal, choice, uniform\n", + ")\n", "import azureml.widgets as widgets\n", "\n", "from utils_cv.classification.data import Urls\n", - "from utils_cv.common.data import unzip_url\n", - "\n", - "print(\"SDK version:\", azureml.core.VERSION)" + "from utils_cv.common.data import unzip_url" ] }, { @@ -98,8 +89,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### 1. Config AzureML workspace\n", - "Below we setup AzureML workspace and get all its details as follows:" + "We now define some parameters which will be used in this notebook:" ] }, { @@ -116,36 +106,37 @@ "subscription_id = \"YOUR_SUBSCRIPTION_ID\"\n", "resource_group = \"YOUR_RESOURCE_GROUP_NAME\" \n", "workspace_name = \"YOUR_WORKSPACE_NAME\" \n", - "workspace_region = \"YOUR_WORKSPACE_REGION\" #Possible values eastus, eastus2 and so on.\n", + "workspace_region = \"YOUR_WORKSPACE_REGION\" #Possible values eastus, eastus2, etc.\n", "\n", - "max_total_runs=50\n" + "# Choose a size for our cluster and the maximum number of nodes\n", + "VM_SIZE = \"STANDARD_NC6\" #\"STANDARD_NC6S_V3\"\n", + "MAX_NODES = 12\n", + "\n", + "# Hyperparameter search space\n", + "IM_SIZES = [150, 300]\n", + "LEARNING_RATE_MAX = 1e-3\n", + "LEARNING_RATE_MIN = 1e-5\n", + "MAX_TOTAL_RUNS = 10 #Set to higher value to test more parameter combinations\n", + "\n", + "# Image data\n", + "DATA = unzip_url(Urls.fridge_objects_path, exist_ok=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1. Config AzureML workspace\n", + "Below we setup (or load an existing) AzureML workspace, and get all its details as follows. Note that the resource group and workspace will get created if they do not yet exist. For more information regaring the AzureML workspace see also the [20_azure_workspace_setup.ipynb](20_azure_workspace_setup.ipynb) notebook.\n", + "\n", + "To simplify clean-up (see end of this notebook), we recommend creating a new resource group to run this notebook." ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING - Warning: Falling back to use azure cli login credentials.\n", - "If you run your code in unattended mode, i.e., where you can't give a user input, then we recommend to use ServicePrincipalAuthentication or MsiAuthentication.\n", - "Please refer to aka.ms/aml-notebook-auth for different authentication mechanisms in azureml-sdk.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Workspace name: smoketestwsnew\n", - "Workspace region: eastus2\n", - "Subscription id: 0ca618d2-22a8-413a-96d0-0f1b531129c3\n", - "Resource group: smoketestnew11\n" - ] - } - ], + "outputs": [], "source": [ "from utils_cv.common.azureml import get_or_create_workspace\n", "\n", @@ -167,9 +158,9 @@ "metadata": {}, "source": [ "### 2. Create Remote Target\n", - "We create a GPU cluster as our remote compute target. If a cluster with the same name already exists in our workspace, the script will load it instead. We can see [here](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-set-up-training-targets#compute-targets-for-training) to learn more about setting up a compute target on different locations.\n", + "We create a GPU cluster as our remote compute target. If a cluster with the same name already exists in our workspace, the script will load it instead. This [link](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-set-up-training-targets#compute-targets-for-training) provides more information about how to set up a compute target on different locations.\n", "\n", - "This notebook selects STANDARD_NC6 virtual machine (VM) and sets its priority as 'lowpriority' to reduce costs." + "By default, the VM size is set to use _STANDARD_NC6_ machines. However, if quota is available, our recommendation is to use _STANDARD_NC6S_V3_ machines which come with the much faster V100 GPU." ] }, { @@ -181,37 +172,32 @@ "name": "stdout", "output_type": "stream", "text": [ - "Found existing compute target.\n", - "{'currentNodeCount': 1, 'targetNodeCount': 0, 'nodeStateCounts': {'preparingNodeCount': 0, 'runningNodeCount': 0, 'idleNodeCount': 0, 'unusableNodeCount': 0, 'leavingNodeCount': 1, 'preemptedNodeCount': 0}, 'allocationState': 'Resizing', 'allocationStateTransitionTime': '2019-07-22T04:40:41.047000+00:00', 'errors': None, 'creationTime': '2019-07-22T02:26:37.808395+00:00', 'modifiedTime': '2019-07-22T02:26:53.969636+00:00', 'provisioningState': 'Succeeded', 'provisioningStateTransitionTime': None, 'scaleSettings': {'minNodeCount': 0, 'maxNodeCount': 4, 'nodeIdleTimeBeforeScaleDown': 'PT120S'}, 'vmPriority': 'Dedicated', 'vmSize': 'STANDARD_NC6'}\n" + "Creating a new compute target...\n", + "Creating\n", + "Succeeded\n", + "AmlCompute wait for completion finished\n", + "Minimum number of nodes requested have been provisioned\n", + "{'currentNodeCount': 0, 'targetNodeCount': 0, 'nodeStateCounts': {'preparingNodeCount': 0, 'runningNodeCount': 0, 'idleNodeCount': 0, 'unusableNodeCount': 0, 'leavingNodeCount': 0, 'preemptedNodeCount': 0}, 'allocationState': 'Steady', 'allocationStateTransitionTime': '2019-08-06T15:57:12.457000+00:00', 'errors': None, 'creationTime': '2019-08-06T15:56:43.315467+00:00', 'modifiedTime': '2019-08-06T15:57:25.740370+00:00', 'provisioningState': 'Succeeded', 'provisioningStateTransitionTime': None, 'scaleSettings': {'minNodeCount': 0, 'maxNodeCount': 12, 'nodeIdleTimeBeforeScaleDown': 'PT120S'}, 'vmPriority': 'Dedicated', 'vmSize': 'STANDARD_NC6'}\n" ] } ], "source": [ - "# choose a name for our cluster\n", - "cluster_name = \"gpu-cluster-nc6\"\n", - "# Remote compute (cluster) configuration. If you want to reduce costs even more, set these to small.\n", - "# For example, using Standard_DS1_v2 instead of using STANDARD_NC6\n", - "VM_SIZE = 'STANDARD_NC6'\n", - "VM_PRIORITY = 'lowpriority'\n", - "\n", - "# Cluster nodes\n", - "MIN_NODES = 0\n", - "MAX_NODES = 4\n", + "CLUSTER_NAME = \"gpu-cluster\"\n", "\n", "try:\n", - " # Retrieve if a compute target with the same cluster_name already exists\n", - " compute_target = ComputeTarget(workspace=ws, name=cluster_name)\n", + " # Retrieve if a compute target with the same cluster name already exists\n", + " compute_target = ComputeTarget(workspace=ws, name=CLUSTER_NAME)\n", " print('Found existing compute target.')\n", + " \n", "except ComputeTargetException:\n", " # If it doesn't already exist, we create a new one with the name provided\n", " print('Creating a new compute target...')\n", " compute_config = AmlCompute.provisioning_configuration(vm_size=VM_SIZE,\n", - " min_nodes=MIN_NODES,\n", + " min_nodes=0,\n", " max_nodes=MAX_NODES)\n", "\n", " # create the cluster\n", - " compute_target = ComputeTarget.create(ws, cluster_name, compute_config)\n", - "\n", + " compute_target = ComputeTarget.create(ws, CLUSTER_NAME, compute_config)\n", " compute_target.wait_for_completion(show_output=True)\n", "\n", "# we can use get_status() to get a detailed status for the current cluster. \n", @@ -228,327 +214,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Uploading an estimated of 138 files\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/cvbp_milk_bottle.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/cvbp_water_bottle.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/example.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects.zip\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/1.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/10.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/11.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/12.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/13.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/14.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/15.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/16.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/17.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/18.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/19.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/2.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/20.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/21.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/22.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/23.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/24.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/25.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/26.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/27.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/28.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/29.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/3.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/30.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/31.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/32.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/4.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/5.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/15.jpg, 1 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/6.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/32.jpg, 2 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/7.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/cvbp_water_bottle.jpg, 3 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/8.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/11.jpg, 4 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/9.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/6.jpg, 5 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/33.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/cvbp_milk_bottle.jpg, 6 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/34.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/14.jpg, 7 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/35.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/5.jpg, 8 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/36.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/example.jpg, 9 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/37.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/18.jpg, 10 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/38.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/7.jpg, 11 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/39.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/9.jpg, 12 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/40.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/36.jpg, 13 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/41.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/13.jpg, 14 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/42.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/34.jpg, 15 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/43.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/39.jpg, 16 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/44.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/33.jpg, 17 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/45.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/41.jpg, 18 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/23.jpg, 19 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/1.jpg, 20 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/46.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/47.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/12.jpg, 21 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/48.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/49.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/16.jpg, 22 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/50.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/31.jpg, 23 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/51.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/26.jpg, 24 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/52.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/47.jpg, 25 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/53.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/44.jpg, 26 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/54.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/49.jpg, 27 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/55.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/53.jpg, 28 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/56.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/29.jpg, 29 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/57.jpg\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/56.jpg, 30 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/58.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/43.jpg, 31 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/59.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/54.jpg, 32 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/60.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/8.jpg, 33 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/61.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/55.jpg, 34 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/62.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/40.jpg, 35 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/63.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/10.jpg, 36 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/64.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/58.jpg, 37 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/100.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/42.jpg, 38 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/101.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/59.jpg, 39 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/65.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/100.jpg, 40 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/66.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/62.jpg, 41 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/67.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/48.jpg, 42 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/68.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/45.jpg, 43 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/69.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/51.jpg, 44 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/70.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/50.jpg, 45 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/71.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/46.jpg, 46 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/72.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/60.jpg, 47 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/73.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/66.jpg, 48 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/74.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/65.jpg, 49 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/75.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/25.jpg, 50 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/76.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/73.jpg, 51 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/77.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/67.jpg, 52 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/78.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/77.jpg, 53 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/79.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/52.jpg, 54 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/80.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/22.jpg, 55 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/81.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/75.jpg, 56 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/82.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/63.jpg, 57 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/83.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/78.jpg, 58 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/84.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/101.jpg, 59 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/85.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/79.jpg, 60 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/86.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/74.jpg, 61 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/87.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/61.jpg, 62 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/88.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/57.jpg, 63 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/89.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/69.jpg, 64 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/90.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/70.jpg, 65 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/91.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/68.jpg, 66 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/92.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/83.jpg, 67 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/93.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/72.jpg, 68 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/94.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/81.jpg, 69 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/95.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/64.jpg, 70 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/3.jpg, 71 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/96.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/97.jpg\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/84.jpg, 72 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/35.jpg, 73 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/98.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/99.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/71.jpg, 74 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/102.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/85.jpg, 75 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/103.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/17.jpg, 76 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/104.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/87.jpg, 77 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/105.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/94.jpg, 78 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/106.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/97.jpg, 79 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/107.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/2.jpg, 80 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/108.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/88.jpg, 81 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/109.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/28.jpg, 82 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/110.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/21.jpg, 83 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/111.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/27.jpg, 84 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/112.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/106.jpg, 85 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/113.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/86.jpg, 86 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/114.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/108.jpg, 87 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/115.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/102.jpg, 88 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/116.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/20.jpg, 89 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/117.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/76.jpg, 90 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/118.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/104.jpg, 91 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/119.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/90.jpg, 92 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/120.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/91.jpg, 93 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/121.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/93.jpg, 94 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/122.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/109.jpg, 95 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/123.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/113.jpg, 96 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/124.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/19.jpg, 97 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/125.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/92.jpg, 98 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/126.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/122.jpg, 99 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/127.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/80.jpg, 100 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/128.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/96.jpg, 101 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/119.jpg, 102 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/24.jpg, 103 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/129.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/130.jpg\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/131.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/116.jpg, 104 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/132.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/120.jpg, 105 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/133.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/121.jpg, 106 files out of an estimated total of 138\n", - "Uploading /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/134.jpg\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/89.jpg, 107 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/103.jpg, 108 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/123.jpg, 109 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/128.jpg, 110 files out of an estimated total of 138\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/118.jpg, 111 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/127.jpg, 112 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/126.jpg, 113 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/38.jpg, 114 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/133.jpg, 115 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/134.jpg, 116 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/124.jpg, 117 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/99.jpg, 118 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/131.jpg, 119 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/132.jpg, 120 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/82.jpg, 121 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/129.jpg, 122 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/98.jpg, 123 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/111.jpg, 124 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/117.jpg, 125 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/107.jpg, 126 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/4.jpg, 127 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/114.jpg, 128 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/milk_bottle/95.jpg, 129 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/125.jpg, 130 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/110.jpg, 131 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/105.jpg, 132 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/112.jpg, 133 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/can/30.jpg, 134 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/130.jpg, 135 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/carton/37.jpg, 136 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects/water_bottle/115.jpg, 137 files out of an estimated total of 138\n", - "Uploaded /Users/richinjain/projects/ComputerVision/data/fridgeObjects.zip, 138 files out of an estimated total of 138\n", - "Uploaded 138 files\n" - ] - }, - { - "data": { - "text/plain": [ - "$AZUREML_DATAREFERENCE_f63fbd85fa17436fa173eb6034cd9eb5" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "# Note, all the files under DATA will be uploaded to the data store\n", - "DATA = unzip_url(Urls.fridge_objects_path, exist_ok=True)\n", - "REPS = 3\n", - "\n", "# Retrieving default datastore that got automatically created when we setup a workspace\n", "ds = ws.get_default_datastore()\n", "\n", @@ -594,7 +263,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Overwriting /Users/richinjain/projects/ComputerVision/classification/notebooks/hyperparameter/train.py\n" + "Overwriting C:\\Users\\pabuehle\\Desktop\\ComputerVision\\classification\\notebooks\\hyperparameter/train.py\n" ] } ], @@ -615,69 +284,66 @@ "\n", "run = Run.get_context()\n", "\n", + "\n", + "#------------------------------------------------------------------\n", "# Define parameters that we are going to use for training\n", - "ARCHITECTURE = models.resnet50\n", + "ARCHITECTURE = models.resnet18\n", + "EPOCHS_HEAD = 4\n", + "EPOCHS_BODY = 12\n", + "BATCH_SIZE = 16\n", + "#------------------------------------------------------------------\n", + "\n", "\n", "# Parse arguments passed by Hyperdrive\n", "parser = argparse.ArgumentParser()\n", "\n", - "\n", "# Data path\n", "parser.add_argument('--data-folder', type=str, dest='DATA_DIR', help=\"Datastore path\")\n", "parser.add_argument('--im_size', type=int, dest='IM_SIZE')\n", "parser.add_argument('--learning_rate', type=float, dest='LEARNING_RATE')\n", - "\n", "args = parser.parse_args()\n", "params = vars(args)\n", "\n", "if params['IM_SIZE'] is None:\n", " raise ValueError(\"Image Size empty\")\n", - " \n", "if params['LEARNING_RATE'] is None:\n", " raise ValueError(\"Learning Rate empty\")\n", - "\n", "if params['DATA_DIR'] is None:\n", " raise ValueError(\"Data folder empty\")\n", - " \n", "\n", + "# Getting training and validation data\n", "path = params['DATA_DIR'] + '/data/fridgeObjects'\n", - "\n", - "# Getting training and validation data and training the CNN as done in 01_training_introduction.ipynb\n", "data = (ImageList.from_folder(path)\n", - " .split_by_rand_pct(valid_pct=0.2, seed=10)\n", + " .split_by_rand_pct(valid_pct=0.5, seed=10)\n", " .label_from_folder() \n", " .transform(size=params['IM_SIZE']) \n", - " .databunch(bs=16) \n", + " .databunch(bs=BATCH_SIZE) \n", " .normalize(imagenet_stats))\n", "\n", + "# Get model and run training\n", "learn = cnn_learner(\n", " data,\n", " ARCHITECTURE,\n", " metrics=[accuracy]\n", ")\n", - "\n", - "epochs=1 # Change the value to 10 to see multiple runs, defaulting to 1 for quick run of notebook.\n", + "learn.fit_one_cycle(EPOCHS_HEAD, params['LEARNING_RATE'])\n", "learn.unfreeze()\n", - "learn.fit(epochs, params['LEARNING_RATE'])\n", + "learn.fit_one_cycle(EPOCHS_BODY, params['LEARNING_RATE'])\n", "\n", + "# Add log entries\n", "training_losses = [x.numpy().ravel()[0] for x in learn.recorder.losses]\n", - "accuracy = [x[0].numpy().ravel()[0] for x in learn.recorder.metrics][-1]\n", - "\n", - "#run.log_list('training_loss', training_losses)\n", - "#run.log_list('validation_loss', learn.recorder.val_losses)\n", - "#run.log_list('error_rate', error_rate)\n", + "accuracy = [100*x[0].numpy().ravel()[0] for x in learn.recorder.metrics][-1]\n", "run.log('data_dir',params['DATA_DIR'])\n", "run.log('im_size', params['IM_SIZE'])\n", "run.log('learning_rate', params['LEARNING_RATE'])\n", "run.log('accuracy', float(accuracy)) # Logging our primary metric 'accuracy'\n", "\n", + "# Save trained model\n", "current_directory = os.getcwd()\n", "output_folder = os.path.join(current_directory, 'outputs')\n", - "MODEL_NAME = 'im_classif_resnet50' # Name we will give our model both locally and on Azure\n", - "PICKLED_MODEL_NAME = MODEL_NAME + '.pkl'\n", + "model_name = 'im_classif_resnet' # Name we will give our model both locally and on Azure\n", "os.makedirs(output_folder, exist_ok=True)\n", - "\n", - "learn.export(os.path.join(output_folder, PICKLED_MODEL_NAME))" + "learn.export(os.path.join(output_folder, model_name + \".pkl\"))" ] }, { @@ -686,8 +352,6 @@ "source": [ "### 5. Setup and run Hyperdrive experiment\n", "\n", - "Next step is to prepare scripts that AzureML Hyperdrive will use to train and evaluate models with selected hyperparameters. To run the model notebook from the Hyperdrive Run, all we need is to prepare an entry script which parses the hyperparameter arguments, passes them to the notebook, and records the results of the notebook to AzureML Run logs. \n", - "\n", "#### 5.1 Create Experiment \n", "Experiment is the main entry point into experimenting with AzureML. To create new Experiment or get the existing one, we pass our experimentation name 'hyperparameter-tuning'.\n" ] @@ -708,10 +372,7 @@ "source": [ "#### 5.2. Define search space\n", "\n", - "Now we define the search space of hyperparameters. For example, if you want to test different batch sizes of {64, 128, 256}, you can use azureml.train.hyperdrive.choice(64, 128, 256). To search from a continuous space, use uniform(start, end). For more options, see [Hyperdrive parameter expressions](https://docs.microsoft.com/en-us/python/api/azureml-train-core/azureml.train.hyperdrive.parameter_expressions?view=azure-ml-py).\n", - "\n", - "In this notebook we use the ResNet50 architecture, and fix the number of epochs to 10.\n", - "In the search space, we set different learning rates and image sizes. Details about the hyperparameters can be found in [11_exploring_hyperparameters.ipynb notebook](https://github.com/microsoft/ComputerVision/blob/master/classification/notebooks/11_exploring_hyperparameters.ipynb).\n", + "Now we define the search space of hyperparameters. As shown below, to test discrete parameter values use 'choice()', and for uniform sampling use 'uniform()'. For more options, see [Hyperdrive parameter expressions](https://docs.microsoft.com/en-us/python/api/azureml-train-core/azureml.train.hyperdrive.parameter_expressions?view=azure-ml-py).\n", "\n", "Hyperdrive provides three different parameter sampling methods: 'RandomParameterSampling', 'GridParameterSampling', and 'BayesianParameterSampling'. Details about each method can be found [here](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-tune-hyperparameters). Here, we use the 'RandomParameterSampling'." ] @@ -722,20 +383,13 @@ "metadata": {}, "outputs": [], "source": [ - "IM_SIZES = [299, 499]\n", - "LEARNING_RATES = [1e-3, 1e-4, 1e-5]\n", - "\n", "# Hyperparameter search space\n", "param_sampling = RandomParameterSampling( {\n", - " '--learning_rate': choice(LEARNING_RATES),\n", + " '--learning_rate': uniform(LEARNING_RATE_MIN, LEARNING_RATE_MAX),\n", " '--im_size': choice(IM_SIZES)\n", " }\n", ")\n", "\n", - "primary_metric_name = 'accuracy'\n", - "primary_metric_goal = PrimaryMetricGoal.MAXIMIZE\n", - "max_concurrent_runs=4\n", - "\n", "early_termination_policy = BanditPolicy(slack_factor=0.15, evaluation_interval=1, delay_evaluation=20)" ] }, @@ -781,7 +435,7 @@ "- early termination policy, in this case we use [Bandit Policy](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-tune-hyperparameters#bandit-policy)\n", "- primary metric name reported by our runs, in this case it is accuracy \n", "- the goal, which determines whether the primary metric has to be maximized/minimized, in this case it is to maximize our accuracy \n", - "- number of total child-runs, in this case it is 4\n", + "- number of total child-runs\n", "\n", "The bigger the search space, the more child-runs get triggered for better results." ] @@ -795,10 +449,10 @@ "hyperdrive_run_config = HyperDriveConfig(estimator=est,\n", " hyperparameter_sampling=param_sampling,\n", " policy=early_termination_policy,\n", - " primary_metric_name=primary_metric_name,\n", - " primary_metric_goal=primary_metric_goal,\n", - " max_total_runs=max_total_runs,\n", - " max_concurrent_runs= max_concurrent_runs)" + " primary_metric_name='accuracy',\n", + " primary_metric_goal=PrimaryMetricGoal.MAXIMIZE,\n", + " max_total_runs=MAX_TOTAL_RUNS,\n", + " max_concurrent_runs=MAX_NODES)" ] }, { @@ -816,7 +470,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "fff89f7fb8284f24a94932ca876cbae2", + "model_id": "5c51804ba4794f3aa163354fef634c59", "version_major": 2, "version_minor": 0 }, @@ -826,25 +480,12 @@ }, "metadata": {}, "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "9a3c69449ea34e48a4e0c884ced34538", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "_UserRunWidget(widget_settings={'childWidgetDisplay': 'popup', 'send_telemetry': False, 'log_level': 'INFO', '…" - ] - }, - "metadata": {}, - "output_type": "display_data" } ], "source": [ "# Now we submit the Run to our experiment. \n", "hyperdrive_run = exp.submit(config=hyperdrive_run_config)\n", + "\n", "# We can see the experiment progress from this notebook by using \n", "widgets.RunDetails(hyperdrive_run).show()" ] @@ -857,18 +498,18 @@ { "data": { "text/plain": [ - "{'runId': 'hyperparameter-tuning_1563770544897',\n", - " 'target': 'gpu-cluster-nc6',\n", + "{'runId': 'hyperparameter-tuning_1565107066432',\n", + " 'target': 'gpu-cluster',\n", " 'status': 'Completed',\n", - " 'startTimeUtc': '2019-07-22T04:42:25.393015Z',\n", - " 'endTimeUtc': '2019-07-22T04:49:58.250673Z',\n", + " 'startTimeUtc': '2019-08-06T15:57:46.90426Z',\n", + " 'endTimeUtc': '2019-08-06T16:13:21.185098Z',\n", " 'properties': {'primary_metric_config': '{\"name\": \"accuracy\", \"goal\": \"maximize\"}',\n", " 'runTemplate': 'HyperDrive',\n", " 'azureml.runsource': 'hyperdrive',\n", " 'platform': 'AML',\n", - " 'baggage': 'eyJvaWQiOiAiNmY1Yjc5M2UtZjhiOS00NGY0LTk0N2YtNTg3N2ZjMDFjZmFjIiwgInRpZCI6ICI3MmY5ODhiZi04NmYxLTQxYWYtOTFhYi0yZDdjZDAxMWRiNDciLCAidW5hbWUiOiAiMDRiMDc3OTUtOGRkYi00NjFhLWJiZWUtMDJmOWUxYmY3YjQ2In0',\n", - " 'ContentSnapshotId': 'a63feca7-742e-49c3-b568-9cf6a53b34c3'},\n", - " 'logFiles': {'azureml-logs/hyperdrive.txt': 'https://smoketesstorage0231aa20c.blob.core.windows.net/azureml/ExperimentRun/dcid.hyperparameter-tuning_1563770544897/azureml-logs/hyperdrive.txt?sv=2018-03-28&sr=b&sig=LL8Fx6UZhJ9jddaqS1xeR%2BHi98wUHPZ%2FYuAxGH3Y39I%3D&st=2019-07-22T04%3A39%3A59Z&se=2019-07-22T12%3A49%3A59Z&sp=r'}}" + " 'baggage': 'eyJvaWQiOiAiNWFlYTJmMzAtZjQxZC00ZDA0LWJiOGUtOWU0NGUyZWQzZGQ2IiwgInRpZCI6ICI3MmY5ODhiZi04NmYxLTQxYWYtOTFhYi0yZDdjZDAxMWRiNDciLCAidW5hbWUiOiAiMDRiMDc3OTUtOGRkYi00NjFhLWJiZWUtMDJmOWUxYmY3YjQ2In0',\n", + " 'ContentSnapshotId': 'c662f56a-ff58-432e-b732-8a3bc6818778'},\n", + " 'logFiles': {'azureml-logs/hyperdrive.txt': 'https://pabuehlestorage1c7e31216.blob.core.windows.net/azureml/ExperimentRun/dcid.hyperparameter-tuning_1565107066432/azureml-logs/hyperdrive.txt?sv=2018-11-09&sr=b&sig=8D2gwxb%2BYn7nbzgGVHE7QSzJ%2FG7C1swzmLD7%2Fior2vE%3D&st=2019-08-06T17%3A36%3A08Z&se=2019-08-07T01%3A46%3A08Z&sp=r'}}" ] }, "execution_count": 14, @@ -877,7 +518,7 @@ } ], "source": [ - "hyperdrive_run.wait_for_completion()\n" + "hyperdrive_run.wait_for_completion()" ] }, { @@ -911,15 +552,15 @@ "name": "stdout", "output_type": "stream", "text": [ - "* Best Run Id:hyperparameter-tuning_1563770544897_0\n", + "* Best Run Id:hyperparameter-tuning_1565107066432_8\n", "Run(Experiment: hyperparameter-tuning,\n", - "Id: hyperparameter-tuning_1563770544897_0,\n", + "Id: hyperparameter-tuning_1565107066432_8,\n", "Type: azureml.scriptrun,\n", "Status: Completed)\n", "\n", "* Best hyperparameters:\n", - "{'--data-folder': '$AZUREML_DATAREFERENCE_workspaceblobstore', '--im_size': '299', '--learning_rate': '0.001'}\n", - "Accuracy = 0.26923078298568726\n" + "{'--data-folder': '$AZUREML_DATAREFERENCE_workspaceblobstore', '--im_size': '150', '--learning_rate': '0.000552896672441507'}\n", + "Accuracy = 92.53731369972229\n" ] } ], @@ -956,8 +597,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Downloading outputs/im_classif_resnet50.pkl..\n", - "119547037146038801333356\n" + "Downloading outputs/im_classif_resnet.pkl..\n" ] } ], @@ -968,11 +608,10 @@ "os.makedirs(output_folder, exist_ok=True)\n", "\n", "for f in best_run.get_file_names():\n", - " if f.startswith('outputs/im_classif_resnet50'):\n", + " if f.startswith('outputs/im_classif_resnet'):\n", " print(\"Downloading {}..\".format(f))\n", - " best_run.download_file('outputs/im_classif_resnet50.pkl')\n", - "saved_model =joblib.load('im_classif_resnet50.pkl')\n", - "print(saved_model)" + " best_run.download_file('outputs/im_classif_resnet.pkl')\n", + "saved_model =joblib.load('im_classif_resnet.pkl')" ] }, { @@ -984,12 +623,27 @@ "saved_model.predict(image)\n", "```" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 7. Clean up\n", + "\n", + "To avoid unnecessary expenses, all resources which were created in this notebook need to get deleted once parameter search is concluded. To simplify this clean-up step, we recommend creating a new resource group to run this notebook. This resource group can then be deleted, e.g. using the Azure Portal, which will remove all created resources." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { - "celltoolbar": "Tags", "kernelspec": { - "display_name": "cv", + "display_name": "Python (cv)", "language": "python", "name": "cv" }, diff --git a/similarity/README.md b/similarity/README.md index 59a76bf..e089ebc 100644 --- a/similarity/README.md +++ b/similarity/README.md @@ -6,6 +6,12 @@ The majority of state-of-the-art systems for image similarity use DNNs to comput A major difference between modern image similarity approaches is how the DNN is trained. A simple but quite powerful approach is to use a standard image classification loss - this is the approach taken in this repository, and explained in the [classification](../classification/README.md) folder. More accurate similarity measures are based on DNNs which are trained explicitly for image similarity, such as the [FaceNet](https://arxiv.org/pdf/1503.03832.pdf) work which uses a Siamese network architecture. FaceNet-like approaches will be added to this repository at a later point. + +## Frequently asked questions + +Answers to Frequently Asked Questions such as "How many images do I need to train a model?" or "How to annotate images?" can be found in the [FAQ.md](FAQ.md) file. For image classification specified questions, see the [FAQ.md](../classification/FAQ.md) in the classification folder. + + ## Notebooks We provide several notebooks to show how image similarity algorithms can be designed and evaluated. @@ -14,11 +20,10 @@ We provide several notebooks to show how image similarity algorithms can be desi | --- | --- | | [00_webcam.ipynb](./notebooks/00_webcam.ipynb)| Quick start notebook which demonstrates how to build an image retrieval system using a single image or webcam as input. | [01_training_and_evaluation_introduction.ipynb](./notebooks/01_training_and_evaluation_introduction.ipynb)| Notebook which explains the basic concepts around model training and evaluation, based on using DNNs trained for image classification.| +| [11_exploring_hyperparameters.ipynb](notebooks/11_exploring_hyperparameters.ipynb)| Finds optimal model parameters using grid search. | + ## Coding guidelines See the [coding guidelines](../classification/#coding-guidelines) in the image classification folder. -## Frequently asked questions - -Answers to Frequently Asked Questions such as "How many images do I need to train a model?" or "How to annotate images?" can be found in the [FAQ.md](FAQ.md) file. For image classification specified questions, see the [FAQ.md](../classification/FAQ.md) in the classification folder. diff --git a/similarity/notebooks/11_exploring_hyperparameters.ipynb b/similarity/notebooks/11_exploring_hyperparameters.ipynb new file mode 100644 index 0000000..60ba418 --- /dev/null +++ b/similarity/notebooks/11_exploring_hyperparameters.ipynb @@ -0,0 +1,974 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Copyright (c) Microsoft Corporation. All rights reserved.\n", + "\n", + "Licensed under the MIT License." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Testing different Hyperparameters and Benchmarking" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this notebook, we'll cover how to test different hyperparameters for a particular dataset and how to benchmark different parameters across a group of datasets. Note that this re-uses functionality which was already introduced and described in the [classification/notebooks/11_exploring_hyperparameters.ipynb](../../classification/notebooks/11_exploring_hyperparameters.ipynb) notebook. **Please refer to that notebook for all explanations, which this notebook will not repeat.**\n", + "\n", + "For an example of how to scale up with remote GPU clusters on Azure Machine Learning, please view [24_exploring_hyperparameters_on_azureml.ipynb](../../classification/notebooks/24_exploring_hyperparameters_on_azureml.ipynb)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Testing hyperparameters" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Ensure edits to libraries are loaded and plotting is shown in the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%reload_ext autoreload\n", + "%autoreload 2\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We start by importing the utilities we need." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'1.0.48'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import sys\n", + "import numpy as np\n", + "import scrapbook as sb\n", + "import fastai\n", + "from fastai.vision import DatasetType\n", + "\n", + "sys.path.append(\"../../\")\n", + "from utils_cv.classification.data import Urls\n", + "from utils_cv.common.data import unzip_url\n", + "from utils_cv.classification.parameter_sweeper import ParameterSweeper, clean_sweeper_df, plot_sweeper_df\n", + "from utils_cv.similarity.data import comparative_set_builder\n", + "from utils_cv.similarity.metrics import positive_image_ranks\n", + "from utils_cv.similarity.model import compute_features_learner\n", + "\n", + "fastai.__version__" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Define the datasets and parameters we will use in this notebook." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "DATA_PATHS = [unzip_url(Urls.fridge_objects_path, exist_ok=True), unzip_url(Urls.fridge_objects_watermark_path, exist_ok=True)]\n", + "REPS = 3\n", + "LEARNING_RATES = [1e-3, 1e-4, 1e-5]\n", + "IM_SIZES = [300, 500]\n", + "EPOCHS = [10]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Similiarity accuracy metric\n", + "\n", + "For image classification, we used the percentage of correctly labeled images to measure accuracy. For image retrieval, our measure is the rank of the positive example among a large number of negatives. This was described in the [01_training_and_evaluation_introduction.ipynb](01_training_and_evaluation_introduction.ipynb) notebook, and we will re-use some of the code from that notebook in the definition of the _retrieval_rank()_ function below." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def retrieval_rank(learn):\n", + " data = learn.data\n", + "\n", + " # Build multiple sets of comparative images from the validation images\n", + " comparative_sets = comparative_set_builder(\n", + " data.valid_ds, num_sets=1000, num_negatives=99\n", + " )\n", + "\n", + " # Compute DNN features for all validation images\n", + " embedding_layer = learn.model[1][6]\n", + " valid_features = compute_features_learner(\n", + " data, DatasetType.Valid, learn, embedding_layer\n", + " )\n", + "\n", + " # For each comparative set compute the distances between the query image and all reference images\n", + " for cs in comparative_sets:\n", + " cs.compute_distances(valid_features)\n", + "\n", + " # Compute the median rank of the positive example over all comparative sets\n", + " ranks = positive_image_ranks(comparative_sets)\n", + " median_rank = np.median(ranks)\n", + " return median_rank" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Using Python " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We start by creating the Parameter Sweeper object. Before we start testing, it's a good idea to see what the default parameters are. We can use a the property `parameters` to easily see those default values." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "OrderedDict([('learning_rate', [0.0001]),\n", + " ('epochs', [15]),\n", + " ('batch_size', [16]),\n", + " ('im_size', [299]),\n", + " ('architecture',\n", + " [)>]),\n", + " ('transform', [True]),\n", + " ('dropout', [0.5]),\n", + " ('weight_decay', [0.01]),\n", + " ('training_schedule',\n", + " []),\n", + " ('discriminative_lr', [False]),\n", + " ('one_cycle_policy', [True])])" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sweeper = ParameterSweeper(metric_name=\"rank\")\n", + "sweeper.parameters" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that we know the defaults, we can pass it the parameters we want to test, and run the parameter sweep." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "this Learner object self-destroyed - it still exists, but no longer usable\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
durationrank
0PARAMETERS [learning_rate: 0.0001]|[epochs: 10]|[batch_size: 16]|[im_size: 300]|[arch: resnet18]|[transforms: True]|[dropout: 0.5]|[weight_decay: 0.01]|[training_schedule: head_first_then_body]|[discriminative_lr: False]|[one_cycle_policy: True]fridgeObjects20.31468711.0
fridgeObjectsWatermark21.80125218.0
PARAMETERS [learning_rate: 0.0001]|[epochs: 10]|[batch_size: 16]|[im_size: 500]|[arch: resnet18]|[transforms: True]|[dropout: 0.5]|[weight_decay: 0.01]|[training_schedule: head_first_then_body]|[discriminative_lr: False]|[one_cycle_policy: True]fridgeObjects29.78926910.0
fridgeObjectsWatermark28.81683821.0
PARAMETERS [learning_rate: 0.001]|[epochs: 10]|[batch_size: 16]|[im_size: 300]|[arch: resnet18]|[transforms: True]|[dropout: 0.5]|[weight_decay: 0.01]|[training_schedule: head_first_then_body]|[discriminative_lr: False]|[one_cycle_policy: True]fridgeObjects21.5935791.0
fridgeObjectsWatermark18.5680011.0
PARAMETERS [learning_rate: 0.001]|[epochs: 10]|[batch_size: 16]|[im_size: 500]|[arch: resnet18]|[transforms: True]|[dropout: 0.5]|[weight_decay: 0.01]|[training_schedule: head_first_then_body]|[discriminative_lr: False]|[one_cycle_policy: True]fridgeObjects30.7427571.0
fridgeObjectsWatermark29.5538851.0
PARAMETERS [learning_rate: 1e-05]|[epochs: 10]|[batch_size: 16]|[im_size: 300]|[arch: resnet18]|[transforms: True]|[dropout: 0.5]|[weight_decay: 0.01]|[training_schedule: head_first_then_body]|[discriminative_lr: False]|[one_cycle_policy: True]fridgeObjects18.39206128.0
fridgeObjectsWatermark18.45886131.0
PARAMETERS [learning_rate: 1e-05]|[epochs: 10]|[batch_size: 16]|[im_size: 500]|[arch: resnet18]|[transforms: True]|[dropout: 0.5]|[weight_decay: 0.01]|[training_schedule: head_first_then_body]|[discriminative_lr: False]|[one_cycle_policy: True]fridgeObjects30.84810728.0
fridgeObjectsWatermark29.85236232.0
1PARAMETERS [learning_rate: 0.0001]|[epochs: 10]|[batch_size: 16]|[im_size: 300]|[arch: resnet18]|[transforms: True]|[dropout: 0.5]|[weight_decay: 0.01]|[training_schedule: head_first_then_body]|[discriminative_lr: False]|[one_cycle_policy: True]fridgeObjects21.64626115.0
fridgeObjectsWatermark21.59038110.0
PARAMETERS [learning_rate: 0.0001]|[epochs: 10]|[batch_size: 16]|[im_size: 500]|[arch: resnet18]|[transforms: True]|[dropout: 0.5]|[weight_decay: 0.01]|[training_schedule: head_first_then_body]|[discriminative_lr: False]|[one_cycle_policy: True]fridgeObjects32.75498314.0
fridgeObjectsWatermark32.17298523.0
PARAMETERS [learning_rate: 0.001]|[epochs: 10]|[batch_size: 16]|[im_size: 300]|[arch: resnet18]|[transforms: True]|[dropout: 0.5]|[weight_decay: 0.01]|[training_schedule: head_first_then_body]|[discriminative_lr: False]|[one_cycle_policy: True]fridgeObjects20.3322461.0
fridgeObjectsWatermark18.1994351.0
PARAMETERS [learning_rate: 0.001]|[epochs: 10]|[batch_size: 16]|[im_size: 500]|[arch: resnet18]|[transforms: True]|[dropout: 0.5]|[weight_decay: 0.01]|[training_schedule: head_first_then_body]|[discriminative_lr: False]|[one_cycle_policy: True]fridgeObjects30.5250141.0
fridgeObjectsWatermark32.1388383.0
PARAMETERS [learning_rate: 1e-05]|[epochs: 10]|[batch_size: 16]|[im_size: 300]|[arch: resnet18]|[transforms: True]|[dropout: 0.5]|[weight_decay: 0.01]|[training_schedule: head_first_then_body]|[discriminative_lr: False]|[one_cycle_policy: True]fridgeObjects21.81931622.0
fridgeObjectsWatermark20.91263722.0
PARAMETERS [learning_rate: 1e-05]|[epochs: 10]|[batch_size: 16]|[im_size: 500]|[arch: resnet18]|[transforms: True]|[dropout: 0.5]|[weight_decay: 0.01]|[training_schedule: head_first_then_body]|[discriminative_lr: False]|[one_cycle_policy: True]fridgeObjects30.99799836.5
fridgeObjectsWatermark31.90061126.0
2PARAMETERS [learning_rate: 0.0001]|[epochs: 10]|[batch_size: 16]|[im_size: 300]|[arch: resnet18]|[transforms: True]|[dropout: 0.5]|[weight_decay: 0.01]|[training_schedule: head_first_then_body]|[discriminative_lr: False]|[one_cycle_policy: True]fridgeObjects18.01549613.0
fridgeObjectsWatermark19.94110515.0
PARAMETERS [learning_rate: 0.0001]|[epochs: 10]|[batch_size: 16]|[im_size: 500]|[arch: resnet18]|[transforms: True]|[dropout: 0.5]|[weight_decay: 0.01]|[training_schedule: head_first_then_body]|[discriminative_lr: False]|[one_cycle_policy: True]fridgeObjects27.32765729.0
fridgeObjectsWatermark27.34250514.0
PARAMETERS [learning_rate: 0.001]|[epochs: 10]|[batch_size: 16]|[im_size: 300]|[arch: resnet18]|[transforms: True]|[dropout: 0.5]|[weight_decay: 0.01]|[training_schedule: head_first_then_body]|[discriminative_lr: False]|[one_cycle_policy: True]fridgeObjects21.6607621.0
fridgeObjectsWatermark21.6554341.0
PARAMETERS [learning_rate: 0.001]|[epochs: 10]|[batch_size: 16]|[im_size: 500]|[arch: resnet18]|[transforms: True]|[dropout: 0.5]|[weight_decay: 0.01]|[training_schedule: head_first_then_body]|[discriminative_lr: False]|[one_cycle_policy: True]fridgeObjects32.0692941.0
fridgeObjectsWatermark34.7714101.0
PARAMETERS [learning_rate: 1e-05]|[epochs: 10]|[batch_size: 16]|[im_size: 300]|[arch: resnet18]|[transforms: True]|[dropout: 0.5]|[weight_decay: 0.01]|[training_schedule: head_first_then_body]|[discriminative_lr: False]|[one_cycle_policy: True]fridgeObjects21.24450225.0
fridgeObjectsWatermark18.23778930.0
PARAMETERS [learning_rate: 1e-05]|[epochs: 10]|[batch_size: 16]|[im_size: 500]|[arch: resnet18]|[transforms: True]|[dropout: 0.5]|[weight_decay: 0.01]|[training_schedule: head_first_then_body]|[discriminative_lr: False]|[one_cycle_policy: True]fridgeObjects28.96686128.0
fridgeObjectsWatermark29.37790434.0
\n", + "
" + ], + "text/plain": [ + " duration \\\n", + "0 PARAMETERS [learning_rate: 0.0001]|[epochs: 10]... fridgeObjects 20.314687 \n", + " fridgeObjectsWatermark 21.801252 \n", + " PARAMETERS [learning_rate: 0.0001]|[epochs: 10]... fridgeObjects 29.789269 \n", + " fridgeObjectsWatermark 28.816838 \n", + " PARAMETERS [learning_rate: 0.001]|[epochs: 10]|... fridgeObjects 21.593579 \n", + " fridgeObjectsWatermark 18.568001 \n", + " PARAMETERS [learning_rate: 0.001]|[epochs: 10]|... fridgeObjects 30.742757 \n", + " fridgeObjectsWatermark 29.553885 \n", + " PARAMETERS [learning_rate: 1e-05]|[epochs: 10]|... fridgeObjects 18.392061 \n", + " fridgeObjectsWatermark 18.458861 \n", + " PARAMETERS [learning_rate: 1e-05]|[epochs: 10]|... fridgeObjects 30.848107 \n", + " fridgeObjectsWatermark 29.852362 \n", + "1 PARAMETERS [learning_rate: 0.0001]|[epochs: 10]... fridgeObjects 21.646261 \n", + " fridgeObjectsWatermark 21.590381 \n", + " PARAMETERS [learning_rate: 0.0001]|[epochs: 10]... fridgeObjects 32.754983 \n", + " fridgeObjectsWatermark 32.172985 \n", + " PARAMETERS [learning_rate: 0.001]|[epochs: 10]|... fridgeObjects 20.332246 \n", + " fridgeObjectsWatermark 18.199435 \n", + " PARAMETERS [learning_rate: 0.001]|[epochs: 10]|... fridgeObjects 30.525014 \n", + " fridgeObjectsWatermark 32.138838 \n", + " PARAMETERS [learning_rate: 1e-05]|[epochs: 10]|... fridgeObjects 21.819316 \n", + " fridgeObjectsWatermark 20.912637 \n", + " PARAMETERS [learning_rate: 1e-05]|[epochs: 10]|... fridgeObjects 30.997998 \n", + " fridgeObjectsWatermark 31.900611 \n", + "2 PARAMETERS [learning_rate: 0.0001]|[epochs: 10]... fridgeObjects 18.015496 \n", + " fridgeObjectsWatermark 19.941105 \n", + " PARAMETERS [learning_rate: 0.0001]|[epochs: 10]... fridgeObjects 27.327657 \n", + " fridgeObjectsWatermark 27.342505 \n", + " PARAMETERS [learning_rate: 0.001]|[epochs: 10]|... fridgeObjects 21.660762 \n", + " fridgeObjectsWatermark 21.655434 \n", + " PARAMETERS [learning_rate: 0.001]|[epochs: 10]|... fridgeObjects 32.069294 \n", + " fridgeObjectsWatermark 34.771410 \n", + " PARAMETERS [learning_rate: 1e-05]|[epochs: 10]|... fridgeObjects 21.244502 \n", + " fridgeObjectsWatermark 18.237789 \n", + " PARAMETERS [learning_rate: 1e-05]|[epochs: 10]|... fridgeObjects 28.966861 \n", + " fridgeObjectsWatermark 29.377904 \n", + "\n", + " rank \n", + "0 PARAMETERS [learning_rate: 0.0001]|[epochs: 10]... fridgeObjects 11.0 \n", + " fridgeObjectsWatermark 18.0 \n", + " PARAMETERS [learning_rate: 0.0001]|[epochs: 10]... fridgeObjects 10.0 \n", + " fridgeObjectsWatermark 21.0 \n", + " PARAMETERS [learning_rate: 0.001]|[epochs: 10]|... fridgeObjects 1.0 \n", + " fridgeObjectsWatermark 1.0 \n", + " PARAMETERS [learning_rate: 0.001]|[epochs: 10]|... fridgeObjects 1.0 \n", + " fridgeObjectsWatermark 1.0 \n", + " PARAMETERS [learning_rate: 1e-05]|[epochs: 10]|... fridgeObjects 28.0 \n", + " fridgeObjectsWatermark 31.0 \n", + " PARAMETERS [learning_rate: 1e-05]|[epochs: 10]|... fridgeObjects 28.0 \n", + " fridgeObjectsWatermark 32.0 \n", + "1 PARAMETERS [learning_rate: 0.0001]|[epochs: 10]... fridgeObjects 15.0 \n", + " fridgeObjectsWatermark 10.0 \n", + " PARAMETERS [learning_rate: 0.0001]|[epochs: 10]... fridgeObjects 14.0 \n", + " fridgeObjectsWatermark 23.0 \n", + " PARAMETERS [learning_rate: 0.001]|[epochs: 10]|... fridgeObjects 1.0 \n", + " fridgeObjectsWatermark 1.0 \n", + " PARAMETERS [learning_rate: 0.001]|[epochs: 10]|... fridgeObjects 1.0 \n", + " fridgeObjectsWatermark 3.0 \n", + " PARAMETERS [learning_rate: 1e-05]|[epochs: 10]|... fridgeObjects 22.0 \n", + " fridgeObjectsWatermark 22.0 \n", + " PARAMETERS [learning_rate: 1e-05]|[epochs: 10]|... fridgeObjects 36.5 \n", + " fridgeObjectsWatermark 26.0 \n", + "2 PARAMETERS [learning_rate: 0.0001]|[epochs: 10]... fridgeObjects 13.0 \n", + " fridgeObjectsWatermark 15.0 \n", + " PARAMETERS [learning_rate: 0.0001]|[epochs: 10]... fridgeObjects 29.0 \n", + " fridgeObjectsWatermark 14.0 \n", + " PARAMETERS [learning_rate: 0.001]|[epochs: 10]|... fridgeObjects 1.0 \n", + " fridgeObjectsWatermark 1.0 \n", + " PARAMETERS [learning_rate: 0.001]|[epochs: 10]|... fridgeObjects 1.0 \n", + " fridgeObjectsWatermark 1.0 \n", + " PARAMETERS [learning_rate: 1e-05]|[epochs: 10]|... fridgeObjects 25.0 \n", + " fridgeObjectsWatermark 30.0 \n", + " PARAMETERS [learning_rate: 1e-05]|[epochs: 10]|... fridgeObjects 28.0 \n", + " fridgeObjectsWatermark 34.0 " + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sweeper.update_parameters(learning_rate=LEARNING_RATES, im_size=IM_SIZES, epochs=EPOCHS)\n", + "df = sweeper.run(datasets=DATA_PATHS, reps=REPS, metric_fct=retrieval_rank); \n", + "df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Visualize Results " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When we read in multi-index dataframe, index 0 represents the run number, index 1 represents a single permutation of parameters, and index 2 represents the dataset. To see the results, show the df using the `clean_sweeper_df` helper function. This will display all the hyperparameters in a nice, readable way." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "df = clean_sweeper_df(df)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Since we've run our benchmarking over 3 repetitions, we may want to just look at the averages across the different __run numbers__." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
P: [learning_rate: 0.0001] [im_size: 300]P: [learning_rate: 0.0001] [im_size: 500]P: [learning_rate: 0.001] [im_size: 300]P: [learning_rate: 0.001] [im_size: 500]P: [learning_rate: 1e-05] [im_size: 300]P: [learning_rate: 1e-05] [im_size: 500]
fridgeObjectsfridgeObjectsWatermarkfridgeObjectsfridgeObjectsWatermarkfridgeObjectsfridgeObjectsWatermarkfridgeObjectsfridgeObjectsWatermarkfridgeObjectsfridgeObjectsWatermarkfridgeObjectsfridgeObjectsWatermark
duration19.99214821.11091329.95730329.44410921.19552919.4742931.11235532.15471120.48529319.20309630.27098930.376959
rank13.00000014.33333317.66666719.3333331.0000001.000001.0000001.66666725.00000027.66666730.83333330.666667
\n", + "
" + ], + "text/plain": [ + " P: [learning_rate: 0.0001] [im_size: 300] \\\n", + " fridgeObjects fridgeObjectsWatermark \n", + "duration 19.992148 21.110913 \n", + "rank 13.000000 14.333333 \n", + "\n", + " P: [learning_rate: 0.0001] [im_size: 500] \\\n", + " fridgeObjects fridgeObjectsWatermark \n", + "duration 29.957303 29.444109 \n", + "rank 17.666667 19.333333 \n", + "\n", + " P: [learning_rate: 0.001] [im_size: 300] \\\n", + " fridgeObjects fridgeObjectsWatermark \n", + "duration 21.195529 19.47429 \n", + "rank 1.000000 1.00000 \n", + "\n", + " P: [learning_rate: 0.001] [im_size: 500] \\\n", + " fridgeObjects fridgeObjectsWatermark \n", + "duration 31.112355 32.154711 \n", + "rank 1.000000 1.666667 \n", + "\n", + " P: [learning_rate: 1e-05] [im_size: 300] \\\n", + " fridgeObjects fridgeObjectsWatermark \n", + "duration 20.485293 19.203096 \n", + "rank 25.000000 27.666667 \n", + "\n", + " P: [learning_rate: 1e-05] [im_size: 500] \n", + " fridgeObjects fridgeObjectsWatermark \n", + "duration 30.270989 30.376959 \n", + "rank 30.833333 30.666667 " + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.mean(level=(1,2)).T" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Print the average accuracy over the different runs for each dataset independently." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAr8AAAIkCAYAAAAXuENuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdeZRdVZn+8echCYYhhCEBkSkRkQQiKYoiEIZggwRsJIS2USLNIDQBERDaBUbaIYptg0Fb5ccUGwQ1IIIoyCAqgzGGKRWKIYS5AzIIITJDIMP7++PeKqpCkrpDVZ3atb+ftWpV3XNP3fOuPMnJW7v23tcRIQAAACAHaxRdAAAAANBTaH4BAACQDZpfAAAAZIPmFwAAANmg+QUAAEA2aH4BAACQjf49ebEhQ4bEsGHDevKSAAAAyFBzc/NLETF0xeM92vwOGzZMc+bM6clLAgAAIEO2n1rZcaY9AAAAIBs0vwAAAMgGzS8AAACy0aNzfldmyZIleuaZZ7R48eKiS0FGBg4cqM0331wDBgwouhQAANCDCm9+n3nmGQ0aNEjDhg2T7aLLQQYiQosWLdIzzzyj4cOHF10OAADoQYVPe1i8eLE22mgjGl/0GNvaaKON+G0DAAAZKrz5lUTjix7H3zkAAPLUK5rfov34xz/WyJEjddhhh3U4PmfOHJ188skr/Z5hw4bppZdeqvma06dP14gRIzRixAiNGTNGs2bN6vS1L7zwQv3sZz+r+lqvvPKKzj///JprBQAA6CsKn/O7omFTbujS11tw1gGdnnP++efrpptu6jD/c+nSpWpqalJTU1OX1iNJ119/vS666CLNmjVLQ4YM0dy5czVx4kTdfffd+uAHP7jK7zv++ONrul5r83vCCSfUWjIAAECfkP3I7/HHH68nn3xSEyZM0ODBgzV58mSNHz9eRxxxhG6//XZ96lOfkiQtWrRI48eP14477qjjjjtOEdH2GmeeeaZGjBihfffdV5MmTdI555wjSXriiSe0//77a6eddtKee+6phx9+WJJ09tlna9q0aRoyZIgkqbGxUUceeaTOO++8ttecNm2axowZozFjxujxxx+XJE2dOrXT137hhRd08MEHa/To0Ro9erRmz56tKVOm6IknnlBDQ4NOO+00Pf/88xo3bpwaGho0atQo/eUvf+nmP2UAAIDeIfvm98ILL9SHPvQh3XbbbTr11FPV3Nysa6+9VpdffnmH8771rW9pjz320L333qsJEybo6aefllSaGvHrX/9a9957r6655poOb988efJknXvuuWpubtY555zTNvI6b9487bTTTh1ev6mpSfPmzWt7vN566+nuu+/WiSeeqFNOOeV9da/qtU8++WTttddeuu+++zR37lxtv/32Ouuss7T11lurpaVF06ZN0+WXX6799ttPLS0tuu+++9TQ0NA1f5gAAAC9XK+b9lC0CRMmaK211nrf8ZkzZ+qaa66RJB1wwAHaYIMNJEmzZs3SQQcd1PY9Bx54oCTpjTfe0OzZs3XIIYe0vcY777yzyutGRIdFWJMmTWr7fOqpp3Y4d3Wvfeutt7bNC+7Xr58GDx6sl19+ucP377zzzjr66KO1ZMkSTZw4keYXAABkg+Z3Beuss84qn1vZDgHtpz+0t3z5cq2//vpqaWl533Pbbbedmpubtffee7cdmzt3rrbbbruVXmvF667utSsxbtw4zZw5UzfccIMOP/xwnXbaaTriiCNqei0AAICU0PxWaNy4cZoxY4a+9rWv6aabbmobTd1jjz103HHH6atf/aqWLl2qG264Qccee6zWW289DR8+XFdddZUOOeQQRYTuv/9+jR49Wqeffrq+8pWv6Pe//7022mgjtbS06NJLL9Vdd93Vdr0rr7xSU6ZM0ZVXXqmxY8d2qGV1r73PPvvoggsu0CmnnKJly5bpzTff1KBBg/T666+3ff9TTz2lzTbbTMcee6zefPNNzZ07l+YXAICCdfWi/85UsilAX0TzW6FvfvObmjRpkhobG7XXXntpyy23lFSaQjBhwgSNHj1aW221lZqamjR48GBJ0owZM/SFL3xB3/nOd7RkyRIdeuihGj16tCZMmKBnn31Wu+22m2xr0KBB+sUvfqFNN9207XrvvPOOdtllFy1fvlxXXHFF2/HWUeBVvfaPfvQjTZ48WRdffLH69eunCy64QGPHjtXuu++uUaNG6ZOf/KRGjRqladOmacCAAVp33XVr2j4NAAAgRV7Vr+27Q1NTU7RfECZJ8+fP18iRI3ushu7wxhtvaN1119Vbb72lcePGafr06WpsbOzy65x00klqbGzU5z//+S5/7Rz1hb97AIC+o8dHfgd+rkevp6mv9ujlbDdHxPv2rM1+t4euMHnyZDU0NKixsVGf/vSnu6Xx/frXv6677rpLEyZM6PLXBgAAyAXTHrrAituidYczzzxTZ555ZrdfBwAAoC/rdOTX9kDbd9u+z/Y8298qHx9u+y7bj9m+0vaa3V8uAAAAULtKpj28I2nviBgtqUHS/rZ3lXS2pP+JiG0kvSzpmO4rEwAAAKhfp81vlLxRfjig/BGS9pZ0dfn4ZZImdkuFAAAAQBepaMGb7X62WyS9KOmPkp6Q9EpELC2f8oykzbqnRAAAAKBrVNT8RsSyiGiQtLmkMZJWtj/USvdMsz3Z9hzbcxYuXFh7pd3oxz/+sUaOHKnDDjusw/E5c+bo5JNPXun3DBs2TC+99FLN15w+fbpGjBihESNGaMyYMZo1a1anr33hhRfWtCfvK6+8ovPPP3+15/zoRz/SKaec0vb4uOOO0yc+8Ym2x+eee+4q/yxaffe73626tu6w7rrrFl0CAADopara7SEiXrF9u6RdJa1vu3959HdzSc+t4numS5oulfb57fQiUwdXU1LnKthT7vzzz9dNN92k4cOHtx1bunSpmpqa1NT0vu3h6nb99dfroosu0qxZszRkyBDNnTtXEydO1N13360PfvCDq/y+448/vqbrtTa/J5xwwirP2W233TRjxoy2xy0tLVq+fLmWLVumfv36afbs2Zo4cfUzW7773e/qjDPOqKq21tfvChGxyrebBgAAkCrb7WGo7fXLX68l6ROS5ku6TdK/lk87UtK13VVkdzr++OP15JNPasKECRo8eLAmT56s8ePH64gjjtDtt9+uT33qU5KkRYsWafz48dpxxx113HHHdWiyzjzzTI0YMUL77ruvJk2apHPOOUeS9MQTT2j//ffXTjvtpD333FMPP/ywJOnss8/WtGnTNGTIEElSY2OjjjzySJ133nltrzlt2jSNGTNGY8aM0eOPPy5Jmjp1aqev/cILL+jggw/W6NGjNXr0aM2ePVtTpkzRE088oYaGBp122ml6/vnnNW7cODU0NGjUqFH6y1/+oh133FGPPvqo3n77bb366qtae+211dDQoAceeECSNHv2bO22226SpIkTJ2qnnXbS9ttvr+nTp0uSpkyZorffflsNDQ1tI+i/+MUvNGbMGDU0NOi4447TsmXLJJVGZr/xjW9ol1120R133KFhw4bpjDPO0NixY9XU1KS5c+dqv/3209Zbb60LL7xQUumNRPbZZx81NjbqYx/7mK69tvTXbcGCBRo5cqROOOEENTY26m9/+1vbn+FLL72ksWPH6oYbenbTcAAA0HtVMu1hU0m32b5f0j2S/hgR10v6iqT/sP24pI0kXdx9ZXafCy+8UB/60Id022236dRTT1Vzc7Ouvfba9+3d+61vfUt77LGH7r33Xk2YMEFPP/20pNLUiF//+te69957dc0116j9O9hNnjxZ5557rpqbm3XOOee0jbzOmzdPO+20U4fXb2pq0rx589oer7feerr77rt14okndpiO0Nlrn3zyydprr7103333ae7cudp+++111llnaeutt1ZLS4umTZumyy+/XPvtt59aWlp03333qaGhQf3791dDQ4Puuece3Xnnndpll1206667avbs2XruuecUEdpiiy0kSZdccomam5s1Z84c/fjHP9aiRYt01llnaa211lJLS4tmzJih+fPn68orr9Rf//pXtbS0qF+/fm0jy2+++aZGjRqlu+66S3vssYckaYstttAdd9yhPffcU0cddZSuvvpq3XnnnfrGN74hSRo4cKB+85vfaO7cubrtttv05S9/ue0HkEceeURHHHGE7r33Xm211VaSSj8EHHDAAfr2t7+tAw7I873LAQDA+3U67SEi7pe040qOP6nS/N8+ZcKECVprrbXed3zmzJm65pprJEkHHHCANthgA0nSrFmzdNBBB7V9z4EHHiipNFI5e/ZsHXLIIW2v8c4776zyuhEh222PJ02a1Pb51FNP7XDu6l771ltvbZsX3K9fPw0ePFgvv/xyh+/feeeddfTRR2vJkiWaOHGiGhoaJEm77767Zs+erbfffltjx47VNttso+9+97saOnRo26ivVJoj/Zvf/EaS9Le//U2PPfaYNtpoow7XuOWWW9Tc3Kydd95ZkvT2229r4403bqvr05/+dIfzW9+57mMf+5jeeOMNDRo0SIMGDdLAgQP1yiuvaJ111tEZZ5yhmTNnao011tCzzz6rF154QZK01VZbadddd217rSVLlmifffbReeedp7322muVf+YAACA/vMPbCtZZZ51VPte+OW21qjmmy5cv1/rrr6+Wlpb3PbfddtupublZe++9d9uxuXPnarvttlvptVa87upeuxLjxo3TzJkzdcMNN+jwww/XaaedpiOOOEK77babLrroIi1evFhf/OIXNXToUD300EMaOnSodt99d0nS7bffrj/96U+64447tPbaa+vjH/+4Fi9e/L5rRISOPPJI/fd///f7nhs4cOD75vl+4AMfkCStscYabV+3Pl66dKlmzJihhQsXqrm5WQMGDNCwYcParrtiZv3799dOO+2km2++meYXAAB0UNFuDyg1jK2/tr/pppvaRlP32GMP/e53v9PixYv1xhtvtM0vXW+99TR8+HBdddVVkkrN4H333SdJOv300/WVr3xFixYtklRaXHbppZd2WJB25ZVXtn0eO3Zsh1pW99r77LOPLrjgAkmlxWSvvfaaBg0apNdff73t+5966iltvPHGOvbYY3XMMcdo7ty5kkqL3u68804tXLhQG2+8sWxr6NChuvbaa9tGfl999VVtsMEGWnvttfXwww/rzjvvbHvdAQMGaMmSJW11XH311XrxxRclSf/4xz/01FNP1frHr1dffVUbb7yxBgwYoNtuu221r2Vbl1xyiR5++GGdddZZNV8TAAD0PYz8Vuib3/ymJk2apMbGRu21117acsstJZWmEEyYMEGjR4/WVlttpaamJg0eXNqxYsaMGfrCF76g73znO1qyZIkOPfRQjR49WhMmTNCzzz6r3XbbTbY1aNAg/eIXv9Cmm27adr133nlHu+yyi5YvX64rrrii7XjrKPCqXvtHP/qRJk+erIsvvlj9+vXTBRdcoLFjx2r33XfXqFGj9MlPflKjRo3StGnTNGDAAK277rpt0yQ22GADDR06VNtvv33b9caOHau//vWvGj16tCRp//3314UXXqgddthB2267bYfpBpMnT9YOO+ygxsZGzZgxQ9/5znc0fvx4LV++XAMGDNB5553XNie3WocddpgOPPBANTU1qaGhQSNGjFjt+f369dMvf/lLHXjggVpvvfVWu9MFAADIh3tya6impqZovyBMkubPn6+RI1e2bXA63njjDa277rp66623NG7cOE2fPl2NjY1dfp2TTjpJjY2N+vznP9/lr52jvvB3DwDQdwyb0rO7Ey0Y+LkevV4l2892JdvNEfG+PWuZ9tAFJk+erIaGBjU2NurTn/50tzS+X//613XXXXe1LQwDAABA9Zj20AVW3BatO5x55pk688wzu/06AAAAfRkjvwAAAMhGr2h+eUta9DT+zgEAkKfCm9+BAwdq0aJFNCPoMRGhRYsWaeDAgUWXAgAAeljhc34333xzPfPMM1q4cGHRpSAjAwcO1Oabb150GQAAoIcV3vwOGDBAw4cPL7oMAAAAZKDwaQ8AAABAT6H5BQAAQDZofgEAAJANml8AAABkg+YXAAAA2aD5BQAAQDZofgEAAJCNwvf5BQAAiZo6uIev92rPXg99EiO/AAAAyAbNLwAAALJB8wsAAIBs0PwCAAAgGzS/AAAAyAbNLwAAALJB8wsAAIBs0PwCAAAgGzS/AAAAyAbNLwAAALJB8wsAAIBs0PwCAAAgGzS/AAAAyAbNLwAAALJB8wsAAIBs0PwCAAAgGzS/AAAAyAbNLwAAALJB8wsAAIBs0PwCAAAgGzS/AAAAyAbNLwAAALLRafNrewvbt9meb3ue7S+Vj0+1/aztlvLHP3d/uQAAAEDt+ldwzlJJX46IubYHSWq2/cfyc/8TEed0X3kAAABA1+m0+Y2I5yU9X/76ddvzJW3W3YUBAAAAXa2qOb+2h0naUdJd5UMn2r7f9iW2N+ji2gAAAIAuVcm0B0mS7XUl/VrSKRHxmu0LJJ0pKcqfvy/p6JV832RJkyVpyy237IqaAQDASgybckOPXm/BwB69HNAlKhr5tT1ApcZ3RkRcI0kR8UJELIuI5ZJ+ImnMyr43IqZHRFNENA0dOrSr6gYAAACqVsluD5Z0saT5EfGDdsc3bXfawZIe7PryAAAAgK5TybSH3SUdLukB2y3lY2dImmS7QaVpDwskHdctFQIAAABdpJLdHmZJ8kqeurHrywEAAAC6D+/wBgAAgGzQ/AIAACAbNL8AAADIBs0vAAAAskHzCwAAgGzQ/AIAACAbNL8AAADIBs0vAAAAskHzCwAAgGzQ/AIAACAbNL8AAADIBs0vAAAAskHzCwAAgGzQ/AIAACAbNL8AAADIBs0vAAAAskHzCwAAgGzQ/AIAACAbNL8AAADIBs0vAAAAskHzCwAAgGzQ/AIAACAbNL8AAADIBs0vAAAAskHzCwAAgGzQ/AIAACAbNL8AAADIBs0vAAAAskHzCwAAgGzQ/AIAACAbNL8AAADIBs0vAAAAskHzCwAAgGzQ/AIAACAbNL8AAADIBs0vAAAAskHzCwAAgGzQ/AIAACAbNL8AAADIBs0vAAAAskHzCwAAgGzQ/AIAACAbnTa/trewfZvt+bbn2f5S+fiGtv9o+7Hy5w26v1wAAACgdpWM/C6V9OWIGClpV0lftL2dpCmSbomIbSTdUn4MAAAA9FqdNr8R8XxEzC1//bqk+ZI2k3SQpMvKp10maWJ3FQkAAAB0harm/NoeJmlHSXdJ2iQinpdKDbKkjVfxPZNtz7E9Z+HChfVVCwAAANSh4ubX9rqSfi3plIh4rdLvi4jpEdEUEU1Dhw6tpUYAAACgS1TU/NoeoFLjOyMirikffsH2puXnN5X0YveUCAAAAHSNSnZ7sKSLJc2PiB+0e+o6SUeWvz5S0rVdXx4AAADQdfpXcM7ukg6X9IDtlvKxMySdJelXto+R9LSkQ7qnRAAAAKBrdNr8RsQsSV7F0/t0bTkAAABA9+Ed3gAAAJCNSqY9AH3f1ME9fL1Xe/Z6AABAEiO/AAAAyAjNLwAAALJB8wsAAIBs0PwCAAAgGzS/AAAAyAbNLwAAALJB8wsAAIBs0PwCAAAgGzS/AAAAyAbNLwAAALJB8wsAAIBs0PwCAAAgGzS/AAAAyAbNLwAAALJB8wsAAIBs0PwCAAAgG/2LLgBYmWFTbujR6y0Y2KOXAwAABWHkFwAAANmg+QUAAEA2aH4BAACQDZpfAAAAZIPmFwAAANmg+QUAAEA2aH4BAACQDZpfAAAAZIPmFwAAANmg+QUAAEA2aH4BAACQDZpfAAAAZIPmFwAAANmg+QUAAEA2aH4BAACQDZpfAAAAZIPmFwAAANmg+QUAAEA2aH4BAACQDZpfAAAAZIPmFwAAANmg+QUAAEA2Om1+bV9i+0XbD7Y7NtX2s7Zbyh//3L1lAgAAAPWrZOT3Ukn7r+T4/0REQ/njxq4tCwAAAOh6nTa/ETFT0j96oBYAAACgW9Uz5/dE2/eXp0Vs0GUVAQAAAN2k1ub3AklbS2qQ9Lyk76/qRNuTbc+xPWfhwoU1Xg4AAACoX03Nb0S8EBHLImK5pJ9IGrOac6dHRFNENA0dOrTWOgEAAIC61dT82t603cODJT24qnMBAACA3qJ/ZyfYvkLSxyUNsf2MpG9K+rjtBkkhaYGk47qxRgAAAKBLdNr8RsSklRy+uBtqAQAAALoV7/AGAACAbND8AgAAIBudTntAFaYO7uHrvdqz1wMAAEgcI78AAADIBs0vAAAAskHzCwAAgGzQ/AIAACAbNL8AAADIBs0vAAAAskHzCwAAgGzQ/AIAACAbNL8AAADIBs0vAAAAskHzCwAAgGzQ/AIAACAb/YsuoDsNm3JDj15vwcAevRwAAACqxMgvAAAAskHzCwAAgGzQ/AIAACAbNL8AAADIBs0vAAAAskHzCwAAgGzQ/AIAACAbNL8AAADIBs0vAAAAskHzCwAAgGzQ/AIAACAbNL8AAADIBs0vAAAAskHzCwAAgGzQ/AIAACAbNL8AAADIBs0vAAAAskHzCwAAgGzQ/AIAACAbNL8AAADIBs0vAAAAskHzCwAAgGzQ/AIAACAbNL8AAADIBs0vAAAAskHzCwAAgGx02vzavsT2i7YfbHdsQ9t/tP1Y+fMG3VsmAAAAUL9KRn4vlbT/CsemSLolIraRdEv5MQAAANCrddr8RsRMSf9Y4fBBki4rf32ZpIldXBcAAADQ5Wqd87tJRDwvSeXPG6/qRNuTbc+xPWfhwoU1Xg4AAACoX7cveIuI6RHRFBFNQ4cO7e7LAQAAAKtUa/P7gu1NJan8+cWuKwkAAADoHrU2v9dJOrL89ZGSru2acgAAAIDuU8lWZ1dIukPStrafsX2MpLMk7Wv7MUn7lh8DAAAAvVr/zk6IiEmreGqfLq4FAAAA6Fa8wxsAAACyQfMLAACAbND8AgAAIBs0vwAAAMgGzS8AAACyQfMLAACAbND8AgAAIBs0vwAAAMgGzS8AAACyQfMLAACAbND8AgAAIBs0vwAAAMgGzS8AAACyQfMLAACAbND8AgAAIBs0vwAAAMgGzS8AAACyQfMLAACAbND8AgAAIBs0vwAAAMgGzS8AAACyQfMLAACAbND8AgAAIBs0vwAAAMgGzS8AAACyQfMLAACAbND8AgAAIBs0vwAAAMgGzS8AAACyQfMLAACAbND8AgAAIBs0vwAAAMgGzS8AAACyQfMLAACAbND8AgAAIBs0vwAAAMgGzS8AAACyQfMLAACAbND8AgAAIBs0vwAAAMgGzS8AAACy0b+eb7a9QNLrkpZJWhoRTV1RFAAAANAd6mp+y/4pIl7qgtcBAAAAuhXTHgAAAJCNepvfkPQH2822J3dFQQAAAEB3qXfaw+4R8ZztjSX90fbDETGz/QnlpniyJG255ZZ1Xg4AAACoXV0jvxHxXPnzi5J+I2nMSs6ZHhFNEdE0dOjQei4HAAAA1KXm5tf2OrYHtX4tabykB7uqMAAAAKCr1TPtYRNJv7Hd+jqXR8Tvu6QqAAAAoBvU3PxGxJOSRndhLQAAAEC3YqszAAAAZIPmFwAAANmg+QUAAEA2aH4BAACQDZpfAAAAZIPmFwAAANmg+QUAAEA2aH4BAACQDZpfAAAAZIPmFwAAANmg+QUAAEA2aH4BAACQDZpfAAAAZIPmFwAAANmg+QUAAEA2aH4BAACQDZpfAAAAZIPmFwAAANmg+QUAAEA2aH4BAACQDZpfAAAAZIPmFwAAANmg+QUAAEA2aH4BAACQDZpfAAAAZIPmFwAAANmg+QUAAEA2aH4BAACQjf5FFwAAyNzUwT14rVd77loAeiVGfgEAAJANml8AAABkg+YXAAAA2aD5BQAAQDZofgEAAJANml8AAABkg63OAHS5YVNu6NHrLRj4uR69Xl/fLqvn8+vRywHIHCO/AAAAyAbNLwAAALJB8wsAAIBs0PwCAAAgGzS/AAAAyEZdza/t/W0/Yvtx21O6qigAAACgO9Tc/NruJ+k8SZ+UtJ2kSba366rCAAAAgK5Wz8jvGEmPR8STEfGupF9KOqhrygIAAAC6Xj3N72aS/tbu8TPlYwAAAECv5Iio7RvtQyTtFxH/Xn58uKQxEXHSCudNljS5/HBbSY/UXm6vN0TSS0UXgZqQXdrIL23kly6yS1tfz2+riBi64sF63t74GUlbtHu8uaTnVjwpIqZLml7HdZJhe05ENBVdB6pHdmkjv7SRX7rILm255lfPtId7JG1je7jtNSUdKum6rikLAAAA6Ho1j/xGxFLbJ0q6WVI/SZdExLwuqwwAAADoYvVMe1BE3Cjpxi6qpS/IYnpHH0V2aSO/tJFfusgubVnmV/OCNwAAACA1vL0xAAAAskHzCwAAgGzUNecXSJHtDSs4bXlEvNLtxQBAIrh3pst2YwWnLYmIB7q9mF6AOb81sP1aZ6dIej4iPtoT9aA6thertCe1V3Nav4jYsodKQoVsV7Kd4j8i4qjurgXVs31/BactjIh9ur0YVI17Z7psv67SFrWry254RAzrmYqKxchvbZ6IiB1Xd4Lte3uqGFRtPvkla6Skf1/N85Z0Xg/Vgur1k/TPq3neYr/43ox7Z7ruiYi9V3eC7Vt7qpiiMfJbA9sfjogn6z0HxbA9MCIW13sOep7tz0TEr+o9B8WwvUdEzKr3HBSDeyf6CprfOtjeRNJmkkLScxHxQsEloUK2LWmM2uUn6e7gHwTQI8rzRyMiXi66FlSOe2e6bA+WtL86ZndzjnO0aX5rYLtB0oWSBkt6tnx4c0mvSDohIuYWVRs6Z3u8pPMlPaaO+X1Epfz+UFRtWL3yzfurkiZKGlo+/KKkayWdleNNPCW2t5T0PUn7qHS/tKT1JN0qaUpELCiuOnSGe2e6bB8h6ZuS/qCO2e0r6VsR8bOiaisCzW8NbLdIOi4i7lrh+K6SLoqI0cVUhkrYni/pkyv+R2t7uKQbI2JkIYWhU7ZvVqlRuiwi/l4+9kFJR0r6RETsW2R9WD3bd0j6oaSrI2JZ+Vg/SYdIOiUidi2yPqwe98502X5E0i4rDhDY3rojSzoAAB6wSURBVEDSXbkt0Gef39qss2LjK0kRcaekdQqoB9XpL+mZlRx/VtKAHq4F1RkWEWe3Nr6SFBF/j4izJbHCvPcbEhFXtja+khQRyyLil5I2KrAuVIZ7Z7qs0lSHFS3X6neA6JPY7aE2N9m+QdLPJP2tfGwLSUdI+n1hVaFSl0i6x/Yv1TG/QyVdXFhVqMRTtk9XaeT3Balt7v1Rei9L9F7Nts+XdJk6/ts7UhK7BPR+3DvT9V+S5tr+g97LbkuVpj2cWVhVBWHaQ41sf1LSQSpNHLdKPw1fFxE3FloYKmJ7O0kT9P78Hiq0MKxW+Vd0U1T6t7dJ+fDfVdoe6+yI+EdRtaFztteUdIxWcu+UdHFEvFNgeagA9850le+f+6ljdjfnuOiU5hdZY8U5AFSPe2ea2KWqhOa3Bu1WnB8kaePyYVacJ6LdivO9Jb1aPjxYrDhPgu39VNrtof12PddGBFOOejnb/VUa+X1ffiqN/C4psDx0gntnulbYpeoZlUZ+s92liua3BqtZcX6UpH1Ycd67seI8XbZ/KOmjKs23b114s7lK8+0fi4gvFVUbOmf7CpX+s71MHfM7UtKGEfHZompD57h3potdqjqi+a2B7UciYttqn0PvYPuxiNim2udQPNuPrmxLnvLG+4+SXe/Wyb1zpdmi9+Dema5Osns8Ij7S0zUVia3OavOU7dPLc2cklebR2P6KWHGegmbb59vexfaHyh+7lFehs+K8d1tse8xKju8sibdU7f1etn2I7bb/e2yvYfuzkpg72vtx70zXTbZvsP1Z27uVPz5b3rkquyljjPzWYIUV561zfl8QK86TwIrzdNlulHSBpEF679fmW0h6TaV5a81F1YbO2R4m6WyV5oy+rNK/vcGSblNpzuj/FVYcOsW9M23sUvUeml8AySnPsW+7gbd/0wukwfZGKv0f9FLRtQDIC9MeauCSz5R/fWfb+9j+se0T2v86D72T7SErPP63cn6Ty3NH0YuVG1+VR3mfkrRbee9RJCQiFkkaZPtfbI8ouh50zvba5Sl/p9keaPtI29fZ/p7tdYuuD6tme4d2Xw+w/bVydt+1vXaRtRWBRq0250n6jKTDJf1c0vGS5kgaJ+l/CqwLlflD6xe2v6ZSjs0qvdPND4oqCp2zfZykOyTdafsLkq6X9ClJv7F9TKHFoVO2f9vu64NU2jXnQEnX2T6qqLpQsUtVenOZ4ZJuUGmu/Tkq/QbmguLKQgUubff1WZI+Iun7ktZSaQu0rDDtoQa2H4iIj9keoNK7S20aEe+W97C8NyI+VnCJWA3b90bEjuWv50raMyLeLOc5l/x6L9sPSNpFpRv2U5I+EhF/L8/Dvy0iGgotEKu1wr+92ZIOi4j/K/825pbctltKje2WiGgo/4bseZX+74vy4/siYodOXgIFWeHfXouknSNiSa7Z9S+6gEQtlaTyX5x7IuLd8uOltpcVWxoqsJbtHVX6zUe/iHhTasuT/Hq3JRHxlqS3bD/ROtc3Il62zU/yvV/7jPq3LnCLiJdsLy+oJlSp3PDeGOXRs/Jj/v31boNtH6zS/3sfaH1DmVyzo/mtzd9trxsRb0TE/q0Hy3MR3y2wLlTmeb03veEftjeNiOfLC3CWFlgXOrfc9oDyjfuA1oO2B4ppXCkYbfs1lX5N/gHbHyyP3K8pqV/BtaFzc9r933d060HbW0t6vcC60Lk/S5pQ/vpO25tExAvlviW7RadMe+hCtteRtE5EvFh0LaieS+9U9IHyyCJ6IZfeXvW5iFi6wvHNJI2MiD8VUxnqYXt9lfK7o+haUBvbDhoKJILmFwAAANng14QAAADIBs0vAAAAskHzCwAAgGzQ/NbJ9vWre4zejfzSZXvq6h6jdyO/tJX3SF/lY/ReZEfz2xWO7eQxejfyS1dzJ4/Ru5FfwiKicXWP0XuRHbs91M32WpK2jIhHiq4F1SM/AKie7a0kbRMRfyrfR/tHBHv9JoDsGPmti+0DJbVI+n35cYPt64qtCpUiv3TZ/qjtW2w/WH68g+2vFV0XKkN+abN9rKSrJV1UPrS5pN8WVxEqRXYlNL/1mSppjKRXJCkiWiQNK7AeVGeqyC9VP5H0VUmtb9F5v6RDC60I1SC/tH1R0u6SXpOkiHhM0saFVoRKkZ1ofuu1NCJeLboI1Iz80rV2RNy9wjHemjod5Je2dyLi3dYHtvtLYg5lGshONL/1etD25yT1s72N7XMlzS66KFSM/NL1ku2tVb5p2/5XSc8XWxKqQH5p+7PtMyStZXtfSVdJ+l3BNaEyZCcWvNXF9tqS/lPS+PKhmyV9JyIWF1cVKkV+6bL9YUnTJe0m6WVJ/yfp3yJiQZF1oTLklzbba0g6RqV7pyXdHBE/KbYqVILsSmh+62B7T0mzI2JZu2ONEZHdnnkpIr/02V5H0hq5rVTuK8gvTbYPl/Tb9rnZ/lREsE96L0d2JUx7qM/Nkm61vUm7Y/9bVDGoGvklyvYy22dJeqv1Jp7jRu2pIr/knSvpL7ZHtjv27aKKQVXITjS/9XpE0jRJt9verXzMBdaD6pBfuuapdP/6g+0Ny8fILh3kl7b/k3S0pKttH1I+Rn5pIDtJ/YsuIHEREdfbfkTSlbYvUYarJhNGfulaGhGn2/6MSqMYR4jsUkJ+aYuImGt7L0lX2N5FUr+ii0JFyE6M/NbLUts+eXtKGidph0IrQjXIL12t2f1K0mck/VTShwutCNUgv7Q9L0kR8ZKk/VT6wWVUoRWhUmQnFrx1OdtbRsTTRdeB2pBfGmzvFBHN7R6vJ2liRPyswLJQIfIDUCSmPdTA9ukR8T3bP17FKSf3aEGoCvmly/beEXGrpK3K70/f3htF1ITKkV/abP8wIk6x/TutZJpKREwooCxUgOw6ovmtzfzy5+bVnoXeivzStZekWyUduJLnQtI1PVsOqkR+aft5+fM5hVaBWpBdO0x76CLljaPXjYjXiq4F1SM/AKie7Q0kbRER9xddC6qTc3YseKuD7cttr1feqP0hSY/YPq3oulAZ8kuX7S+Vs7Pt/7U91/b4zr8TvQH5pc327eX8NpR0n6Sf2v5B0XWhc2RXQvNbn+3KI4UTJd0oaUtJhxdbEqpAfuk6upzdeEkbS/q8pLOKLQlVIL+0DS7n9y+SfhoRO0n6RME1oTJkJ5rfeg2wPUCl5unaiFgi9qpMCfmlq3VT9n9W6QZ+nzLcqD1h5Je2/rY3VWmbuqzeFrcPIDvR/NbrIkkLJK0jaWZ59TJzRtNBfulqtv0HlZqnm20PkrS84JpQOfJL27dVenv4xyPiHtsflvRYwTWhMmQnFrx1KduW1C8ilpYfHxkRlxVcFipEfukoL1BskPRkRLxieyNJm7Uu3LC9fUTMK7RIrBL59W22vxoR/110HaheLtnR/HYj23MjorHoOlAb8ksX2aWN/NJGfunKJTumPXQv5rCljfzSRXZpI7+0kV+6ssiO5rd7MayeNvJLF9mljfzSRn7pyiI7mt/ulcVPUH0Y+QFA9bh3piuL7Gh+u9dfiy4AdSG/dL1bdAGoC/ml7aqiC0DNssiOBW91sL2+pCMkDZPUv/V4RJxcVE2oHPmlzfYOen921xRWEKpCfumyPVzSSXp/fhOKqgmVIbuS/p2fgtW4UdKdkh4Qe1SmiPwSZfsSSTtImqf3sgtJNE8JIL/k/VbSxZJ+J+6dqSE7MfJbl1y2BOmryC9dth+KiO2KrgO1Ib+02b4rInYpug5Uj+xKaH7rYPtUSW+o9BaB77Qej4h/FFYUKkZ+6bJ9saTvR8RDRdeC6pFf2mx/TtI2kv6gjvfOuYUVhYqQXQnTHurzrqRpkv5T720PEpI+XFhFqAb5pesySXfY/rtKN3BLiojYodiyUCHyS9vHJB0uaW91nLayd2EVoVJkJ0Z+62L7CUm7RMRLRdeC6pFfumw/Luk/tMJ87Yh4qrCiUDHyS5vthyXtEBHsypEYsith5Lc+8yS9VXQRqBn5pevpiLiu6CJQM/JL232S1pf0YtGFoGpkJ5rfei2T1GL7NnWcO8NWWWkgv3Q9bPtylVYst8+O3QLSQH5p20SlDO9Rx/yy2i4rUWQnmt96/bb8gTSRX7rWUunGPb7dMbbKSgf5pe2bRReAmpGdmPMLAACAjDDyWwPbv4qIz9h+QO/tEtCGFcu9G/mly/bpEfE92+dq5dkxZaUXI7+02Z4VEXvYfl0d82vdrWO9gkpDJ8iuI5rf2nyp/PlThVaBWpFfuuaXP88ptArUivwSFhF7lD8PKroWVIfsOmLaAwAAALKxRtEFpMj29V1xDopBfumyPbUrzkExyC9ttjt9F7BKzkHPI7uOGPmtge1XJM1c3SmSto8I3imsFyK/dNl+RtIPVneKpGMjYkQPlYQqkF/abL8t6bHVnSJpcERs2UMloUJk1xFzfmtzUAXnZP3uKb0c+aXrJ5I6m7P2k54oBDUhv7RV8kPJsm6vArUgu3YY+QUAAEA2mPMLAACAbND8AgAAIBvM+QUAAJ2yvWEFpy2PiFe6vRhUhew6Ys5vDWy/1tkpkp6PiI/2RD2oDvmly/Z1FZz2j4g4qrtrQfXIL222F0t6TqV75Kr0y2XHgJSQXUeM/NbmiYjYcXUn2L63p4pB1cgvXSMl/ftqnrek83qoFlSP/NI2n3tnssiuHUZ+a2D7wxHxZL3noBjkly7bn4mIX9V7DopBfmmzPTAiFtd7Dnoe2XVE8wsAAGpie8OI+EfRdaB6OWfHbg9dzPYDRdeA2pFfumxPL7oG1I78ej/bX2v39Xa2H5XUbHuB7V0KLA2dILuOGPmtge1/WdVTki6MiKE9WQ+qQ37pWs2KZUu6LyI278l6UB3yS5vtuRHRWP76Bkn/LyJusj1G0g8jYrdiK8SqkF1HLHirzZWSZkha2U8OA3u4FlSP/NK1UNJT6rhiOcqPNy6kIlSD/PqOD0XETZIUEXfbXqvoglCx7LOj+a3N/ZLOiYgHV3zC9icKqAfVIb90PSlpn4h4esUnbP+tgHpQHfJL24fL29VZ0ua2146It8rPDSiwLnSO7Nqh+a3NKZJWtVfswT1ZCGpCfun6oaQNJL2veZL0vR6uBdUjv7QdtMLjfpJkexNJF/R8OagC2bXDnF8AAABkg5HfGtjuL+kYlUYJP6TSnLXnJF0r6eKIWFJgeegE+aXN9giVRjE203vZXRcR8wstDBUhv3TZHizpq5ImSmpdGPyiSvfOs3J5a9wUkV1HbHVWm59LapA0VdI/SzpA0rckjZb0i+LKQoXIL1G2vyLplyrNW7tb0j3lr6+wPaXI2tA58kveryS9LOnjEbFRRGwk6Z/Kx64qtDJ0huzaYdpDDWw/EhHbruK5RyPioz1dEypHfukq7025/Yqj87bXlDQvIrYppjJUgvzS1sm9c5XPoXhk1xEjv7V52fYhttv+/GyvYfuzKv0Uhd6N/NK1XKWpKivatPwcejfyS9tTtk8vL5KSVFowVR7RZ7eO3o3s2mHOb20OlXS2pPNtv6zSr+3Wl3Rr+Tn0buSXrlMk3WL7Mb13w95S0kcknVhYVagU+aXts5KmSPqz7dZ9mV+QdJ2kzxRWFSpBdu0w7aFOtjdS6c/xpaJrQfXILz3lEfsxKi2YsqRnJN0TEcsKLQwVIT8ARWPkt0Yrrli2/ZykayPi4WIrQyXIL2nR7mN5u89IA/n1Ibavj4hPFV0Hqpdzdsz5rcFqViz/khXLvR/5pcv2eEmP6f07dTxWfg69GPn1SZsVXQBqlm12THuoASuW00Z+6bI9X9InI2LBCseHS7oxIkYWUhgqQn59j+1LIuLooutA9XLOjpHf2rBiOW3kl67+Ks0RXdGzyvD96RNEfn2E7bVsb5tr85QysmPOb61YsZw28kvXJZLusf1LvZfdFirt0nFxYVWhUuTXB9g+UNI5ktaUNNx2g6RvR8SEYitDZ8iuhGkPNWLFctrIL122R+q9xYqt2V0XEQ8VWhgqQn7ps90saW9Jt0fEjuVj90fEDsVWhs6QXQkjv7VjxXLayC9RETFf0vyi60BtyK9PWBoRr9ouug5Uj+xE81uT8qrk81Vatfxs+fDmkj5i+4SI+ENhxaFT5Nc32Z4aEVOLrgO1Ib+kPGj7c5L62d5G0smSZhdcEypDdmLaQ01YsZw28uubbB8YEb8rug7UhvzSYXttSf8pqXV7upslfSciFhdXFSpBdiU0vzUoL5QaGRFLVzi+pqSHIuIjxVSGSpAfAAD5YtpDbVixnDbyS5Tt/pKOkXSwStvVhaTnJF0r6eIV925G70J+fZft6RExueg6UL0cs2Pkt0asWE4b+aXJ9hWSXpF0md7bL3ZzSUdK2jAiPltUbegc+aXN9oarekrSfRGxeU/Wg8qRXUc0vwCSYfuRiNh2Fc89GhEf7emaUDnyS5vtZZKeUqlhahXlx5tFxJqFFIZOkV1HvMNbF7M9tegaUDvy6/Vetn1IeZ9mSaU9m21/VtLLBdaFypBf2p6U9PGIGN7u48MRMVzSC0UXh9Uiu3Zofrtec9EFoC7k17sdKulfJb1g+1Hbj0r6u6R/KT+H3o380vZDSRus4rnv9WQhqBrZtcO0BwBJsr2RSvewl4quBdUjPwBFofmtASuW00Z+AADki+a3BqxYThv5AQCQL5rfGrBiOW3kBwBAvljwVhtWLKeN/AAAyBTNb21YsZw28usDbM9d3WP0buSXNvJLF9kx7aFurFhOG/kBAJAXRn7rFBGLaJzSRX7psr2V7U+Uv17L9qCia0LlyC9t5JcusqP5BZAg28dKulrSReVDm0v6bXEVoRrklzbySxfZldD8AkjRFyXtLuk1SYqIxyRtXGhFqAb5pY380kV2ovkFkKZ3IuLd1gflNy5hAUM6yC9t5JcushPNb91YNZk28kvWn22fIWkt2/tKukrS7wquCZUjv7SRX7rITuz2ACBB5T2aj5E0XpIl3RwRPym2KlSK/NJGfukiuxKa3zrZ3krSNhHxJ9trSeofEa8XXRcqQ35psv2liPhRZ8fQO5Ff2sgvXWRXwrSHOrBqMm3kl7QjV3LsqJ4uAjUjv7SRX7rITlL/ogtI3BcljZF0l1RaNWk7u1WTCSO/xNieJOlzkobbvq7dU4MkLSqmKlSK/NJGfukiu45ofuvzTkS8a1tSvqsmE0Z+6Zkt6XlJQyR9v93x1yXdX0hFqAb5pY380kV27TDntw62vyfpFUlHSDpJ0gmSHoqI/yy0MFSE/AAAyA9zfuszRdJCSQ9IOk7SjTROSSG/RNne1fY9tt+w/a7tZbZfK7ouVIb80kZ+6SK7EqY91Oek8grJtm1Cclw1mTDyS9f/k3SoSntUNqk0ev+RQitCNcgvbeSXLrITI7/1YtVk2sgvYRHxuKR+EbEsIn4q6Z+KrgmVI7+0kV+6yI6R35qwajJt5NcnvGV7TUkt5bnbz0tap+CaUDnySxv5pYvsxIK3mpTfGGG4pP9Wad5oq9cl3R8RSwspDBUhv/SVM3xB0pqSTpU0WNL55REN9HLklzbySxfZldD8AkiK7X6SLouIfyu6FlSP/NJGfukiu/cw57cOrJpMG/mlKSKWSRpa/tUdEkN+aSO/dJHde5jzWx9WTaaN/NK1QNJfy3O232w9GBE/KKwiVGOByC9lC0R+qVogsqP5rVdEPG67X/knqp/anl10Tagc+SXrufLHGiotVERayC9t5JcushNzfutie6akT0j6X0l/V2nV5FERMbrQwlAR8uu7bJ8bEScVXQdqQ35pI7905ZIdc37rc7hKf4YnqvTrgy0kfbrQilAN8uu7di+6ANSF/NJGfunKIjumPdSovGryv8qrJhdL+lbBJaEK5AcAQJ4Y+a0RqybTRn4AAOSJkd/6LBCrJlO2QOTXV7noAlAX8ksb+aUri+xofuvDqsm0kV/f9aOiC0BdyC9t5JeuLLJjt4dulMuqyb6K/Hov27+TtOLN61VJcyRdFBGLe74qVIr80kZ+6SK7Eub8dq8sVk32YeTXez0p6Q1JPyl/vKbS+9V/tPwYvRv5pY380kV2YtoDgDTtGBHj2j3+ne2ZETHO9rzCqkKlyC9t5JcushMjvwDSNNT2lq0Pyl8PKT98t5iSUAXySxv5pYvsxMhvd8ti1WQfRn6915clzbL9hEo5DZd0gu11JF1WaGWoBPmljfzSRXZiwVu3sn1URFxadB2oDfn1brY/IGmESjfwh3NZqNFXkF/ayC9dZEfzWxdWTaaN/NJmezdJw9TuN1gR8bPCCkJVyC9t5JcusmPaQ72elDRU0hXlx59Vx1WThxdUFypDfomy/XNJW0tqkbSsfDgkZXUDTxX5pY380kV2JYz81qF1heTKjtmeFxHbF1UbOkd+6bI9X9J2wQ0sSeSXNvJLF9mVsNtDfVg1mTbyS9eDkj5YdBGoGfmljfzSRXZi2kO9WDWZNvJL1xBJD9m+W9I7rQcjYkJxJaEK5Jc28ksX2YlpD3Vj1WTayC9Ntvda2fGI+HNP14LqkV/ayC9dZFdC81snVk2mjfwAAMgL0x7qwKrJtJFfemzPiog9bL+ujtvUWVJExHoFlYYKkF/ayC9dZNcRI791YNVk2sgPAID8MPJbn9ZVk88XXQhqQn4Js91P0ibqOGXl6eIqQjXIL23kly6yo/mtF6sm00Z+ibJ9kqRvqvSmJMvLh0PSDoUVhYqRX9rIL11kV8K0hzqwajJt5Jcu249L2iUiFhVdC6pHfmkjv3SRXQkjv3WgSUob+SXtb5JeLboI1Iz80kZ+6SI70fzWhFWTaSO/PuFJSbfbvkEdp6z8oLiSUAXySxv5pYvsRPNbk4jYo/x5UNG1oHrk1yc8Xf5Ys/yBtJBf2sgvXWQnmt+6sWoybeSXnnJm60bEaUXXguqRX9rIL11k9x6a3zqwajJt5JemiFhmu7HoOlAb8ksb+aWL7N7Dbg91YNVk2sgvXba/L2kbSVdJerP1eERcU1hRqBj5pY380kV2JYz81odVk2kjv3RtKGmRpL3bHQtJWd3AE0Z+aSO/dJGdGPmti+2LJW0rKetVk6kiPwAA8sPIb31YNZk28kuU7YGSjpG0vaSBrccj4ujCikLFyC9t5Jcusiuh+a0RqybTRn7J+7mkhyXtJ+nbkg6TNL/QilAN8ksb+aWL7MS0h7rYviUi9im6DtSG/NJl+96I2NH2/RGxg+0Bkm6OiL07/WYUjvzSRn7pIrsSRn7r02L7OmW+ajJh5JeuJeXPr9geJenvkoYVVw6qRH5pI790kZ1ofuvFqsm0kV+6ptveQNLXJV0naV1J3yi2JFSB/NJGfukiOzHtAQAAABlh5LcOrJpMG/mly/Ymkr4r6UMR8Unb20kaGxEXF1waKkB+aSO/dJFdyRpFF5C4n0v6oEqrJv8saXNJrxdaEapBfum6VNLNkj5UfvyopFMKqwbVulTkl7JLRX6pulRkR/Nbp49ExNclvRkRl0k6QNLHCq4JlSO/dA2JiF9JWi5JEbFU0rJiS0IVyC9t5JcushPNb71WXDU5WBmumkwY+aXrTdsbqbRAUbZ3FW9VnRLySxv5pYvsxJzferFqMm3kl67/UCmzrW3/VdJQSf9abEmoAvmljfzSRXZitwcAibLdX9K2kizpkYhY0sm3oBchv7SRX7rIjua3LqyaTBv5pcf2v6zued6gpHcjv7SRX7rIriOa3zrYvknSTyX9Z0SMLv80dW9EsGgqAeSXHts/Xc3TwTZ1vRv5pY380kV2HdH81sH2PRGxc+t7ZZePtUREQ9G1oXPkBwBAftjtoT6smkwb+SXG9qe64hwUg/zSRn7pIruOGPmtg+1GSedKGiXpwf/f3h3iRhVFcRz+nwQBBt0FYEGwC0JQOFaAY0cEywLwJDhkEWgcCgcGyEEwEEa0tK9JZ07v96l5qbnJz5zc6XmT3dZkd58e9GBciH7zVNXHJM/ye1HjLK+6+8E1HYlL0G82/ebSbp/h94psTc6m3yxV9Ta7m/pzfOnup9dwHC5Jv9n0m0u7fYbfDWxNzqYfAKzLj1xs8+Scv3USw9Nx0w8AFuXmFwCAZXjbwwa2JmfTDwDW5eZ3A1uTs+l3M1TVSXd/PuuZ46bfbPrNpZ3hdxNbk7PpdzNU1ZvufnzWM8dNv9n0m0s7wy8AAAvxtgdgjKq6neR5kntJTpO87O4fhz0VF6XfbPrNpd0+N7/AGFX1Osn3JO+SPEryqbtfHPZUXJR+s+k3l3b7DL/AGFX1obvv7z7fSvK+ux8e+FhckH6z6TeXdvu86uyKqurkvGeOm37j/P356ZW/shtMv9n0m0u7f7j5vSJbk7PpN0tV/Uzy9c9jkjtJvu0+d3ffPdTZ+D/9ZtNvLu32GX4BAFiGtz1sYGtyNv0AYF1ufjewNTmbfgCwLsPvBrYmZ9MPANblbQ/b2JqcTT8AWJSb3w1sTc6mHwCsy/ALAMAy/NsDAADLMPwCALAMwy8AAMsw/AIAsAzDLwAAyzD8AgCwjF8VDe3dqTmqtAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "ax = df.mean(level=(1,2))[\"rank\"].unstack().plot(kind='bar', figsize=(12, 6))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Additionally, we may want simply to see which set of hyperparameters perform the best across the different __datasets__. We can do that by averaging the results of the different datasets." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
P: [learning_rate: 0.0001] [im_size: 300]P: [learning_rate: 0.0001] [im_size: 500]P: [learning_rate: 0.001] [im_size: 300]P: [learning_rate: 0.001] [im_size: 500]P: [learning_rate: 1e-05] [im_size: 300]P: [learning_rate: 1e-05] [im_size: 500]
duration20.55153029.70070620.3349131.63353319.84419430.323974
rank13.66666718.5000001.000001.33333326.33333330.750000
\n", + "
" + ], + "text/plain": [ + " P: [learning_rate: 0.0001] [im_size: 300] \\\n", + "duration 20.551530 \n", + "rank 13.666667 \n", + "\n", + " P: [learning_rate: 0.0001] [im_size: 500] \\\n", + "duration 29.700706 \n", + "rank 18.500000 \n", + "\n", + " P: [learning_rate: 0.001] [im_size: 300] \\\n", + "duration 20.33491 \n", + "rank 1.00000 \n", + "\n", + " P: [learning_rate: 0.001] [im_size: 500] \\\n", + "duration 31.633533 \n", + "rank 1.333333 \n", + "\n", + " P: [learning_rate: 1e-05] [im_size: 300] \\\n", + "duration 19.844194 \n", + "rank 26.333333 \n", + "\n", + " P: [learning_rate: 1e-05] [im_size: 500] \n", + "duration 30.323974 \n", + "rank 30.750000 " + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.mean(level=(1)).T" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To make it easier to see which permutation did the best, we can plot the results using the `plot_sweeper_df` helper function. This plot will help us easily see which parameters offer the highest accuracies." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAs0AAAKhCAYAAACreZOgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzde5xWdb33/9cn8BgqkYM3iEQeEuQ06gh6222SG1MzFTwUWWFiZI/d3cE02fUrtbu9szuP7dy5Md2SleZhK9xlbA9kqW3EUUdFCS1FOW2Bgjwrh8/vj7lgjzDDGnHWdc0Mr+fjMQ+u9V3ru9bnmiXOm+981/eKzESSJElS295V6wIkSZKkzs7QLEmSJBUwNEuSJEkFDM2SJElSAUOzJEmSVMDQLEmSJBUwNEvqtiKiR0S8HBEDO/LYrioiJkbEb0o69/YR8ceI6FvG+TtSRPxdRCxox3FnRcR3q1CSpC7A0Cyp06iE1vVf6yLitRbbp77d82Xm2szslZnPd+Sxb1dEfDciVkfES5Wv+RHxw4j4Hx19rRbX3Dsi3rIQf2ZOy8yjS7rkF4C7MnNZSeevhSuBz0bEe2tdiKTaMzRL6jQqobVXZvYCngc+1qLt5xsfHxE9q1/lFvt5Zu4EvBc4EdgDaIyI3bbkZBHRoyOL6wCfB66rdREdKTNfBe4APl3rWiTVnqFZUpdRGbH9ZURcHxEvAZ+KiEMiYnZErIqIpZUR3G0qx/eMiIyIQZXtn1X2/6Yy4vufEfH+t3tsZf/REfFURPwtIv45Iu6PiNOK3kNmvpmZc4GTgVXAVyvnOyMi7mlx/tbquSIiZkbEK8D/iojjIqKpUt/zEfGtFpf6faXf+pH6g1q5xgcjorHyHuZExOgW++6LiAsi4g+V88+MiD5t3Jc9qfwjoEXbsRExr9J3UUR8tcW+4yLi0co9uy8ihrXY976IuC0ilkfEioi4vNL+roj4dkQ8FxHLIuLaiNi5sm/vyvfqM5VrLY+IKS3OuWNEXBcRKyPiCeDAjer/RkQsiYgXo3mKyeEtdt8DfLS19y1p62JoltTVjAN+AewC/BJYA3wZ2BU4FDiK5lHPtnwS+BbQh+bR7P/zdo+N5nm7NwLnVK77LDDq7byJzFwDzAD+19vo9kngAmAn4D+Bl4FP0fy9+Bjw5Yg4tnLsYZXrrB+pf7DliSJiV+DXwMU0j37/ELg9It6z0fUmArsB7wbOaqOu4cCfMnNti7Z/AyZVRtdHAL+rXPcg4CrgjMp1rwGmR8S2ld8c/Br4EzCI5iB+Y+V8Z1Te6+HAXsB7gMs3quN/AnsDHwEuiIh9Ku3fqZxrT+CYynta/30YSvN/Lwdk5s7A0TTf6/XmASPbeN+StiKGZkldzX2Z+f8yc11mvpaZD2bmA5m5JjOfAaYCH9pM/5szszEzVwM/B+q34NhjgabMnF7ZdymwYgveyxKaA3l73ZqZ/1l5729k5qzMnFvZfhS4gc2/95Y+BjyRmddXvnc/A57hraOqV2fm05VpCjfR9veqN/DSRm2rgf0iYqfM/GtmPlxpnwz8S+W+rc3MayrtBwGH0PyPkHMz85XK/b2/sv9U4KLMfDYzXwK+AXwyIlr+HDs/M1+vXOsJ/jvsngJ8NzNXZuZzwI9a9FkDbA8MjYielfM/02L/S5X3J2krZ2iW1NUsbLkREYMj4tcR8V8R8SLNo4q7bqb/f7V4/SrQawuO7d+yjsxMYFE7at/Y7sBf38bxG7/3QyLinsp0hL/RPBq7uffeUn/guY3anqvUtF57v1craR79bmkccBzwfKXG9VM/3gecW5masSoiVgH9KtfdA1iw0Yh1W/U+B2wL1K1vyMy26u3HW793z7XoMx/4Gs3/3SyrTP1p+YDmTjRPo5G0lTM0S+pqcqPtfwXmAntXfr3+bSBKrmEpMGD9RkQEbw2bhaL5Qb6PAfdWml4BdmxxSGsra2z83m8AbgH2yMxdgJ/w3+9942M3toTmANvSQGBxQb/WPAbsFS0eTqyM/h8H9AV+VakVmsPrBZnZu8XXjpl5Y2Xf+6L1hxw3rncg8CawvB31/RfNgbxl3w0y82eZeSjwfqAH8L0Wu4cAj7bjGpK6OUOzpK5uJ+BvwCsRMYTNz2fuKL8CDoiIj1Xm4X6ZFiOemxMR20TEfjSHyD7AZZVdjwIjImJ4ROwAnNeO0+0E/DUzX4+Ig4FPtNi3DMjKQ3ptvYehEfHxykOHn6R5PvDt7XkfLWXmAprnAR8IEBE7RMQnI2LnyvSVl4D1o8dTgb+vPJgYEdGr8n18N83ztP8C/FPl4b0dIuLQSr/rgbMiYlBE7AT8I3B9Zq5rR4k3At+IiN7RvA73F9fviIghETEmIrYDXqt8tRzp/hBQytrWkroWQ7Okru5rND/Y9RLNo86/LPuCmfkC8HHgEppD3l7AI8Abm+l2ajSv+LESmA68ADSsn1KQmU8C/0Tzag3zqax+UeALwPcq5/0G//3QHJV5v98DHqhMg2jY6D0sp3n6xLmV9/BV4NjMfDvTRVr6V966NNtE4LnKlJlJ6/dl5gOVun9M8/fiKZof8Fv/cOSxNI/uLqQ5iJ9UOd9VNN/be2mee/0Szf9YaY/zaP7twAKaA/BPW+zbDvi/NM9J/y+aHzD8/6A5/NP8YGnL4yVtpaJ5Kp4kaUtVphMsAU7KzHuLju+OImJ7mv/h8KHu8gEnlWXy6jLzG7WuRVLtGZolaQtExFE0Tyd4HfgH4HPAnpm5udFmSVIX5fQMSdoyH6R5msAKmn+Ff4KBWZK6L0eaJUmSpAKONEuSJEkFDM2SJElSgZ61LqA9dt111xw0aFCty5AkSVI399BDD63IzE3W3u8SoXnQoEE0NjbWugxJkiR1cxHxXGvtTs+QJEmSChiaJUmSpAKGZkmSJKmAoVmSJEkqYGiWJEmSChiaJUmSpAKGZkmSJKmAoVmSJEkqYGiWJEmSChiaJUmSpAKlheaI2D4i5kTEoxHxRERcUGm/NiKejYimyld9WTVIkiRJHaFnied+A/hwZr4cEdsA90XEbyr7zsnMm0u8tiRJktRhSgvNmZnAy5XNbSpfWdb1JEmSpLKUOqc5InpERBOwDLgzMx+o7PrHiHgsIi6NiO3a6Ds5IhojonH58uVllilJkiRtVqmhOTPXZmY9MAAYFRHDgH8ABgMHAX2Ac9voOzUzGzKzoa6urswyJUmSpM2qyuoZmbkKuAc4KjOXZrM3gH8DRlWjBkmSJGlLlbl6Rl1E9K683gH4O+CPEdGv0hbACcDcsmqQttTrr7/OqFGjGDlyJEOHDuW8884D4Ec/+hF77703EcGKFSva7P/8889z5JFHMmTIEPbbbz8WLFhQpcolSVIZylw9ox8wLSJ60BzOb8zMX0XErIioAwJoAs4ssQZpi2y33XbMmjWLXr16sXr1aj74wQ9y9NFHc+ihh3Lsscdy+OGHb7b/Zz7zGb75zW8yduxYXn75Zd71LpdElySpKytz9YzHgP1baf9wWdeUOkpE0KtXLwBWr17N6tWriQj233+T/6Q38eSTT7JmzRrGjh0LsOE8kiSp63L4S2rD2rVrqa+vp2/fvowdO5bRo0e3q99TTz1F7969GT9+PPvvvz/nnHMOa9euLblaSZJUJkOz1IYePXrQ1NTEokWLmDNnDnPntm/6/Zo1a7j33nu56KKLePDBB3nmmWe49tpryy1WkiSVytAsFejduzeHH344M2fObNfxAwYMYP/992fPPfekZ8+enHDCCTz88MMlVylJkspkaJZasXz5clatWgXAa6+9xl133cXgwYPb1feggw5i5cqVrP9QnlmzZrHffvuVVqskSSqfoVlqxdKlSxkzZgwjRozgoIMOYuzYsRx77LH88Ic/ZMCAASxatIgRI0ZwxhlnANDY2LjhdY8ePbjooos44ogjGD58OJnJ5z73uVq+HUmSNqutpVafffZZRo8ezT777MPHP/5x3nzzzU36zpkzh/r6eurr6xk5ciS33nprtcuvisjMWtdQqKGhIRsbG2tdhiRJUreUmbzyyitvWWr18ssv55JLLmH8+PF84hOf4Mwzz2TkyJF84QtfeEvfV199lW233ZaePXuydOlSRo4cyZIlS+jZs8yVjcsTEQ9lZsPG7Y40S5IkbeXaWmp11qxZnHTSSQBMnDiR2267bZO+O+6444aA/Prrr9P8+XXdj6FZktRpLFy4kDFjxjBkyBCGDh3K5ZdfDsCjjz7KIYccwvDhw/nYxz7Giy++uEnf+fPnb/gVcX19PTvvvDOXXXZZtd+C1GVtvNTqXnvtRe/evTcE4gEDBrB48eJW+z7wwAMMHTqU4cOHc+WVV3bZUebNMTRLkjqNnj17cvHFFzNv3jxmz57NFVdcwZNPPskZZ5zBhRdeyOOPP864ceP4wQ9+sEnffffdl6amJpqamnjooYfYcccdGTduXA3ehdQ1bbzU6rx58zY5pq1R5NGjR/PEE0/w4IMP8r3vfY/XX3+97HKrztAsSeo0+vXrxwEHHADATjvtxJAhQ1i8eDHz58/nsMMOA2Ds2LHccsstmz3P3XffzV577cX73ve+0muWupv1S63Onj2bVatWsWbNGgAWLVpE//79N9t3yJAhvPvd7273Zxt0JYZmSVKntGDBAh555BFGjx7NsGHDmDFjBgA33XQTCxcu3GzfG264gQkTJlSjTKlbaG2p1SFDhjBmzBhuvvlmAKZNm8bxxx+/Sd9nn312Q7B+7rnnmD9/PoMGDapa7dViaJYkdTovv/wyJ554Ipdddhk777wz11xzDVdccQUHHnggL730Ettuu22bfd98801mzJjBySefXMWKpa6traVWv//973PJJZew995785e//IVJkyYBMGPGDL797W8DcN999zFy5Ejq6+sZN24c//Iv/8Kuu+5ay7dTCpecU7cyaMqva11CqRZc+NFalyCVbvXq1Rx77LF85CMf4ayzztpk/1NPPcWnPvUp5syZ02r/6dOnc8UVV3DHHXeUXaqkbsgl5yRJnV5mMmnSJIYMGfKWwLxs2TIA1q1bx3e/+13OPPPMNs9x/fXXOzVDUoczNEuSOo3777+f6667jlmzZm1YOu7222/n+uuv5wMf+ACDBw+mf//+fPaznwVgyZIlHHPMMRv6v/rqq9x5552MHz++Vm9BUjfl9Ax1K07PkCRJ74TTMyRJkqQtVFpojojtI2JORDwaEU9ExAWV9vdHxAMR8XRE/DIi2n4EWpIkSeoEyhxpfgP4cGaOBOqBoyLiYOD7wKWZuQ+wEphUYg2SJEnSO1baB4Nn82Tplyub21S+Evgw8MlK+zTgfODHZdUhSZLUnfk8T3WUOqc5InpERBOwDLgT+DOwKjPXVA5ZBOzeRt/JEdEYEY3Lly8vs0xJkiRps0oNzZm5NjPrgQHAKGBIa4e10XdqZjZkZkNdXV2ZZUqSJEmbVdr0jJYyc1VE3AMcDPSOiJ6V0eYBwJJq1CBJKld3/hVxZ/n1sKTaKXP1jLqI6F15vQPwd8A84LfASZXDJgLTy6pBkiRJ6ghljjT3A6ZFRA+aw/mNmfmriHgSuCEivgs8AlxdYg2SJEnSO1bm6hmPAfu30v4MzfObJUmSpC7BTwSUJEmSChiaJUmSpAKGZkmSJKmAoVmSJEkqYGiWJEmSChiaJUmSpAKGZkmSJKmAoVmSJEkqYGiWJEmSChiaJUmSpAKGZkmSJKmAoVmSJEkqYGiWJEmSChiaJUmSpAKGZkmSJKmAoVmSJEkqYGiWJEmSCpQWmiNij4j4bUTMi4gnIuLLlfbzI2JxRDRVvo4pqwZJkiSpI/Qs8dxrgK9l5sMRsRPwUETcWdl3aWZeVOK1JUmSpA5TWmjOzKXA0srrlyJiHrB7WdeTJEmSylKVOc0RMQjYH3ig0vTFiHgsIq6JiPdUowZJkiRpS5UemiOiF3AL8JXMfBH4MbAXUE/zSPTFbfSbHBGNEdG4fPnyssuUJEmS2lRqaI6IbWgOzD/PzH8HyMwXMnNtZq4DrgJGtdY3M6dmZkNmNtTV1ZVZpiRJkrRZZa6eEcDVwLzMvKRFe78Wh40D5pZVgyRJktQRylw941Dg08DjEdFUafsGMCEi6oEEFgCfL7EGSZIk6R0rc/WM+4BoZdftZV1TkiRJKoOfCChJkiQVMDRLkiRJBQzNkiRJUgFDs6RuZ+HChYwZM4YhQ4YwdOhQLr/8cgD++te/MnbsWPbZZx/Gjh3LypUrW+3//PPPc+SRRzJkyBD2228/FixYUMXqJUmdkaG5JO/kh/Zzzz3HgQceSH19PUOHDuXKK6+sdvlSl9azZ08uvvhi5s2bx+zZs7niiit48sknufDCCzniiCN4+umnOeKII7jwwgtb7f+Zz3yGc845h3nz5jFnzhz69u1b5XcgSepsDM0leSc/tPv168cf/vAHmpqaeOCBB7jwwgtZsmRJDd6F1DX169ePAw44AICddtqJIUOGsHjxYqZPn87EiRMBmDhxIrfddtsmfZ988knWrFnD2LFjAejVqxc77rhj9YqXJHVKhuaSvJMf2ttuuy3bbbcdAG+88Qbr1q2rXuFSN7NgwQIeeeQRRo8ezQsvvEC/fs2fr9SvXz+WLVu2yfFPPfUUvXv3Zvz48ey///6cc845rF27ttplS5I6mXaF5oh4d0S8q/L6AxFxXOUjstUOb/eHNjRP7xgxYgR77LEH5557Lv37969myVK38PLLL3PiiSdy2WWXsfPOO7erz5o1a7j33nu56KKLePDBB3nmmWe49tpryy1U6iZOP/10+vbty7Bhwza0PfrooxxyyCEMHz6cj33sY7z44out9r300ksZOnQow4YNY8KECbz++uvVKltql/aONP8e2D4idgfuBj4LXFtWUd3JlvzQBthjjz147LHH+NOf/sS0adN44YUXSqxS6n5Wr17NiSeeyKmnnsr48eMB2G233Vi6dCkAS5cubXWu8oABA9h///3Zc8896dmzJyeccAIPP/xwVWuXuqrTTjuNmTNnvqXtjDPO4MILL+Txxx9n3Lhx/OAHP9ik3+LFi/nhD39IY2Mjc+fOZe3atdxwww3VKltql/aG5sjMV4HxwD9n5jhgv/LK6h629Id2S/3792fo0KHce++9pdcrdReZyaRJkxgyZAhnnXXWhvbjjjuOadOmATBt2jSOP/74TfoedNBBrFy5kuXLlwMwa9Ys9tvP/91J7XHYYYfRp0+ft7TNnz+fww47DICxY8dyyy23tNp3zZo1vPbaa6xZs4ZXX33V37Cq02l3aI6IQ4BTgV9X2kr7CO7u4J380F60aBGvvfYaACtXruT+++9n3333rU7hUjdw//33c9111zFr1izq6+upr6/n9ttvZ8qUKdx5553ss88+3HnnnUyZMgWAxsZGzjjjDAB69OjBRRddxBFHHMHw4cPJTD73uc/V8u1IXdqwYcOYMWMGADfddBMLFy7c5Jjdd9+ds88+m4EDB9KvXz922WUXjjzyyGqXKm1We4PvV4B/AG7NzCciYk/gt+WV1fWt/6E9fPhw6uvrAfinf/onpkyZwimnnMLVV1/NwIEDuemmm4DmH9pXXnklP/nJT5g3bx5f+9rXiAgyk7PPPpvhw4fX8u1IXcoHP/hBMrPVfXffffcmbQ0NDfzkJz/ZsD127Fgee+yx0uqTtibXXHMNX/rSl/jOd77Dcccdx7bbbrvJMStXrmT69Ok8++yz9O7dm5NPPpmf/exnfOpTn6pBxVLr2hWaM/N3wO9abD8DfKmsorqDd/JD2x/YkqTuYvDgwdxxxx1A8+o0v/71rzc55q677uL9738/dXV1AIwfP54//OEPhmZ1KpsNzRHx/4DWkx+Qmcd1eEWSJKnbWLZsGX379mXdunV897vf5cwzz9zkmIEDBzJ79mxeffVVdthhB+6++24aGhpqUK3UtqI5zRcBFwPPAq8BV1W+XgbmlluaJEnqSiZMmMAhhxzC/PnzGTBgAFdffTXXX389H/jABxg8eDD9+/fns5/9LABLlizhmGOOAWD06NGcdNJJHHDAAQwfPpx169YxefLkWr4VaRPR1hSCtxwU8fvMPKyorSwNDQ3Z2NhYjUupixs0ZdNf+3UnCy78aK1LkNrUnf/++XdPnVl3/rsH1f/7FxEPZeYmv+po7+oZdZWH/9af7P1AXUcVJ0mSJHVm7V0946vAPRHxTGV7EPD5zXWIiD2AnwL/A1gHTM3MyyOiD/DLyjkWAKdk5sq3XbkkSZJUJe1dPWNmROwDDK40/TEz3yjotgb4WmY+HBE7AQ9FxJ3AacDdmXlhREwBpgDnbln5kroTf8UoSeqs3s4HlBxI8+hwT2BkZQ3hn7Z1cGYuBZZWXr8UEfOA3YHjgcMrh00D7qGThWZ/cEuSJKmldoXmiLgO2AtoAtZWmpPm6Rft6T8I2B94ANitEqjJzKURsfnPkZYkSZJqrL0jzQ3AftmepTY2EhG9gFuAr2TmixHR3n6TgcnQvH6jJEkqh79hlYq1d/WMuTQ/0Pe2RMQ2NAfmn2fmv1eaX4iIfpX9/YBlrfXNzKmZ2ZCZDes/IUiSJEmqhfaONO8KPBkRc4ANDwBu7hMBo3lI+WpgXmZe0mLXDGAicGHlz+lvt2hJkiSpmtobms/fgnMfCnwaeDwimipt36A5LN8YEZOA54GTt+DckiRJUtW0d8m530XEbsBBlaY5mdnqtIoWfe4D2prAfET7S5QkSZJqq11zmiPiFGAOzaPCpwAPRMRJZRYmSZIkdRbtnZ7xTeCg9aPLEVEH3AXcXFZhkiRJUmfR3tUz3rXRdIy/vI2+kiRJUpfW3pHmmRHxH8D1le2PA78ppyRJkiSpc2nvg4DnRMR44IM0P9w3NTNvLbUySZIkqZNo78dovx+4ff0HlETEDhExKDMXlFmcJEmS1Bm0d17yTcC6FttrK22SJElSt9fe0NwzM99cv1F5vW05JUmSJEmdS3tD8/KI2PCR2RFxPLCinJIkSZKkzqW9q2ecCfw8Iq4AElgEfKa0qiRJkqROpL2rZ/wZODgiegGRmS+VW5YkSZLUebT3Y7R3i4irgZsy86WI2C8iJpVcmyRJktQptHdO87XAfwD9K9tPAV8poyBJkiSps2lvaN41M2+ksuxcZq6hedk5SZIkqdtrb2h+JSLeS/NDgETEwcDfSqtKkiRJ6kTau3rGWcAMYK+IuB+oA04qrSpJkiSpE2nvSPNewNHA/6R5bvPTtD9wS5IkSV1ae0PztzLzReA9wN8BU4Efb65DRFwTEcsiYm6LtvMjYnFENFW+jtniyiVJkqQqaW9oXv/Q30eBKzNzOsUfo30tcFQr7ZdmZn3l6/Z2Xl+SJEmqmfaG5sUR8a/AKcDtEbFdUd/M/D3w13dYnyRJklRz7Q3Np9A8l/mozFwF9AHO2cJrfjEiHqtM33jPFp5DkiRJqpp2hebMfDUz/z0zn65sL83MO7bgej+m+aHCemApcHFbB0bE5IhojIjG5cuXb8GlJEmSpI7R3pHmDpGZL2Tm2sxcB1wFjNrMsVMzsyEzG+rq6qpXpCRJkrSRqobmiOjXYnMcMLetYyVJkqTOorS1liPieuBwYNeIWAScBxweEfU0f7LgAuDzZV1fkiRJ6iilhebMnNBK89VlXU+SJEkqS1WnZ0iSJEldkaFZkiRJKmBoliRJkgoYmiVJkqQChmZJkiSpgKFZkiRJKmBoliRJkgoYmiVJkqQChmZJkiSpgKFZkiRJKmBoliRJkgoYmiVJkqQChmZJkiSpgKFZkiRJKmBoliRJkgoYmiVJkqQChmZJkiSpQGmhOSKuiYhlETG3RVufiLgzIp6u/Pmesq4vSZIkdZQyR5qvBY7aqG0KcHdm7gPcXdmWJEmSOrXSQnNm/h7460bNxwPTKq+nASeUdX1JkiSpo1R7TvNumbkUoPJn3ypfX5IkSXrbOu2DgBExOSIaI6Jx+fLltS5HkiRJW7Fqh+YXIqIfQOXPZW0dmJlTM7MhMxvq6uqqVqAkSZK0sWqH5hnAxMrricD0Kl9fkiRJetvKXHLueuA/gX0jYlFETAIuBMZGxNPA2Mq2JEmS1Kn1LOvEmTmhjV1HlHVNSZIkqQyd9kFASZIkqbMwNEuSJEkFDM2SJElSAUOzJEmSVMDQLEmSJBUwNEuSJEkFDM2SJElSAUOzJEmSVMDQLEmSJBUwNEuSJEkFDM2SJElSAUOzJEmSVMDQLEmSJBUwNEuSJEkFDM2SJElSAUOzJEmSVMDQLEmSJBXoWYuLRsQC4CVgLbAmMxtqUYckSZLUHjUJzRVjMnNFDa8vSZIktYvTMyRJkqQCtQrNCdwREQ9FxOQa1SBJkiS1S62mZxyamUsioi9wZ0T8MTN/3/KASpieDDBw4MBa1ChJkiQBNRppzswllT+XAbcCo1o5ZmpmNmRmQ11dXbVLlCRJkjaoemiOiHdHxE7rXwNHAnOrXYckSZLUXrWYnrEbcGtErL/+LzJzZg3qkCRJktql6qE5M58BRlb7upIkSdKWcsk5SZIkqYChWZIkSSpgaJYkSZIKGJolSZKkAoZmSZIkqYChWZIkSSpgaJYkSZIKGJolSZKkAoZmSZIkqYChWZIkSSpgaJYkSZIKGJolSZKkAoZmSZIkqYChWZIkSSpgaJYkSZIKGJolSZKkAoZmSZIkqUBNQnNEHBUR8yPiTxExpRY1SJIkSe1V9dAcET2AK4Cjgf2ACRGxX7XrkCRJktqrFiPNo4A/ZeYzmfkmcANwfA3qkCRJktqlFqF5d2Bhi+1FlTZJkiSpU4rMrO4FI04GPpKZZ1S2Pw2Mysz/vdFxk4HJlc19gflVLbS6dgVW1LoIbRHvXdfm/evavH9dl/eua+vu9+99mVm3cWPPGhSyCNijxfYAYMnGB2XmVGBqtYqqpYhozMyGWteht89717V5/7o271/X5b3r2rbW+1eL6RkPAvtExPsjYlvgE8CMGtQhSZIktUvVR5ozc01EfBH4D6AHcE1mPlHtOiRJkqT2qsX0DDLzduD2Wly7k9oqpqF0U967rs3717V5/7ou713XtlXev6o/CChJkiR1NX6MtiRJklTA0CxJkiQVMDRLksnpkoAAACAASURBVCRJBQzNkiRJUgFDsyRJklTA0CxJkiQVMDRLkiRJBQzNkiRJUgFDsyRJklTA0CxJkiQVMDRLkiRJBQzNkiRJUgFDsyRJklTA0CxJkiQVMDRLkiRJBQzNkiRJUgFDsyRJklTA0CxJW5mIuCcizqh1HZLUlRiaJUmSpAKGZknqYiKiZ61rkKStjaFZkrqAiFgQEedGxGPAKxHx/0XEnyPipYh4MiLGtTj2tIi4LyIuioiVEfFsRBzdxnn7RcRjEXF21d6MJHVBhmZJ6jomAB8FegPzgf8F7AJcAPwsIvq1OHZ05Zhdgf8LXB0R0fJkETEI+B3wo8y8qOziJakrMzRLUtfxw8xcmJmvZeZNmbkkM9dl5i+Bp4FRLY59LjOvysy1wDSgH7Bbi/37AfcA52Xm1Gq9AUnqqgzNktR1LFz/IiI+ExFNEbEqIlYBw2geVV7vv9a/yMxXKy97tdh/KrAYuLnEeiWp2zA0S1LXkQAR8T7gKuCLwHszszcwF4jN9N3Y+cAK4BcR0aOD65SkbsfQLEldz7tpDtDLASLiszSPNL8dq4GTK+e6LiL8eSBJm+H/JCWpi8nMJ4GLgf8EXgCGA/dvwXneBMYDfYFrDM6S1LbIzFrXIEmSJHVqjipIkiRJBQzNkiRJUgFDsyRJklTA0CxJkiQVMDRLkiRJBXrWuoD22HXXXXPQoEG1LkOSJEnd3EMPPbQiM+s2bu8SoXnQoEE0NjbWugxJkiR1cxHxXGvtTs+QJEmSChiaJUmSpAKGZkmSJKmAoVmSJEkqYGiWJEmSCpQWmiNi+4iYExGPRsQTEXFBpf3aiHg2IpoqX/Vl1SBJkiR1hDKXnHsD+HBmvhwR2wD3RcRvKvvOycybS7y2JEmS1GFKC82ZmcDLlc1tKl9Z1vUkSZKkspQ6pzkiekREE7AMuDMzH6js+seIeCwiLo2I7droOzkiGiOicfny5WWWKUmSJG1WqaE5M9dmZj0wABgVEcOAfwAGAwcBfYBz2+g7NTMbMrOhrm6TTzKUJEmSqqYqq2dk5irgHuCozFyazd4A/g0YVY0aJEmSpC1V5uoZdRHRu/J6B+DvgD9GRL9KWwAnAHPLqkGSJEnqCGWuntEPmBYRPWgO5zdm5q8iYlZE1AEBNAFnlliDJEmS9I6VuXrGY8D+rbR/uKxrSpIkSWXwEwElSZKkAoZmSZIkqYChWZIkaSv3+uuvM2rUKEaOHMnQoUM577zzAHj22WcZPXo0++yzDx//+Md58803N+m7evVqJk6cyPDhwxkyZAjf+973ql1+VRiaJUmStnLbbbcds2bN4tFHH6WpqYmZM2cye/Zszj33XL761a/y9NNP8573vIerr756k7433XQTb7zxBo8//jgPPfQQ//qv/8qCBQuq/yZKZmiWJEnaykUEvXr1AppHjlevXk1EMGvWLE466SQAJk6cyG233dZq31deeYU1a9bw2muvse2227LzzjtXtf5qMDRLkiSJtWvXUl9fT9++fRk7dix77bUXvXv3pmfP5sXWBgwYwOLFizfpd9JJJ/Hud7+bfv36MXDgQM4++2z69OlT7fJLZ2iWJEkSPXr0oKmpiUWLFjFnzhzmzZu3yTHNn033VnPmzKFHjx4sWbKEZ599losvvphnnnmmGiVXlaFZkiRJG/Tu3ZvDDz+c2bNns2rVKtasWQPAokWL6N+//ybH/+IXv+Coo45im222oW/fvhx66KE0NjZWu+zSGZolSZK2csuXL2fVqlUAvPbaa9x1110MGTKEMWPGcPPNNwMwbdo0jj/++E36Dhw4kFmzZpGZvPLKK8yePZvBgwdXtf5qMDRLkiRt5ZYuXcqYMWMYMWIEBx10EGPHjuXYY4/l+9//Ppdccgl77703f/nLX5g0aRIAM2bM4Nvf/jYAf//3f8/LL7/MsGHDOOigg/jsZz/LiBEjavl2ShGZWesaCjU0NGR3HOaXJElS5xIRD2Vmw8btjjRLkiRJBQzNkiRJUgFDsyRJklTA0CxJkiQVMDRLkiRJBQzNkiRJUoGeZZ04IrYHfg9sV7nOzZl5XkS8H7gB6AM8DHw6M98sqw5JkqRu7fxdal1Buc7/W60rAModaX4D+HBmjgTqgaMi4mDg+8ClmbkPsBKYVGINkiRJ0jtWWmjOZi9XNrepfCXwYeDmSvs04ISyapAkSZI6QqlzmiOiR0Q0AcuAO4E/A6syc03lkEXA7mXWIEmSJL1TpYbmzFybmfXAAGAUMKS1w1rrGxGTI6IxIhqXL19eZpmSJEnSZlVl9YzMXAXcAxwM9I6I9Q8gDgCWtNFnamY2ZGZDXV1dNcqUJEmSWlVaaI6IuojoXXm9A/B3wDzgt8BJlcMmAtPLqkGSJEnqCKUtOQf0A6ZFRA+aw/mNmfmriHgSuCEivgs8AlxdYg2SJEnSO1ZaaM7Mx4D9W2l/hub5zZIkSVKX4CcCSpKkDrFw4ULGjBnDkCFDGDp0KJdffvmGff/8z//Mvvvuy9ChQ/n617++Sd/XX3+dUaNGMXLkSIYOHcp5551XzdKlQmVOz5AkSVuRnj17cvHFF3PAAQfw0ksvceCBBzJ27FheeOEFpk+fzmOPPcZ2223HsmXLNum73XbbMWvWLHr16sXq1av54Ac/yNFHH83BBx9cg3cibcrQLEmSOkS/fv3o168fADvttBNDhgxh8eLFXHXVVUyZMoXtttsOgL59+27SNyLo1asXAKtXr2b16tVERPWKlwo4PUOSJHW4BQsW8MgjjzB69Gieeuop7r33XkaPHs2HPvQhHnzwwVb7rF27lvr6evr27cvYsWMZPXp0lauW2mZoliRJHerll1/mxBNP5LLLLmPnnXdmzZo1rFy5ktmzZ/ODH/yAU045hcxNP9usR48eNDU1sWjRIubMmcPcuXNrUL3UOkOzJEnqMKtXr+bEE0/k1FNPZfz48QAMGDCA8ePHExGMGjWKd73rXaxYsaLNc/Tu3ZvDDz+cmTNnVqtsqZChWZIkdYjMZNKkSQwZMoSzzjprQ/sJJ5zArFmzAHjqqad488032XXXXd/Sd/ny5axatQqA1157jbvuuovBgwdXr3ipgKFZkiR1iPvvv5/rrruOWbNmUV9fT319Pbfffjunn346zzzzDMOGDeMTn/gE06ZNIyJYsmQJxxxzDABLly5lzJgxjBgxgoMOOoixY8dy7LHH1vgdSf8tWptT1Nk0NDRkY2NjrcuQJEnqfM7fpdYVlOv8v1X1chHxUGY2bNzuSLMkSZJUwNAsSZIkFTA0S5IkSQUMzZIkSVIBQ7MkSZJUwNAsSZIkFehZ6wIkSVKNuWSZVMiRZkmSJKmAoVmSJEkqUFpojog9IuK3ETEvIp6IiC9X2s+PiMUR0VT5OqasGiRJkqSOUOac5jXA1zLz4YjYCXgoIu6s7Ls0My8q8dqSJElShyktNGfmUmBp5fVLETEP2L2s60mSJEllqcqc5ogYBOwPPFBp+mJEPBYR10TEe6pRgyRJkrSlSg/NEdELuAX4Sma+CPwY2Auop3kk+uI2+k2OiMaIaFy+fHnZZUqSJEltKjU0R8Q2NAfmn2fmvwNk5guZuTYz1wFXAaNa65uZUzOzITMb6urqyixTkiRJ2qwyV88I4GpgXmZe0qK9X4vDxgFzy6pBkiRJ6ghlrp5xKPBp4PGIaKq0fQOYEBH1QAILgM+XWIMkSZL0jpW5esZ9QLSy6/ayrilJkiSVwU8ElCRJkgoYmiVJkqQChmZJkiSpgKFZkiRJKmBoliRJkgoYmiVJkqQChmZJkiSpgKFZkiRJKmBoliRJkgoYmiVJkqQChmZJkiSpgKFZkiRJKmBoliRJkgoYmiVJkqQChmZJkiSpgKFZktSpnH766fTt25dhw4ZtaGtqauLggw+mvr6ehoYG5syZ02rfHj16UF9fT319Pccdd1y1Spa0FTA0S5I6ldNOO42ZM2e+pe3rX/865513Hk1NTXznO9/h61//eqt9d9hhB5qammhqamLGjBnVKFfSVsLQLEnqVA477DD69OnzlraI4MUXXwTgb3/7G/37969FaZK2Yj3LOnFE7AH8FPgfwDpgamZeHhF9gF8Cg4AFwCmZubKsOiRJXd9ll13GRz7yEc4++2zWrVvHH/7wh1aPe/3112loaKBnz55MmTKFE044ocqVSuquyhxpXgN8LTOHAAcDfx8R+wFTgLszcx/g7sq2JElt+vGPf8yll17KwoULufTSS5k0aVKrxz3//PM0Njbyi1/8gq985Sv8+c9/rnKlkrqr0kJzZi7NzIcrr18C5gG7A8cD0yqHTQMcBpAkbda0adMYP348ACeffHKbDwKun7ax5557cvjhh/PII49UrUZJ3VtV5jRHxCBgf+ABYLfMXArNwRroW40aJEldV//+/fnd734HwKxZs9hnn302OWblypW88cYbAKxYsYL777+f/fbbr6p1Suq+SpvTvF5E9AJuAb6SmS9GRHv7TQYmAwwcOLC8AiVJncqECRO45557WLFiBQMGDOCCCy7gqquu4stf/jJr1qxh++23Z+rUqQA0NjZy5ZVX8pOf/IR58+bx+c9/nne9612sW7eOKVOmGJoldZjIzPJOHrEN8CvgPzLzkkrbfODwzFwaEf2AezJz382dp6GhIRsbG0urU5Kkrdr5u9S6gnKd/7daV1Au71+HioiHMrNh4/bSpmdE85Dy1cC89YG5YgYwsfJ6IjC9rBokSZKkjlDm9IxDgU8Dj0dEU6XtG8CFwI0RMQl4Hji5xBokSZKkd6y00JyZ9wFtTWA+oqzrSpIkSR3NTwSUJEmSChiaJUmSpAKGZkmSJKlA6es0S5K2Et152avuvmSZpEKONEuSJEkFDM2SJElSAUOzJEmSVMDQLEmSJBUwNEuSJEkFDM2SJElSAUOzJEmSVMDQLEmSJBUwNEuSJEkFDM2SJElSAUOzpG7n9NNPp2/fvgwbNmxD27e+9S1GjBhBfX09Rx55JEuWLNmk329/+1vq6+s3fG2//fbcdttt1SxdktRJGZoldTunnXYaM2fOfEvbOeecw2OPPUZTUxPHHnss3/nOdzbpN2bMGJqammhqamLWrFnsuOOOHHnkkdUqW5LUiRmaJXU7hx12GH369HlL284777zh9SuvvEJEbPYcN998M0cffTQ77rhjKTVKkrqWnrUuQJKq5Zvf/CY//elP2WWXXfjtb3+72WNvuOEGzjrrrCpVJknq7EobaY6IayJiWUTMbdF2fkQsjoimytcxZV1fkjb2j//4jyxcuJBTTz2VH/3oR20et3TpUh5//HE+8pGPVLE6SVJnVub0jGuBo1ppvzQz6ytft5d4fUlq1Sc/+UluueWWNvffeOONjBs3jm222aaKVUmSOrPSQnNm/h74a1nnl6S34+mnn97wesaMGQwePLjNY6+//nomTJhQjbIkSV1ELR4E/GJEPFaZvvGeGlxfUjc3YcIEDjnkEObPn8+AAQO4+uqrmTJlCsOGDWPEiBHccccdXH755QA0NjZyxhlnbOi7YMECFi5cyIc+9KFalS9J6oQiM8s7ecQg4FeZOayyvRuwAkjg/wD9MvP0NvpOBiYDDBw48MDnnnuutDolSR3g/F1qXUF5zv9brSsoV3e+d+D96+qqfP8i4qHMbNi4vaojzZn5Qmauzcx1wFXAqM0cOzUzGzKzoa6urnpFSpIkSRupamiOiH4tNscBc9s6VpIkSeosSlunOSKuBw4Hdo2IRcB5wOERUU/z9IwFwOfLur4kSZLUUUoLzZnZ2qPnV5d1PUmSJKksfoy2JEmSVMDQLEmSJBUobXqGJL1tLpskSeqkHGmWJEmSChiaJUmSpAKGZkmSJKmAoVmSJEkqYGiWJEmSChiaJUmSpAKGZkmSJKmAoVmSJEkqYGiWJEmSChiaJUmSpAKGZkmSJKmAoVmSJEkqYGiWJEmSChiaJUmSpAKGZkmSJKlAaaE5Iq6JiGURMbdFW5+IuDMinq78+Z6yri9JkiR1lDJHmq8FjtqobQpwd2buA9xd2ZYkSZI6tdJCc2b+HvjrRs3HA9Mqr6cBJ5R1fUmSJKmjVHtO826ZuRSg8mfftg6MiMkR0RgRjcuXL69agZIkSdLGOu2DgJk5NTMbMrOhrq6u1uVIkiRpK1bt0PxCRPQDqPy5rMrXlyRJkt62aofmGcDEyuuJwPQqX1+SJEl628pccu564D+BfSNiUURMAi4ExkbE08DYyrYkSZLUqfUs68SZOaGNXUeUdU1JkiSpDJ32QUBJkiSpszA0S5IkSQUMzZIkSVIBQ7MkSZJUwNAsSZIkFTA0S5IkSQUMzZIkSVIBQ7MkSZJUwNAsSZIkFTA0S5IkSQUMzZIkSVIBQ7MkSZJUwNAsSZIkFTA0S5IkSQUMzZIkSVIBQ7MkSZJUwNAsSZIkFehZi4tGxALgJWAtsCYzG2pRhyRJktQeNQnNFWMyc0UNry9JkiS1i9MzJEmSpAK1Cs0J3BERD0XE5BrVIEmSJLVLraZnHJqZSyKiL3BnRPwxM3/f8oBKmJ4MMHDgwFrUKEmSJAE1GmnOzCWVP5cBtwKjWjlmamY2ZGZDXV1dtUuUJEmSNqh6aI6Id0fETutfA0cCc6tdhyRJktRetZiesRtwa0Ssv/4vMnNmDeqQJEmS2qXqoTkznwFGVvu6kiRJ0pZyyTmpHU4//XT69u3LsGHDWt0/ffp0RowYQX19PQ0NDdx3331VrlCSJJXJ0Cy1w2mnncbMmW3PIjriiCN49NFHaWpq4pprruGMM86oYnWSJKlshuYqKRqpzEy+9KUvsffeezNixAgefvjhKleozTnssMPo06dPm/t79epFZZ4+r7zyyobXkiSpezA0V0nRSOVvfvMbnn76aZ5++mmmTp3KF77whSpWp45w6623MnjwYD760Y9yzTXX1LocSZLUgQzNVVI0Ujl9+nQ+85nPEBEcfPDBrFq1iqVLl1axQr1T48aN449//CO33XYb3/rWt2pdjiRJ6kCG5k5i8eLF7LHHHhu2BwwYwOLFi2tYkbbUYYcdxp///GdWrFhR61IkSVIHMTR3Epm5SZvzYruOP/3pTxvu4cMPP8ybb77Je9/73hpXJUmSOkotPtxErRgwYAALFy7csL1o0SL69+9fw4rU0oQJE7jnnntYsWIFAwYM4IILLmD16tUAnHnmmdxyyy389Kc/ZZtttmGHHXbgl7/8pf/okSSpGzE0dxLHHXccP/rRj/jEJz7BAw88wC677EK/fv1qXZYqrr/++s3uP/fcczn33HOrVI0kSao2Q3OVFI1UHnPMMdx+++3svffe7Ljjjvzbv/1bjSuWJEnSeobmKikaqYwIrrjiiipVI0mSpLfDBwElSZKkAoZmSZIkqYChWZIkSSpgaJYkSZIK+CCgupfzd6l1BeU6/2+1rkCSpK2SI82SJElSAUeaW+NopSRJklqoyUhzRBwVEfMj4k8RMaUWNUiSJEntVfXQHBE9gCuAo4H9gAkRsV+165AkSZLaqxYjzaOAP2XmM5n5JnADcHwN6pAkSZLapRaheXdgYYvtRZU2SZIkqVOKzKzuBSNOBj6SmWdUtj8NjMrM/73RcZOByZXNfYH5VS20unYFVtS6CG0R713X5v3r2rx/XZf3rmvr7vfvfZlZt3FjLVbPWATs0WJ7ALBk44MycyowtVpF1VJENGZmQ63r0NvnvevavH9dm/ev6/LedW1b6/2rxfSMB4F9IuL9EbEt8An4/9u7+2jbq7re4++P54AgKgoqPiAKgoomAt3Ea2aFT5mPWYLXUrma1bVM645M027k7UFNvZZlqaGg9+bj6CoqBiNNzXxCHgQBAUUwFTAVBOUicPjeP36/zdn7yNlrrYOsueZe79cYe+y9fmuhn8Fnn8k8vzXnXBzfIIckSZI0lbnfaa6q65L8FnAisAl4c1WdNe8ckiRJ0rSafLhJVZ0AnNDi/3tBLcUylA3K7vpmf32zv37ZXd+Wsr+5bwSUJEmSetPkEwElSZKknjhpliRJkiZosqZZkqRpJdljipddX1WX3+xhNDP761eSQ6d42bVVdebNHmYBuKZ5jpJMc7Ted6rqqJs7i2aX5IwpXvYfVfXwmz2MZpLkikkvAS6uqnvPI49mk+RqhvP8s87LNlXVPnOKpBnYX7+SXMlwVPB63e1bVfecT6K2vNM8XwcCv7rO8wH+Zk5ZNLtNwM+v83zwzPFF9eWqOmS9FyQ5bV5hNLNz7K9r9tevk6vq8PVekOQj8wrTmnea5yjJEVX1rpv6GrWR5KFV9Ymb+hrNX5L9quqCm/oatZFkl6q6+qa+Rm3YnzYKJ83SDhjX6FVVXdY6i6aXZC/gbkAB36iqSxtH0pSSBHgQq/oDPlv+R6wL9tevJLsDP8fa7k5cxjXoTprnaPzFezHwJOCO4+VvAu8DXr6Mv4A9SbIP8Erg4cDlDMsxbgt8BHhRVV3YLp3Wk+Rg4O+A3YGvj5f3ZujxuVV1aqtsmizJo4DXA+eztr/9Gfo7qVU2TWZ//UryDOCPgJNY290jgT+uqre2ytaCk+Y5SnIiwwTruKq6ZLx2Z+CZwCOq6pEt82l9ST4FvBZ4T1VtGa9tAp4CvKCqHtwyn7YvyenAr1fVZ7a5/mDgDVX1wDbJNI0k5wCP2fYvpkn2BU6oqgObBNNU7K9fSc4FDtv2pl6S2wOfWbbN057TPF/3rKpXrEyYAarqkqp6BeCu4cV3h6p658qEGaCqtlTVO4A9G+bSZLttO2EGqKpPA7s1yKPZbAa+diPXvw7sNOcsmp399SsMSzK2dT3rn6ixIXl6xnxdlOSFDHeaL4Ub1lgeBfx7y2CayilJXg8cx9a+7s7wToE7vxfbh5J8EHgra7t7BvBPzVJpWm8GTk7yDtb291TgmGapNC3769efAqcmOYmt3e3DsDzjfzZL1YjLM+ZofDvjRcATgb3Gy5cwHFP2iqr6TqtsmizJzsCzGfq7G8Pfsr/G0N8xVfWDhvE0QZLHcCPdVdUJTYNpKknuBzyBH+7v7KbBNBX769c4d3k0a7s7cRk3wjtpliR1w5Nr+mZ/ffLkoYGT5jlL8miG0zNWH93yvqryLeIFl2Qzw53mH+qP4U7ztQ3jaR2rTq55InCn8bIn13Ri1ck1hwPfHS/vjifXdMH++rXNyUNfY7jTvLQnDzlpnqMkrwXuzbCucmVTxN4M6yrPr6rnt8qmyZK8nWGgOI61/T0T2KOqjmyVTetb5+Sao4CHe3LNYvPkmr7ZX788eWgtJ81zlOS8GzueZTz0/byqOqBBLE0pyblVdZ/tPHej3WoxTOhuu89pMSQ5f3vj43rPaTHYX78mdPelqtp/3pla8si5+bo6yYNu5PpPAH586OK7LMlTktzw5ybJLZIcCbg+b7FdlOSF47o8YFijl+T38eSaHpyS5PVJDkty1/HrsPE0G0+uWXz2168PJflgkiOTPGT8OnI8jWjplpV6p3mOkhwK/C1wG7a+vX934AqGtUGntMqmyZLcE3gFw7q8yxjWdu0O/AvDuryvNAundW1zcs3KmuZL8eSaLnhyTd/sr2+ePLSVk+YGxrWUN/zyrf6wE/UhyZ4Mf36+1TqLJEm6+bk8Y87GCTPjXeWLgIeM51eqI1X1beA2SZ6c5L6t82h9GRwxLq9Jkocn+askz1293EaLKckdtnn8K2N/vzbuCdECS3KrcXnU7yXZJckzkxyf5JVJbt06n7YvyUGrft4pyUvH7v4sya1aZmvB/1jMUZJfBz4FfDrJfwM+ADwO+L9Jnt00nCZK8t5VPz+R4TSGxwPHJzmqVS5N5W+AI4CnA28DfgP4HPAw4H81zKXpnLTyQ5KXMvR4CsOnkr2mVShN7ViGD/TaF/ggwz6eVzG82/q37WJpCseu+vnlwP7Aq4FdGY6iWyouz5ijJGcChzH8sl0E7F9Vl4zrLf+lqg5uGlDrSnJaVR0y/vxJ4Jer6ivjXbAPL9vROz1JcmZVPSDJTgyfwnmXqrpmPHv7tKp6QOOIWsc2f/ZOBX6qqr4/9nmq/S22JKdX1cHjuwIXM/z5q/Hx56vqoAn/E2pkmz97pwM/UVXXLmt3m1sHWDLXVtVVwFVJvryylrmqLkvi314W3+qONq9s/KuqbyW5vlEmTec6gHGwP7mqrhkfX5dkS9tomsKuSQ5heHd0U1V9H27o0/46MU6UT6jxbt342P/2Lbbdk/wCw5+9W658iNeyduekeb6uT7LT+Ev32JWLSXbBpTI9eGCSKxjeUrxlkjuP7xTsDGxqnE3ruyTJravqe1X1cysXxz0G1zTMpelczNZlGN9JcpequnjckHtdw1yazudW/fl71srFJPcCrmyYS5N9DHjC+POnk+xVVZeOY+fSbYR3ecYcZfgo0W9U1XXbXL8bcGBV/XObZLopktyOob9Ptc6i2STZDditqr7ZOotml+FT5W45voOnDiVJORFRJ5w0S5IkSRO4JECSJEmawEmzJEmSNIGTZkmSJGkCJ82NJDl6vcdabPbXryQfWO+xFpv99W08Z3u7j7W47M5Jc0unTHisxWZ//XrOhMdabPbXsao6dL3HWlx25+kZkpZMkl2Bfarq3NZZNDv761uSewAHVNU/j11urirPau6A3XmnuYkk907y4SRfGB8flOSlrXNpOvbXrySPB04H/ml8fHCS49um0rTsr29JngO8B3jDeGlv4L3tEmladjdw0tzGm4AXAysfR3kG8NSmiTQL++vX0cCDgMsBqup04J4N82g2R2N/PftN4CeBKwCq6nzgTk0TaVp2h5PmVm5VVZ/d5pofBdsP++vXdVX13dYhtMPsr28/qKobPrY+yWbANaJ9sDucNLfyrST3YvyFS/JLwMVtI2kG9tevLyR5GrApyQFJXgd8snUoTc3++vaxJH8A7JrkkcC7gfc3zqTp2B1uBGwiyX7AG4GHzoUGQgAAF75JREFUAJcBXwF+paoubJlL07G/fiW5FfAS4FHjpROBP6mqq9ul0rTsr29JbgE8m6G/ACdW1ZvaptI07G7gpLmhJLsBt1i23acbhf31J8lPAZ+sqi2rrh1aVUt33miP7K9vSZ4OvHf1mJnkcVXlWdsLzu4GLs9oIMmWJC8Hrlr5BVzGQ8J7ZX9dOxH4SJK9Vl37+1ZhNDP769vrgH9NcuCqay9rFUYzsTucNLdyFsO/+5OS7DFeS8M8mo399etc4C+AjyZ5yHjN7vphf337CvAs4D1JnjJes78+2B2wuXWAJXVdVb0wyREMf3N7Bku4C7Vj9tevqqoPJDkXeGeSN2N3PbG/vlVVnZrkp4G3JzkM2NQ6lKZid3inuZUAVNW7gCOAtwD7NU2kWdhfv1a6Ox/4KeBhwEFNE2kW9te3iwGq6lvAoxn+wvNjTRNpWnaHGwGbSPLjVXXKqse3BZ5UVW9tGEtTsr+NJck+VfXV1jm0Y+xP0ry4PGOOkhxeVR8B7jF+hvtq32uRSdOzv34leWFVvTLJX23nJb8910Caif31Lclrq+oFSd7PjSynqaonNIilKdjdWk6a5+ungY8Aj7+R5wr4x/nG0Yzsr1/njN9PWfdVWlT217e3jd9f1TSFdoTdreLyDElLaTys/9ZVdUXrLJqd/fUtye2Bu1fVGa2zaDbL3J0bARtI8vwkt83g75OcmuRRk/9JLQL761eSfxi72w04Gzg3ye+1zqXp2F/fknx07G8P4PPAW5K8pnUuTWZ3AyfNbTxrvDvyKOBOwH8FXt42kmZgf/2639jdk4ATgH2Ap7eNpBnYX992H/t7MvCWqvpx4BGNM2k6doeT5lZWDgT/eYZfvs+zhIeEd8z++rVTkp0YJl3vq6pr8Zzfnthf3zYnuQvDUZ1L9fHLG4Dd4aS5lVOSnMQw6ToxyW2A6xtn0vTsr19vAC4EdgM+Pp6C4prYfthf317G8FHoX6qqk5PsB5zfOJOmY3e4EbCJcQPLwcAFVXV5kj2Bu60sqk9y/6o6q2lIbZf9bRxJAmyqquvGx8+squMax9KU7G9jSfLiqvrz1jk0u2XpzknzAkpyalUd2jqHdoz99cvu+mZ/fbO/fi1Ldy7PWEyuj+2b/fXL7vpmf32zv34tRXdOmheTt//7Zn/9sru+2V/f7K9fS9Gdk2ZJ2mop7pZsYPbXN/vr11J056R5MV3TOoBuEvvr17+1DqCbxP769u7WAbTDlqI7NwI2kuQg4J7A5pVrVfWPzQJpJvbXpyS3A57BD3f3260yaXr217ck+wLP44f7e0KrTJqO3Q02T36JftSSvBk4CDiLref7FuCkqwP217UTgE8DZ+LZ2j2yv769FzgGeD/21xu7wzvNTSQ5u6ru1zqHdoz99WtZjkXaqOyvb0k+U1WHtc6h2dndwElzA0mOAV5dVWe3zqLZ2V+/kvwO8D2Gj4H9wcr1qvpOs1Camv31LcnTgAOAk1jb36nNQmkqdjdweUYbxwGfSnIJwy9fgKqqg9rG0pTsr1/XAH8BvIStRyQVsF+zRJqF/fXtAcDTgcNZu7Tt8GaJNC27wzvNTST5EvC7bLMur6ouahZKU7O/fiX5MnBYVX2rdRbNzv76luSLwEFV5QlDnbG7gXea2/hqVR3fOoR2mP316yzgqtYhtMPsr2+fB24HfLN1EM3M7nDS3MoXk/wDwy7U1WuDPH2hD/bXry3A6Un+hbXdeWRZH+yvb3sxjJ8ns7a/pTq2rFN2h5PmVnZl+KV71KprHlnWD/vr13vHL/XJ/vr2R60DaIfZHa5pliRJkibyTvMcJXlhVb0yyevYuvP7Br7FuNjsr19J3lVVRyQ5kxvvzpNPFpj99S3JJ6rqoUmuZG1/KycP3bZRNE1gd2s5aZ6vc8bvn2uaQjvK/vr1/PH745qm0I6yv45V1UPH77dpnUWzsbu1XJ4hSZIkTXCL1gGWSZKjfxSvURv2168kH/hRvEZt2F/fkkz81LhpXqP5s7u1vNM8R0m+BrxmvZcAz6mq+84pkmZgf/1Kcjnw8fVeAty/qvxkuQVkf31L8v+A89d7CbB7Ve0zp0iakt2t5Zrm+XoTMGld0JvmEUQ7xP769cQpXrPUn3S14Oyvb9PcSNhys6fQjrC7VbzTLEmSJE3gmmZJkiRpAifNkiRJ0gSuaZYkSTebJHtM8bLrq+rymz2MZmJ3a7mmeY6SHD/Fy75TVUfd3Fk0O/vrV5IrJr0EuLiq7j2PPJqN/fUtydXANxh62p5Ny3ICQ0/sbi3vNM/XgcCvrvN8gL+ZUxbNzv769eWqOmS9FyQ5bV5hNDP769s59tctu1vFO81zlOSIqnrXTX2N2rC/fiXZr6ouuKmvURv217cku1TV1Tf1NZo/u1vLSbMkSZqrJHtU1Xda59Dslrk7T89YEEne2DqDdpz99SvJma0zaMfZ3+JL8tJVP98vyXnAKUkuTHJYw2iawO7W8k7zHK2zCzXA56tq73nm0Wzsr19Jnry9p4C/q6o7zjOPZmN/fUtyalUdOv78QeCvq+pDSR4EvLaqHtI2obbH7tZyI+B8/QdwEWt3odb4+E5NEmkW9tevdwL/h6Gvbe0y5yyanf1tHHetqg8BVNVnk+zaOpCmtvTdOWmerwuAh1fVV7d9Ism/N8ij2dhfv84AXlVVX9j2iSSPaJBHs7G/vu03HtkZYO8kt6qqq8bndmqYS5PZ3SpOmufrtcDtgR+adAGvnHMWzc7++vUCYHtn/f7CPINoh9hf3564zeNNAEn2Av52/nE0A7tbxTXNkiRJ0gTeaZ6zJPdl+Jvb3RjW530DOL6qzmkaTFOxvz4l2Qw8m+Gu5F3Z2t37gGOq6tqG8TSB/fUtye7Ai4EnASubNr/J0N/Ll+UjmHtkd2t55NwcJfl94B0Ma4M+C5w8/vz2JC9qmU2T2V/X3gYcDBwN/DzwWOCPgQcC/7tdLE3J/vr2LuAy4Geqas+q2hP42fHau5sm0yR2t4rLM+ZoPN/w/tveFUmyM3BWVR3QJpmmYX/9SnJuVd1nO8+dV1X3nncmTc/++jahv+0+p/bsbi3vNM/X9QxvLW7rLuNzWmz216/LkjwlyQ1jXpJbJDmS4Y6JFpv99e2iJC8cN48Bw0ay8d07Tx5abHa3imua5+sFwIeTnM/WX7Z9gP2B32qWStOyv349FXgF8PoklzEsq7kd8JHxOS02++vbkcCLgI8lWTnT/lLgeOCIZqk0DbtbxeUZczbeKXkQw0ayAF8DTq6qLU2DaSr2178kezKMfd9qnUWzsz9JrXinef5q1df1q76rD/bXqW1PPknyDeB9VfXFtsk0DfvbWJJ8oKoe1zqHZrfM3bmmeY6SPAo4nx/eAX7++JwWmP31a52TT97hySeLz/42pLu1DqAdtrTduTxjjpKcAzymqi7c5vq+wAlVdWCTYJqK/fXLk0/6Zn8bT5I3V9WzWufQ7Ja5O+80z9dmhjWw2/o6S/gZ7h2yv3558knf7G+DSLJrkvss66SrZ3bnmuZ5ezNwcpJ3sPX0hbsz7P4+plkqTcv++uXJJ32zvw0gyeOBVwE7A/smORh4WVU9oW0yTWJ3A5dnzFmSA9m6mWXl9IXjq+rspsE0Ffvrlyef9M3++pfkFOBw4KNVdch47YyqOqhtMk1idwPvNM9ZVZ0DnNM6h3aM/XXNk0/6Zn/9u66qvpukdQ7Nzu5w0rwwkhxdVUe3zqEdY3+LbTzd5PUMp598fby8N7B/kudW1UnNwmki+9swvpDkacCmJAcAvw18snEmTcfucHnGwkjy+Kp6f+sc2jH2t9g8+aRv9rcxJLkV8BJg5YjOE4E/qaqr26XSNOxu4KRZ0oY3biA7sKqu2+b6zsDZVbV/m2Sahv1JWgQuz5ijJJuBZwO/wHB8UgHfAN4HHLPtGaRaLPbXNU8+6Zv9bVBJ3lhVv9Y6h2a3jN15p3mOkrwduBw4jq3n/e4NPBPYo6qObJVNk9lf3zz5pG/2168ke2zvKeDzVbX3PPNoena3lpPmOUpyblXdZzvPnVdV9553Jk3P/iRpdkm2ABcxTLRW1Pj4blW1c5Ngmsju1vITAefrsiRPGc8bBYazR5McCVzWMJemY38bUJKjW2fQjrO/LlwA/ExV7bvqa7+q2he4tHU4rcvuVnHSPF9PBX4JuDTJeUnOAy4Bnjw+p8VmfxvTKa0D6Caxv8X3WuD223nulfMMopnZ3Souz2gkyZ4M//6/1TqLZmd/kiQtFyfNkjY8Tz7pm/1JWgROmiVteJ580jf7k7QInDRL2vA8+aRv9idpEbgRUNIy8OSTvtmfpOacNDeS5NT1Hmux2V93PPmkb/a3QTh29svuXJ4hacl48knf7E9SK95pbiTJPZI8Yvx51yS3aZ1J07O/flXVt51w9cv++ubY2S+7c9LcRJLnAO8B3jBe2ht4b7tEmoX9SdLsHDv7ZXcDJ81t/Cbwk8AVAFV1PnCnpok0C/uTpNk5dvbL7nDS3MoPquqalQfjwf0uLu+H/UnS7Bw7+2V3OGlu5WNJ/gDYNckjgXcD72+cSdOzv465A7xv9tc1x85+2R2entHEeNbos4FHAQFOrKo3tU2ladmfJM3OsbNfdjdw0txAkudX1V9OuqbFZH99S3IP4ICq+uckuwKbq+rK1rk0Hfvrl2Nnv+xu4PKMNp55I9eOmncI7TD765Q7wPtmf91z7OyX3QGbWwdYJkn+C/A0YN8kx6966jbAt9uk0rTsb0P4TeBBwGdg2AGeZOl2gHfM/jrk2Nkvu1vLSfN8fRK4GLgD8OpV168EzmiSSLOwv/79oKquSQIs7w7wjtlfnxw7+2V3q7imWdLSSPJK4HLgGcDzgOcCZ1fVS5oG01TsT1JLrmluIMmDk5yc5HtJrkmyJckVrXNpOvbXtRcB/wGcCfw6cIITrq7YX8ccO/tldwOXZ7Tx18BTGc45/E8Md032b5pIs7C/fj1v3O19w1FJy7gDvGP21zfHzn7ZHd5pbqaqvgRsqqotVfUW4GdbZ9L07K9b7gDvm/11zrGzX3bnneZWrkqyM3D6uEbvYmC3xpk0PfvrjDvA+2Z/G4ZjZ7/sDjcCNjEezn8psDPwO8DuwOvHv8Vpwdlff8bO9gX+nGFd7IorgTOq6romwTQV+9sYHDv7ZXcDJ81zlmQTcFxV/UrrLJqd/UnS7Bw7+2V3W7mmec6qagtwx/FtDnXG/vrmDvC+2V+/HDv7ZXdbuaa5jQuBfxvX5n1/5WJVvaZZIs3iQuyvV+4A75v99e1CHDt7dSF256S5kW+MX7dg2Miivthfx6rqS0k2jXdP3pLkk60zaXr21zXHzn7ZHa5pXkhJXldVz2udQzvG/hZXko8DjwD+HriEYQf4UVX1wKbBNBX729gcO/u1LN25pnkx/WTrALpJ7G9xPZ1h3PsthrcY7w78YtNEmoX9bWyOnf1aiu5cniFpKYw7wP903AF+NfDHjSNpBvYnqTXvNEtaCu4A75v9SWrNO82LKa0D6Caxv8V1Ie4A79mF2N9G5tjZr6XozknzYvrL1gF0k9jf4nIHeN/sb2Nz7OzXUnTn6RkNJHk/sO2/+O8CnwPeUFVXzz+VpmV/G9ey7ADfqOxvsTl29svuBq5pbuMC4HvAm8avKxg+0/3e42MtNvvbuJZiB/gGZn+LzbGzX3aHyzNaOaSqHrbq8fuTfLyqHpbkrGapNC37k6TZOXb2y+7wTnMrd0yyz8qD8ec7jA+vaRNJM7A/SZqdY2e/7A7vNLfy34FPJPkyw47TfYHnJtkNOK5pMk3D/jaupdgBvoHZ32Jz7OyX3eFGwGaS3BK4L8Mv3xeXZRH9RmF/G1OSo6rq2NY5tGPsb/E5dvbL7pw0N5PkIcA9WXW3v6re2iyQZmJ/fXIHeN/sr3+Onf2yO5dnNJHkbcC9gNOBLePlApbql69X9te1C4A7Am8fHx/J2h3gT2+US9Oxv445dvbL7gbeaW4gyTnA/cp/+V2yv36t7Pa+sWtJzqqq+7fKpsnsr2+Onf2yu4GnZ7TxBeDOrUNoh9lfv9wB3jf765tjZ7/sDpdntHIH4OwknwV+sHKxqp7QLpJmYH/9cgd43+yvb46d/bI7XJ7RRJKfvrHrVfWxeWfR7Oyvb+4A75v99cuxs192N3DSLGmpuAO8b/YnqRWXZ8xRkk9U1UOTXMnaY5MCVFXdtlE0TcH++ucO8L7ZX58cO/tld2t5p1nS0nAHeN/sT1JL3mluJMkmYC/WvsX41XaJNAv769bKDvCLWwfRDrG/zjl29svunDQ3keR5wB8xHMp//Xi5gIOahdLU7K9r7gDvm/11zLGzX3Y3cHlGA0m+BBxWVd9unUWzs79+uQO8b/bXN8fOftndwDvNbfw78N3WIbTD7K9TTq76Zn/dc+zsl93hpLmVC4CPJvkga99ifE27SJqB/XXGHeB9s78Nw7GzX3aHk+ZWvjp+7Tx+qS/215mqeuj4/Tats2h29rdhOHb2y+5w0jx34+7TW1fV77XOotnZX//cAd43++uTY2e/7G4rJ81zVlVbkhzaOod2jP31zR3gfbO/fjl29svutvL0jAaSvBo4AHg38P2V61X1j81CaWr21y93gPfN/vrm2Nkvuxt4p7mNPYBvA4evulbAUv3ydcz++uUO8L7ZX98cO/tld3inWdISSXIMcB9gqXeA98r+JLXkneYGkuwCPBu4P7DLyvWqelazUJqa/XXNHeB9s7+OOXb2y+4GTprbeBvwReDRwMuAXwbOaZpIs7C/DrkDvG/2tyE4dvbL7nB5RhNJTquqQ5KcUVUHJdkJOLGqDp/4D6s5++tXkg9X1cNb59COsb++OXb2y+4G3mlu49rx++VJfgy4BLhnuziakf316/Qkx7PkO8A7Zn99c+zsl93hpLmVNya5PfCHwPHArYH/0TaSZmB//XIHeN/sr2+Onf2yO1yeIUmSJE3kneYGkuwF/Blw16p6TJL7Af+5qo5pHE1TsL9+uQO8b/bXN8fOftnd4BatAyypY4ETgbuOj88DXtAsjWZ1LPbXq7cBd2bYAf4xYG/gyqaJNAv769uxOHb26ljszklzI3eoqncB1wNU1XXAlraRNAP769f+VfWHwPer6jjgscADGmfS9Oyvb46d/bI7nDS38v0kezJsYCHJg/GjYXtif/3adgf47izhDvCO2V/fHDv7ZXe4prmV32XYfXqvJP8G3BH4pbaRNAP765c7wPtmf31z7OyX3eHpGc0k2QzcBwhwblVdO+Ef0QKxP0manWNnv+zOSfNcJXnyes97QP9is7/+uQO8b/bXJ8fOftndWk6a5yjJW9Z5ujw2abHZX/+SfAh4C/CSqnrgeOfktKpyM1kH7K9Pjp39sru1nDRLWhpJTq6qn0hyWlUdMl47vaoObp1Nk9mfpJY8PWOOkjzuR/EatWF/G4I7wPtmfx1y7OyX3a3lneY5SnIO8DSGRfTbc2xVHTSnSJqB/fUvyaHA64AfA77AuAO8qs5oGkxTsb8+OXb2y+7WctI8R0k+yniHZB3fqapfnEMczcj+NgZ3gPfN/vrj2Nkvu1vLSbOkDc8d4H2zP0mLwA83kbQMHr/OcwU46Vps9iepOe80S5IkSRN4eoakDc8d4H2zP0mLwDvNjSS5c1Vdsr3HWmz21xd3gPfN/jYOx85+2Z2T5maSfLCqHru9x1ps9tcXd4D3zf42DsfOftmdk2ZJkiRpIk/PmKMkuwC/AewPnAG8uaqua5tK07I/SZqdY2e/7G4t7zTPUZJ3AtcC/wo8Brioqp7fNpWmZX+SNDvHzn7Z3VpOmucoyZlV9YDx583AZ6vq0MaxNCX7k6TZOXb2y+7W8si5+brh416X+e2NjtnfBpDkzus91mKzvy45dvbL7lbxTvMcJdkCfH/lIbArcNX4c1XVbVtl02T2tzG4A7xv9tcfx85+2d1aTpolSZKkCTw9Q9KG5w7wvtmfpEXgnWZJG547wPtmf5IWgZNmSRueO8D7Zn+SFoGnZ0haBu4A75v9SWrOO82SNjx3gPfN/iQtAifNkiRJ0gQuz5AkSZImcNIsSZIkTeCkWZIkSZrASbMkSZI0gZNmSZIkaQInzZIkSdIE/x+6H2drX6hlkwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plot_sweeper_df(df.mean(level=(1)), sort_by=\"rank\")" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "application/scrapbook.scrap.json+json": { + "data": 36, + "encoder": "json", + "name": "nr_elements", + "version": 1 + } + }, + "metadata": { + "scrapbook": { + "data": true, + "display": false, + "name": "nr_elements" + } + }, + "output_type": "display_data" + }, + { + "data": { + "application/scrapbook.scrap.json+json": { + "data": [ + 13.666666666666666, + 18.5, + 1, + 1.3333333333333333, + 26.333333333333332, + 30.75 + ], + "encoder": "json", + "name": "ranks", + "version": 1 + } + }, + "metadata": { + "scrapbook": { + "data": true, + "display": false, + "name": "ranks" + } + }, + "output_type": "display_data" + }, + { + "data": { + "application/scrapbook.scrap.json+json": { + "data": 34.771409999999996, + "encoder": "json", + "name": "max_duration", + "version": 1 + } + }, + "metadata": { + "scrapbook": { + "data": true, + "display": false, + "name": "max_duration" + } + }, + "output_type": "display_data" + }, + { + "data": { + "application/scrapbook.scrap.json+json": { + "data": 18.015496, + "encoder": "json", + "name": "min_duration", + "version": 1 + } + }, + "metadata": { + "scrapbook": { + "data": true, + "display": false, + "name": "min_duration" + } + }, + "output_type": "display_data" + } + ], + "source": [ + "# Preserve some of the notebook outputs\n", + "sb.glue(\"nr_elements\", len(df))\n", + "sb.glue(\"ranks\", list(df.mean(level=(1))[\"rank\"]))\n", + "sb.glue(\"max_duration\", df.max().duration)\n", + "sb.glue(\"min_duration\", df.min().duration)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python (cv)", + "language": "python", + "name": "cv" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/conftest.py b/tests/conftest.py index 835c559..731666c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -85,7 +85,7 @@ def classification_notebooks(): ), "24_exploring_hyperparameters_on_azureml": os.path.join( folder_notebooks, "24_exploring_hyperparameters_on_azureml.ipynb" - ) + ), } return paths @@ -100,6 +100,9 @@ def similarity_notebooks(): "01": os.path.join( folder_notebooks, "01_training_and_evaluation_introduction.ipynb" ), + "11": os.path.join( + folder_notebooks, "11_exploring_hyperparameters.ipynb" + ), } return paths @@ -252,14 +255,16 @@ def testing_databunch(tmp_session): def pytest_addoption(parser): - parser.addoption("--subscription_id", - help="Azure Subscription Id to create resources in") - parser.addoption("--resource_group", - help="Name of the resource group") - parser.addoption("--workspace_name", - help="Name of Azure ML Workspace") - parser.addoption("--workspace_region", - help="Azure region to create the workspace in") + parser.addoption( + "--subscription_id", + help="Azure Subscription Id to create resources in", + ) + parser.addoption("--resource_group", help="Name of the resource group") + parser.addoption("--workspace_name", help="Name of Azure ML Workspace") + parser.addoption( + "--workspace_region", help="Azure region to create the workspace in" + ) + @pytest.fixture def subscription_id(request): diff --git a/tests/integration/similarity/test_integration_similarity_notebooks.py b/tests/integration/similarity/test_integration_similarity_notebooks.py index f261e16..a3fa929 100644 --- a/tests/integration/similarity/test_integration_similarity_notebooks.py +++ b/tests/integration/similarity/test_integration_similarity_notebooks.py @@ -23,3 +23,23 @@ def test_01_notebook_run(similarity_notebooks): nb_output = sb.read_notebook(OUTPUT_NOTEBOOK) assert nb_output.scraps["median_rank"].data <= 10 + + +@pytest.mark.notebooks +@pytest.mark.linuxgpu +def test_11_notebook_run(similarity_notebooks, tiny_ic_data_path): + notebook_path = similarity_notebooks["11"] + pm.execute_notebook( + notebook_path, + OUTPUT_NOTEBOOK, + parameters=dict( + PM_VERSION=pm.__version__, + # Speed up testing since otherwise would take ~12 minutes on V100 + DATA_PATHS=[tiny_ic_data_path], + REPS=1, + IM_SIZES=[60, 100], + ), + kernel_name=KERNEL_NAME, + ) + nb_output = sb.read_notebook(OUTPUT_NOTEBOOK) + assert min(nb_output.scraps["ranks"].data) <= 30 \ No newline at end of file diff --git a/tests/smoke/test_azureml_notebooks.py b/tests/smoke/test_azureml_notebooks.py index cf623bf..0b63b08 100644 --- a/tests/smoke/test_azureml_notebooks.py +++ b/tests/smoke/test_azureml_notebooks.py @@ -113,9 +113,11 @@ def test_24_notebook_run( subscription_id, resource_group, workspace_name, - workspace_region + workspace_region, ): - notebook_path = classification_notebooks["24_exploring_hyperparameters_on_azureml"] + notebook_path = classification_notebooks[ + "24_exploring_hyperparameters_on_azureml" + ] pm.execute_notebook( notebook_path, OUTPUT_NOTEBOOK, @@ -125,8 +127,9 @@ def test_24_notebook_run( resource_group=resource_group, workspace_name=workspace_name, workspace_region=workspace_region, - epochs=1, - max_total_runs=1 + MAX_NODES=2, + MAX_TOTAL_RUNS=1, + IM_SIZES=[30, 40], ), kernel_name=KERNEL_NAME, ) diff --git a/tests/unit/similarity/test_similarity_notebooks.py b/tests/unit/similarity/test_similarity_notebooks.py index ab1bd3b..e281d64 100644 --- a/tests/unit/similarity/test_similarity_notebooks.py +++ b/tests/unit/similarity/test_similarity_notebooks.py @@ -47,4 +47,21 @@ def test_01_notebook_run(similarity_notebooks, tiny_ic_data_path): ), kernel_name=KERNEL_NAME, ) - nb_output = sb.read_notebook(OUTPUT_NOTEBOOK) + + +@pytest.mark.notebooks +def test_11_notebook_run(similarity_notebooks, tiny_ic_data_path): + notebook_path = similarity_notebooks["11"] + pm.execute_notebook( + notebook_path, + OUTPUT_NOTEBOOK, + parameters=dict( + PM_VERSION=pm.__version__, + DATA_PATHS=[tiny_ic_data_path], + REPS=1, + LEARNING_RATES=[1e-4], + IM_SIZES=[30], + EPOCHS=[1], + ), + kernel_name=KERNEL_NAME, + ) diff --git a/tools/repo_metrics/README.md b/tools/repo_metrics/README.md index 102529d..6f684b9 100644 --- a/tools/repo_metrics/README.md +++ b/tools/repo_metrics/README.md @@ -1,8 +1,8 @@ # Repository Metrics -[![Build Status](https://dev.azure.com/best-practices/computervision/_apis/build/status/repo-metrics?branchName=master)](https://dev.azure.com/best-practices/computervision/_build/latest?definitionId=27&branchName=master) +[![Build Status](https://dev.azure.com/best-practices/computervision/_apis/build/status/repo-metrics?branchName=staging)](https://dev.azure.com/best-practices/computervision/_build/latest?definitionId=27&branchName=staging) -We developed a script that allows us to track the metrics of the ComputerVisionBestPractices repo. Some of the metrics we can track are listed here: +We developed a script that allows us to track the repo metrics. Some of the metrics we can track are listed here: * Number of stars * Number of forks @@ -10,17 +10,27 @@ We developed a script that allows us to track the metrics of the ComputerVisionB * Number of views * Number of lines of code -To see the full list of metrics, see [git_stats.py](scripts/repo_metrics/git_stats.py) +To see the full list of metrics, see [git_stats.py](git_stats.py) The first step is to set up the credentials, copy the configuration file and fill up the credentials of GitHub and CosmosDB: - cp scripts/repo_metrics/config_template.py scripts/repo_metrics/config.py + cp tools/repo_metrics/config_template.py tools/repo_metrics/config.py To track the current state of the repository and save it to CosmosDB: - python scripts/repo_metrics/track_metrics.py --github_repo "https://github.com/Microsoft/ComputerVision" --save_to_database + python tools/repo_metrics/track_metrics.py --github_repo "https://github.com/Microsoft/ComputerVision" --save_to_database To track an event related to this repository and save it to CosmosDB: - python scripts/repo_metrics/track_metrics.py --event "Today we did our first blog of the project" --event_date 2018-12-01 --save_to_database + python tools/repo_metrics/track_metrics.py --event "Today we did our first blog of the project" --event_date 2018-12-01 --save_to_database + + +### Setting up Azure CosmosDB + +The API that we is used to track the GitHub metrics is the [Mongo API](https://docs.microsoft.com/en-us/azure/cosmos-db/mongodb-introduction). + +The database name and collections name are defined in the [config file](config_template.py). There are two main collections, defined as `COLLECTION_GITHUB_STATS` and `COLLECTION_EVENTS` to store the information defined on the previous section. + +**IMPORTANT NOTE**: If the database and the collections are created directly through the portal, a common partition key should be defined. We recommend to use `date` as partition key. + diff --git a/tools/repo_metrics/config_template.py b/tools/repo_metrics/config_template.py index 03efb45..1b6b42b 100644 --- a/tools/repo_metrics/config_template.py +++ b/tools/repo_metrics/config_template.py @@ -3,10 +3,12 @@ # Github token # More info: https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/ -GITHUB_TOKEN = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" +GITHUB_TOKEN = "" # CosmosDB Mongo API -CONNECTION_STRING = "mongodb://XXXXXXXXXXXXXXXXXXXXXXXXX.documents.azure.com:10255/?ssl=true&replicaSet=globaldb" +# * Azure Portal: Settings -> Connection String -> PRIMARY CONNECTION STRING +# * For example, 'mongodb://:@:/?ssl=true&replicaSet=globaldb' +CONNECTION_STRING = "" DATABASE = "cv_stats" COLLECTION_GITHUB_STATS = "github_stats" COLLECTION_EVENTS = "events" diff --git a/tools/repo_metrics/track_metrics.py b/tools/repo_metrics/track_metrics.py index 5f5b181..b0e40fd 100644 --- a/tools/repo_metrics/track_metrics.py +++ b/tools/repo_metrics/track_metrics.py @@ -14,7 +14,6 @@ import logging from datetime import datetime from dateutil.parser import isoparse from pymongo import MongoClient -from datetime import datetime from tools.repo_metrics.git_stats import Github from tools.repo_metrics.config import ( GITHUB_TOKEN, @@ -32,6 +31,7 @@ log = logging.getLogger() def parse_args(): """Argument parser. + Returns: obj: Parser. """ @@ -61,12 +61,14 @@ def parse_args(): def connect(uri="mongodb://localhost"): """Mongo connector. + Args: uri (str): Connection string. + Returns: obj: Mongo client. """ - client = MongoClient(uri, serverSelectionTimeoutMS=1000) + client = MongoClient(uri, serverSelectionTimeoutMS=5000) # Send a query to the server to see if the connection is working. try: @@ -78,9 +80,11 @@ def connect(uri="mongodb://localhost"): def event_as_dict(event, date): """Encodes an string event input as a dictionary with the date. + Args: event (str): Details of a event. date (datetime): Date of the event. + Returns: dict: Dictionary with the event and the date. """ @@ -89,8 +93,10 @@ def event_as_dict(event, date): def github_stats_as_dict(github): """Encodes Github statistics as a dictionary with the date. + Args: obj: Github object. + Returns: dict: Dictionary with Github details and the date. """ @@ -125,6 +131,7 @@ def github_stats_as_dict(github): def tracker(args): """Main function to track metrics. + Args: args (obj): Parsed arguments. """ diff --git a/utils_cv/classification/parameter_sweeper.py b/utils_cv/classification/parameter_sweeper.py index 63ee471..8ce8ad7 100644 --- a/utils_cv/classification/parameter_sweeper.py +++ b/utils_cv/classification/parameter_sweeper.py @@ -195,7 +195,7 @@ class ParameterSweeper: one_cycle_policy=True, ) - def __init__(self, **kwargs) -> None: + def __init__(self, metric_name="accuracy", **kwargs) -> None: """ Initialize class with default params if kwargs is empty. Otherwise, initialize params with kwargs. @@ -214,6 +214,8 @@ class ParameterSweeper: one_cycle_policy=[self.default_params.get("one_cycle_policy")], ) + self.metric_name = metric_name + self.param_order = tuple(self.params.keys()) self.update_parameters(**kwargs) @@ -411,8 +413,8 @@ class ParameterSweeper: Otherwise overwrite the corresponding self.params key. """ for k, v in kwargs.items(): - if k not in self.params.keys(): - raise Exception("Parameter {k} is invalid.") + if k not in set(self.params.keys()): + raise Exception(f"Parameter {k} is invalid.") if v is None: continue self.params[k] = v @@ -420,7 +422,11 @@ class ParameterSweeper: return self def run( - self, datasets: List[Path], reps: int = 3, early_stopping: bool = False + self, + datasets: List[Path], + reps: int = 3, + early_stopping: bool = False, + metric_fct=None, ) -> pd.DataFrame: """ Performs the experiment. Iterates through the number of specified , the list permutations @@ -440,8 +446,8 @@ class ParameterSweeper: res = dict() for rep in range(reps): - res[rep] = dict() + for i, permutation in enumerate(self.permutations): print( f"Running {i+1} of {len(self.permutations)} permutations. " @@ -462,15 +468,20 @@ class ParameterSweeper: dataset, permutation, early_stopping ) - _, metric = learn.validate( - learn.data.valid_dl, metrics=[accuracy] - ) + if metric_fct is None: + _, metric = learn.validate( + learn.data.valid_dl, metrics=[accuracy] + ) + + else: + metric = metric_fct(learn) res[rep][stringified_permutation][data_name][ "duration" ] = duration + res[rep][stringified_permutation][data_name][ - "accuracy" + self.metric_name ] = float(metric) learn.destroy()