From 89e986fe2ce354d6657f111170630c28072fb85e Mon Sep 17 00:00:00 2001 From: Chenhui Hu Date: Mon, 23 Mar 2020 13:23:32 -0400 Subject: [PATCH] Revert "Chenhui/add ci tests (#150)" (#151) This reverts commit 21846168a78e15f084e5edef1d83193211270f91. --- .gitignore | 21 - .lintr | 18 - forecasting.Rproj => R/forecasting.Rproj | 1 + R/orange_juice/01_dataprep.Rmd | 83 + .../R => R/orange_juice}/01_dataprep.nb.html | 123 +- R/orange_juice/02_simplemodels.Rmd | 74 + .../orange_juice/02_simplemodels.nb.html | 116 +- R/orange_juice/02a_simplereg_models.Rmd | 39 + .../orange_juice/02a_simplereg_models.nb.html | 92 +- R/orange_juice/03_model_eval.Rmd | 58 + .../orange_juice/03_model_eval.nb.html | 123 +- R/orange_juice/README.md | 32 + R/orange_juice/ojdata_forecast_settings.json | 5 + README.md | 85 +- R_utils/cluster.R | 36 - R_utils/model_eval.R | 50 - R_utils/save_objects.R | 25 - .../feature_engineering/feature_utils.py | 1004 --------- .../readme_generator/readme_generator.py | 140 -- docs/SETUP.md | 59 +- .../auto_arima_forecasting.ipynb} | 13 +- .../azure_automl_forecast.ipynb | 756 +++++++ .../lightgbm_point_forecast.ipynb} | 12 +- .../ojdata_exploration_retail.ipynb} | 0 .../ojdata_preparation_retail.ipynb} | 4 +- ...ilatedcnn_point_forecast_multiround.ipynb} | 17 +- .../lightgbm_point_forecast_multiround.ipynb} | 14 +- examples/README.md | 16 +- examples/grocery_sales/R/01_dataprep.Rmd | 96 - examples/grocery_sales/R/02_basic_models.Rmd | 87 - examples/grocery_sales/R/02a_reg_models.Rmd | 86 - .../grocery_sales/R/02b_prophet_models.Rmd | 67 - examples/grocery_sales/R/README.md | 45 - .../grocery_sales/R/forecast_settings.yaml | 6 - examples/grocery_sales/README.md | 26 - .../azure_automl_single_round.ipynb | 1790 ----------------- .../02_model/autoarima_multi_round.ipynb | 588 ------ .../aml_scripts/train_validate.py | 275 --- .../azure_hyperdrive_lightgbm.ipynb | 1185 ----------- examples/grocery_sales/python/README.md | 16 - fclib/README.md | 39 +- fclib/fclib/azureml/azureml_utils.py | 147 +- .../{load_oj_data.R => download_oj_data.R} | 23 +- fclib/fclib/dataset/ojdata.py | 91 +- .../fclib/evaluation}/evaluate.py | 0 .../fclib/evaluation}/train_utils.py | 0 .../feature_engineering/base_ts_estimators.py | 0 .../feature_engineering.py | 0 .../feature_engineering/feature_utils.py | 92 +- .../fclib}/feature_engineering/lag.py | 0 .../feature_engineering/normalization.py | 0 .../feature_engineering/rolling_window.py | 0 .../fclib}/feature_engineering/stats.py | 0 .../fclib}/feature_engineering/temporal.py | 0 .../feature_engineering/us_holidays.csv | 0 .../fclib}/feature_engineering/utils.py | 0 .../fclib/tuning}/__init__.py | 0 .../fclib/tuning}/back_test_utils.py | 0 fclib/requirements.txt | 3 +- tests/ci/component_governance.yml | 49 - tests/ci/cpu_integration_tests_linux.yml | 4 +- tests/ci/cpu_unit_tests_linux.yml | 2 +- tests/conftest.py | 15 +- tests/integration/test_notebooks_python.py | 32 +- tools/environment.yml | 30 +- tools/environment_setup.bat | 24 - tools/generate_conda_file.py | 165 -- tools/generate_requirements_txt.py | 43 - .../readme_generator/Benchmarks.csv | 0 .../readme_generator/TSPerfBoard-Energy.csv | 0 .../readme_generator/TSPerfBoard-Retail.csv | 0 tools/readme_generator/readme_generator.py | 119 ++ tools/repo_metrics/placeholder.txt | 1 + 73 files changed, 1443 insertions(+), 6719 deletions(-) delete mode 100644 .lintr rename forecasting.Rproj => R/forecasting.Rproj (99%) create mode 100644 R/orange_juice/01_dataprep.Rmd rename {examples/grocery_sales/R => R/orange_juice}/01_dataprep.nb.html (69%) create mode 100644 R/orange_juice/02_simplemodels.Rmd rename examples/grocery_sales/R/02b_prophet_models.nb.html => R/orange_juice/02_simplemodels.nb.html (98%) create mode 100644 R/orange_juice/02a_simplereg_models.Rmd rename examples/grocery_sales/R/02a_reg_models.nb.html => R/orange_juice/02a_simplereg_models.nb.html (98%) create mode 100644 R/orange_juice/03_model_eval.Rmd rename examples/grocery_sales/R/02_basic_models.nb.html => R/orange_juice/03_model_eval.nb.html (98%) create mode 100644 R/orange_juice/README.md create mode 100644 R/orange_juice/ojdata_forecast_settings.json delete mode 100644 R_utils/cluster.R delete mode 100644 R_utils/model_eval.R delete mode 100644 R_utils/save_objects.R delete mode 100644 contrib/tsperf/energy_utils/feature_engineering/feature_utils.py delete mode 100644 contrib/tsperf/readme_generator/readme_generator.py rename examples/{grocery_sales/python/00_quick_start/autoarima_single_round.ipynb => 00_quick_start/auto_arima_forecasting.ipynb} (99%) create mode 100644 examples/00_quick_start/azure_automl_forecast.ipynb rename examples/{grocery_sales/python/00_quick_start/lightgbm_single_round.ipynb => 00_quick_start/lightgbm_point_forecast.ipynb} (99%) rename examples/{grocery_sales/python/01_prepare_data/ojdata_exploration.ipynb => 01_prepare_data/ojdata_exploration_retail.ipynb} (100%) rename examples/{grocery_sales/python/01_prepare_data/ojdata_preparation.ipynb => 01_prepare_data/ojdata_preparation_retail.ipynb} (99%) rename examples/{grocery_sales/python/02_model/dilatedcnn_multi_round.ipynb => 02_model/dilatedcnn_point_forecast_multiround.ipynb} (99%) rename examples/{grocery_sales/python/02_model/lightgbm_multi_round.ipynb => 02_model/lightgbm_point_forecast_multiround.ipynb} (99%) delete mode 100644 examples/grocery_sales/R/01_dataprep.Rmd delete mode 100644 examples/grocery_sales/R/02_basic_models.Rmd delete mode 100644 examples/grocery_sales/R/02a_reg_models.Rmd delete mode 100644 examples/grocery_sales/R/02b_prophet_models.Rmd delete mode 100644 examples/grocery_sales/R/README.md delete mode 100644 examples/grocery_sales/R/forecast_settings.yaml delete mode 100644 examples/grocery_sales/README.md delete mode 100644 examples/grocery_sales/python/00_quick_start/azure_automl_single_round.ipynb delete mode 100644 examples/grocery_sales/python/02_model/autoarima_multi_round.ipynb delete mode 100644 examples/grocery_sales/python/03_model_tune_deploy/aml_scripts/train_validate.py delete mode 100644 examples/grocery_sales/python/03_model_tune_deploy/azure_hyperdrive_lightgbm.ipynb delete mode 100644 examples/grocery_sales/python/README.md rename fclib/fclib/dataset/{load_oj_data.R => download_oj_data.R} (57%) rename {contrib/tsperf/scripts => fclib/fclib/evaluation}/evaluate.py (100%) rename {contrib/tsperf/energy_utils => fclib/fclib/evaluation}/train_utils.py (100%) rename {contrib/tsperf/energy_utils => fclib/fclib}/feature_engineering/base_ts_estimators.py (100%) rename {contrib/tsperf/energy_utils => fclib/fclib}/feature_engineering/feature_engineering.py (100%) rename {contrib/tsperf/energy_utils => fclib/fclib}/feature_engineering/lag.py (100%) rename {contrib/tsperf/energy_utils => fclib/fclib}/feature_engineering/normalization.py (100%) rename {contrib/tsperf/energy_utils => fclib/fclib}/feature_engineering/rolling_window.py (100%) rename {contrib/tsperf/energy_utils => fclib/fclib}/feature_engineering/stats.py (100%) rename {contrib/tsperf/energy_utils => fclib/fclib}/feature_engineering/temporal.py (100%) rename {contrib/tsperf/energy_utils => fclib/fclib}/feature_engineering/us_holidays.csv (100%) rename {contrib/tsperf/energy_utils => fclib/fclib}/feature_engineering/utils.py (100%) rename {contrib/tsperf/energy_utils/feature_engineering => fclib/fclib/tuning}/__init__.py (100%) rename {contrib/tsperf/energy_utils => fclib/fclib/tuning}/back_test_utils.py (100%) delete mode 100644 tests/ci/component_governance.yml delete mode 100644 tools/environment_setup.bat delete mode 100644 tools/generate_conda_file.py delete mode 100644 tools/generate_requirements_txt.py rename {contrib/tsperf => tools}/readme_generator/Benchmarks.csv (100%) rename {contrib/tsperf => tools}/readme_generator/TSPerfBoard-Energy.csv (100%) rename {contrib/tsperf => tools}/readme_generator/TSPerfBoard-Retail.csv (100%) create mode 100644 tools/readme_generator/readme_generator.py create mode 100644 tools/repo_metrics/placeholder.txt diff --git a/.gitignore b/.gitignore index 708a92cb..d6dc00f7 100644 --- a/.gitignore +++ b/.gitignore @@ -2,27 +2,6 @@ **/.ipynb_checkpoints *.egg-info/ .vscode/ -*.pkl -*.h5 -# Data ojdata/* *.Rdata - -# AML Config -aml_config/ -.azureml/ -.config/ - -# Pytests -.pytest_cache/ - -# File for model deployment -score.py - -# Environments -myenv.yml - -# Logs -logs/ -*.log \ No newline at end of file diff --git a/.lintr b/.lintr deleted file mode 100644 index 269fffb5..00000000 --- a/.lintr +++ /dev/null @@ -1,18 +0,0 @@ -linters: with_defaults( - infix_spaces_linter = NULL, - spaces_left_parentheses_linter = NULL, - open_curly_linter = NULL, - line_length_linter = NULL, - camel_case_linter = NULL, - object_name_linter = NULL, - object_usage_linter = NULL, - object_length_linter = NULL, - trailing_blank_lines_linter = NULL, - absolute_paths_linter = NULL, - commented_code_linter = NULL, - implicit_integer_linter = NULL, - extraction_operator_linter = NULL, - single_quotes_linter = NULL, - pipe_continuation_linter = NULL, - cyclocomp_linter = NULL - ) diff --git a/forecasting.Rproj b/R/forecasting.Rproj similarity index 99% rename from forecasting.Rproj rename to R/forecasting.Rproj index f06cf89a..5dfd7033 100644 --- a/forecasting.Rproj +++ b/R/forecasting.Rproj @@ -10,3 +10,4 @@ NumSpacesForTab: 4 Encoding: UTF-8 RnwWeave: knitr + diff --git a/R/orange_juice/01_dataprep.Rmd b/R/orange_juice/01_dataprep.Rmd new file mode 100644 index 00000000..cd1194a9 --- /dev/null +++ b/R/orange_juice/01_dataprep.Rmd @@ -0,0 +1,83 @@ +--- +title: Data preparation +output: html_notebook +--- + +```{r, echo=FALSE, results="hide", message=FALSE} +library(tidyr) +library(dplyr) +library(tsibble) +library(feasts) +library(fable) +``` + +In this notebook, we generate the datasets that will be used for model training and validating. The experiment parameters are obtained from the file `ojdata_forecast_settings.json`; you can modify that file to vary the experimental setup, or just edit the values in this notebook. + +The orange juice dataset comes from the bayesm package, and gives pricing and sales figures over time for a variety of orange juice brands in several stores in Florida. + +A complicating factor is that the data is in a hybrid of long and wide format: while the sales figures are long (one column of sales data for every store and brand), the prices are wide (one price column for each brand). Therefore we need to reshape the data if we want to use prices for modelling. As part of this, we also compute a new column `maxpricediff`: this represents the log-ratio of the price of this brand compared to the best competing price. A positive `maxpricediff` means this brand is cheaper than all the other brands, and a negative `maxpricediff` means it is more expensive. + +```{r} +settings <- jsonlite::fromJSON("ojdata_forecast_settings.json") + +train_periods <- seq(settings$TRAIN_WINDOW, 160 - settings$STEP - 1, settings$STEP) +start_date <- as.Date(settings$START_DATE) + +data(orangeJuice, package="bayesm") + +oj_data <- orangeJuice$yx %>% + complete(store, brand, week) %>% + group_by(store, brand) %>% + group_modify(~ { + pricevars <- grep("price", names(.x), value=TRUE) + thispricevar <- paste0("price", .y$brand) + best_other_price <- do.call(pmin, .x[setdiff(pricevars, thispricevar)]) + .x$price <- .x[[thispricevar]] + .x$maxpricediff <- log(best_other_price/.x$price) + select(.x, week, logmove, deal, feat, price, maxpricediff) + }) %>% + ungroup() %>% + mutate(week=yearweek(start_date + week*7)) %>% # do this separately because of tsibble/vctrs issues + as_tsibble(index=week, key=c(store, brand)) +``` + +Here are some glimpses of what the data looks like. The dependent variable is `logmove`, the logarithm of the total sales for a given brand and store, in a particular week. Note that we do _not_ fill in the missing values in the data, as (with the exception of `ETS`) the modelling functions in the fable package can handle this innately. + +```{r} +head(oj_data) +``` + +The time series plots for a small subset of brands and stores are shown below. It is clear that the statistical behaviour of the data varies by store and brand. + +```{r} +library(ggplot2) + +oj_data %>% + filter(store < 10, brand < 5) %>% + ggplot(aes(x=week, y=logmove)) + + geom_line() + + scale_x_date(labels=NULL) + + facet_grid(vars(store), vars(brand), labeller="label_both") +``` + +Finally, we split the dataset into separate samples for training and testing. The schema used is broadly time series cross-validation, whereby we train a model on data up to time $t$, test it on data for times $t+1$ to $t+k$, then train on data up to time $t+k$, test it on data for times $t+k+1$ to $t+2k$, and so on. + +In this specific case study we introduce a small extra piece of complexity. We train a model on data up to month $t$, then test it on months $t+2$ to $t+3$. Then we train on data up to month $t+2$, and test it on months $t+4$ to $t+5$, and so on. Thus there is always a gap of one month between the training and test samples, a complicating factor introduced after discussions with domain experts. + +```{r} +subset_oj_data <- function(start, end) +{ + start <- yearweek(start_date + start*7) + end <- yearweek(start_date + end*7) + filter(oj_data, week >= start, week <= end) +} + +oj_train <- lapply(train_periods, function(i) subset_oj_data(40, i)) +oj_test <- lapply(train_periods, function(i) subset_oj_data(i + 2, i + settings$STEP + 1)) + +save(oj_train, oj_test, file="oj_data.Rdata") + +head(oj_train[[1]]) + +head(oj_test[[1]]) +``` diff --git a/examples/grocery_sales/R/01_dataprep.nb.html b/R/orange_juice/01_dataprep.nb.html similarity index 69% rename from examples/grocery_sales/R/01_dataprep.nb.html rename to R/orange_juice/01_dataprep.nb.html index bf37aa54..8378a894 100644 --- a/examples/grocery_sales/R/01_dataprep.nb.html +++ b/R/orange_juice/01_dataprep.nb.html @@ -226,91 +226,41 @@ summary { -

Copyright (c) Microsoft Corporation.
Licensed under the MIT License.

-

In this notebook, we generate the datasets that will be used for model training and validating.

-

The orange juice dataset comes from the bayesm package, and gives pricing and sales figures over time for a variety of orange juice brands in several stores in Florida. Rather than installing the entire package (which is very complex), we download the dataset itself from the GitHub mirror of the CRAN repository.

- -
# download the data from the GitHub mirror of the bayesm package source
-ojfile <- tempfile(fileext=".rda")
-download.file("https://github.com/cran/bayesm/raw/master/data/orangeJuice.rda", ojfile)
-load(ojfile)
-file.remove(ojfile)
- -

The dataset generation parameters are obtained from the file ojdata_forecast_settings.yaml; you can modify that file to vary the experimental setup. The settings are

- ----- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ParameterDescriptionDefault
N_SPLITSThe number of splits to make.10
HORIZONThe forecast horizon for the test dataset for each split.2
GAPThe gap in weeks from the end of the training period to the start of the testing period; see below.2
FIRST_WEEKThe first week of data to use.40
LAST_WEEKThe last week of data to use.156
START_DATEThe actual calendar date for the start of the first week in the data.1989-09-14
-

A complicating factor is that the data does not include every possible combination of store, brand and date, so we have to pad out the missing rows with complete. In addition, one store/brand combination has no data beyond week 156; we therefore end the analysis at this week. We also do not fill in the missing values in the data, as many of the modelling functions in the fable package can handle this innately.

+

In this notebook, we generate the datasets that will be used for model training and validating. The experiment parameters are obtained from the file ojdata_forecast_settings.json; you can modify that file to vary the experimental setup, or just edit the values in this notebook.

+

The orange juice dataset comes from the bayesm package, and gives pricing and sales figures over time for a variety of orange juice brands in several stores in Florida.

+

A complicating factor is that the data is in a hybrid of long and wide format: while the sales figures are long (one column of sales data for every store and brand), the prices are wide (one price column for each brand). Therefore we need to reshape the data if we want to use prices for modelling. As part of this, we also compute a new column maxpricediff: this represents the log-ratio of the price of this brand compared to the best competing price. A positive maxpricediff means this brand is cheaper than all the other brands, and a negative maxpricediff means it is more expensive.

- -
library(tidyr)
-library(dplyr)
-library(tsibble)
-library(feasts)
-library(fable)
+
+
settings <- jsonlite::fromJSON("ojdata_forecast_settings.json")
 
-settings <- yaml::read_yaml(here::here("examples/grocery_sales/R/forecast_settings.yaml"))
+train_periods <- seq(settings$TRAIN_WINDOW, 160 - settings$STEP - 1, settings$STEP)
 start_date <- as.Date(settings$START_DATE)
-train_periods <- seq(to=settings$LAST_WEEK - settings$HORIZON - settings$GAP + 1,
-                     by=settings$HORIZON,
-                     length.out=settings$N_SPLITS)
+
+data(orangeJuice, package="bayesm")
 
 oj_data <- orangeJuice$yx %>%
     complete(store, brand, week) %>%
-    mutate(week=yearweek(start_date + week*7)) %>%
+    group_by(store, brand) %>%
+    group_modify(~ {
+        pricevars <- grep("price", names(.x), value=TRUE)
+        thispricevar <- paste0("price", .y$brand)
+        best_other_price <- do.call(pmin, .x[setdiff(pricevars, thispricevar)])
+        .x$price <- .x[[thispricevar]]
+        .x$maxpricediff <- log(best_other_price/.x$price)
+        select(.x, week, logmove, deal, feat, price, maxpricediff)
+    }) %>%
+    ungroup() %>%
+    mutate(week=yearweek(start_date + week*7)) %>%  # do this separately because of tsibble/vctrs issues
     as_tsibble(index=week, key=c(store, brand))

 
 
 
-
Here are some glimpses of what the data looks like. The dependent variable is logmove, the logarithm of the total sales for a given brand and store, in a particular week.

+
Here are some glimpses of what the data looks like. The dependent variable is logmove, the logarithm of the total sales for a given brand and store, in a particular week. Note that we do not fill in the missing values in the data, as (with the exception of ETS) the modelling functions in the fable package can handle this innately.

 
 
 
@@ -318,39 +268,34 @@ oj_data <- orangeJuice$yx %>%
 
 

 
 

 
 
-
The time series plots for a small subset of brands and stores are shown below. We can make the following observations:

-
    
-
  • There appears to be little seasonal variation in sales (probably because Florida is a state without very different seasons). In any case, with less than 2 years of observations, the time series is not long enough for many model-fitting functions in the fable package to automatically estimate seasonal parameters.
    • 
-
  • While some store/brand combinations show weak trends over time, this is far from universal.
    • 
-
  • Different brands can exhibit very different behaviour, especially in terms of variation about the mean.
    • 
-
  • Many of the time series have missing values, indicating that the dataset is incomplete.
    • 
-

+
The time series plots for a small subset of brands and stores are shown below. It is clear that the statistical behaviour of the data varies by store and brand.

 
 
-
+
 
library(ggplot2)
 
 oj_data %>%
-    filter(store < 25, brand < 5) %>%
+    filter(store < 10, brand < 5) %>%
     ggplot(aes(x=week, y=logmove)) +
         geom_line() +
         scale_x_date(labels=NULL) +
         facet_grid(vars(store), vars(brand), labeller="label_both")

 
 
-

+

 
 
 
-
Finally, we split the dataset into separate samples for training and testing. The schema used is broadly time series cross-validation, whereby we train a model on data up to time \(t\), test it on data for times \(t+1\) to \(t+k\), then train on data up to time \(t+k\), test it on data for times \(t+k+1\) to \(t+2k\), and so on. In this specific case study, however, we introduce a small extra piece of complexity based on discussions with domain experts. We train a model on data up to week \(t\), then test it on week \(t+2\) to \(t+3\). Then we train on data up to week \(t+2\), and test it on weeks \(t+4\) to \(t+5\), and so on. There is thus always a gap of one week between the training and test samples. The reason for this is because in reality, inventory planning always takes some time; the gap allows store managers to prepare the stock based on the forecasted demand.

+
Finally, we split the dataset into separate samples for training and testing. The schema used is broadly time series cross-validation, whereby we train a model on data up to time \(t\), test it on data for times \(t+1\) to \(t+k\), then train on data up to time \(t+k\), test it on data for times \(t+k+1\) to \(t+2k\), and so on.

+
In this specific case study we introduce a small extra piece of complexity. We train a model on data up to month \(t\), then test it on months \(t+2\) to \(t+3\). Then we train on data up to month \(t+2\), and test it on months \(t+4\) to \(t+5\), and so on. Thus there is always a gap of one month between the training and test samples, a complicating factor introduced after discussions with domain experts.

 
 
-
+
 
subset_oj_data <- function(start, end)
 {
     start <- yearweek(start_date + start*7)
@@ -358,16 +303,16 @@ oj_data %>%
     filter(oj_data, week >= start, week <= end)
 }
 
-oj_train <- lapply(train_periods, function(i) subset_oj_data(settings$FIRST_WEEK, i))
-oj_test <- lapply(train_periods, function(i) subset_oj_data(i + settings$GAP, i + settings$GAP + settings$HORIZON - 1))
+oj_train <- lapply(train_periods, function(i) subset_oj_data(40, i))
+oj_test <- lapply(train_periods, function(i) subset_oj_data(i + 2, i + settings$STEP + 1))
 
-save(oj_train, oj_test, file=here::here("examples/grocery_sales/R/data.Rdata"))
+save(oj_train, oj_test, file="oj_data.Rdata")
 
 head(oj_train[[1]])

 
 

 
 

 
@@ -375,12 +320,12 @@ head(oj_train[[1]])
-
LS0tCnRpdGxlOiBEYXRhIHByZXBhcmF0aW9uCm91dHB1dDogaHRtbF9ub3RlYm9vawotLS0KCl9Db3B5cmlnaHQgKGMpIE1pY3Jvc29mdCBDb3Jwb3JhdGlvbi5fPGJyLz4KX0xpY2Vuc2VkIHVuZGVyIHRoZSBNSVQgTGljZW5zZS5fCgpJbiB0aGlzIG5vdGVib29rLCB3ZSBnZW5lcmF0ZSB0aGUgZGF0YXNldHMgdGhhdCB3aWxsIGJlIHVzZWQgZm9yIG1vZGVsIHRyYWluaW5nIGFuZCB2YWxpZGF0aW5nLiAKClRoZSBvcmFuZ2UganVpY2UgZGF0YXNldCBjb21lcyBmcm9tIHRoZSBiYXllc20gcGFja2FnZSwgYW5kIGdpdmVzIHByaWNpbmcgYW5kIHNhbGVzIGZpZ3VyZXMgb3ZlciB0aW1lIGZvciBhIHZhcmlldHkgb2Ygb3JhbmdlIGp1aWNlIGJyYW5kcyBpbiBzZXZlcmFsIHN0b3JlcyBpbiBGbG9yaWRhLiBSYXRoZXIgdGhhbiBpbnN0YWxsaW5nIHRoZSBlbnRpcmUgcGFja2FnZSAod2hpY2ggaXMgdmVyeSBjb21wbGV4KSwgd2UgZG93bmxvYWQgdGhlIGRhdGFzZXQgaXRzZWxmIGZyb20gdGhlIEdpdEh1YiBtaXJyb3Igb2YgdGhlIENSQU4gcmVwb3NpdG9yeS4KCmBgYHtyLCByZXN1bHRzPSJoaWRlIiwgbWVzc2FnZT1GQUxTRX0KIyBkb3dubG9hZCB0aGUgZGF0YSBmcm9tIHRoZSBHaXRIdWIgbWlycm9yIG9mIHRoZSBiYXllc20gcGFja2FnZSBzb3VyY2UKb2pmaWxlIDwtIHRlbXBmaWxlKGZpbGVleHQ9Ii5yZGEiKQpkb3dubG9hZC5maWxlKCJodHRwczovL2dpdGh1Yi5jb20vY3Jhbi9iYXllc20vcmF3L21hc3Rlci9kYXRhL29yYW5nZUp1aWNlLnJkYSIsIG9qZmlsZSkKbG9hZChvamZpbGUpCmZpbGUucmVtb3ZlKG9qZmlsZSkKYGBgCgpUaGUgZGF0YXNldCBnZW5lcmF0aW9uIHBhcmFtZXRlcnMgYXJlIG9idGFpbmVkIGZyb20gdGhlIGZpbGUgYG9qZGF0YV9mb3JlY2FzdF9zZXR0aW5ncy55YW1sYDsgeW91IGNhbiBtb2RpZnkgdGhhdCBmaWxlIHRvIHZhcnkgdGhlIGV4cGVyaW1lbnRhbCBzZXR1cC4gVGhlIHNldHRpbmdzIGFyZQoKfCBQYXJhbWV0ZXIgfCBEZXNjcmlwdGlvbiB8IERlZmF1bHQgfCAKfC0tLS0tLS0tLS0tfC0tLS0tLS0tLS0tLS18LS0tLS0tLS0tfAp8IGBOX1NQTElUU2AgfCBUaGUgbnVtYmVyIG9mIHNwbGl0cyB0byBtYWtlLiB8IDEwIHwKfCBgSE9SSVpPTmAgfCBUaGUgZm9yZWNhc3QgaG9yaXpvbiBmb3IgdGhlIHRlc3QgZGF0YXNldCBmb3IgZWFjaCBzcGxpdC4gfCAyIHwKfCBgR0FQYCB8IFRoZSBnYXAgaW4gd2Vla3MgZnJvbSB0aGUgZW5kIG9mIHRoZSB0cmFpbmluZyBwZXJpb2QgdG8gdGhlIHN0YXJ0IG9mIHRoZSB0ZXN0aW5nIHBlcmlvZDsgc2VlIGJlbG93LiB8IDIgfAp8IGBGSVJTVF9XRUVLYCB8IFRoZSBmaXJzdCB3ZWVrIG9mIGRhdGEgdG8gdXNlLiB8IDQwIHwKfCBgTEFTVF9XRUVLYCB8IFRoZSBsYXN0IHdlZWsgb2YgZGF0YSB0byB1c2UuIHwgMTU2IHwKfCBgU1RBUlRfREFURWAgfCBUaGUgYWN0dWFsIGNhbGVuZGFyIGRhdGUgZm9yIHRoZSBzdGFydCBvZiB0aGUgZmlyc3Qgd2VlayBpbiB0aGUgZGF0YS4gfCBgMTk4OS0wOS0xNGAgfAoKQSBjb21wbGljYXRpbmcgZmFjdG9yIGlzIHRoYXQgdGhlIGRhdGEgZG9lcyBub3QgaW5jbHVkZSBldmVyeSBwb3NzaWJsZSBjb21iaW5hdGlvbiBvZiBzdG9yZSwgYnJhbmQgYW5kIGRhdGUsIHNvIHdlIGhhdmUgdG8gcGFkIG91dCB0aGUgbWlzc2luZyByb3dzIHdpdGggYGNvbXBsZXRlYC4gSW4gYWRkaXRpb24sIG9uZSBzdG9yZS9icmFuZCBjb21iaW5hdGlvbiBoYXMgbm8gZGF0YSBiZXlvbmQgd2VlayAxNTY7IHdlIHRoZXJlZm9yZSBlbmQgdGhlIGFuYWx5c2lzIGF0IHRoaXMgd2Vlay4gV2UgYWxzbyBkbyBfbm90XyBmaWxsIGluIHRoZSBtaXNzaW5nIHZhbHVlcyBpbiB0aGUgZGF0YSwgYXMgbWFueSBvZiB0aGUgbW9kZWxsaW5nIGZ1bmN0aW9ucyBpbiB0aGUgZmFibGUgcGFja2FnZSBjYW4gaGFuZGxlIHRoaXMgaW5uYXRlbHkuCgpgYGB7ciwgcmVzdWx0cz0iaGlkZSIsIG1lc3NhZ2U9RkFMU0V9CmxpYnJhcnkodGlkeXIpCmxpYnJhcnkoZHBseXIpCmxpYnJhcnkodHNpYmJsZSkKbGlicmFyeShmZWFzdHMpCmxpYnJhcnkoZmFibGUpCgpzZXR0aW5ncyA8LSB5YW1sOjpyZWFkX3lhbWwoaGVyZTo6aGVyZSgiZXhhbXBsZXMvZ3JvY2VyeV9zYWxlcy9SL2ZvcmVjYXN0X3NldHRpbmdzLnlhbWwiKSkKc3RhcnRfZGF0ZSA8LSBhcy5EYXRlKHNldHRpbmdzJFNUQVJUX0RBVEUpCnRyYWluX3BlcmlvZHMgPC0gc2VxKHRvPXNldHRpbmdzJExBU1RfV0VFSyAtIHNldHRpbmdzJEhPUklaT04gLSBzZXR0aW5ncyRHQVAgKyAxLAogICAgICAgICAgICAgICAgICAgICBieT1zZXR0aW5ncyRIT1JJWk9OLAogICAgICAgICAgICAgICAgICAgICBsZW5ndGgub3V0PXNldHRpbmdzJE5fU1BMSVRTKQoKb2pfZGF0YSA8LSBvcmFuZ2VKdWljZSR5eCAlPiUKICAgIGNvbXBsZXRlKHN0b3JlLCBicmFuZCwgd2VlaykgJT4lCiAgICBtdXRhdGUod2Vlaz15ZWFyd2VlayhzdGFydF9kYXRlICsgd2Vlayo3KSkgJT4lCiAgICBhc190c2liYmxlKGluZGV4PXdlZWssIGtleT1jKHN0b3JlLCBicmFuZCkpCmBgYAoKSGVyZSBhcmUgc29tZSBnbGltcHNlcyBvZiB3aGF0IHRoZSBkYXRhIGxvb2tzIGxpa2UuIFRoZSBkZXBlbmRlbnQgdmFyaWFibGUgaXMgYGxvZ21vdmVgLCB0aGUgbG9nYXJpdGhtIG9mIHRoZSB0b3RhbCBzYWxlcyBmb3IgYSBnaXZlbiBicmFuZCBhbmQgc3RvcmUsIGluIGEgcGFydGljdWxhciB3ZWVrLgoKYGBge3J9CmhlYWQob2pfZGF0YSkKYGBgCgpUaGUgdGltZSBzZXJpZXMgcGxvdHMgZm9yIGEgc21hbGwgc3Vic2V0IG9mIGJyYW5kcyBhbmQgc3RvcmVzIGFyZSBzaG93biBiZWxvdy4gV2UgY2FuIG1ha2UgdGhlIGZvbGxvd2luZyBvYnNlcnZhdGlvbnM6CgotIFRoZXJlIGFwcGVhcnMgdG8gYmUgbGl0dGxlIHNlYXNvbmFsIHZhcmlhdGlvbiBpbiBzYWxlcyAocHJvYmFibHkgYmVjYXVzZSBGbG9yaWRhIGlzIGEgc3RhdGUgd2l0aG91dCB2ZXJ5IGRpZmZlcmVudCBzZWFzb25zKS4gSW4gYW55IGNhc2UsIHdpdGggbGVzcyB0aGFuIDIgeWVhcnMgb2Ygb2JzZXJ2YXRpb25zLCB0aGUgdGltZSBzZXJpZXMgaXMgbm90IGxvbmcgZW5vdWdoIGZvciBtYW55IG1vZGVsLWZpdHRpbmcgZnVuY3Rpb25zIGluIHRoZSBmYWJsZSBwYWNrYWdlIHRvIGF1dG9tYXRpY2FsbHkgZXN0aW1hdGUgc2Vhc29uYWwgcGFyYW1ldGVycy4KLSBXaGlsZSBzb21lIHN0b3JlL2JyYW5kIGNvbWJpbmF0aW9ucyBzaG93IHdlYWsgdHJlbmRzIG92ZXIgdGltZSwgdGhpcyBpcyBmYXIgZnJvbSB1bml2ZXJzYWwuCi0gRGlmZmVyZW50IGJyYW5kcyBjYW4gZXhoaWJpdCB2ZXJ5IGRpZmZlcmVudCBiZWhhdmlvdXIsIGVzcGVjaWFsbHkgaW4gdGVybXMgb2YgdmFyaWF0aW9uIGFib3V0IHRoZSBtZWFuLgotIE1hbnkgb2YgdGhlIHRpbWUgc2VyaWVzIGhhdmUgbWlzc2luZyB2YWx1ZXMsIGluZGljYXRpbmcgdGhhdCB0aGUgZGF0YXNldCBpcyBpbmNvbXBsZXRlLgoKCmBgYHtyLCBmaWcuaGVpZ2h0PTEwfQpsaWJyYXJ5KGdncGxvdDIpCgpval9kYXRhICU+JQogICAgZmlsdGVyKHN0b3JlIDwgMjUsIGJyYW5kIDwgNSkgJT4lCiAgICBnZ3Bsb3QoYWVzKHg9d2VlaywgeT1sb2dtb3ZlKSkgKwogICAgICAgIGdlb21fbGluZSgpICsKICAgICAgICBzY2FsZV94X2RhdGUobGFiZWxzPU5VTEwpICsKICAgICAgICBmYWNldF9ncmlkKHZhcnMoc3RvcmUpLCB2YXJzKGJyYW5kKSwgbGFiZWxsZXI9ImxhYmVsX2JvdGgiKQpgYGAKCkZpbmFsbHksIHdlIHNwbGl0IHRoZSBkYXRhc2V0IGludG8gc2VwYXJhdGUgc2FtcGxlcyBmb3IgdHJhaW5pbmcgYW5kIHRlc3RpbmcuIFRoZSBzY2hlbWEgdXNlZCBpcyBicm9hZGx5IHRpbWUgc2VyaWVzIGNyb3NzLXZhbGlkYXRpb24sIHdoZXJlYnkgd2UgdHJhaW4gYSBtb2RlbCBvbiBkYXRhIHVwIHRvIHRpbWUgJHQkLCB0ZXN0IGl0IG9uIGRhdGEgZm9yIHRpbWVzICR0KzEkIHRvICR0K2skLCB0aGVuIHRyYWluIG9uIGRhdGEgdXAgdG8gdGltZSAkdCtrJCwgdGVzdCBpdCBvbiBkYXRhIGZvciB0aW1lcyAkdCtrKzEkIHRvICR0KzJrJCwgYW5kIHNvIG9uLiBJbiB0aGlzIHNwZWNpZmljIGNhc2Ugc3R1ZHksIGhvd2V2ZXIsIHdlIGludHJvZHVjZSBhIHNtYWxsIGV4dHJhIHBpZWNlIG9mIGNvbXBsZXhpdHkgYmFzZWQgb24gZGlzY3Vzc2lvbnMgd2l0aCBkb21haW4gZXhwZXJ0cy4gV2UgdHJhaW4gYSBtb2RlbCBvbiBkYXRhIHVwIHRvIHdlZWsgJHQkLCB0aGVuIHRlc3QgaXQgb24gd2VlayAkdCsyJCB0byAkdCszJC4gVGhlbiB3ZSB0cmFpbiBvbiBkYXRhIHVwIHRvIHdlZWsgJHQrMiQsIGFuZCB0ZXN0IGl0IG9uIHdlZWtzICR0KzQkIHRvICR0KzUkLCBhbmQgc28gb24uIFRoZXJlIGlzIHRodXMgYWx3YXlzIGEgZ2FwIG9mIG9uZSB3ZWVrIGJldHdlZW4gdGhlIHRyYWluaW5nIGFuZCB0ZXN0IHNhbXBsZXMuIFRoZSByZWFzb24gZm9yIHRoaXMgaXMgYmVjYXVzZSBpbiByZWFsaXR5LCBpbnZlbnRvcnkgcGxhbm5pbmcgYWx3YXlzIHRha2VzIHNvbWUgdGltZTsgdGhlIGdhcCBhbGxvd3Mgc3RvcmUgbWFuYWdlcnMgdG8gcHJlcGFyZSB0aGUgc3RvY2sgYmFzZWQgb24gdGhlIGZvcmVjYXN0ZWQgZGVtYW5kLgoKYGBge3J9CnN1YnNldF9val9kYXRhIDwtIGZ1bmN0aW9uKHN0YXJ0LCBlbmQpCnsKICAgIHN0YXJ0IDwtIHllYXJ3ZWVrKHN0YXJ0X2RhdGUgKyBzdGFydCo3KQogICAgZW5kIDwtIHllYXJ3ZWVrKHN0YXJ0X2RhdGUgKyBlbmQqNykKICAgIGZpbHRlcihval9kYXRhLCB3ZWVrID49IHN0YXJ0LCB3ZWVrIDw9IGVuZCkKfQoKb2pfdHJhaW4gPC0gbGFwcGx5KHRyYWluX3BlcmlvZHMsIGZ1bmN0aW9uKGkpIHN1YnNldF9val9kYXRhKHNldHRpbmdzJEZJUlNUX1dFRUssIGkpKQpval90ZXN0IDwtIGxhcHBseSh0cmFpbl9wZXJpb2RzLCBmdW5jdGlvbihpKSBzdWJzZXRfb2pfZGF0YShpICsgc2V0dGluZ3MkR0FQLCBpICsgc2V0dGluZ3MkR0FQICsgc2V0dGluZ3MkSE9SSVpPTiAtIDEpKQoKc2F2ZShval90cmFpbiwgb2pfdGVzdCwgZmlsZT1oZXJlOjpoZXJlKCJleGFtcGxlcy9ncm9jZXJ5X3NhbGVzL1IvZGF0YS5SZGF0YSIpKQoKaGVhZChval90cmFpbltbMV1dKQoKaGVhZChval90ZXN0W1sxXV0pCmBgYAo=
+
LS0tCnRpdGxlOiBEYXRhIHByZXBhcmF0aW9uCm91dHB1dDogaHRtbF9ub3RlYm9vawotLS0KCmBgYHtyLCBlY2hvPUZBTFNFLCByZXN1bHRzPSJoaWRlIiwgbWVzc2FnZT1GQUxTRX0KbGlicmFyeSh0aWR5cikKbGlicmFyeShkcGx5cikKbGlicmFyeSh0c2liYmxlKQpsaWJyYXJ5KGZlYXN0cykKbGlicmFyeShmYWJsZSkKYGBgCgpJbiB0aGlzIG5vdGVib29rLCB3ZSBnZW5lcmF0ZSB0aGUgZGF0YXNldHMgdGhhdCB3aWxsIGJlIHVzZWQgZm9yIG1vZGVsIHRyYWluaW5nIGFuZCB2YWxpZGF0aW5nLiBUaGUgZXhwZXJpbWVudCBwYXJhbWV0ZXJzIGFyZSBvYnRhaW5lZCBmcm9tIHRoZSBmaWxlIGBvamRhdGFfZm9yZWNhc3Rfc2V0dGluZ3MuanNvbmA7IHlvdSBjYW4gbW9kaWZ5IHRoYXQgZmlsZSB0byB2YXJ5IHRoZSBleHBlcmltZW50YWwgc2V0dXAsIG9yIGp1c3QgZWRpdCB0aGUgdmFsdWVzIGluIHRoaXMgbm90ZWJvb2suCgpUaGUgb3JhbmdlIGp1aWNlIGRhdGFzZXQgY29tZXMgZnJvbSB0aGUgYmF5ZXNtIHBhY2thZ2UsIGFuZCBnaXZlcyBwcmljaW5nIGFuZCBzYWxlcyBmaWd1cmVzIG92ZXIgdGltZSBmb3IgYSB2YXJpZXR5IG9mIG9yYW5nZSBqdWljZSBicmFuZHMgaW4gc2V2ZXJhbCBzdG9yZXMgaW4gRmxvcmlkYS4KCkEgY29tcGxpY2F0aW5nIGZhY3RvciBpcyB0aGF0IHRoZSBkYXRhIGlzIGluIGEgaHlicmlkIG9mIGxvbmcgYW5kIHdpZGUgZm9ybWF0OiB3aGlsZSB0aGUgc2FsZXMgZmlndXJlcyBhcmUgbG9uZyAob25lIGNvbHVtbiBvZiBzYWxlcyBkYXRhIGZvciBldmVyeSBzdG9yZSBhbmQgYnJhbmQpLCB0aGUgcHJpY2VzIGFyZSB3aWRlIChvbmUgcHJpY2UgY29sdW1uIGZvciBlYWNoIGJyYW5kKS4gVGhlcmVmb3JlIHdlIG5lZWQgdG8gcmVzaGFwZSB0aGUgZGF0YSBpZiB3ZSB3YW50IHRvIHVzZSBwcmljZXMgZm9yIG1vZGVsbGluZy4gQXMgcGFydCBvZiB0aGlzLCB3ZSBhbHNvIGNvbXB1dGUgYSBuZXcgY29sdW1uIGBtYXhwcmljZWRpZmZgOiB0aGlzIHJlcHJlc2VudHMgdGhlIGxvZy1yYXRpbyBvZiB0aGUgcHJpY2Ugb2YgdGhpcyBicmFuZCBjb21wYXJlZCB0byB0aGUgYmVzdCBjb21wZXRpbmcgcHJpY2UuIEEgcG9zaXRpdmUgYG1heHByaWNlZGlmZmAgbWVhbnMgdGhpcyBicmFuZCBpcyBjaGVhcGVyIHRoYW4gYWxsIHRoZSBvdGhlciBicmFuZHMsIGFuZCBhIG5lZ2F0aXZlIGBtYXhwcmljZWRpZmZgIG1lYW5zIGl0IGlzIG1vcmUgZXhwZW5zaXZlLgoKYGBge3J9CnNldHRpbmdzIDwtIGpzb25saXRlOjpmcm9tSlNPTigib2pkYXRhX2ZvcmVjYXN0X3NldHRpbmdzLmpzb24iKQoKdHJhaW5fcGVyaW9kcyA8LSBzZXEoc2V0dGluZ3MkVFJBSU5fV0lORE9XLCAxNjAgLSBzZXR0aW5ncyRTVEVQIC0gMSwgc2V0dGluZ3MkU1RFUCkKc3RhcnRfZGF0ZSA8LSBhcy5EYXRlKHNldHRpbmdzJFNUQVJUX0RBVEUpCgpkYXRhKG9yYW5nZUp1aWNlLCBwYWNrYWdlPSJiYXllc20iKQoKb2pfZGF0YSA8LSBvcmFuZ2VKdWljZSR5eCAlPiUKICAgIGNvbXBsZXRlKHN0b3JlLCBicmFuZCwgd2VlaykgJT4lCiAgICBncm91cF9ieShzdG9yZSwgYnJhbmQpICU+JQogICAgZ3JvdXBfbW9kaWZ5KH4gewogICAgICAgIHByaWNldmFycyA8LSBncmVwKCJwcmljZSIsIG5hbWVzKC54KSwgdmFsdWU9VFJVRSkKICAgICAgICB0aGlzcHJpY2V2YXIgPC0gcGFzdGUwKCJwcmljZSIsIC55JGJyYW5kKQogICAgICAgIGJlc3Rfb3RoZXJfcHJpY2UgPC0gZG8uY2FsbChwbWluLCAueFtzZXRkaWZmKHByaWNldmFycywgdGhpc3ByaWNldmFyKV0pCiAgICAgICAgLngkcHJpY2UgPC0gLnhbW3RoaXNwcmljZXZhcl1dCiAgICAgICAgLngkbWF4cHJpY2VkaWZmIDwtIGxvZyhiZXN0X290aGVyX3ByaWNlLy54JHByaWNlKQogICAgICAgIHNlbGVjdCgueCwgd2VlaywgbG9nbW92ZSwgZGVhbCwgZmVhdCwgcHJpY2UsIG1heHByaWNlZGlmZikKICAgIH0pICU+JQogICAgdW5ncm91cCgpICU+JQogICAgbXV0YXRlKHdlZWs9eWVhcndlZWsoc3RhcnRfZGF0ZSArIHdlZWsqNykpICU+JSAgIyBkbyB0aGlzIHNlcGFyYXRlbHkgYmVjYXVzZSBvZiB0c2liYmxlL3ZjdHJzIGlzc3VlcwogICAgYXNfdHNpYmJsZShpbmRleD13ZWVrLCBrZXk9YyhzdG9yZSwgYnJhbmQpKQpgYGAKCkhlcmUgYXJlIHNvbWUgZ2xpbXBzZXMgb2Ygd2hhdCB0aGUgZGF0YSBsb29rcyBsaWtlLiBUaGUgZGVwZW5kZW50IHZhcmlhYmxlIGlzIGBsb2dtb3ZlYCwgdGhlIGxvZ2FyaXRobSBvZiB0aGUgdG90YWwgc2FsZXMgZm9yIGEgZ2l2ZW4gYnJhbmQgYW5kIHN0b3JlLCBpbiBhIHBhcnRpY3VsYXIgd2Vlay4gTm90ZSB0aGF0IHdlIGRvIF9ub3RfIGZpbGwgaW4gdGhlIG1pc3NpbmcgdmFsdWVzIGluIHRoZSBkYXRhLCBhcyAod2l0aCB0aGUgZXhjZXB0aW9uIG9mIGBFVFNgKSB0aGUgbW9kZWxsaW5nIGZ1bmN0aW9ucyBpbiB0aGUgZmFibGUgcGFja2FnZSBjYW4gaGFuZGxlIHRoaXMgaW5uYXRlbHkuCgpgYGB7cn0KaGVhZChval9kYXRhKQpgYGAKClRoZSB0aW1lIHNlcmllcyBwbG90cyBmb3IgYSBzbWFsbCBzdWJzZXQgb2YgYnJhbmRzIGFuZCBzdG9yZXMgYXJlIHNob3duIGJlbG93LiBJdCBpcyBjbGVhciB0aGF0IHRoZSBzdGF0aXN0aWNhbCBiZWhhdmlvdXIgb2YgdGhlIGRhdGEgdmFyaWVzIGJ5IHN0b3JlIGFuZCBicmFuZC4KCmBgYHtyfQpsaWJyYXJ5KGdncGxvdDIpCgpval9kYXRhICU+JQogICAgZmlsdGVyKHN0b3JlIDwgMTAsIGJyYW5kIDwgNSkgJT4lCiAgICBnZ3Bsb3QoYWVzKHg9d2VlaywgeT1sb2dtb3ZlKSkgKwogICAgICAgIGdlb21fbGluZSgpICsKICAgICAgICBzY2FsZV94X2RhdGUobGFiZWxzPU5VTEwpICsKICAgICAgICBmYWNldF9ncmlkKHZhcnMoc3RvcmUpLCB2YXJzKGJyYW5kKSwgbGFiZWxsZXI9ImxhYmVsX2JvdGgiKQpgYGAKCkZpbmFsbHksIHdlIHNwbGl0IHRoZSBkYXRhc2V0IGludG8gc2VwYXJhdGUgc2FtcGxlcyBmb3IgdHJhaW5pbmcgYW5kIHRlc3RpbmcuIFRoZSBzY2hlbWEgdXNlZCBpcyBicm9hZGx5IHRpbWUgc2VyaWVzIGNyb3NzLXZhbGlkYXRpb24sIHdoZXJlYnkgd2UgdHJhaW4gYSBtb2RlbCBvbiBkYXRhIHVwIHRvIHRpbWUgJHQkLCB0ZXN0IGl0IG9uIGRhdGEgZm9yIHRpbWVzICR0KzEkIHRvICR0K2skLCB0aGVuIHRyYWluIG9uIGRhdGEgdXAgdG8gdGltZSAkdCtrJCwgdGVzdCBpdCBvbiBkYXRhIGZvciB0aW1lcyAkdCtrKzEkIHRvICR0KzJrJCwgYW5kIHNvIG9uLgoKSW4gdGhpcyBzcGVjaWZpYyBjYXNlIHN0dWR5IHdlIGludHJvZHVjZSBhIHNtYWxsIGV4dHJhIHBpZWNlIG9mIGNvbXBsZXhpdHkuIFdlIHRyYWluIGEgbW9kZWwgb24gZGF0YSB1cCB0byBtb250aCAkdCQsIHRoZW4gdGVzdCBpdCBvbiBtb250aHMgJHQrMiQgdG8gJHQrMyQuIFRoZW4gd2UgdHJhaW4gb24gZGF0YSB1cCB0byBtb250aCAkdCsyJCwgYW5kIHRlc3QgaXQgb24gbW9udGhzICR0KzQkIHRvICR0KzUkLCBhbmQgc28gb24uIFRodXMgdGhlcmUgaXMgYWx3YXlzIGEgZ2FwIG9mIG9uZSBtb250aCBiZXR3ZWVuIHRoZSB0cmFpbmluZyBhbmQgdGVzdCBzYW1wbGVzLCBhIGNvbXBsaWNhdGluZyBmYWN0b3IgaW50cm9kdWNlZCBhZnRlciBkaXNjdXNzaW9ucyB3aXRoIGRvbWFpbiBleHBlcnRzLgoKYGBge3J9CnN1YnNldF9val9kYXRhIDwtIGZ1bmN0aW9uKHN0YXJ0LCBlbmQpCnsKICAgIHN0YXJ0IDwtIHllYXJ3ZWVrKHN0YXJ0X2RhdGUgKyBzdGFydCo3KQogICAgZW5kIDwtIHllYXJ3ZWVrKHN0YXJ0X2RhdGUgKyBlbmQqNykKICAgIGZpbHRlcihval9kYXRhLCB3ZWVrID49IHN0YXJ0LCB3ZWVrIDw9IGVuZCkKfQoKb2pfdHJhaW4gPC0gbGFwcGx5KHRyYWluX3BlcmlvZHMsIGZ1bmN0aW9uKGkpIHN1YnNldF9val9kYXRhKDQwLCBpKSkKb2pfdGVzdCA8LSBsYXBwbHkodHJhaW5fcGVyaW9kcywgZnVuY3Rpb24oaSkgc3Vic2V0X29qX2RhdGEoaSArIDIsIGkgKyBzZXR0aW5ncyRTVEVQICsgMSkpCgpzYXZlKG9qX3RyYWluLCBval90ZXN0LCBmaWxlPSJval9kYXRhLlJkYXRhIikKCmhlYWQob2pfdHJhaW5bWzFdXSkKCmhlYWQob2pfdGVzdFtbMV1dKQpgYGAK
diff --git a/R/orange_juice/02_simplemodels.Rmd b/R/orange_juice/02_simplemodels.Rmd new file mode 100644 index 00000000..65ff50d2 --- /dev/null +++ b/R/orange_juice/02_simplemodels.Rmd @@ -0,0 +1,74 @@ +--- +title: Simple models +output: html_notebook +encoding: utf8 +--- + +```{r, echo=FALSE, results="hide", message=FALSE} +library(tidyr) +library(dplyr) +library(tsibble) +library(feasts) +library(fable) +``` + +We fit some simple models to the orange juice data. One model is fit for each combination of store and brand. + +- `mean`: This is just a simple mean. +- `naive`: A random walk model without any other components. This amounts to setting all forecast values to the last observed value. +- `drift`: This adjusts the `naive` model to incorporate a trend. +- `arima`: An ARIMA model with the parameter values estimated from the data. +- `ets`: An exponentially weighted model, again with parameter values estimated from the data. + +Note that the model training process is embarrassingly parallel on 3 levels: + +- We have multiple independent training datasets; +- For which we fit multiple independent models; +- Within which we have independent sub-models for each store and brand. + +This lets us speed up the training significantly. While the `fable::model` function can fit multiple models in parallel, we will run it sequentially here and instead parallelise by dataset. This avoids contention for cores, and also results in the simplest code. + +```{r, results="hide"} +load("oj_data.Rdata") + +ncores <- max(2, parallel::detectCores(logical=FALSE) - 2) +cl <- parallel::makeCluster(ncores) +parallel::clusterEvalQ(cl, +{ + library(tidyr) + library(feasts) + library(fable) + library(tsibble) +}) +``` + +First, we fit the models that can innately handle missing values. + +```{r} +oj_modelset <- parallel::parLapply(cl, oj_train, function(df) +{ + model(df, + mean=MEAN(logmove), + naive=NAIVE(logmove), + drift=RW(logmove ~ drift()), + arima=ARIMA(logmove ~ pdq() + PDQ(0, 0, 0)) + ) +}) +``` +Next, we fit models that require manual imputation (ETS). + +```{r} +oj_modelset_ets <- parallel::parLapply(cl, oj_train, function(df) +{ + df %>% + fill(everything()) %>% + model(ets=ETS(logmove ~ error("A") + trend("A") + season("N"))) +}) + +parallel::stopCluster(cl) +save(oj_modelset, oj_modelset_ets, file="oj_modelset.Rdata") + +head(oj_modelset[[1]]) +head(oj_modelset_ets[[1]]) +``` + diff --git a/examples/grocery_sales/R/02b_prophet_models.nb.html b/R/orange_juice/02_simplemodels.nb.html similarity index 98% rename from examples/grocery_sales/R/02b_prophet_models.nb.html rename to R/orange_juice/02_simplemodels.nb.html index 4bf9ae60..735d3223 100644 --- a/examples/grocery_sales/R/02b_prophet_models.nb.html +++ b/R/orange_juice/02_simplemodels.nb.html @@ -11,7 +11,7 @@ -Prophet models +Simple models @@ -220,76 +220,98 @@ summary { -

Prophet models

+

Simple models

-

Copyright (c) Microsoft Corporation.
Licensed under the MIT License.

-

This notebook builds a forecasting model using the Prophet algorithm. Prophet is a time series model developed by Facebook that is designed to be simple for non-experts to use, yet flexible and powerful.

-
-

Prophet is a procedure for forecasting time series data based on an additive model where non-linear trends are fit with yearly, weekly, and daily seasonality, plus holiday effects. It works best with time series that have strong seasonal effects and several seasons of historical data. Prophet is robust to missing data and shifts in the trend, and typically handles outliers well.

-
-

Here, we will use the fable.prophet package which provides a tidyverts frontend to the prophet package itself. As with ETS, prophet does not support time series with missing values, so we again impute them using the ARIMA model forecasts.

+

We fit some simple models to the orange juice data. One model is fit for each combination of store and brand.

+ +

Note that the model training process is embarrassingly parallel on 3 levels:

+ +

This lets us speed up the training significantly. While the fable::model function can fit multiple models in parallel, we will run it sequentially here and instead parallelise by dataset. This avoids contention for cores, and also results in the simplest code.

- -
srcdir <- here::here("R_utils")
-for(src in dir(srcdir, full.names=TRUE)) source(src)
+
+
load("oj_data.Rdata")
 
-load_objects("grocery_sales", "data_reg.Rdata")
-load_objects("grocery_sales", "model_basic.Rdata")
-
-cl <- make_cluster(libs=c("tidyr", "dplyr", "fable", "tsibble", "feasts", "prophet", "fable.prophet"))
-
-oj_modelset_pr <- parallel::clusterMap(cl, function(df, basicmod)
+ncores <- max(2, parallel::detectCores(logical=FALSE) - 2)
+cl <- parallel::makeCluster(ncores)
+parallel::clusterEvalQ(cl,
+{
+    library(tidyr)
+    library(feasts)
+    library(fable)
+    library(tsibble)
+})

+
+
+
+
First, we fit the models that can innately handle missing values.

+
+
+
+
oj_modelset <- parallel::parLapply(cl, oj_train, function(df)
+{
+    model(df,
+        mean=MEAN(logmove),
+        naive=NAIVE(logmove),
+        drift=RW(logmove ~ drift()),
+        arima=ARIMA(logmove ~ pdq() + PDQ(0, 0, 0))
+    )
+})

+
+
+
+
Next, we fit models that require manual imputation (ETS).

+
+
+
+
oj_modelset_ets <- parallel::parLapply(cl, oj_train, function(df)
 {
-    df$logmove <- interpolate(select(basicmod, -c(mean, naive, drift)), df)$logmove
     df %>%
-        group_by(store, brand) %>%
-        fill(deal:maxpricediff, .direction="downup") %>%
-        model(
-            pr=prophet(logmove ~ deal + feat + price + maxpricediff),
+        fill(everything()) %>%
+        model(ets=ETS(logmove ~ error("A") + trend("A") + season("N")))
+})
 
-            pr_tune=prophet(logmove ~ deal + feat + price + maxpricediff +
-                growth(n_changepoints=2) + season(period=52, order=5, prior_scale=2)),
+parallel::stopCluster(cl)
+save(oj_modelset, oj_modelset_ets, file="oj_modelset.Rdata")
 
-            .safely=FALSE
-        )
-}, oj_trainreg, oj_modelset_basic)
-
-oj_fcast_pr <- parallel::clusterMap(cl, function(mable, newdata, fcast_func)
-{
-    newdata <- newdata %>%
-        fill(deal:maxpricediff, .direction="downup")
-    fcast_func(mable, newdata)
-}, oj_modelset_pr, oj_testreg, MoreArgs=list(fcast_func=get_forecasts))
-
-destroy_cluster(cl)
-
-save_objects(oj_modelset_pr, oj_fcast_pr,
-             example="grocery_sales", file="model_pr.Rdata")
-
-do.call(rbind, oj_fcast_pr) %>%
-    mutate_at(-(1:3), exp) %>%
-    eval_forecasts()

+head(oj_modelset[[1]])
+
+ +
head(oj_modelset_ets[[1]])
+ +
+
-

It appears that Prophet does not do better than the simple ARIMA model with regression variables. This is possibly because the dataset does not have a strong time series nature: there is no seasonality, and only weak or nonexistent trends. These are features which the Prophet algorithm is designed to detect, and their absence means that there would be little advantage in using it.

-
LS0tCnRpdGxlOiBQcm9waGV0IG1vZGVscwpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sKLS0tCgpfQ29weXJpZ2h0IChjKSBNaWNyb3NvZnQgQ29ycG9yYXRpb24uXzxici8+Cl9MaWNlbnNlZCB1bmRlciB0aGUgTUlUIExpY2Vuc2UuXwoKYGBge3IsIGVjaG89RkFMU0UsIHJlc3VsdHM9ImhpZGUiLCBtZXNzYWdlPUZBTFNFfQpsaWJyYXJ5KHRpZHlyKQpsaWJyYXJ5KGRwbHlyKQpsaWJyYXJ5KHRzaWJibGUpCmxpYnJhcnkoZmVhc3RzKQpsaWJyYXJ5KGZhYmxlKQpsaWJyYXJ5KHByb3BoZXQpCmxpYnJhcnkoZmFibGUucHJvcGhldCkKYGBgCgpUaGlzIG5vdGVib29rIGJ1aWxkcyBhIGZvcmVjYXN0aW5nIG1vZGVsIHVzaW5nIHRoZSBbUHJvcGhldF0oaHR0cHM6Ly9mYWNlYm9vay5naXRodWIuaW8vcHJvcGhldC8pIGFsZ29yaXRobS4gUHJvcGhldCBpcyBhIHRpbWUgc2VyaWVzIG1vZGVsIGRldmVsb3BlZCBieSBGYWNlYm9vayB0aGF0IGlzIGRlc2lnbmVkIHRvIGJlIHNpbXBsZSBmb3Igbm9uLWV4cGVydHMgdG8gdXNlLCB5ZXQgZmxleGlibGUgYW5kIHBvd2VyZnVsLgoKPiBQcm9waGV0IGlzIGEgcHJvY2VkdXJlIGZvciBmb3JlY2FzdGluZyB0aW1lIHNlcmllcyBkYXRhIGJhc2VkIG9uIGFuIGFkZGl0aXZlIG1vZGVsIHdoZXJlIG5vbi1saW5lYXIgdHJlbmRzIGFyZSBmaXQgd2l0aCB5ZWFybHksIHdlZWtseSwgYW5kIGRhaWx5IHNlYXNvbmFsaXR5LCBwbHVzIGhvbGlkYXkgZWZmZWN0cy4gSXQgd29ya3MgYmVzdCB3aXRoIHRpbWUgc2VyaWVzIHRoYXQgaGF2ZSBzdHJvbmcgc2Vhc29uYWwgZWZmZWN0cyBhbmQgc2V2ZXJhbCBzZWFzb25zIG9mIGhpc3RvcmljYWwgZGF0YS4gUHJvcGhldCBpcyByb2J1c3QgdG8gbWlzc2luZyBkYXRhIGFuZCBzaGlmdHMgaW4gdGhlIHRyZW5kLCBhbmQgdHlwaWNhbGx5IGhhbmRsZXMgb3V0bGllcnMgd2VsbC4KCkhlcmUsIHdlIHdpbGwgdXNlIHRoZSBmYWJsZS5wcm9waGV0IHBhY2thZ2Ugd2hpY2ggcHJvdmlkZXMgYSB0aWR5dmVydHMgZnJvbnRlbmQgdG8gdGhlIHByb3BoZXQgcGFja2FnZSBpdHNlbGYuIEFzIHdpdGggRVRTLCBwcm9waGV0IGRvZXMgbm90IHN1cHBvcnQgdGltZSBzZXJpZXMgd2l0aCBtaXNzaW5nIHZhbHVlcywgc28gd2UgYWdhaW4gaW1wdXRlIHRoZW0gdXNpbmcgdGhlIEFSSU1BIG1vZGVsIGZvcmVjYXN0cy4KCmBgYHtyfQpzcmNkaXIgPC0gaGVyZTo6aGVyZSgiUl91dGlscyIpCmZvcihzcmMgaW4gZGlyKHNyY2RpciwgZnVsbC5uYW1lcz1UUlVFKSkgc291cmNlKHNyYykKCmxvYWRfb2JqZWN0cygiZ3JvY2VyeV9zYWxlcyIsICJkYXRhX3JlZy5SZGF0YSIpCmxvYWRfb2JqZWN0cygiZ3JvY2VyeV9zYWxlcyIsICJtb2RlbF9iYXNpYy5SZGF0YSIpCgpjbCA8LSBtYWtlX2NsdXN0ZXIobGlicz1jKCJ0aWR5ciIsICJkcGx5ciIsICJmYWJsZSIsICJ0c2liYmxlIiwgImZlYXN0cyIsICJwcm9waGV0IiwgImZhYmxlLnByb3BoZXQiKSkKCm9qX21vZGVsc2V0X3ByIDwtIHBhcmFsbGVsOjpjbHVzdGVyTWFwKGNsLCBmdW5jdGlvbihkZiwgYmFzaWNtb2QpCnsKICAgIGRmJGxvZ21vdmUgPC0gaW50ZXJwb2xhdGUoc2VsZWN0KGJhc2ljbW9kLCAtYyhtZWFuLCBuYWl2ZSwgZHJpZnQpKSwgZGYpJGxvZ21vdmUKICAgIGRmICU+JQogICAgICAgIGdyb3VwX2J5KHN0b3JlLCBicmFuZCkgJT4lCiAgICAgICAgZmlsbChkZWFsOm1heHByaWNlZGlmZiwgLmRpcmVjdGlvbj0iZG93bnVwIikgJT4lCiAgICAgICAgbW9kZWwoCiAgICAgICAgICAgIHByPXByb3BoZXQobG9nbW92ZSB+IGRlYWwgKyBmZWF0ICsgcHJpY2UgKyBtYXhwcmljZWRpZmYpLAoKICAgICAgICAgICAgcHJfdHVuZT1wcm9waGV0KGxvZ21vdmUgfiBkZWFsICsgZmVhdCArIHByaWNlICsgbWF4cHJpY2VkaWZmICsKICAgICAgICAgICAgICAgIGdyb3d0aChuX2NoYW5nZXBvaW50cz0yKSArIHNlYXNvbihwZXJpb2Q9NTIsIG9yZGVyPTUsIHByaW9yX3NjYWxlPTIpKSwKCiAgICAgICAgICAgIC5zYWZlbHk9RkFMU0UKICAgICAgICApCn0sIG9qX3RyYWlucmVnLCBval9tb2RlbHNldF9iYXNpYykKCm9qX2ZjYXN0X3ByIDwtIHBhcmFsbGVsOjpjbHVzdGVyTWFwKGNsLCBmdW5jdGlvbihtYWJsZSwgbmV3ZGF0YSwgZmNhc3RfZnVuYykKewogICAgbmV3ZGF0YSA8LSBuZXdkYXRhICU+JQogICAgICAgIGZpbGwoZGVhbDptYXhwcmljZWRpZmYsIC5kaXJlY3Rpb249ImRvd251cCIpCiAgICBmY2FzdF9mdW5jKG1hYmxlLCBuZXdkYXRhKQp9LCBval9tb2RlbHNldF9wciwgb2pfdGVzdHJlZywgTW9yZUFyZ3M9bGlzdChmY2FzdF9mdW5jPWdldF9mb3JlY2FzdHMpKQoKZGVzdHJveV9jbHVzdGVyKGNsKQoKc2F2ZV9vYmplY3RzKG9qX21vZGVsc2V0X3ByLCBval9mY2FzdF9wciwKICAgICAgICAgICAgIGV4YW1wbGU9Imdyb2Nlcnlfc2FsZXMiLCBmaWxlPSJtb2RlbF9wci5SZGF0YSIpCgpkby5jYWxsKHJiaW5kLCBval9mY2FzdF9wcikgJT4lCiAgICBtdXRhdGVfYXQoLSgxOjMpLCBleHApICU+JQogICAgZXZhbF9mb3JlY2FzdHMoKQpgYGAKCkl0IGFwcGVhcnMgdGhhdCBQcm9waGV0IGRvZXMgX25vdF8gZG8gYmV0dGVyIHRoYW4gdGhlIHNpbXBsZSBBUklNQSBtb2RlbCB3aXRoIHJlZ3Jlc3Npb24gdmFyaWFibGVzLiBUaGlzIGlzIHBvc3NpYmx5IGJlY2F1c2UgdGhlIGRhdGFzZXQgZG9lcyBub3QgaGF2ZSBhIHN0cm9uZyB0aW1lIHNlcmllcyBuYXR1cmU6IHRoZXJlIGlzIG5vIHNlYXNvbmFsaXR5LCBhbmQgb25seSB3ZWFrIG9yIG5vbmV4aXN0ZW50IHRyZW5kcy4gVGhlc2UgYXJlIGZlYXR1cmVzIHdoaWNoIHRoZSBQcm9waGV0IGFsZ29yaXRobSBpcyBkZXNpZ25lZCB0byBkZXRlY3QsIGFuZCB0aGVpciBhYnNlbmNlIG1lYW5zIHRoYXQgdGhlcmUgd291bGQgYmUgbGl0dGxlIGFkdmFudGFnZSBpbiB1c2luZyBpdC4K
+
LS0tCnRpdGxlOiBTaW1wbGUgbW9kZWxzCm91dHB1dDogaHRtbF9ub3RlYm9vawplbmNvZGluZzogdXRmOAotLS0KCmBgYHtyLCBlY2hvPUZBTFNFLCByZXN1bHRzPSJoaWRlIiwgbWVzc2FnZT1GQUxTRX0KbGlicmFyeSh0aWR5cikKbGlicmFyeShkcGx5cikKbGlicmFyeSh0c2liYmxlKQpsaWJyYXJ5KGZlYXN0cykKbGlicmFyeShmYWJsZSkKYGBgCgpXZSBmaXQgc29tZSBzaW1wbGUgbW9kZWxzIHRvIHRoZSBvcmFuZ2UganVpY2UgZGF0YS4gT25lIG1vZGVsIGlzIGZpdCBmb3IgZWFjaCBjb21iaW5hdGlvbiBvZiBzdG9yZSBhbmQgYnJhbmQuCgotIGBtZWFuYDogVGhpcyBpcyBqdXN0IGEgc2ltcGxlIG1lYW4uCi0gYG5haXZlYDogQSByYW5kb20gd2FsayBtb2RlbCB3aXRob3V0IGFueSBvdGhlciBjb21wb25lbnRzLiBUaGlzIGFtb3VudHMgdG8gc2V0dGluZyBhbGwgZm9yZWNhc3QgdmFsdWVzIHRvIHRoZSBsYXN0IG9ic2VydmVkIHZhbHVlLgotIGBkcmlmdGA6IFRoaXMgYWRqdXN0cyB0aGUgYG5haXZlYCBtb2RlbCB0byBpbmNvcnBvcmF0ZSBhIHRyZW5kLgotIGBhcmltYWA6IEFuIEFSSU1BIG1vZGVsIHdpdGggdGhlIHBhcmFtZXRlciB2YWx1ZXMgZXN0aW1hdGVkIGZyb20gdGhlIGRhdGEuCi0gYGV0c2A6IEFuIGV4cG9uZW50aWFsbHkgd2VpZ2h0ZWQgbW9kZWwsIGFnYWluIHdpdGggcGFyYW1ldGVyIHZhbHVlcyBlc3RpbWF0ZWQgZnJvbSB0aGUgZGF0YS4KCk5vdGUgdGhhdCB0aGUgbW9kZWwgdHJhaW5pbmcgcHJvY2VzcyBpcyBlbWJhcnJhc3NpbmdseSBwYXJhbGxlbCBvbiAzIGxldmVsczoKCi0gV2UgaGF2ZSBtdWx0aXBsZSBpbmRlcGVuZGVudCB0cmFpbmluZyBkYXRhc2V0czsKLSBGb3Igd2hpY2ggd2UgZml0IG11bHRpcGxlIGluZGVwZW5kZW50IG1vZGVsczsKLSBXaXRoaW4gd2hpY2ggd2UgaGF2ZSBpbmRlcGVuZGVudCBzdWItbW9kZWxzIGZvciBlYWNoIHN0b3JlIGFuZCBicmFuZC4KClRoaXMgbGV0cyB1cyBzcGVlZCB1cCB0aGUgdHJhaW5pbmcgc2lnbmlmaWNhbnRseS4gV2hpbGUgdGhlIGBmYWJsZTo6bW9kZWxgIGZ1bmN0aW9uIGNhbiBmaXQgbXVsdGlwbGUgbW9kZWxzIGluIHBhcmFsbGVsLCB3ZSB3aWxsIHJ1biBpdCBzZXF1ZW50aWFsbHkgaGVyZSBhbmQgaW5zdGVhZCBwYXJhbGxlbGlzZSBieSBkYXRhc2V0LiBUaGlzIGF2b2lkcyBjb250ZW50aW9uIGZvciBjb3JlcywgYW5kIGFsc28gcmVzdWx0cyBpbiB0aGUgc2ltcGxlc3QgY29kZS4KCmBgYHtyLCByZXN1bHRzPSJoaWRlIn0KbG9hZCgib2pfZGF0YS5SZGF0YSIpCgpuY29yZXMgPC0gbWF4KDIsIHBhcmFsbGVsOjpkZXRlY3RDb3Jlcyhsb2dpY2FsPUZBTFNFKSAtIDIpCmNsIDwtIHBhcmFsbGVsOjptYWtlQ2x1c3RlcihuY29yZXMpCnBhcmFsbGVsOjpjbHVzdGVyRXZhbFEoY2wsCnsKICAgIGxpYnJhcnkodGlkeXIpCiAgICBsaWJyYXJ5KGZlYXN0cykKICAgIGxpYnJhcnkoZmFibGUpCiAgICBsaWJyYXJ5KHRzaWJibGUpCn0pCmBgYAoKRmlyc3QsIHdlIGZpdCB0aGUgbW9kZWxzIHRoYXQgY2FuIGlubmF0ZWx5IGhhbmRsZSBtaXNzaW5nIHZhbHVlcy4KCmBgYHtyfQpval9tb2RlbHNldCA8LSBwYXJhbGxlbDo6cGFyTGFwcGx5KGNsLCBval90cmFpbiwgZnVuY3Rpb24oZGYpCnsKICAgIG1vZGVsKGRmLAogICAgICAgIG1lYW49TUVBTihsb2dtb3ZlKSwKICAgICAgICBuYWl2ZT1OQUlWRShsb2dtb3ZlKSwKICAgICAgICBkcmlmdD1SVyhsb2dtb3ZlIH4gZHJpZnQoKSksCiAgICAgICAgYXJpbWE9QVJJTUEobG9nbW92ZSB+IHBkcSgpICsgUERRKDAsIDAsIDApKQogICAgKQp9KQpgYGAKTmV4dCwgd2UgZml0IG1vZGVscyB0aGF0IHJlcXVpcmUgbWFudWFsIGltcHV0YXRpb24gKEVUUykuCgpgYGB7cn0Kb2pfbW9kZWxzZXRfZXRzIDwtIHBhcmFsbGVsOjpwYXJMYXBwbHkoY2wsIG9qX3RyYWluLCBmdW5jdGlvbihkZikKewogICAgZGYgJT4lCiAgICAgICAgZmlsbChldmVyeXRoaW5nKCkpICU+JQogICAgICAgIG1vZGVsKGV0cz1FVFMobG9nbW92ZSB+IGVycm9yKCJBIikgKyB0cmVuZCgiQSIpICsgc2Vhc29uKCJOIikpKQp9KQoKcGFyYWxsZWw6OnN0b3BDbHVzdGVyKGNsKQpzYXZlKG9qX21vZGVsc2V0LCBval9tb2RlbHNldF9ldHMsIGZpbGU9Im9qX21vZGVsc2V0LlJkYXRhIikKCmhlYWQob2pfbW9kZWxzZXRbWzFdXSkKaGVhZChval9tb2RlbHNldF9ldHNbWzFdXSkKYGBgCgo=
@@ -336,7 +358,7 @@ $(document).ready(function () { diff --git a/R/orange_juice/02a_simplereg_models.Rmd b/R/orange_juice/02a_simplereg_models.Rmd new file mode 100644 index 00000000..038bed53 --- /dev/null +++ b/R/orange_juice/02a_simplereg_models.Rmd @@ -0,0 +1,39 @@ +--- +title: Regression models +output: html_notebook +--- + +```{r, echo=FALSE, results="hide", message=FALSE} +library(tidyr) +library(dplyr) +library(tsibble) +library(feasts) +library(fable) +``` + +This notebook builds on the output from "Simple models" by including regressor variables in the ARIMA model(s). + +```{r, results="hide"} +load("oj_data.Rdata") + +ncores <- max(2, parallel::detectCores(logical=FALSE) - 2) +cl <- parallel::makeCluster(ncores) +parallel::clusterEvalQ(cl, +{ + library(feasts) + library(fable) + library(tsibble) +}) + +oj_modelset_reg <- parallel::parLapply(cl, oj_train, function(df) +{ + model(df, + ar_reg=ARIMA(logmove ~ pdq() + PDQ(0, 0, 0) + deal + feat + price + maxpricediff), + ar_trend=ARIMA(logmove ~ pdq() + PDQ(0, 0, 0) + trend()), + ar_regtrend=ARIMA(logmove ~ pdq() + PDQ(0, 0, 0) + trend() + deal + feat + price + maxpricediff) + ) +}) + +parallel::stopCluster(cl) +save(oj_modelset_reg, file="oj_modelset_reg.Rdata") +``` diff --git a/examples/grocery_sales/R/02a_reg_models.nb.html b/R/orange_juice/02a_simplereg_models.nb.html similarity index 98% rename from examples/grocery_sales/R/02a_reg_models.nb.html rename to R/orange_juice/02a_simplereg_models.nb.html index f5e186af..4aece593 100644 --- a/examples/grocery_sales/R/02a_reg_models.nb.html +++ b/R/orange_juice/02a_simplereg_models.nb.html @@ -11,7 +11,7 @@ -ARIMA-Regression models +Regression models @@ -220,97 +220,47 @@ summary { -

ARIMA-Regression models

+

Regression models

-

Copyright (c) Microsoft Corporation.
Licensed under the MIT License.

-

This notebook builds on the output from “Basic models” by including regressor variables in the ARIMA model(s). We fit the following model types:

- -

As part of the modelling, we also compute a new independent variable maxpricediff, the log-ratio of the price of this brand compared to the best competing price. A positive maxpricediff means this brand is cheaper than all the other brands, and a negative maxpricediff means it is more expensive.

+

This notebook builds on the output from “Simple models” by including regressor variables in the ARIMA model(s).

- -
srcdir <- here::here("R_utils")
-for(src in dir(srcdir, full.names=TRUE)) source(src)
+
+
load("oj_data.Rdata")
 
-load_objects("grocery_sales", "data.Rdata")
-
-cl <- make_cluster(libs=c("tidyr", "dplyr", "fable", "tsibble", "feasts"))
-
-# add extra regression variables to training and test datasets
-add_regvars <- function(df)
+ncores <- max(2, parallel::detectCores(logical=FALSE) - 2)
+cl <- parallel::makeCluster(ncores)
+parallel::clusterEvalQ(cl,
 {
-    df %>%
-        group_by(store, brand) %>%
-        group_modify(~ {
-            pricevars <- grep("price", names(.x), value=TRUE)
-            thispricevar <- unique(paste0("price", .y$brand))
-            best_other_price <- do.call(pmin, .x[setdiff(pricevars, thispricevar)])
-            .x$price <- .x[[thispricevar]]
-            .x$maxpricediff <- log(best_other_price/.x$price)
-            .x
-        }) %>%
-        ungroup() %>%
-        mutate(week=yearweek(week)) %>%  # need to recreate this variable because of tsibble/vctrs issues
-        as_tsibble(week, key=c(store, brand))
-}
-
-oj_trainreg <- parallel::parLapply(cl, oj_train, add_regvars)
-oj_testreg <- parallel::parLapply(cl, oj_test, add_regvars)
-
-save_objects(oj_trainreg, oj_testreg,
-             example="grocery_sales", file="data_reg.Rdata")
-
-oj_modelset_reg <- parallel::parLapply(cl, oj_trainreg, function(df)
+    library(feasts)
+    library(fable)
+    library(tsibble)
+})

+
+
+
oj_modelset_reg <- parallel::parLapply(cl, oj_train, function(df)
 {
     model(df,
+        ar_reg=ARIMA(logmove ~ pdq() + PDQ(0, 0, 0) + deal + feat + price + maxpricediff),
         ar_trend=ARIMA(logmove ~ pdq() + PDQ(0, 0, 0) + trend()),
-
-        ar_reg=ARIMA(logmove ~ pdq() + PDQ(0, 0, 0) + deal + feat + maxpricediff +
-            price1 + price2 + price3 + price4 + price5 + price6 + price7 + price8 + price9 + price10 + price11),
-
-        ar_reg_price=ARIMA(logmove ~ pdq() + PDQ(0, 0, 0) + deal + feat + maxpricediff + price),
-
-        ar_reg_price_trend=ARIMA(logmove ~ pdq() + PDQ(0, 0, 0) + trend() + deal + feat + maxpricediff + price),
-
-        .safely=FALSE
+        ar_regtrend=ARIMA(logmove ~ pdq() + PDQ(0, 0, 0) + trend() + deal + feat + price + maxpricediff)
     )
 })
 
-oj_fcast_reg <- parallel::clusterMap(cl, get_forecasts, oj_modelset_reg, oj_testreg)
-
-destroy_cluster(cl)
-
-save_objects(oj_modelset_reg, oj_fcast_reg,
-             example="grocery_sales", file="model_reg.Rdata")
-
-do.call(rbind, oj_fcast_reg) %>%
-    mutate_at(-(1:3), exp) %>%
-    eval_forecasts()

+parallel::stopCluster(cl)
+save(oj_modelset_reg, file="oj_modelset_reg.Rdata")
-
- -
- -

This shows that the models incorporating price are a significant improvement over the previous naive models. The model that uses stepwise selection to choose the best price variable does worse than the one where we choose the price beforehand, confirming the suspicion that stepwise leads to overfitting in this case.

- -
LS0tCnRpdGxlOiBBUklNQS1SZWdyZXNzaW9uIG1vZGVscwpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sKLS0tCgpfQ29weXJpZ2h0IChjKSBNaWNyb3NvZnQgQ29ycG9yYXRpb24uXzxici8+Cl9MaWNlbnNlZCB1bmRlciB0aGUgTUlUIExpY2Vuc2UuXwoKYGBge3IsIGVjaG89RkFMU0UsIHJlc3VsdHM9ImhpZGUiLCBtZXNzYWdlPUZBTFNFfQpsaWJyYXJ5KHRpZHlyKQpsaWJyYXJ5KGRwbHlyKQpsaWJyYXJ5KHRzaWJibGUpCmxpYnJhcnkoZmVhc3RzKQpsaWJyYXJ5KGZhYmxlKQpgYGAKClRoaXMgbm90ZWJvb2sgYnVpbGRzIG9uIHRoZSBvdXRwdXQgZnJvbSAiQmFzaWMgbW9kZWxzIiBieSBpbmNsdWRpbmcgcmVncmVzc29yIHZhcmlhYmxlcyBpbiB0aGUgQVJJTUEgbW9kZWwocykuIFdlIGZpdCB0aGUgZm9sbG93aW5nIG1vZGVsIHR5cGVzOgoKLSBgYXJfdHJlbmRgIGluY2x1ZGVzIG9ubHkgYSBsaW5lYXIgdHJlbmQgb3ZlciB0aW1lLgotIGBhcl9yZWdgIGFsbG93cyBzdGVwd2lzZSBzZWxlY3Rpb24gb2YgaW5kZXBlbmRlbnQgcmVncmVzc29ycy4KLSBgYXJfcmVnX3ByaWNlYDogcmF0aGVyIHRoYW4gYWxsb3dpbmcgdGhlIGFsZ29yaXRobSB0byBzZWxlY3QgZnJvbSB0aGUgMTEgcHJpY2UgdmFyaWFibGVzLCB3ZSB1c2Ugb25seSB0aGUgcHJpY2UgcmVsZXZhbnQgdG8gZWFjaCBicmFuZC4gVGhpcyBpcyB0byBndWFyZCBhZ2FpbnN0IHBvc3NpYmxlIG92ZXJmaXR0aW5nLCBzb21ldGhpbmcgdGhhdCBjbGFzc2ljYWwgc3RlcHdpc2UgcHJvY2VkdXJlcyBhcmUgd29udCB0byBkby4KLSBgYXJfcmVnX3ByaWNlX3RyZW5kYCBpcyB0aGUgc2FtZSBhcyBgYXJfcmVnX3ByaWNlYCwgYnV0IGluY2x1ZGluZyBhIGxpbmVhciB0cmVuZC4KCkFzIHBhcnQgb2YgdGhlIG1vZGVsbGluZywgd2UgYWxzbyBjb21wdXRlIGEgbmV3IGluZGVwZW5kZW50IHZhcmlhYmxlIGBtYXhwcmljZWRpZmZgLCB0aGUgbG9nLXJhdGlvIG9mIHRoZSBwcmljZSBvZiB0aGlzIGJyYW5kIGNvbXBhcmVkIHRvIHRoZSBiZXN0IGNvbXBldGluZyBwcmljZS4gQSBwb3NpdGl2ZSBgbWF4cHJpY2VkaWZmYCBtZWFucyB0aGlzIGJyYW5kIGlzIGNoZWFwZXIgdGhhbiBhbGwgdGhlIG90aGVyIGJyYW5kcywgYW5kIGEgbmVnYXRpdmUgYG1heHByaWNlZGlmZmAgbWVhbnMgaXQgaXMgbW9yZSBleHBlbnNpdmUuCgpgYGB7cn0Kc3JjZGlyIDwtIGhlcmU6OmhlcmUoIlJfdXRpbHMiKQpmb3Ioc3JjIGluIGRpcihzcmNkaXIsIGZ1bGwubmFtZXM9VFJVRSkpIHNvdXJjZShzcmMpCgpsb2FkX29iamVjdHMoImdyb2Nlcnlfc2FsZXMiLCAiZGF0YS5SZGF0YSIpCgpjbCA8LSBtYWtlX2NsdXN0ZXIobGlicz1jKCJ0aWR5ciIsICJkcGx5ciIsICJmYWJsZSIsICJ0c2liYmxlIiwgImZlYXN0cyIpKQoKIyBhZGQgZXh0cmEgcmVncmVzc2lvbiB2YXJpYWJsZXMgdG8gdHJhaW5pbmcgYW5kIHRlc3QgZGF0YXNldHMKYWRkX3JlZ3ZhcnMgPC0gZnVuY3Rpb24oZGYpCnsKICAgIGRmICU+JQogICAgICAgIGdyb3VwX2J5KHN0b3JlLCBicmFuZCkgJT4lCiAgICAgICAgZ3JvdXBfbW9kaWZ5KH4gewogICAgICAgICAgICBwcmljZXZhcnMgPC0gZ3JlcCgicHJpY2UiLCBuYW1lcygueCksIHZhbHVlPVRSVUUpCiAgICAgICAgICAgIHRoaXNwcmljZXZhciA8LSB1bmlxdWUocGFzdGUwKCJwcmljZSIsIC55JGJyYW5kKSkKICAgICAgICAgICAgYmVzdF9vdGhlcl9wcmljZSA8LSBkby5jYWxsKHBtaW4sIC54W3NldGRpZmYocHJpY2V2YXJzLCB0aGlzcHJpY2V2YXIpXSkKICAgICAgICAgICAgLngkcHJpY2UgPC0gLnhbW3RoaXNwcmljZXZhcl1dCiAgICAgICAgICAgIC54JG1heHByaWNlZGlmZiA8LSBsb2coYmVzdF9vdGhlcl9wcmljZS8ueCRwcmljZSkKICAgICAgICAgICAgLngKICAgICAgICB9KSAlPiUKICAgICAgICB1bmdyb3VwKCkgJT4lCiAgICAgICAgbXV0YXRlKHdlZWs9eWVhcndlZWsod2VlaykpICU+JSAgIyBuZWVkIHRvIHJlY3JlYXRlIHRoaXMgdmFyaWFibGUgYmVjYXVzZSBvZiB0c2liYmxlL3ZjdHJzIGlzc3VlcwogICAgICAgIGFzX3RzaWJibGUod2Vlaywga2V5PWMoc3RvcmUsIGJyYW5kKSkKfQoKb2pfdHJhaW5yZWcgPC0gcGFyYWxsZWw6OnBhckxhcHBseShjbCwgb2pfdHJhaW4sIGFkZF9yZWd2YXJzKQpval90ZXN0cmVnIDwtIHBhcmFsbGVsOjpwYXJMYXBwbHkoY2wsIG9qX3Rlc3QsIGFkZF9yZWd2YXJzKQoKc2F2ZV9vYmplY3RzKG9qX3RyYWlucmVnLCBval90ZXN0cmVnLAogICAgICAgICAgICAgZXhhbXBsZT0iZ3JvY2VyeV9zYWxlcyIsIGZpbGU9ImRhdGFfcmVnLlJkYXRhIikKCm9qX21vZGVsc2V0X3JlZyA8LSBwYXJhbGxlbDo6cGFyTGFwcGx5KGNsLCBval90cmFpbnJlZywgZnVuY3Rpb24oZGYpCnsKICAgIG1vZGVsKGRmLAogICAgICAgIGFyX3RyZW5kPUFSSU1BKGxvZ21vdmUgfiBwZHEoKSArIFBEUSgwLCAwLCAwKSArIHRyZW5kKCkpLAoKICAgICAgICBhcl9yZWc9QVJJTUEobG9nbW92ZSB+IHBkcSgpICsgUERRKDAsIDAsIDApICsgZGVhbCArIGZlYXQgKyBtYXhwcmljZWRpZmYgKwogICAgICAgICAgICBwcmljZTEgKyBwcmljZTIgKyBwcmljZTMgKyBwcmljZTQgKyBwcmljZTUgKyBwcmljZTYgKyBwcmljZTcgKyBwcmljZTggKyBwcmljZTkgKyBwcmljZTEwICsgcHJpY2UxMSksCgogICAgICAgIGFyX3JlZ19wcmljZT1BUklNQShsb2dtb3ZlIH4gcGRxKCkgKyBQRFEoMCwgMCwgMCkgKyBkZWFsICsgZmVhdCArIG1heHByaWNlZGlmZiArIHByaWNlKSwKCiAgICAgICAgYXJfcmVnX3ByaWNlX3RyZW5kPUFSSU1BKGxvZ21vdmUgfiBwZHEoKSArIFBEUSgwLCAwLCAwKSArIHRyZW5kKCkgKyBkZWFsICsgZmVhdCArIG1heHByaWNlZGlmZiArIHByaWNlKSwKCiAgICAgICAgLnNhZmVseT1GQUxTRQogICAgKQp9KQoKb2pfZmNhc3RfcmVnIDwtIHBhcmFsbGVsOjpjbHVzdGVyTWFwKGNsLCBnZXRfZm9yZWNhc3RzLCBval9tb2RlbHNldF9yZWcsIG9qX3Rlc3RyZWcpCgpkZXN0cm95X2NsdXN0ZXIoY2wpCgpzYXZlX29iamVjdHMob2pfbW9kZWxzZXRfcmVnLCBval9mY2FzdF9yZWcsCiAgICAgICAgICAgICBleGFtcGxlPSJncm9jZXJ5X3NhbGVzIiwgZmlsZT0ibW9kZWxfcmVnLlJkYXRhIikKCmRvLmNhbGwocmJpbmQsIG9qX2ZjYXN0X3JlZykgJT4lCiAgICBtdXRhdGVfYXQoLSgxOjMpLCBleHApICU+JQogICAgZXZhbF9mb3JlY2FzdHMoKQpgYGAKClRoaXMgc2hvd3MgdGhhdCB0aGUgbW9kZWxzIGluY29ycG9yYXRpbmcgcHJpY2UgYXJlIGEgc2lnbmlmaWNhbnQgaW1wcm92ZW1lbnQgb3ZlciB0aGUgcHJldmlvdXMgbmFpdmUgbW9kZWxzLiBUaGUgbW9kZWwgdGhhdCB1c2VzIHN0ZXB3aXNlIHNlbGVjdGlvbiB0byBjaG9vc2UgdGhlIGJlc3QgcHJpY2UgdmFyaWFibGUgZG9lcyB3b3JzZSB0aGFuIHRoZSBvbmUgd2hlcmUgd2UgY2hvb3NlIHRoZSBwcmljZSBiZWZvcmVoYW5kLCBjb25maXJtaW5nIHRoZSBzdXNwaWNpb24gdGhhdCBzdGVwd2lzZSBsZWFkcyB0byBvdmVyZml0dGluZyBpbiB0aGlzIGNhc2UuCg==
+
LS0tCnRpdGxlOiBSZWdyZXNzaW9uIG1vZGVscwpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sKLS0tCgpgYGB7ciwgZWNobz1GQUxTRSwgcmVzdWx0cz0iaGlkZSIsIG1lc3NhZ2U9RkFMU0V9CmxpYnJhcnkodGlkeXIpCmxpYnJhcnkoZHBseXIpCmxpYnJhcnkodHNpYmJsZSkKbGlicmFyeShmZWFzdHMpCmxpYnJhcnkoZmFibGUpCmBgYAoKVGhpcyBub3RlYm9vayBidWlsZHMgb24gdGhlIG91dHB1dCBmcm9tICJTaW1wbGUgbW9kZWxzIiBieSBpbmNsdWRpbmcgcmVncmVzc29yIHZhcmlhYmxlcyBpbiB0aGUgQVJJTUEgbW9kZWwocykuCgpgYGB7ciwgcmVzdWx0cz0iaGlkZSJ9CmxvYWQoIm9qX2RhdGEuUmRhdGEiKQoKbmNvcmVzIDwtIG1heCgyLCBwYXJhbGxlbDo6ZGV0ZWN0Q29yZXMobG9naWNhbD1GQUxTRSkgLSAyKQpjbCA8LSBwYXJhbGxlbDo6bWFrZUNsdXN0ZXIobmNvcmVzKQpwYXJhbGxlbDo6Y2x1c3RlckV2YWxRKGNsLAp7CiAgICBsaWJyYXJ5KGZlYXN0cykKICAgIGxpYnJhcnkoZmFibGUpCiAgICBsaWJyYXJ5KHRzaWJibGUpCn0pCgpval9tb2RlbHNldF9yZWcgPC0gcGFyYWxsZWw6OnBhckxhcHBseShjbCwgb2pfdHJhaW4sIGZ1bmN0aW9uKGRmKQp7CiAgICBtb2RlbChkZiwKICAgICAgICBhcl9yZWc9QVJJTUEobG9nbW92ZSB+IHBkcSgpICsgUERRKDAsIDAsIDApICsgZGVhbCArIGZlYXQgKyBwcmljZSArIG1heHByaWNlZGlmZiksCiAgICAgICAgYXJfdHJlbmQ9QVJJTUEobG9nbW92ZSB+IHBkcSgpICsgUERRKDAsIDAsIDApICsgdHJlbmQoKSksCiAgICAgICAgYXJfcmVndHJlbmQ9QVJJTUEobG9nbW92ZSB+IHBkcSgpICsgUERRKDAsIDAsIDApICsgdHJlbmQoKSArIGRlYWwgKyBmZWF0ICsgcHJpY2UgKyBtYXhwcmljZWRpZmYpCiAgICApCn0pCgpwYXJhbGxlbDo6c3RvcENsdXN0ZXIoY2wpCnNhdmUob2pfbW9kZWxzZXRfcmVnLCBmaWxlPSJval9tb2RlbHNldF9yZWcuUmRhdGEiKQpgYGAK
@@ -357,7 +307,7 @@ $(document).ready(function () { diff --git a/R/orange_juice/03_model_eval.Rmd b/R/orange_juice/03_model_eval.Rmd new file mode 100644 index 00000000..fdce453e --- /dev/null +++ b/R/orange_juice/03_model_eval.Rmd @@ -0,0 +1,58 @@ +--- +title: Model evaluation +output: html_notebook +encoding: utf8 +--- + +```{r, echo=FALSE, results="hide", message=FALSE} +library(tidyr) +library(dplyr) +library(tsibble) +library(feasts) +library(fable) +``` + +Having fit the models, let's examine their rolling goodness of fit, using the MAPE (mean absolute percentage error) metric. + +First, we compute the forecasts for each dataset and model, again in parallel. + +```{r, results="hide"} +for(f in dir(pattern="Rdata$")) + load(f) + +ncores <- max(2, parallel::detectCores(logical=FALSE) - 2) +cl <- parallel::makeCluster(ncores) +parallel::clusterEvalQ(cl, +{ + library(feasts) + library(fable) + library(tsibble) +}) + +fcast_sets <- lapply(ls(pattern="^oj_modelset"), function(mod) + parallel::clusterMap(cl, function(mod, df) forecast(mod, df), get(mod), oj_test) +) + +parallel::stopCluster(cl) +``` + +Next, we compute the MAPE for each model. It is apparent that adding independent variables as regressors improves the quality of the fit substantially. Adding a simple trend does _not_ improve the fit, indicating that the level of sales does not appear to change over time (at least over the period included in the data). + +```{r} +orig <- do.call(rbind, oj_test) %>% + as_tibble() %>% + select(store, brand, week, logmove) %>% + mutate(move=exp(logmove)) + +gof <- function(fcast_data) +{ + fcast_data <- do.call(rbind, fcast_data) %>% + as_tibble() %>% + select(store, brand, week, .model, logmove) %>% + pivot_wider(id_cols=c(store, brand, week), names_from=.model, values_from=logmove) %>% + select(-store, -brand, -week) %>% + summarise_all(function(x) MAPE(exp(x) - orig$move, orig$move)) +} + +lapply(fcast_sets, gof) %>% bind_cols() +``` diff --git a/examples/grocery_sales/R/02_basic_models.nb.html b/R/orange_juice/03_model_eval.nb.html similarity index 98% rename from examples/grocery_sales/R/02_basic_models.nb.html rename to R/orange_juice/03_model_eval.nb.html index 6f91ee65..3540448f 100644 --- a/examples/grocery_sales/R/02_basic_models.nb.html +++ b/R/orange_juice/03_model_eval.nb.html @@ -11,7 +11,7 @@ -Basic models +Model evaluation @@ -220,104 +220,71 @@ summary { -

Basic models

+

Model evaluation

-

Copyright (c) Microsoft Corporation.
Licensed under the MIT License.

-

We fit some simple models to the orange juice data for illustrative purposes. Here, each model is actually a group of models, one for each combination of store and brand. This is the standard approach taken in statistical forecasting, and is supported out-of-the-box by the tidyverts framework.

- -

Note that the model training process is embarrassingly parallel on 3 levels:

- -

This lets us speed up the training significantly. While the fable::model function can fit multiple models in parallel, we will run it sequentially here and instead parallelise by dataset. This avoids contention for cores, and also results in the simplest code. As a guard against returning invalid results, we also specify the argument .safely=FALSE; this forces model to throw an error if a model algorithm fails.

+

Having fit the models, let’s examine their rolling goodness of fit, using the MAPE (mean absolute percentage error) metric.

+

First, we compute the forecasts for each dataset and model, again in parallel.

- -
srcdir <- here::here("R_utils")
-for(src in dir(srcdir, full.names=TRUE)) source(src)
+
+
for(f in dir(pattern="Rdata$"))
+    load(f)
 
-load_objects("grocery_sales", "data.Rdata")
-
-cl <- make_cluster(libs=c("tidyr", "dplyr", "fable", "tsibble", "feasts"))
-
-oj_modelset_basic <- parallel::parLapply(cl, oj_train, function(df)
+ncores <- max(2, parallel::detectCores(logical=FALSE) - 2)
+cl <- parallel::makeCluster(ncores)
+parallel::clusterEvalQ(cl,
 {
-    model(df,
-        mean=MEAN(logmove),
-        naive=NAIVE(logmove),
-        drift=RW(logmove ~ drift()),
-        arima=ARIMA(logmove ~ pdq() + PDQ(0, 0, 0)),
-        .safely=FALSE
-    )
-})
-oj_fcast_basic <- parallel::clusterMap(cl, get_forecasts, oj_modelset_basic, oj_test)
+    library(feasts)
+    library(fable)
+    library(tsibble)
+})

+
+
+
fcast_sets <- lapply(ls(pattern="^oj_modelset"), function(mod)
+    parallel::clusterMap(cl, function(mod, df) forecast(mod, df), get(mod), oj_test)
+)
 
-save_objects(oj_modelset_basic, oj_fcast_basic,
-             example="grocery_sales", file="model_basic.Rdata")
+parallel::stopCluster(cl)

+
+
+
+
Next, we compute the MAPE for each model. It is apparent that adding independent variables as regressors improves the quality of the fit substantially. Adding a simple trend does not improve the fit, indicating that the level of sales does not appear to change over time (at least over the period included in the data).

+
+
+
+
orig <- do.call(rbind, oj_test) %>%
+    as_tibble() %>%
+    select(store, brand, week, logmove) %>%
+    mutate(move=exp(logmove))
 
-do.call(rbind, oj_fcast_basic) %>%
-    mutate_at(-(1:3), exp) %>%
-    eval_forecasts()

+gof <- function(fcast_data)
+{
+    fcast_data <- do.call(rbind, fcast_data) %>%
+        as_tibble() %>%
+        select(store, brand, week, .model, logmove) %>%
+        pivot_wider(id_cols=c(store, brand, week), names_from=.model, values_from=logmove) %>%
+        select(-store, -brand, -week) %>%
+        summarise_all(function(x) MAPE(exp(x) - orig$move, orig$move))
+}
+
+lapply(fcast_sets, gof) %>% bind_cols()
- -

The ARIMA model does the best of the simple models, but not any better than a simple mean.

-

Having fit some basic models, we can also try an exponential smoothing model, fit using the ETS function. Unlike the others, ETS does not currently support time series with missing values; we therefore have to use one of the other models to impute missing values first via the interpolate function.

- - - -
oj_modelset_ets <- parallel::clusterMap(cl, function(df, basicmod)
-{
-    df %>%
-        interpolate(object=select(basicmod, -c(mean, naive, drift))) %>%
-        model(
-            ets=ETS(logmove ~ error("A") + trend("A") + season("N")),
-            .safely=FALSE
-        )
-}, oj_train, oj_modelset_basic)
 
-oj_fcast_ets <- parallel::clusterMap(cl, get_forecasts, oj_modelset_ets, oj_test)
-
-destroy_cluster(cl)
-
-save_objects(oj_modelset_ets, oj_fcast_ets,
-             example="grocery_sales", file="model_ets.Rdata")
-
-do.call(rbind, oj_fcast_ets) %>%
-    mutate_at(-(1:3), exp) %>%
-    eval_forecasts()
- -
- -
- - -

The ETS model does worse than the ARIMA model, something that should not be a surprise given the lack of strong seasonality and trend in this dataset. We conclude that any simple univariate approach is unlikely to do well.

- - -
LS0tCnRpdGxlOiBCYXNpYyBtb2RlbHMKb3V0cHV0OiBodG1sX25vdGVib29rCi0tLQoKX0NvcHlyaWdodCAoYykgTWljcm9zb2Z0IENvcnBvcmF0aW9uLl88YnIvPgpfTGljZW5zZWQgdW5kZXIgdGhlIE1JVCBMaWNlbnNlLl8KCmBgYHtyLCBlY2hvPUZBTFNFLCByZXN1bHRzPSJoaWRlIiwgbWVzc2FnZT1GQUxTRX0KbGlicmFyeSh0aWR5cikKbGlicmFyeShkcGx5cikKbGlicmFyeSh0c2liYmxlKQpsaWJyYXJ5KGZlYXN0cykKbGlicmFyeShmYWJsZSkKYGBgCgpXZSBmaXQgc29tZSBzaW1wbGUgbW9kZWxzIHRvIHRoZSBvcmFuZ2UganVpY2UgZGF0YSBmb3IgaWxsdXN0cmF0aXZlIHB1cnBvc2VzLiBIZXJlLCBlYWNoIG1vZGVsIGlzIGFjdHVhbGx5IGEgX2dyb3VwXyBvZiBtb2RlbHMsIG9uZSBmb3IgZWFjaCBjb21iaW5hdGlvbiBvZiBzdG9yZSBhbmQgYnJhbmQuIFRoaXMgaXMgdGhlIHN0YW5kYXJkIGFwcHJvYWNoIHRha2VuIGluIHN0YXRpc3RpY2FsIGZvcmVjYXN0aW5nLCBhbmQgaXMgc3VwcG9ydGVkIG91dC1vZi10aGUtYm94IGJ5IHRoZSB0aWR5dmVydHMgZnJhbWV3b3JrLgoKLSBgbWVhbmA6IFRoaXMgaXMganVzdCBhIHNpbXBsZSBtZWFuLgotIGBuYWl2ZWA6IEEgcmFuZG9tIHdhbGsgbW9kZWwgd2l0aG91dCBhbnkgb3RoZXIgY29tcG9uZW50cy4gVGhpcyBhbW91bnRzIHRvIHNldHRpbmcgYWxsIGZvcmVjYXN0IHZhbHVlcyB0byB0aGUgbGFzdCBvYnNlcnZlZCB2YWx1ZS4KLSBgZHJpZnRgOiBUaGlzIGFkanVzdHMgdGhlIGBuYWl2ZWAgbW9kZWwgdG8gaW5jb3Jwb3JhdGUgYSBzdHJhaWdodC1saW5lIHRyZW5kLgotIGBhcmltYWA6IEFuIEFSSU1BIG1vZGVsIHdpdGggdGhlIHBhcmFtZXRlciB2YWx1ZXMgZXN0aW1hdGVkIGZyb20gdGhlIGRhdGEuCgpOb3RlIHRoYXQgdGhlIG1vZGVsIHRyYWluaW5nIHByb2Nlc3MgaXMgZW1iYXJyYXNzaW5nbHkgcGFyYWxsZWwgb24gMyBsZXZlbHM6CgotIFdlIGhhdmUgbXVsdGlwbGUgaW5kZXBlbmRlbnQgdHJhaW5pbmcgZGF0YXNldHM7Ci0gRm9yIHdoaWNoIHdlIGZpdCBtdWx0aXBsZSBpbmRlcGVuZGVudCBtb2RlbHM7Ci0gV2l0aGluIHdoaWNoIHdlIGhhdmUgaW5kZXBlbmRlbnQgc3ViLW1vZGVscyBmb3IgZWFjaCBzdG9yZSBhbmQgYnJhbmQuCgpUaGlzIGxldHMgdXMgc3BlZWQgdXAgdGhlIHRyYWluaW5nIHNpZ25pZmljYW50bHkuIFdoaWxlIHRoZSBgZmFibGU6Om1vZGVsYCBmdW5jdGlvbiBjYW4gZml0IG11bHRpcGxlIG1vZGVscyBpbiBwYXJhbGxlbCwgd2Ugd2lsbCBydW4gaXQgc2VxdWVudGlhbGx5IGhlcmUgYW5kIGluc3RlYWQgcGFyYWxsZWxpc2UgYnkgZGF0YXNldC4gVGhpcyBhdm9pZHMgY29udGVudGlvbiBmb3IgY29yZXMsIGFuZCBhbHNvIHJlc3VsdHMgaW4gdGhlIHNpbXBsZXN0IGNvZGUuIEFzIGEgZ3VhcmQgYWdhaW5zdCByZXR1cm5pbmcgaW52YWxpZCByZXN1bHRzLCB3ZSBhbHNvIHNwZWNpZnkgdGhlIGFyZ3VtZW50IGAuc2FmZWx5PUZBTFNFYDsgdGhpcyBmb3JjZXMgYG1vZGVsYCB0byB0aHJvdyBhbiBlcnJvciBpZiBhIG1vZGVsIGFsZ29yaXRobSBmYWlscy4KCmBgYHtyfQpzcmNkaXIgPC0gaGVyZTo6aGVyZSgiUl91dGlscyIpCmZvcihzcmMgaW4gZGlyKHNyY2RpciwgZnVsbC5uYW1lcz1UUlVFKSkgc291cmNlKHNyYykKCmxvYWRfb2JqZWN0cygiZ3JvY2VyeV9zYWxlcyIsICJkYXRhLlJkYXRhIikKCmNsIDwtIG1ha2VfY2x1c3RlcihsaWJzPWMoInRpZHlyIiwgImRwbHlyIiwgImZhYmxlIiwgInRzaWJibGUiLCAiZmVhc3RzIikpCgpval9tb2RlbHNldF9iYXNpYyA8LSBwYXJhbGxlbDo6cGFyTGFwcGx5KGNsLCBval90cmFpbiwgZnVuY3Rpb24oZGYpCnsKICAgIG1vZGVsKGRmLAogICAgICAgIG1lYW49TUVBTihsb2dtb3ZlKSwKICAgICAgICBuYWl2ZT1OQUlWRShsb2dtb3ZlKSwKICAgICAgICBkcmlmdD1SVyhsb2dtb3ZlIH4gZHJpZnQoKSksCiAgICAgICAgYXJpbWE9QVJJTUEobG9nbW92ZSB+IHBkcSgpICsgUERRKDAsIDAsIDApKSwKICAgICAgICAuc2FmZWx5PUZBTFNFCiAgICApCn0pCm9qX2ZjYXN0X2Jhc2ljIDwtIHBhcmFsbGVsOjpjbHVzdGVyTWFwKGNsLCBnZXRfZm9yZWNhc3RzLCBval9tb2RlbHNldF9iYXNpYywgb2pfdGVzdCkKCnNhdmVfb2JqZWN0cyhval9tb2RlbHNldF9iYXNpYywgb2pfZmNhc3RfYmFzaWMsCiAgICAgICAgICAgICBleGFtcGxlPSJncm9jZXJ5X3NhbGVzIiwgZmlsZT0ibW9kZWxfYmFzaWMuUmRhdGEiKQoKZG8uY2FsbChyYmluZCwgb2pfZmNhc3RfYmFzaWMpICU+JQogICAgbXV0YXRlX2F0KC0oMTozKSwgZXhwKSAlPiUKICAgIGV2YWxfZm9yZWNhc3RzKCkKYGBgCgpUaGUgQVJJTUEgbW9kZWwgZG9lcyB0aGUgYmVzdCBvZiB0aGUgc2ltcGxlIG1vZGVscywgYnV0IG5vdCBhbnkgYmV0dGVyIHRoYW4gYSBzaW1wbGUgbWVhbi4KCkhhdmluZyBmaXQgc29tZSBiYXNpYyBtb2RlbHMsIHdlIGNhbiBhbHNvIHRyeSBhbiBleHBvbmVudGlhbCBzbW9vdGhpbmcgbW9kZWwsIGZpdCB1c2luZyB0aGUgYEVUU2AgZnVuY3Rpb24uIFVubGlrZSB0aGUgb3RoZXJzLCBgRVRTYCBkb2VzIG5vdCBjdXJyZW50bHkgc3VwcG9ydCB0aW1lIHNlcmllcyB3aXRoIG1pc3NpbmcgdmFsdWVzOyB3ZSB0aGVyZWZvcmUgaGF2ZSB0byB1c2Ugb25lIG9mIHRoZSBvdGhlciBtb2RlbHMgdG8gaW1wdXRlIG1pc3NpbmcgdmFsdWVzIGZpcnN0IHZpYSB0aGUgYGludGVycG9sYXRlYCBmdW5jdGlvbi4KCmBgYHtyfQpval9tb2RlbHNldF9ldHMgPC0gcGFyYWxsZWw6OmNsdXN0ZXJNYXAoY2wsIGZ1bmN0aW9uKGRmLCBiYXNpY21vZCkKewogICAgZGYgJT4lCiAgICAgICAgaW50ZXJwb2xhdGUob2JqZWN0PXNlbGVjdChiYXNpY21vZCwgLWMobWVhbiwgbmFpdmUsIGRyaWZ0KSkpICU+JQogICAgICAgIG1vZGVsKAogICAgICAgICAgICBldHM9RVRTKGxvZ21vdmUgfiBlcnJvcigiQSIpICsgdHJlbmQoIkEiKSArIHNlYXNvbigiTiIpKSwKICAgICAgICAgICAgLnNhZmVseT1GQUxTRQogICAgICAgICkKfSwgb2pfdHJhaW4sIG9qX21vZGVsc2V0X2Jhc2ljKQoKb2pfZmNhc3RfZXRzIDwtIHBhcmFsbGVsOjpjbHVzdGVyTWFwKGNsLCBnZXRfZm9yZWNhc3RzLCBval9tb2RlbHNldF9ldHMsIG9qX3Rlc3QpCgpkZXN0cm95X2NsdXN0ZXIoY2wpCgpzYXZlX29iamVjdHMob2pfbW9kZWxzZXRfZXRzLCBval9mY2FzdF9ldHMsCiAgICAgICAgICAgICBleGFtcGxlPSJncm9jZXJ5X3NhbGVzIiwgZmlsZT0ibW9kZWxfZXRzLlJkYXRhIikKCmRvLmNhbGwocmJpbmQsIG9qX2ZjYXN0X2V0cykgJT4lCiAgICBtdXRhdGVfYXQoLSgxOjMpLCBleHApICU+JQogICAgZXZhbF9mb3JlY2FzdHMoKQpgYGAKClRoZSBFVFMgbW9kZWwgZG9lcyBfd29yc2VfIHRoYW4gdGhlIEFSSU1BIG1vZGVsLCBzb21ldGhpbmcgdGhhdCBzaG91bGQgbm90IGJlIGEgc3VycHJpc2UgZ2l2ZW4gdGhlIGxhY2sgb2Ygc3Ryb25nIHNlYXNvbmFsaXR5IGFuZCB0cmVuZCBpbiB0aGlzIGRhdGFzZXQuIFdlIGNvbmNsdWRlIHRoYXQgYW55IHNpbXBsZSB1bml2YXJpYXRlIGFwcHJvYWNoIGlzIHVubGlrZWx5IHRvIGRvIHdlbGwuCg==
+
LS0tCnRpdGxlOiBNb2RlbCBldmFsdWF0aW9uCm91dHB1dDogaHRtbF9ub3RlYm9vawplbmNvZGluZzogdXRmOAotLS0KCmBgYHtyLCBlY2hvPUZBTFNFLCByZXN1bHRzPSJoaWRlIiwgbWVzc2FnZT1GQUxTRX0KbGlicmFyeSh0aWR5cikKbGlicmFyeShkcGx5cikKbGlicmFyeSh0c2liYmxlKQpsaWJyYXJ5KGZlYXN0cykKbGlicmFyeShmYWJsZSkKYGBgCgpIYXZpbmcgZml0IHRoZSBtb2RlbHMsIGxldCdzIGV4YW1pbmUgdGhlaXIgcm9sbGluZyBnb29kbmVzcyBvZiBmaXQsIHVzaW5nIHRoZSBNQVBFIChtZWFuIGFic29sdXRlIHBlcmNlbnRhZ2UgZXJyb3IpIG1ldHJpYy4KCkZpcnN0LCB3ZSBjb21wdXRlIHRoZSBmb3JlY2FzdHMgZm9yIGVhY2ggZGF0YXNldCBhbmQgbW9kZWwsIGFnYWluIGluIHBhcmFsbGVsLgoKYGBge3IsIHJlc3VsdHM9ImhpZGUifQpmb3IoZiBpbiBkaXIocGF0dGVybj0iUmRhdGEkIikpCiAgICBsb2FkKGYpCgpuY29yZXMgPC0gbWF4KDIsIHBhcmFsbGVsOjpkZXRlY3RDb3Jlcyhsb2dpY2FsPUZBTFNFKSAtIDIpCmNsIDwtIHBhcmFsbGVsOjptYWtlQ2x1c3RlcihuY29yZXMpCnBhcmFsbGVsOjpjbHVzdGVyRXZhbFEoY2wsCnsKICAgIGxpYnJhcnkoZmVhc3RzKQogICAgbGlicmFyeShmYWJsZSkKICAgIGxpYnJhcnkodHNpYmJsZSkKfSkKCmZjYXN0X3NldHMgPC0gbGFwcGx5KGxzKHBhdHRlcm49Il5val9tb2RlbHNldCIpLCBmdW5jdGlvbihtb2QpCiAgICBwYXJhbGxlbDo6Y2x1c3Rlck1hcChjbCwgZnVuY3Rpb24obW9kLCBkZikgZm9yZWNhc3QobW9kLCBkZiksIGdldChtb2QpLCBval90ZXN0KQopCgpwYXJhbGxlbDo6c3RvcENsdXN0ZXIoY2wpCmBgYAoKTmV4dCwgd2UgY29tcHV0ZSB0aGUgTUFQRSBmb3IgZWFjaCBtb2RlbC4gSXQgaXMgYXBwYXJlbnQgdGhhdCBhZGRpbmcgaW5kZXBlbmRlbnQgdmFyaWFibGVzIGFzIHJlZ3Jlc3NvcnMgaW1wcm92ZXMgdGhlIHF1YWxpdHkgb2YgdGhlIGZpdCBzdWJzdGFudGlhbGx5LiBBZGRpbmcgYSBzaW1wbGUgdHJlbmQgZG9lcyBfbm90XyBpbXByb3ZlIHRoZSBmaXQsIGluZGljYXRpbmcgdGhhdCB0aGUgbGV2ZWwgb2Ygc2FsZXMgZG9lcyBub3QgYXBwZWFyIHRvIGNoYW5nZSBvdmVyIHRpbWUgKGF0IGxlYXN0IG92ZXIgdGhlIHBlcmlvZCBpbmNsdWRlZCBpbiB0aGUgZGF0YSkuCgpgYGB7cn0Kb3JpZyA8LSBkby5jYWxsKHJiaW5kLCBval90ZXN0KSAlPiUKICAgIGFzX3RpYmJsZSgpICU+JQogICAgc2VsZWN0KHN0b3JlLCBicmFuZCwgd2VlaywgbG9nbW92ZSkgJT4lCiAgICBtdXRhdGUobW92ZT1leHAobG9nbW92ZSkpCgpnb2YgPC0gZnVuY3Rpb24oZmNhc3RfZGF0YSkKewogICAgZmNhc3RfZGF0YSA8LSBkby5jYWxsKHJiaW5kLCBmY2FzdF9kYXRhKSAlPiUKICAgICAgICBhc190aWJibGUoKSAlPiUKICAgICAgICBzZWxlY3Qoc3RvcmUsIGJyYW5kLCB3ZWVrLCAubW9kZWwsIGxvZ21vdmUpICU+JQogICAgICAgIHBpdm90X3dpZGVyKGlkX2NvbHM9YyhzdG9yZSwgYnJhbmQsIHdlZWspLCBuYW1lc19mcm9tPS5tb2RlbCwgdmFsdWVzX2Zyb209bG9nbW92ZSkgJT4lCiAgICAgICAgc2VsZWN0KC1zdG9yZSwgLWJyYW5kLCAtd2VlaykgJT4lCiAgICAgICAgc3VtbWFyaXNlX2FsbChmdW5jdGlvbih4KSBNQVBFKGV4cCh4KSAtIG9yaWckbW92ZSwgb3JpZyRtb3ZlKSkKfQoKbGFwcGx5KGZjYXN0X3NldHMsIGdvZikgJT4lIGJpbmRfY29scygpCmBgYAo=
@@ -364,7 +331,7 @@ $(document).ready(function () { diff --git a/R/orange_juice/README.md b/R/orange_juice/README.md new file mode 100644 index 00000000..dead2f56 --- /dev/null +++ b/R/orange_juice/README.md @@ -0,0 +1,32 @@ +## Orange juice dataset + +### Package installation + +You'll need the following packages to run the notebooks in this directory: + +- bayesm (the source of the data) +- ggplot2 +- dplyr +- tidyr +- jsonlite +- tsibble +- urca +- fable +- fabletools +- feasts + +The easiest way to install them is to run + +```r +install.packages("bayesm") +install.packages("tidyverse") # installs all tidyverse packages +install.packages(c("fable", "feasts", "urca")) +``` + +The Rmarkdown notebooks in this directory are as follows. You should run them in sequence, as each will create output objects (datasets/models) that are used in later notebooks. + +- [`01_dataprep.Rmd`](01_dataprep.Rmd) creates the training and test datasets +- [`02_simplemodels.Rmd`](02_simplemodels.Rmd) fits a range of simple time series models to the data, including ARIMA and ETS models. +- [`02a_simplereg_models.Rmd`](02a_simplereg_models.Rmd) adds independent variables as regressors to the ARIMA model. +- [`03_model_eval.Rmd`](03_model_eval.Rmd) evaluates the goodness of fit of the models on the test data. + diff --git a/R/orange_juice/ojdata_forecast_settings.json b/R/orange_juice/ojdata_forecast_settings.json new file mode 100644 index 00000000..5bc46a25 --- /dev/null +++ b/R/orange_juice/ojdata_forecast_settings.json @@ -0,0 +1,5 @@ +{ + "STEP": 2, + "TRAIN_WINDOW": 135, + "START_DATE": "1989-09-14" +} diff --git a/README.md b/README.md index 1c14a9cd..e854f207 100644 --- a/README.md +++ b/README.md @@ -1,87 +1,16 @@ # Forecasting Best Practices -Time series forecasting is one of the most important topics in data science. Almost every business needs to predict the future in order to make better decisions and allocate resources more effectively. +This repository contains examples and best practices for building Forecasting solutions and systems, provided as [Jupyter notebooks](examples) and [a library of utility functions](fclib). The focus of the repository is on state-of-the-art methods and common scenarios that are popular among researchers and practitioners working on forecasting problems. -This repository provides examples and best practice guidelines for building forecasting solutions. The goal of this repository is to build a comprehensive set of tools and examples that leverage recent advances in forecasting algorithms to build solutions and operationalize them. Rather than creating implementations from scratch, we draw from existing state-of-the-art libraries and build additional utilities around processing and featurizing the data, optimizing and evaluating models, and scaling up to the cloud. +## Getting Started -The examples and best practices are provided as [Python Jupyter notebooks and R markdown files](examples) and [a library of utility functions](fclib). We hope that these examples and utilities can significantly reduce the “time to market” by simplifying the experience from defining the business problem to the development of solutions by orders of magnitude. In addition, the example notebooks would serve as guidelines and showcase best practices and usage of the tools in a wide variety of languages. - - -## Content - -The following is a summary of the examples related to the process of building forecasting solutions covered in this repository. The [examples](examples) are organized according to use cases. Currently, we focus on a retail sales forecasting use case as it is widely used in [assortment planning](https://repository.upenn.edu/cgi/viewcontent.cgi?article=1569&context=edissertations), [inventory optimization](https://en.wikipedia.org/wiki/Inventory_optimization), and [price optimization](https://en.wikipedia.org/wiki/Price_optimization). - -| Example | Models/Methods | Description | Language | -|----------------------------------|-------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------|-----------| -| Quick Start | Auto ARIMA, Azure AutoML, Linear Regression, LightGBM | Quick start notebooks that demonstrate workflow of developing a forecast model using one-round training and testing data | Python | -| Data Exploration and Preparation | Statistical Analysis and Data Transformation | Data exploration and preparation examples | Python, R | -| Model Training and Evaluation | Auto ARIMA, LightGBM, Dilated CNN | Deep dive notebooks that perform multi-round training and testing of various classical and deep learning forecast algorithms | Python | -| Model Tuning and Deployment | HyperDrive, LightGBM | Example notebook for model tuning using Azure Machine Learning Service and deploying the best model on Azure | Python | -| R Models | Mean Forecast, ARIMA, ETS, Prophet | Popular statistical forecast models and Prophet model implmented in R | R | - - -## Getting Started in Python - -To quickly get started with the repository on your local machine, use the following commands. - -1. Install Anaconda with Python >= 3.6. [Miniconda](https://conda.io/miniconda.html) is a quick way to get started. - -2. Clone the repository - ``` - git clone https://github.com/microsoft/forecasting - cd forecasting/ - ``` - -3. Run setup scripts to create conda environment. Please execute one of the following commands from the root of Forecasting repo based on your operating system. - - - Linux - ``` - ./tools/environment_setup.sh - ``` - - - Windows - ``` - tools\environment_setup.bat - ``` - - Note that for Windows you need to run the batch script from Anaconda Prompt. The script creates a conda environment `forecasting_env` and installs the forecasting utility library `fclib`. - -4. Start the Jupyter notebook server - ``` - jupyter notebook - ``` - -5. Run the [LightGBM single-round](examples/oj_retail/python/00_quick_start/lightgbm_single_round.ipynb) notebook under the `00_quick_start` folder. Make sure that the selected Jupyter kernel is `forecasting_env`. - -If you have any issues with the above setup, or want to find more detailed instructions on how to set up your environment and run examples provided in the repository, on local or a remote machine, please navigate to the [Setup Guide](./docs/SETUP.md). - -## Getting Started in R - -We assume you already have R installed on your machine. If not, simply follow the [instructions on CRAN](https://cloud.r-project.org/) to download and install R. - -The recommended editor is [RStudio](https://rstudio.com), which supports interactive editing and previewing of R notebooks. However, you can use any editor or IDE that supports RMarkdown. In particular, [Visual Studio Code](https://code.visualstudio.com) with the [R extension](https://marketplace.visualstudio.com/items?itemName=Ikuyadeu.r) can be used to edit and render the notebook files. The rendered `.nb.html` files can be viewed in any modern web browser. - -The examples use the [Tidyverts](https://tidyverts.org) family of packages, which is a modern framework for time series analysis that builds on the widely-used [Tidyverse](https://tidyverse.org) family. The Tidyverts framework is still under active development, so it's recommended that you update your packages regularly to get the latest bug fixes and features. - -## Target Audience -Our target audience for this repository includes data scientists and machine learning engineers with varying levels of knowledge in forecasting as our content is source-only and targets custom machine learning modelling. The utilities and examples provided are intended to be solution accelerators for real-world forecasting problems. +To get started, navigate to the [Setup Guide](./docs/SETUP.md), which lists instructions on how to set up your environment and dependencies, download the data and run examples provided in the repository. ## Contributing We hope that the open source community would contribute to the content and bring in the latest SOTA algorithm. This project welcomes contributions and suggestions. Before contributing, please see our [Contributing Guide](./docs/CONTRIBUTING.md). -## Reference - -The following is a list of related repositories that you may find helpful. - -| | | -|------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------| -| [Deep Learning for Time Series Forecasting](https://github.com/Azure/DeepLearningForTimeSeriesForecasting) | A collection of examples for using deep neural networks for time series forecasting with Keras. | -| [Demand Forecasting and Price Optimization Solution](https://github.com/Azure/cortana-intelligence-price-optimization) | A Cortana Intelligence solution how-to guide for demand forecasting and price optimization. | - - - ## Build Status -| Build | Branch | Status | -|---------------|---------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Linux CPU** | master | [![Build Status](https://dev.azure.com/best-practices/forecasting/_apis/build/status/cpu_unit_tests_linux?branchName=master)](https://dev.azure.com/best-practices/forecasting/_build/latest?definitionId=128&branchName=master) | -| **Linux CPU** | staging | [![Build Status](https://dev.azure.com/best-practices/forecasting/_apis/build/status/cpu_unit_tests_linux?branchName=staging)](https://dev.azure.com/best-practices/forecasting/_build/latest?definitionId=128&branchName=staging) | +| Build | Branch | Status | +| --- | --- | --- | +| **Linux CPU** | master | [![Build Status](https://dev.azure.com/best-practices/forecasting/_apis/build/status/cpu_unit_tests_linux?branchName=master)](https://dev.azure.com/best-practices/forecasting/_build/latest?definitionId=128&branchName=master) | +| **Linux CPU** | staging | [![Build Status](https://dev.azure.com/best-practices/forecasting/_apis/build/status/cpu_unit_tests_linux?branchName=staging)](https://dev.azure.com/best-practices/forecasting/_build/latest?definitionId=128&branchName=staging) | \ No newline at end of file diff --git a/R_utils/cluster.R b/R_utils/cluster.R deleted file mode 100644 index 949101d0..00000000 --- a/R_utils/cluster.R +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -#' Creates a local background cluster for parallel computations -#' -#' @param ncores The number of nodes (cores) for the cluster. The default is 2 less than the number of physical cores. -#' @param libs The packages to load on each node, as a character vector. -#' @param useXDR For most platforms, this can be left at its default `FALSE` value. -#' @return -#' A cluster object. -make_cluster <- function(ncores=NULL, libs=character(0), useXDR=FALSE) -{ - if(is.null(ncores)) - ncores <- max(2, parallel::detectCores(logical=FALSE) - 2) - cl <- parallel::makeCluster(ncores, type="PSOCK", useXDR=useXDR) - res <- try(parallel::clusterCall( - cl, - function(libs) - { - for(lib in libs) library(lib, character.only=TRUE) - }, - libs - ), silent=TRUE) - if(inherits(res, "try-error")) - parallel::stopCluster(cl) - else cl -} - - -#' Deletes a local background cluster -#' -#' @param cl The cluster object, as returned from `make_cluster`. -destroy_cluster <- function(cl) -{ - try(parallel::stopCluster(cl), silent=TRUE) -} diff --git a/R_utils/model_eval.R b/R_utils/model_eval.R deleted file mode 100644 index a321ad58..00000000 --- a/R_utils/model_eval.R +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -#' Computes forecast values on a dataset -#' -#' @param mable A mable (model table) as returned by `fabletools::model`. -#' @param newdata The dataset for which to compute forecasts. -#' @param ... Further arguments to `fabletools::forecast`. -#' @return -#' A tsibble, with one column per model type in `mable`, and one column named `.response` containing the response variable from `newdata`. -get_forecasts <- function(mable, newdata, ...) -{ - fcast <- forecast(mable, new_data=newdata, ...) - keyvars <- key_vars(fcast) - keyvars <- keyvars[-length(keyvars)] - indexvar <- index_var(fcast) - fcastvar <- as.character(attr(fcast, "response")[[1]]) - fcast <- fcast %>% - as_tibble() %>% - pivot_wider( - id_cols=all_of(c(keyvars, indexvar)), - names_from=.model, - values_from=all_of(fcastvar)) - select(newdata, !!keyvars, !!indexvar, !!fcastvar) %>% - rename(.response=!!fcastvar) %>% - inner_join(fcast) -} - - -#' Evaluate quality of forecasts given a criterion -#' -#' @param fcast_df A tsibble as returned from `get_forecasts`. -#' @param gof A goodness-of-fit function. The default is to use `fabletools::MAPE`, which computes the mean absolute percentage error. -#' @return -#' A single-row data frame with the computed goodness-of-fit statistic for each model. -eval_forecasts <- function(fcast_df, gof=fabletools::MAPE) -{ - if(!is.function(gof)) - gof <- get(gof, mode="function") - resp <- fcast_df$.response - keyvars <- key_vars(fcast_df) - indexvar <- index_var(fcast_df) - fcast_df %>% - as_tibble() %>% - select(-all_of(c(keyvars, indexvar, ".response"))) %>% - summarise_all( - function(x, .actual) gof(x - .actual, .actual=.actual), - .actual=resp - ) -} diff --git a/R_utils/save_objects.R b/R_utils/save_objects.R deleted file mode 100644 index b9492491..00000000 --- a/R_utils/save_objects.R +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -#' Loads serialised objects relating to a given forecasting example into the current workspace -#' -#' @param example The particular forecasting example. -#' @param file The name of the file (with extension). -#' @return -#' This function is run for its side effect, namely loading the given file into the global environment. -load_objects <- function(example, file) -{ - examp_dir <- here::here("examples", example, "R") - load(file.path(examp_dir, file), envir=globalenv()) -} - -#' Saves R objects for a forecasting example to a file -#' -#' @param ... Objects to save, as unquoted names. -#' @param example The particular forecasting example. -#' @param file The name of the file (with extension). -save_objects <- function(..., example, file) -{ - examp_dir <- here::here("examples", example, "R") - save(..., file=file.path(examp_dir, file)) -} diff --git a/contrib/tsperf/energy_utils/feature_engineering/feature_utils.py b/contrib/tsperf/energy_utils/feature_engineering/feature_utils.py deleted file mode 100644 index 55ffcc7e..00000000 --- a/contrib/tsperf/energy_utils/feature_engineering/feature_utils.py +++ /dev/null @@ -1,1004 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -""" -This file contains utility functions for creating features for time -series forecasting applications. All functions defined assume that -there is no missing data. -""" - -import calendar -import itertools -import pandas as pd -import numpy as np -from datetime import timedelta -from sklearn.preprocessing import MinMaxScaler - -from fclib.feature_engineering.utils import is_datetime_like - -# 0: Monday, 2: T/W/TR, 4: F, 5:SA, 6: S -WEEK_DAY_TYPE_MAP = {1: 2, 3: 2} # Map for converting Wednesday and -# Thursday to have the same code as Tuesday -HOLIDAY_CODE = 7 -SEMI_HOLIDAY_CODE = 8 # days before and after a holiday - -DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S" - - -def day_type(datetime_col, holiday_col=None, semi_holiday_offset=timedelta(days=1)): - """ - Convert datetime_col to 7 day types - 0: Monday - 2: Tuesday, Wednesday, and Thursday - 4: Friday - 5: Saturday - 6: Sunday - 7: Holiday - 8: Days before and after a holiday - - Args: - datetime_col: Datetime column. - holiday_col: Holiday code column. Default value None. - semi_holiday_offset: Time difference between the date before (or after) - the holiday and the holiday. Default value timedelta(days=1). - - Returns: - A numpy array containing converted datatime_col into day types. - """ - - datetype = pd.DataFrame({"DayType": datetime_col.dt.dayofweek}) - datetype.replace({"DayType": WEEK_DAY_TYPE_MAP}, inplace=True) - - if holiday_col is not None: - holiday_mask = holiday_col > 0 - datetype.loc[holiday_mask, "DayType"] = HOLIDAY_CODE - - # Create a temporary Date column to calculate dates near the holidays - datetype["Date"] = pd.to_datetime(datetime_col.dt.date, format=DATETIME_FORMAT) - holiday_dates = set(datetype.loc[holiday_mask, "Date"]) - - semi_holiday_dates = [ - pd.date_range(start=d - semi_holiday_offset, end=d + semi_holiday_offset, freq="D") for d in holiday_dates - ] - - # Flatten the list of lists - semi_holiday_dates = [d for dates in semi_holiday_dates for d in dates] - - semi_holiday_dates = set(semi_holiday_dates) - semi_holiday_dates = semi_holiday_dates.difference(holiday_dates) - - datetype.loc[datetype["Date"].isin(semi_holiday_dates), "DayType"] = SEMI_HOLIDAY_CODE - - return datetype["DayType"].values - - -def hour_of_day(datetime_col): - """Returns the hour from a datetime column.""" - return datetime_col.dt.hour - - -def time_of_year(datetime_col): - """ - Time of year is a cyclic variable that indicates the annual position and - repeats each year. It is each year linearly increasing over time going - from 0 on January 1 at 00:00 to 1 on December 31st at 23:00. The values - are normalized to be between [0; 1]. - - Args: - datetime_col: Datetime column. - - Returns: - A numpy array containing converted datatime_col into time of year. - """ - - time_of_year = pd.DataFrame( - {"DayOfYear": datetime_col.dt.dayofyear, "HourOfDay": datetime_col.dt.hour, "Year": datetime_col.dt.year} - ) - time_of_year["TimeOfYear"] = (time_of_year["DayOfYear"] - 1) * 24 + time_of_year["HourOfDay"] - - time_of_year["YearLength"] = time_of_year["Year"].apply(lambda y: 366 if calendar.isleap(y) else 365) - - time_of_year["TimeOfYear"] = time_of_year["TimeOfYear"] / (time_of_year["YearLength"] * 24 - 1) - - return time_of_year["TimeOfYear"].values - - -def week_of_year(datetime_col): - """Returns the week from a datetime column.""" - return datetime_col.dt.week - - -def week_of_month(date_time): - """Returns the week of the month for a specified date. - - Args: - dt (Datetime): Input date - - Returns: - wom (Integer): Week of the month of the input date - """ - - def _week_of_month(date_time): - from math import ceil - - first_day = date_time.replace(day=1) - dom = date_time.day - adjusted_dom = dom + first_day.weekday() - wom = int(ceil(adjusted_dom / 7.0)) - return wom - - if isinstance(date_time, pd.Series): - return date_time.apply(lambda x: _week_of_month(x)) - else: - return _week_of_month(date_time) - - -def month_of_year(date_time_col): - """Returns the month from a datetime column.""" - return date_time_col.dt.month - - -def day_of_week(date_time_col): - """Returns the day of week from a datetime column.""" - return date_time_col.dt.dayofweek - - -def day_of_month(date_time_col): - """Returns the day of month from a datetime column.""" - return date_time_col.dt.day - - -def day_of_year(date_time_col): - """Returns the day of year from a datetime column.""" - return date_time_col.dt.dayofyear - - -def encoded_month_of_year(month_of_year): - """ - Create one hot encoding of month of year. - """ - month_of_year = pd.get_dummies(month_of_year, prefix="MonthOfYear") - - return month_of_year - - -def encoded_day_of_week(day_of_week): - """ - Create one hot encoding of day_of_week. - """ - day_of_week = pd.get_dummies(day_of_week, prefix="DayOfWeek") - - return day_of_week - - -def encoded_day_of_month(day_of_month): - """ - Create one hot encoding of day_of_month. - """ - day_of_month = pd.get_dummies(day_of_month, prefix="DayOfMonth") - - return day_of_month - - -def encoded_day_of_year(day_of_year): - """ - Create one hot encoding of day_of_year. - """ - day_of_year = pd.get_dummies(day_of_year) - - return day_of_year - - -def encoded_hour_of_day(hour_of_day): - """ - Create one hot encoding of hour_of_day. - """ - hour_of_day = pd.get_dummies(hour_of_day, prefix="HourOfDay") - - return hour_of_day - - -def encoded_week_of_year(week_of_year): - """ - Create one hot encoding of week_of_year. - """ - week_of_year = pd.get_dummies(week_of_year, prefix="WeekOfYear") - - return week_of_year - - -def normalized_current_year(datetime_col, min_year, max_year): - """ - Temporal feature indicating the position of the year of a record in the - entire time period under consideration, normalized to be between 0 and 1. - - Args: - datetime_col: Datetime column. - min_year: minimum value of year. - max_year: maximum value of year. - - Returns: - float: the position of the current year in the min_year:max_year range - """ - year = datetime_col.dt.year - - if max_year != min_year: - current_year = (year - min_year) / (max_year - min_year) - elif max_year == min_year: - current_year = 0 - - return current_year - - -def normalized_current_date(datetime_col, min_date, max_date): - """ - Temporal feature indicating the position of the date of a record in the - entire time period under consideration, normalized to be between 0 and 1. - - Args: - datetime_col: Datetime column. - min_date: minimum value of date. - max_date: maximum value of date. - - Returns: - float: the position of the current date in the min_date:max_date range - """ - date = datetime_col.dt.date - current_date = (date - min_date).apply(lambda x: x.days) - - if max_date != min_date: - current_date = current_date / (max_date - min_date).days - elif max_date == min_date: - current_date = 0 - - return current_date - - -def normalized_current_datehour(datetime_col, min_datehour, max_datehour): - """ - Temporal feature indicating the position of the hour of a record in the - entire time period under consideration, normalized to be between 0 and 1. - - Args: - datetime_col: Datetime column. - min_datehour: minimum value of datehour. - max_datehour: maximum value of datehour. - - Returns: - float: the position of the current datehour in the min_datehour:max_datehour range - """ - current_datehour = (datetime_col - min_datehour).apply(lambda x: x.days * 24 + x.seconds / 3600) - - max_min_diff = max_datehour - min_datehour - - if max_min_diff != 0: - current_datehour = current_datehour / (max_min_diff.days * 24 + max_min_diff.seconds / 3600) - elif max_min_diff == 0: - current_datehour = 0 - - return current_datehour - - -def normalized_columns(datetime_col, value_col, mode="log", output_colname="normalized_columns"): - """ - Creates columns normalized to be log of input columns devided by global average of each columns, - or normalized using maximum and minimum. - - Args: - datetime_col: Datetime column. - value_col: Value column to be normalized. - mode: Normalization mode, - accepted values are 'log' and 'minmax'. Default value 'log'. - - Returns: - Normalized value column. - """ - - if not is_datetime_like(datetime_col): - datetime_col = pd.to_datetime(datetime_col, format=DATETIME_FORMAT) - - df = pd.DataFrame({"Datetime": datetime_col, "value": value_col}) - df.set_index("Datetime", inplace=True) - - if not df.index.is_monotonic: - df.sort_index(inplace=True) - - if mode == "log": - mean_value = df["value"].mean() - if mean_value != 0: - df[output_colname] = np.log(df["value"] / mean_value) - elif mean_value == 0: - df[output_colname] = 0 - elif mode == "minmax": - min_value = min(df["value"]) - max_value = max(df["value"]) - if min_value != max_value: - df[output_colname] = (df["value"] - min_value) / (max_value - min_value) - elif min_value == max_value: - df[output_colname] = 0 - else: - raise ValueError("Valid values for mode are 'log' and 'minmax'") - - return df[[output_colname]] - - -def fourier_approximation(t, n, period): - """ - Generic helper function to create Fourier Series at different harmonies (n) and periods. - - Args: - t: Datetime column. - n: Harmonies, n=0, 1, 2, 3,... - period: Period of the datetime variable t. - - Returns: - float: Sine component - float: Cosine component - """ - x = n * 2 * np.pi * t / period - x_sin = np.sin(x) - x_cos = np.cos(x) - - return x_sin, x_cos - - -def annual_fourier(datetime_col, n_harmonics): - """ - Creates Annual Fourier Series at different harmonies (n). - - Args: - datetime_col: Datetime column. - n_harmonics: Harmonies, n=0, 1, 2, 3,... - - Returns: - dict: Output dictionary containing sine and cosine components of - the Fourier series for all harmonies. - """ - day_of_year = datetime_col.dt.dayofyear - - output_dict = {} - for n in range(1, n_harmonics + 1): - sin, cos = fourier_approximation(day_of_year, n, 365.24) - - output_dict["annual_sin_" + str(n)] = sin - output_dict["annual_cos_" + str(n)] = cos - - return output_dict - - -def weekly_fourier(datetime_col, n_harmonics): - """ - Creates Weekly Fourier Series at different harmonies (n). - - Args: - datetime_col: Datetime column. - n_harmonics: Harmonies, n=0, 1, 2, 3,... - - Returns: - dict: Output dictionary containing sine and cosine components of - the Fourier series for all harmonies. - """ - day_of_week = datetime_col.dt.dayofweek + 1 - - output_dict = {} - for n in range(1, n_harmonics + 1): - sin, cos = fourier_approximation(day_of_week, n, 7) - - output_dict["weekly_sin_" + str(n)] = sin - output_dict["weekly_cos_" + str(n)] = cos - - return output_dict - - -def daily_fourier(datetime_col, n_harmonics): - """ - Creates Daily Fourier Series at different harmonies (n). - - Args: - datetime_col: Datetime column. - n_harmonics: Harmonies, n=0, 1, 2, 3,... - - Returns: - dict: Output dictionary containing sine and cosine components of - the Fourier series for all harmonies. - """ - hour_of_day = datetime_col.dt.hour + 1 - - output_dict = {} - for n in range(1, n_harmonics + 1): - sin, cos = fourier_approximation(hour_of_day, n, 24) - - output_dict["daily_sin_" + str(n)] = sin - output_dict["daily_cos_" + str(n)] = cos - - return output_dict - - -def same_week_day_hour_lag( - datetime_col, value_col, n_years=3, week_window=1, agg_func="mean", q=None, output_colname="SameWeekHourLag" -): - """ - Creates a lag feature by calculating quantiles, mean and std of values of and - around the same week, same day of week, and same hour of day, of previous years. - - Args: - datetime_col: Datetime column. - value_col: Feature value column to create lag feature from. - n_years: Number of previous years data to use. Default value 3. - week_window: Number of weeks before and after the same week to use, - which should help reduce noise in the data. Default value 1. - agg_func: Aggregation function to apply on multiple previous values, - accepted values are 'mean', 'quantile', 'std'. Default value 'mean'. - q: If agg_func is 'quantile', taking value between 0 and 1. - output_colname: name of the output lag feature column. - Default value 'SameWeekHourLag'. - - Returns: - pd.DataFrame: data frame containing the newly created lag - feature as a column. - """ - - if not is_datetime_like(datetime_col): - datetime_col = pd.to_datetime(datetime_col, format=DATETIME_FORMAT) - min_time_stamp = min(datetime_col) - max_time_stamp = max(datetime_col) - - df = pd.DataFrame({"Datetime": datetime_col, "value": value_col}) - df.set_index("Datetime", inplace=True) - - week_lag_base = 52 - week_lag_last_year = list(range(week_lag_base - week_window, week_lag_base + week_window + 1)) - week_lag_all = [] - for y in range(n_years): - week_lag_all += [x + y * 52 for x in week_lag_last_year] - - week_lag_cols = [] - for w in week_lag_all: - if (max_time_stamp - timedelta(weeks=w)) >= min_time_stamp: - col_name = "week_lag_" + str(w) - week_lag_cols.append(col_name) - - lag_datetime = df.index.get_level_values(0) - timedelta(weeks=w) - valid_lag_mask = lag_datetime >= min_time_stamp - - df[col_name] = np.nan - - df.loc[valid_lag_mask, col_name] = df.loc[lag_datetime[valid_lag_mask], "value"].values - - # Additional aggregation options will be added as needed - if agg_func == "mean" and q is None: - df[output_colname] = round(df[week_lag_cols].mean(axis=1)) - elif agg_func == "quantile" and q is not None: - df[output_colname] = round(df[week_lag_cols].quantile(q, axis=1)) - elif agg_func == "std" and q is None: - df[output_colname] = round(df[week_lag_cols].std(axis=1)) - - return df[[output_colname]] - - -def same_day_hour_lag( - datetime_col, value_col, n_years=3, day_window=1, agg_func="mean", q=None, output_colname="SameDayHourLag" -): - """ - Creates a lag feature by calculating quantiles, mean, and std of values of - and around the same day of year, and same hour of day, of previous years. - - Args: - datetime_col: Datetime column. - value_col: Feature value column to create lag feature from. - n_years: Number of previous years data to use. Default value 3. - day_window: Number of days before and after the same day to use, - which should help reduce noise in the data. Default value 1. - agg_func: Aggregation function to apply on multiple previous values, - accepted values are 'mean', 'quantile', 'std'. Default value 'mean'. - q: If agg_func is 'quantile', taking value between 0 and 1. - output_colname: name of the output lag feature column. - Default value 'SameDayHourLag'. - - Returns: - pd.DataFrame: data frame containing the newly created lag - feature as a column. - """ - - if not is_datetime_like(datetime_col): - datetime_col = pd.to_datetime(datetime_col, format=DATETIME_FORMAT) - min_time_stamp = min(datetime_col) - max_time_stamp = max(datetime_col) - - df = pd.DataFrame({"Datetime": datetime_col, "value": value_col}) - df.set_index("Datetime", inplace=True) - - day_lag_base = 365 - day_lag_last_year = list(range(day_lag_base - day_window, day_lag_base + day_window + 1)) - day_lag_all = [] - for y in range(n_years): - day_lag_all += [x + y * 365 for x in day_lag_last_year] - - day_lag_cols = [] - for d in day_lag_all: - if (max_time_stamp - timedelta(days=d)) >= min_time_stamp: - col_name = "day_lag_" + str(d) - day_lag_cols.append(col_name) - - lag_datetime = df.index.get_level_values(0) - timedelta(days=d) - valid_lag_mask = lag_datetime >= min_time_stamp - - df[col_name] = np.nan - - df.loc[valid_lag_mask, col_name] = df.loc[lag_datetime[valid_lag_mask], "value"].values - - # Additional aggregation options will be added as needed - if agg_func == "mean" and q is None: - df[output_colname] = round(df[day_lag_cols].mean(axis=1)) - elif agg_func == "quantile" and q is not None: - df[output_colname] = round(df[day_lag_cols].quantile(q, axis=1)) - elif agg_func == "std" and q is None: - df[output_colname] = round(df[day_lag_cols].std(axis=1)) - - return df[[output_colname]] - - -def same_day_hour_moving_average( - datetime_col, - value_col, - window_size, - start_week, - average_count, - forecast_creation_time, - output_col_prefix="moving_average_lag_", -): - """ - Creates moving average features by averaging values of the same day of - week and same hour of day of previous weeks. - - Args: - datetime_col: Datetime column - value_col: Feature value column to create moving average features from. - window_size: Number of weeks used to compute the average. - start_week: First week of the first moving average feature. - average_count: Number of moving average features to create. - forecast_creation_time: The time point when the feature is created. - This value is used to prevent using data that are not available - at forecast creation time to compute features. - output_col_prefix: Prefix of the output columns. The start week of each - moving average feature is added at the end. Default value 'moving_average_lag_'. - - Returns: - pd.DataFrame: data frame containing the newly created lag features as - columns. - - For example, start_week = 9, window_size=4, and average_count = 3 will - create three moving average features. - 1) moving_average_lag_9: average the same day and hour values of the 9th, - 10th, 11th, and 12th weeks before the current week. - 2) moving_average_lag_10: average the same day and hour values of the - 10th, 11th, 12th, and 13th weeks before the current week. - 3) moving_average_lag_11: average the same day and hour values of the - 11th, 12th, 13th, and 14th weeks before the current week. - """ - - df = pd.DataFrame({"Datetime": datetime_col, "value": value_col}) - df.set_index("Datetime", inplace=True) - - df = df.asfreq("H") - - if not df.index.is_monotonic: - df.sort_index(inplace=True) - - df["fct_diff"] = df.index - forecast_creation_time - df["fct_diff"] = df["fct_diff"].apply(lambda x: x.days * 24 + x.seconds / 3600) - max_diff = max(df["fct_diff"]) - - for i in range(average_count): - output_col = output_col_prefix + str(start_week + i) - week_lag_start = start_week + i - hour_lags = [(week_lag_start + w) * 24 * 7 for w in range(window_size)] - hour_lags = [h for h in hour_lags if h > max_diff] - if len(hour_lags) > 0: - tmp_df = df[["value"]].copy() - tmp_col_all = [] - for h in hour_lags: - tmp_col = "tmp_lag_" + str(h) - tmp_col_all.append(tmp_col) - tmp_df[tmp_col] = tmp_df["value"].shift(h) - - df[output_col] = round(tmp_df[tmp_col_all].mean(axis=1)) - df.drop(["fct_diff", "value"], inplace=True, axis=1) - - return df - - -def same_day_hour_moving_quantile( - datetime_col, - value_col, - window_size, - start_week, - quantile_count, - q, - forecast_creation_time, - output_col_prefix="moving_quatile_lag_", -): - """ - Creates a series of quantiles features by calculating quantiles of values of - the same day of week and same hour of day of previous weeks. - - Args: - datetime_col: Datetime column - value_col: Feature value column to create quantile features from. - window_size: Number of weeks used to compute the quantile. - start_week: First week of the first moving quantile feature. - quantile_count: Number of quantile features to create. - q: quantile to compute from history values, should be between 0 and 1. - forecast_creation_time: The time point when the feature is created. - This value is used to prevent using data that are not available - at forecast creation time to compute features. - output_col_prefix: Prefix of the output columns. The start week of each - moving average feature is added at the end. Default value 'moving_quatile_lag_'. - - Returns: - pd.DataFrame: data frame containing the newly created lag features as - columns. - - For example, start_week = 9, window_size=4, and quantile_count = 3 will - create three quantiles features. - 1) moving_quantile_lag_9: calculate quantile of the same day and hour values of the 9th, - 10th, 11th, and 12th weeks before the current week. - 2) moving_quantile_lag_10: calculate quantile of average the same day and hour values of the - 10th, 11th, 12th, and 13th weeks before the current week. - 3) moving_quantile_lag_11: calculate quantile of average the same day and hour values of the - 11th, 12th, 13th, and 14th weeks before the current week. - """ - - df = pd.DataFrame({"Datetime": datetime_col, "value": value_col}) - df.set_index("Datetime", inplace=True) - - df = df.asfreq("H") - - if not df.index.is_monotonic: - df.sort_index(inplace=True) - - df["fct_diff"] = df.index - forecast_creation_time - df["fct_diff"] = df["fct_diff"].apply(lambda x: x.days * 24 + x.seconds / 3600) - max_diff = max(df["fct_diff"]) - - for i in range(quantile_count): - output_col = output_col_prefix + str(start_week + i) - week_lag_start = start_week + i - hour_lags = [(week_lag_start + w) * 24 * 7 for w in range(window_size)] - hour_lags = [h for h in hour_lags if h > max_diff] - if len(hour_lags) > 0: - tmp_df = df[["value"]].copy() - tmp_col_all = [] - for h in hour_lags: - tmp_col = "tmp_lag_" + str(h) - tmp_col_all.append(tmp_col) - tmp_df[tmp_col] = tmp_df["value"].shift(h) - - df[output_col] = round(tmp_df[tmp_col_all].quantile(q, axis=1)) - - df.drop(["fct_diff", "value"], inplace=True, axis=1) - - return df - - -def same_day_hour_moving_std( - datetime_col, - value_col, - window_size, - start_week, - std_count, - forecast_creation_time, - output_col_prefix="moving_std_lag_", -): - """ - Creates standard deviation features by calculating std of values of the - same day of week and same hour of day of previous weeks. - - Args: - datetime_col: Datetime column - value_col: Feature value column to create moving std features from. - window_size: Number of weeks used to compute the std. - start_week: First week of the first moving std feature. - std_count: Number of moving std features to create. - forecast_creation_time: The time point when the feature is created. - This value is used to prevent using data that are not available at - forecast creation time to compute features. - output_col_prefix: Prefix of the output columns. The start week of each - moving average feature is added at the end. Default value 'moving_std_lag_'. - - Returns: - pd.DataFrame: data frame containing the newly created lag features as - columns. - - For example, start_week = 9, window_size=4, and std_count = 3 will - create three moving std features. - 1) moving_std_lag_9: calculate std of the same day and hour values of the 9th, - 10th, 11th, and 12th weeks before the current week. - 2) moving_std_lag_10: calculate std of the same day and hour values of the - 10th, 11th, 12th, and 13th weeks before the current week. - 3) moving_std_lag_11: calculate std of the same day and hour values of the - 11th, 12th, 13th, and 14th weeks before the current week. - """ - - df = pd.DataFrame({"Datetime": datetime_col, "value": value_col}) - df.set_index("Datetime", inplace=True) - - df = df.asfreq("H") - - if not df.index.is_monotonic: - df.sort_index(inplace=True) - - df["fct_diff"] = df.index - forecast_creation_time - df["fct_diff"] = df["fct_diff"].apply(lambda x: x.days * 24 + x.seconds / 3600) - max_diff = max(df["fct_diff"]) - - for i in range(std_count): - output_col = output_col_prefix + str(start_week + i) - week_lag_start = start_week + i - hour_lags = [(week_lag_start + w) * 24 * 7 for w in range(window_size)] - hour_lags = [h for h in hour_lags if h > max_diff] - if len(hour_lags) > 0: - tmp_df = df[["value"]].copy() - tmp_col_all = [] - for h in hour_lags: - tmp_col = "tmp_lag_" + str(h) - tmp_col_all.append(tmp_col) - tmp_df[tmp_col] = tmp_df["value"].shift(h) - - df[output_col] = round(tmp_df[tmp_col_all].std(axis=1)) - - df.drop(["value", "fct_diff"], inplace=True, axis=1) - - return df - - -def same_day_hour_moving_agg( - datetime_col, - value_col, - window_size, - start_week, - count, - forecast_creation_time, - agg_func="mean", - q=None, - output_col_prefix="moving_agg_lag_", -): - """ - Creates a series of aggregation features by calculating mean, quantiles, - or std of values of the same day of week and same hour of day of previous weeks. - - Args: - datetime_col: Datetime column - value_col: Feature value column to create aggregation features from. - window_size: Number of weeks used to compute the aggregation. - start_week: First week of the first aggregation feature. - count: Number of aggregation features to create. - forecast_creation_time: The time point when the feature is created. - This value is used to prevent using data that are not available - at forecast creation time to compute features. - agg_func: Aggregation function to apply on multiple previous values, - accepted values are 'mean', 'quantile', 'std'. - q: If agg_func is 'quantile', taking value between 0 and 1. - output_col_prefix: Prefix of the output columns. The start week of each - moving average feature is added at the end. Default value 'moving_agg_lag_'. - - Returns: - pd.DataFrame: data frame containing the newly created lag features as - columns. - - For example, start_week = 9, window_size=4, and count = 3 will - create three aggregation of features. - 1) moving_agg_lag_9: aggregate the same day and hour values of the 9th, - 10th, 11th, and 12th weeks before the current week. - 2) moving_agg_lag_10: aggregate the same day and hour values of the - 10th, 11th, 12th, and 13th weeks before the current week. - 3) moving_agg_lag_11: aggregate the same day and hour values of the - 11th, 12th, 13th, and 14th weeks before the current week. - """ - - df = pd.DataFrame({"Datetime": datetime_col, "value": value_col}) - df.set_index("Datetime", inplace=True) - - df = df.asfreq("H") - - if not df.index.is_monotonic: - df.sort_index(inplace=True) - - df["fct_diff"] = df.index - forecast_creation_time - df["fct_diff"] = df["fct_diff"].apply(lambda x: x.days * 24 + x.seconds / 3600) - max_diff = max(df["fct_diff"]) - - for i in range(count): - output_col = output_col_prefix + str(start_week + i) - week_lag_start = start_week + i - hour_lags = [(week_lag_start + w) * 24 * 7 for w in range(window_size)] - hour_lags = [h for h in hour_lags if h > max_diff] - if len(hour_lags) > 0: - tmp_df = df[["value"]].copy() - tmp_col_all = [] - for h in hour_lags: - tmp_col = "tmp_lag_" + str(h) - tmp_col_all.append(tmp_col) - tmp_df[tmp_col] = tmp_df["value"].shift(h) - - if agg_func == "mean" and q is None: - df[output_col] = round(tmp_df[tmp_col_all].mean(axis=1)) - elif agg_func == "quantile" and q is not None: - df[output_col] = round(tmp_df[tmp_col_all].quantile(q, axis=1)) - elif agg_func == "std" and q is None: - df[output_col] = round(tmp_df[tmp_col_all].std(axis=1)) - - df.drop(["fct_diff", "value"], inplace=True, axis=1) - - return df - - -def df_from_cartesian_product(dict_in): - """Generate a Pandas dataframe from Cartesian product of lists. - - Args: - dict_in (Dictionary): Dictionary containing multiple lists, e.g. {"fea1": list1, "fea2": list2} - - Returns: - df (Dataframe): Dataframe corresponding to the Caresian product of the lists - """ - from itertools import product - - cart = list(product(*dict_in.values())) - df = pd.DataFrame(cart, columns=dict_in.keys()) - return df - - -def lagged_features(df, lags): - """Create lagged features based on time series data. - - Args: - df (Dataframe): Input time series data sorted by time - lags (List): Lag lengths - - Returns: - fea (Dataframe): Lagged features - """ - df_list = [] - for lag in lags: - df_shifted = df.shift(lag) - df_shifted.columns = [x + "_lag" + str(lag) for x in df_shifted.columns] - df_list.append(df_shifted) - fea = pd.concat(df_list, axis=1) - return fea - - -def moving_averages(df, start_step, window_size=None): - """Compute averages of every feature over moving time windows. - - Args: - df (Dataframe): Input features as a dataframe - start_step (Integer): Starting time step of rolling mean - window_size (Integer): Windows size of rolling mean - - Returns: - fea (Dataframe): Dataframe consisting of the moving averages - """ - if window_size is None: - # Use a large window to compute average over all historical data - window_size = df.shape[0] - fea = df.shift(start_step).rolling(min_periods=1, center=False, window=window_size).mean() - fea.columns = fea.columns + "_mean" - return fea - - -def combine_features(df, lag_fea, lags, window_size, used_columns): - """Combine lag features, moving average features, and orignal features in the data. - - Args: - df (Dataframe): Time series data including the target series and external features - lag_fea (List): A list of column names for creating lagged features - lags (Numpy Array): Numpy array including all the lags - window_size (Integer): Window size of rolling mean - used_columns (List): A list containing the names of columns that are needed in the - input dataframe (including the target column) - - Returns: - fea_all (Dataframe): Dataframe including all the features - """ - lagged_fea = lagged_features(df[lag_fea], lags) - moving_avg = moving_averages(df[lag_fea], 2, window_size) - fea_all = pd.concat([df[used_columns], lagged_fea, moving_avg], axis=1) - return fea_all - - -def gen_sequence(df, seq_len, seq_cols, start_timestep=0, end_timestep=None): - """Reshape time series features into an array of dimension (# of time steps, # of - features). - - Args: - df (pd.Dataframe): Dataframe including time series data for a specific grain of a - multi-granular time series, e.g., data of a specific store-brand combination for - time series data involving multiple stores and brands - seq_len (int): Number of previous time series values to be used to form feature - sequences which can be used for model training - seq_cols (list[str]): A list of names of the feature columns - start_timestep (int): First time step you can use to create feature sequences - end_timestep (int): Last time step you can use to create feature sequences - - Returns: - object: A generator object for iterating all the feature sequences - """ - data_array = df[seq_cols].values - if end_timestep is None: - end_timestep = df.shape[0] - for start, stop in zip( - range(start_timestep, end_timestep - seq_len + 2), range(start_timestep + seq_len, end_timestep + 2) - ): - yield data_array[start:stop, :] - - -def gen_sequence_array(df_all, seq_len, seq_cols, grain1_name, grain2_name, start_timestep=0, end_timestep=None): - """Combine feature sequences for all the combinations of (grain1_name, grain2_name) into a - 3-dimensional array. - - Args: - df_all (pd.Dataframe): Time series data of all the grains for multi-granular data - seq_len (int): Number of previous time series values to be used to form sequences - seq_cols (list[str]): A list of names of the feature columns - grain1_name (str): Name of the 1st column indicating the time series graunularity - grain2_name (str): Name of the 2nd column indicating the time series graunularity - start_timestep (int): First time step you can use to create feature sequences - end_timestep (int): Last time step you can use to create feature sequences - - Returns: - seq_array (np.array): An array of feature sequences for all combinations of granularities - """ - seq_gen = ( - list( - gen_sequence( - df_all[(df_all[grain1_name] == grain1) & (df_all[grain2_name] == grain2)], - seq_len, - seq_cols, - start_timestep, - end_timestep, - ) - ) - for grain1, grain2 in itertools.product(df_all[grain1_name].unique(), df_all[grain2_name].unique()) - ) - seq_array = np.concatenate(list(seq_gen)).astype(np.float32) - return seq_array - - -def static_feature_array(df_all, total_timesteps, seq_cols, grain1_name, grain2_name): - """Generate an arary which encodes all the static features. - - Args: - df_all (pd.DataFrame): Time series data of all the grains for multi-granular data - total_timesteps (int): Total number of training samples for modeling - seq_cols (list[str]): A list of names of the static feature columns, e.g. store ID - grain1_name (str): Name of the 1st column indicating the time series graunularity - grain2_name (str): Name of the 2nd column indicating the time series graunularity - - Return: - fea_array (np.array): An array of static features of all the grains, e.g. all the - combinations of stores and brands in retail sale forecasting - """ - fea_df = ( - df_all.groupby([grain1_name, grain2_name]).apply(lambda x: x.iloc[:total_timesteps, :]).reset_index(drop=True) - ) - fea_array = fea_df[seq_cols].values - return fea_array - - -def normalize_columns(df, seq_cols, scaler=MinMaxScaler()): - """Normalize a subset of columns of a dataframe. - - Args: - df (pd.DataFrame): Input dataframe - seq_cols (list[str]): A list of names of columns to be normalized - scaler (object): A scikit learn scaler object - - Returns: - pd.DataFrame: Normalized dataframe - object: Scaler object - """ - cols_fixed = df.columns.difference(seq_cols) - df_scaled = pd.DataFrame(scaler.fit_transform(df[seq_cols]), columns=seq_cols, index=df.index) - df_scaled = pd.concat([df[cols_fixed], df_scaled], axis=1) - return df_scaled, scaler diff --git a/contrib/tsperf/readme_generator/readme_generator.py b/contrib/tsperf/readme_generator/readme_generator.py deleted file mode 100644 index 6e5e057a..00000000 --- a/contrib/tsperf/readme_generator/readme_generator.py +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -import csvtomd -import matplotlib.pyplot as plt -import pandas as pd - - -### Generating performance charts -################################################# - -# Function to plot a performance chart -def plot_perf(x, y, df): - - # extract submission name from submission URL - labels = df.apply(lambda x: x["Submission Name"][1:].split("]")[0], axis=1) - - fig = plt.scatter(x=df[x], y=df[y], label=labels, s=150, alpha=0.5, c=["b", "g", "r", "c", "m", "y", "k"]) - plt.xlabel(x) - plt.ylabel(y) - plt.title(y + " by " + x) - offset = (max(df[y]) - min(df[y])) / 50 - for i, name in enumerate(labels): - ax = df[x][i] - ay = df[y][i] + offset * (-2.5 + i % 5) - plt.text(ax, ay, name, fontsize=10) - - return fig - - -### Printing the Readme.md file -############################################ -readmefile = "../../Readme.md" -# Write header -# print(file=open(readmefile)) -print("# TSPerf\n", file=open(readmefile, "w")) - -print( - "TSPerf is a collection of implementations of time-series forecasting algorithms in Azure cloud and comparison of their performance over benchmark datasets. \ -Algorithm implementations are compared by model accuracy, training and scoring time and cost. Each implementation includes all the necessary \ -instructions and tools that ensure its reproducibility.", - file=open(readmefile, "a"), -) - -print("The following table summarizes benchmarks that are currently included in TSPerf.\n", file=open(readmefile, "a")) - -# Read the benchmark table the CSV file and converrt to a table in md format -with open("Benchmarks.csv", "r") as f: - table = csvtomd.csv_to_table(f, ",") -print(csvtomd.md_table(table), file=open(readmefile, "a")) -print("\n\n\n", file=open(readmefile, "a")) - -print( - "A complete documentation of TSPerf, along with the instructions for submitting and reviewing implementations, \ -can be found [here](./docs/tsperf_rules.md). The tables below show performance of implementations that are developed so far. Source code of \ -implementations and instructions for reproducing their performance can be found in submission folders, which are linked in the first column.\n", - file=open(readmefile, "a"), -) - -### Write the Energy section -# ============================ - -print("## Probabilistic energy forecasting performance board\n\n", file=open(readmefile, "a")) -print( - "The following table lists the current submision for the energy forecasting and their respective performances.\n\n", - file=open(readmefile, "a"), -) - -# Read the energy perfromane board from the CSV file and converrt to a table in md format -with open("TSPerfBoard-Energy.csv", "r") as f: - table = csvtomd.csv_to_table(f, ",") -print(csvtomd.md_table(table), file=open(readmefile, "a")) - -# Read Energy Performance Board CSV file -df = pd.read_csv("TSPerfBoard-Energy.csv", engine="python") -# df - -# Plot ,'Pinball Loss' by 'Training and Scoring Cost($)' chart -fig4 = plt.figure(figsize=(12, 8), dpi=80, facecolor="w", edgecolor="k") # this sets the plotting area size -fig4 = plot_perf("Training and Scoring Cost($)", "Pinball Loss", df) -plt.savefig("../../docs/images/Energy-Cost.png") - - -# insetting the performance charts -print( - "\n\nThe following chart compares the submissions performance on accuracy in Pinball Loss vs. Training and Scoring cost in $:\n\n ", - file=open(readmefile, "a"), -) -print("![EnergyPBLvsTime](./docs/images/Energy-Cost.png)", file=open(readmefile, "a")) -print("\n\n\n", file=open(readmefile, "a")) - - -# print the retail sales forcsating section -# ======================================== -print("## Retail sales forecasting performance board\n\n", file=open(readmefile, "a")) -print( - "The following table lists the current submision for the retail forecasting and their respective performances.\n\n", - file=open(readmefile, "a"), -) - -# Read the energy perfromane board from the CSV file and converrt to a table in md format -with open("TSPerfBoard-Retail.csv", "r") as f: - table = csvtomd.csv_to_table(f, ",") -print(csvtomd.md_table(table), file=open(readmefile, "a")) -print("\n\n\n", file=open(readmefile, "a")) - -# Read Retail Performane Board CSV file -df = pd.read_csv("TSPerfBoard-Retail.csv", engine="python") -# df - -# Plot MAPE (%) by Training and Scoring Cost ($) chart -fig2 = plt.figure(figsize=(12, 8), dpi=80, facecolor="w", edgecolor="k") # this sets the plotting area size -fig2 = plot_perf("Training and Scoring Cost ($)", "MAPE (%)", df) -plt.savefig("../../docs/images/Retail-Cost.png") - - -# insetting the performance charts -print( - "\n\nThe following chart compares the submissions performance on accuracy in %MAPE vs. Training and Scoring cost in $:\n\n ", - file=open(readmefile, "a"), -) -print("![EnergyPBLvsTime](./docs/images/Retail-Cost.png)", file=open(readmefile, "a")) -print("\n\n\n", file=open(readmefile, "a")) - -# insertting build status badge -print("## Build Status\n\n", file=open(readmefile, "a")) -print("| Build Type | Branch | Status | | Branch | Status |", file=open(readmefile, "a")) -print("| --- | --- | --- | --- | --- | --- |", file=open(readmefile, "a")) -print( - "| **Python Linux CPU** | master | [![Build Status](https://dev.azure.com/best-practices/forecasting/_apis/build/status/python_unit_tests_base?branchName=master)](https://dev.azure.com/best-practices/forecasting/_build/latest?definitionId=12&branchName=master) | | staging | [![Build Status](https://dev.azure.com/best-practices/forecasting/_apis/build/status/python_unit_tests_base?branchName=chenhui/python_test_pipeline)](https://dev.azure.com/best-practices/forecasting/_build/latest?definitionId=12&branchName=chenhui/python_test_pipeline) |", - file=open(readmefile, "a"), -) -print( - "| **R Linux CPU** | master | [![Build Status](https://dev.azure.com/best-practices/forecasting/_apis/build/status/Forecasting/r_unit_tests_prototype?branchName=master)](https://dev.azure.com/best-practices/forecasting/_build/latest?definitionId=9&branchName=master) | | staging | [![Build Status](https://dev.azure.com/best-practices/forecasting/_apis/build/status/Forecasting/r_unit_tests_prototype?branchName=zhouf/r_test_pipeline)](https://dev.azure.com/best-practices/forecasting/_build/latest?definitionId=9&branchName=zhouf/r_test_pipeline) |", - file=open(readmefile, "a"), -) -print("\n\n\n", file=open(readmefile, "a")) - - -print("A new Readme.md file has been generated successfully.") diff --git a/docs/SETUP.md b/docs/SETUP.md index a59aae0c..e60c4e64 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -4,12 +4,9 @@ Please follow these instructions to read about the preferred compute environment ### Compute environment -The code in this repo has been developed and tested on an Azure Linux VM. Therefore, we recommend using an [Azure Data Science Virtual Machine (DSVM) for Linux (Ubuntu)](https://docs.microsoft.com/en-us/azure/machine-learning/data-science-virtual-machine/dsvm-ubuntu-intro) to run the example notebooks and scripts. This VM will come installed with all the system requirements that are needed to create the conda environment described below and then run the notebooks in this repository. If you are using a Linux machine without conda installed, please install Miniconda by following the instructions in this [link](https://docs.conda.io/projects/conda/en/latest/user-guide/install/linux.html). - -You can also use a Windows machine to run the example notebooks and scripts. In this case, you may either work with a [Windows Server 2019 Data Science Virtual Machine on Azure](https://docs.microsoft.com/en-us/azure/machine-learning/data-science-virtual-machine/provision-vm) or a local Windows machine. Azure Windows VW comes with conda pre-installed. If conda is not installed on your machine, please follow the instructions in this [link](https://docs.conda.io/projects/conda/en/latest/user-guide/install/windows.html) to install Miniconda. +The code in this repo has been developed and tested on an Azure Linux VM. Therefore, we recommend using an [Azure Data Science Virtual Machine (DSVM) for Linux (Ubuntu)](https://docs.microsoft.com/en-us/azure/machine-learning/data-science-virtual-machine/dsvm-ubuntu-intro) to run the example notebooks and scripts. This VM will come installed with all the system requirements that are needed to create the conda environment described below and then run the notebooks in this repository. ### Clone the repository - To clone the Forecasting repository to your local machine, please run: ``` @@ -17,51 +14,27 @@ git clone https://github.com/microsoft/forecasting.git cd forecasting/ ``` -Next, follow the instruction below to install all dependencies required to run the examples provided in the repository. Follow [Automated environment setup](#automated-environment-setup) section to set up the environment automatically using a script. Alternatively, follow the [Manual environment setup](#manual-environment-setup) section for a step-by-step guide to setting up the environment. +Next, follow the instruction below to install all dependencies required to run the examples provided in the repository. Follow [Automated environment setup](#automated-environment-setup) section to setup the environment automatically using a script. Alternatively, follow the [Manual environment setup](#manual-environment-setup) section for a step-by-step guide to setting up the environment. ### Automated environment setup -We provide scripts to install all dependencies automatically on a Linux machine as well as on a Windows machine. +We provide a script to install all dependencies automatically on a Linux machine. To execute the script, please run: -#### Linux - -If you are using a Linux machine, please run the following command to execute the shell script for Linux ``` ./tools/environment_setup.sh ``` -from the root of Forecasting repo. +from the root of Forecasting repo. If you have issues with running the setup script, please follow the [Manual environment setup](#manual-environment-setup) instructions below. -#### Windows - -Similarly, if you are using a Windows machine, please run the batch script for Windows via -``` -tools\environment_setup.bat -``` -from the root of Forecasting repo. Note that you need to run the above command from Anaconda Prompt (a terminal with conda available), which can be started by opening the Windows Start menu and clicking `Anaconda Prompt (Miniconda3)` as follows - -

- -

- -Once you've executed the setup script, please activate the newly created conda environment: - -``` -conda activate forecasting_env -``` - ->!NOTE: If you have issues with running the setup script, please follow the [Manual environment setup](#manual-environment-setup) instructions below. - -Next, navigate to [Starting the Jupyter Notebook Server](#starting-the-jupyter-notebook-server) section below to start the Jupyter server necessary for running the examples. +Once you've executed the setup script, you can run example notebooks under [examples/](./examples) directory. ### Manual environment setup - #### Conda environment To install the package contained in this repository, navigate to the directory where you pulled the Forecasting repo to run: ```bash conda update conda -conda env create -f tools/environment.yml +conda env create -f tools/environment.yaml ``` This will create the appropriate conda environment to run experiments. Next activate the installed environment: ```bash @@ -90,24 +63,6 @@ In order to run the example notebooks, make sure to run the notebooks in the con python -m ipykernel install --user --name forecasting_env ``` -### Starting the Jupyter Notebook Server -In order to run the example notebooks provided in this repository, you will have to start a Jupyter notebook server. +Once you've set up the environment, you can run example notebooks under [examples/](./examples) directory. -For running examples on your **local machine**, please open your terminal application and run the following command: -``` -jupyter notebook -``` - -If you are working on a remote VM, you can start the notebook server with the following command: -``` -jupyter notebook --no-browser --port=8889 -``` -and forward the port where the notebooks are running (e.g., 8889) to the local machine via running the following command from the local machine: -``` -ssh -L localhost:8889:localhost:8889 @ -``` - -To access the notebooks, type `localhost:8889/` in the browser on your local machine. - -Now you're ready to run the examples provided in the `examples/`, by simply opening and executing the notebooks in the Jupyter server. Please also navigate to the [examples README file](../examples/README.md) to read about the available notebooks. diff --git a/examples/grocery_sales/python/00_quick_start/autoarima_single_round.ipynb b/examples/00_quick_start/auto_arima_forecasting.ipynb similarity index 99% rename from examples/grocery_sales/python/00_quick_start/autoarima_single_round.ipynb rename to examples/00_quick_start/auto_arima_forecasting.ipynb index c403d8a7..06106a7e 100644 --- a/examples/grocery_sales/python/00_quick_start/autoarima_single_round.ipynb +++ b/examples/00_quick_start/auto_arima_forecasting.ipynb @@ -164,16 +164,19 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Process training data" + "Our data preparation for the training and test set include the following steps:\n", + "\n", + "- The unit sales of orange juice are give in logarithmic scale. We will transfrom them back into the unit scale by applying `math.exp()`\n", + "- Our time series data is not complete, since we have missing sales for some stores/products and weeks. We will fill in those missing values by propagating the last valid observation forward to next available value.\n", + "\n", + "Note that our time series are grouped by `store` and `brand`, while `week` represents a time step, and `move` represents the value to predict." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Our time series data is not complete, since we have missing sales for some stores/products and weeks. We will fill in those missing values by propagating the last valid observation forward to next available value. We will define functions for data frame processing, then use these functions within a loop that loops over each forecasting rounds.\n", - "\n", - "Note that our time series are grouped by `store` and `brand`, while `week` represents a time step, and `logmove` represents the value to predict." + "### Process training data" ] }, { @@ -474,7 +477,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Let's now process the test data. Note that the test data runs from `LAST_WEEK - HORIZON + 1` to `LAST_WEEK`. Note that, in addition to filling out missing values, we also convert unit sales from logarithmic scale to the counts. We will do model training on the log scale, due to improved performance, however, we will transfrom the test data back into the unit scale (counts) by applying `math.exp()`, so that we can evaluate the performance on the unit scale." + "Let's now process the test data. Note that the test data runs from `LAST_WEEK - HORIZON + 1` to `LAST_WEEK`. Note that we are converting unit sales below from logarithmic scale to the counts, as we will be using counts to calculate the evaluation metrics." ] }, { diff --git a/examples/00_quick_start/azure_automl_forecast.ipynb b/examples/00_quick_start/azure_automl_forecast.ipynb new file mode 100644 index 00000000..cce1a92a --- /dev/null +++ b/examples/00_quick_start/azure_automl_forecast.ipynb @@ -0,0 +1,756 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Copyright (c) Microsoft Corporation.\n", + "\n", + "Licensed under the MIT License." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Automated Machine Learning (AutoML) on Azure for Retail Sales Forecasting\n", + "\n", + "This notebook demonstrates how to apply [AutoML in Azure Machine Learning services](https://docs.microsoft.com/en-us/azure/machine-learning/concept-automated-ml) to train and tune machine learning models for forecasting product sales in retail. We will use the Orange Juice dataset to illustrate the steps of utilizing AutoML as well as how to combine an AutoML model with a custom model for better performance.\n", + "\n", + "AutoML is a process of automating the tasks of machine learning model development. It helps data scientists and other practioners build machine learning models with high scalability and quality in less amount of time. AutoML in Azure Machine Learning allows you to train and tune a model using a target metric that you specify. This service iterates through machine learning algorithms and feature selection approaches, producing a score that measures the quality of each machine learning pipeline. The best model will then be selected based on the scores. For more technical details about Azure AutoML, please check [this paper](https://papers.nips.cc/paper/7595-probabilistic-matrix-factorization-for-automated-machine-learning.pdf).\n", + "\n", + "This notebook uses [Azure ML SDK](https://docs.microsoft.com/en-us/python/api/overview/azureml-sdk/?view=azure-ml-py) which is included in the `forecasting_env` conda environment. If you are running in Azure Notebooks or another Microsoft managed environment, the SDK is already installed. On the other hand, if you are running this notebook in your own environment, please follow [SDK installation instructions](https://docs.microsoft.com/azure/machine-learning/service/how-to-configure-environment) to install the SDK." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Global Settings and Imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "import math\n", + "import datetime\n", + "import logging\n", + "import azureml.core\n", + "import azureml.automl\n", + "import pandas as pd\n", + "\n", + "from matplotlib import pyplot as plt\n", + "from fclib.common.utils import git_repo_path\n", + "from fclib.evaluation.evaluation_utils import MAPE\n", + "from fclib.dataset.ojdata import download_ojdata, FIRST_WEEK_START\n", + "from fclib.common.utils import align_outputs\n", + "from fclib.models.multiple_linear_regression import fit, predict\n", + "\n", + "from azureml.core import Workspace\n", + "from azureml.core.dataset import Dataset\n", + "from azureml.core.experiment import Experiment\n", + "from automl.client.core.common import constants\n", + "from azureml.train.automl import AutoMLConfig\n", + "from azureml.core.compute import ComputeTarget, AmlCompute\n", + "from azureml.core.compute_target import ComputeTargetException\n", + "from azureml.automl.core._vendor.automl.client.core.common import metrics\n", + "\n", + "print(\"System version: {}\".format(sys.version))\n", + "print(\"This notebook was created using version 1.0.85 of the Azure ML SDK\")\n", + "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Use False if you've already downloaded and split the data\n", + "DOWNLOAD_SPLIT_DATA = True\n", + "\n", + "# Data directory\n", + "DATA_DIR = os.path.join(git_repo_path(), \"ojdata\")\n", + "\n", + "# Forecasting settings\n", + "GAP = 2\n", + "LAST_WEEK = 138\n", + "\n", + "# Number of test periods\n", + "NUM_TEST_PERIODS = 3\n", + "\n", + "# Column names\n", + "time_column_name = \"week_start\"\n", + "target_column_name = \"move\"\n", + "grain_column_names = [\"store\", \"brand\"]\n", + "index_column_names = [time_column_name] + grain_column_names\n", + "\n", + "# Subset of stores used in the notebook\n", + "USE_STORES = [2, 5, 8]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Set up Azure Machine Learning Workspace\n", + "\n", + "An Azure ML workspace is an Azure resource that organizes and coordinates the actions of many other Azure resources to assist in executing and sharing machine learning workflows. In particular, an Azure ML workspace coordinates storage, databases, and compute resources providing added functionality for machine learning experimentation, deployment, inference, and the monitoring of deployed models. To create an Azure ML workspace, first you need access to an Azure subscription. An Azure subscription allows you to manage storage, compute, and other assets in the Azure cloud. You can [create a new subscription](https://azure.microsoft.com/en-us/free/) or access existing subscription information from the [Azure portal](https://portal.azure.com/). Given that you have access to your Azure subscription, you can further create an Azure ML workspace by following the instructions [here](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-manage-workspace). You can also do so [using Azure CLI](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-manage-workspace-cli) or the `Workspace.create()` method in Azure SDK.\n", + "\n", + "In the following cell, please replace the value of each parameter with the value of the corresponding attribute of your workspace." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "subscription_id = \"\"\n", + "resource_group = \"\"\n", + "workspace_name = \"\"\n", + "workspace_region = \"eastus2\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Access Azure ML Workspace\n", + "\n", + "In what follows, we use Azure ML SDK to attempt to load the workspace specified by your parameters. The cell can fail if the specified workspace doesn't exist or you don't have permissions to access it. Hence, you may need to log into your Azure account and change the default subscription to the one which the workspace belongs to using Azure CLI `az account set --subscription `." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " ws = Workspace.create(subscription_id=subscription_id, resource_group=resource_group, \n", + " name=workspace_name, create_resource_group=True, exist_ok=True, \n", + " location=workspace_region)\n", + " # write the details of the workspace to a configuration file to the notebook library\n", + " ws.write_config()\n", + " print(\"Workspace configuration succeeded. Skip the workspace creation steps below\")\n", + "except ValueError:\n", + " raise Exception(\"Workspace not accessible. Change your parameters or create a new workspace below\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create compute resources for your experiments\n", + "\n", + "We run AutoML on a dynamically scalable compute cluster. To create a compute cluster, you need to specify a compute configuration that specifies the type of machine to be used and the scalability behaviors. Then you choose a name for the cluster that is unique within the workspace that can be used to address the cluster later." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Choose a name for your CPU cluster\n", + "cpu_cluster_name = \"cpu-cluster\"\n", + "\n", + "# Verify that cluster does not exist already\n", + "workspace_compute = ws.compute_targets\n", + "if cpu_cluster_name in workspace_compute:\n", + " print(\"Found existing cpu-cluster\")\n", + " cpu_cluster = ComputeTarget(workspace=ws, name=cpu_cluster_name)\n", + "else: \n", + " print(\"Creating new cpu-cluster\")\n", + "\n", + " # Specify the configuration for the new cluster\n", + " compute_config = AmlCompute.provisioning_configuration(vm_size=\"STANDARD_D2_V2\", min_nodes=4, max_nodes=4)\n", + "\n", + " # Create the cluster with the specified name and configuration\n", + " cpu_cluster = ComputeTarget.create(ws, cpu_cluster_name, compute_config)\n", + "\n", + " # Wait for the cluster to complete, show the output log\n", + " cpu_cluster.wait_for_completion(show_output=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Define Experiment\n", + "\n", + "To run AutoML, you need to create an Experiment. An Experiment corresponds to a prediction problem you are trying to solve, while a Run corresponds to a specific approach to the problem." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# choose a name for the run history container in the workspace\n", + "experiment_name = \"automl-ojforecasting\"\n", + "\n", + "experiment = Experiment(ws, experiment_name)\n", + "\n", + "output = {}\n", + "output[\"SDK version\"] = azureml.core.VERSION\n", + "output[\"Workspace\"] = ws.name\n", + "output[\"SKU\"] = ws.sku\n", + "output[\"Resource Group\"] = ws.resource_group\n", + "output[\"Location\"] = ws.location\n", + "output[\"Run History Name\"] = experiment_name\n", + "pd.set_option(\"display.max_colwidth\", -1)\n", + "outputDf = pd.DataFrame(data=output, index=[\"\"])\n", + "outputDf.T" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Data Preparation\n", + "\n", + "We need to download the Orange Juice data and split it into training and test sets. By default, the following cell will download and spit the data. If you've already done so, you may skip this part by switching `DOWNLOAD_SPLIT_DATA` to `False`.\n", + "\n", + "We store the training data and test data using dataframes. The training data includes `train_df` and `aux_df` with `train_df` containing the historical sales up to week 135 (the time we make forecasts) and `aux_df` containing price/promotion information up until week 138. We assume that future price and promotion information up to a certain number of weeks ahead is predetermined and known. The test data is stored in `test_df` which contains the sales of each product in week 137 and 138. Assuming the current week is week 135, our goal is to forecast the sales in week 137 and 138 using the training data. There is a one-week gap between the current week and the first target week of forecasting as we want to leave time for planning inventory in practice." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Data download and split" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if DOWNLOAD_SPLIT_DATA:\n", + " download_ojdata(DATA_DIR)\n", + " df = pd.read_csv(os.path.join(DATA_DIR, \"yx.csv\"))\n", + " df = df.loc[df.week <= LAST_WEEK]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Convert logarithm of the unit sales to unit sales\n", + "df[\"move\"] = df[\"logmove\"].apply(lambda x: round(math.exp(x)))\n", + "# Add timestamp column\n", + "df[\"week_start\"] = df[\"week\"].apply(lambda x: FIRST_WEEK_START + datetime.timedelta(days=(x - 1) * 7))\n", + "# Select a subset of stores for demo purpose\n", + "df_sub = df[df.store.isin(USE_STORES)]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Split data into training and test sets\n", + "def split_last_n_by_grain(df, n):\n", + " \"\"\"Group df by grain and split on last n rows for each group.\"\"\"\n", + " df_grouped = df.sort_values(time_column_name).groupby( # Sort by ascending time\n", + " grain_column_names, group_keys=False\n", + " )\n", + " df_head = df_grouped.apply(lambda dfg: dfg.iloc[:-n])\n", + " df_tail = df_grouped.apply(lambda dfg: dfg.iloc[-n:])\n", + " return df_head, df_tail\n", + "\n", + "\n", + "train_df, test_df = split_last_n_by_grain(df_sub, NUM_TEST_PERIODS)\n", + "train_df.reset_index(drop=True)\n", + "test_df.reset_index(drop=True)\n", + "\n", + "# Save data locally\n", + "local_data_pathes = [\n", + " os.path.join(DATA_DIR, \"train.csv\"),\n", + " os.path.join(DATA_DIR, \"test.csv\"),\n", + "]\n", + "\n", + "train_df.to_csv(local_data_pathes[0], index=None, header=True)\n", + "test_df.to_csv(local_data_pathes[1], index=None, header=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Upload data to datastore\n", + "\n", + "The [Machine Learning service workspace](https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-workspace), is paired with the storage account, which contains the default data store. We will use it to upload the train and test data and create [tabular datasets](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.data.tabulardataset?view=azure-ml-py) for training and testing. A tabular dataset defines a series of lazily-evaluated, immutable operations to load data from the data source into tabular representation.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "datastore = ws.get_default_datastore()\n", + "datastore.upload_files(files=local_data_pathes, target_path=\"dataset/\", overwrite=True, show_progress=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create dataset for training" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "train_dataset = Dataset.Tabular.from_delimited_files(path=datastore.path(\"dataset/train.csv\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "train_dataset.to_pandas_dataframe().tail()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Modeling\n", + "\n", + "For forecasting tasks, AutoML uses pre-processing and estimation steps that are specific to time-series. AutoML will undertake the following pre-processing steps:\n", + "* Detect time-series sample frequency (e.g. hourly, daily, weekly) and create new records for absent time points to make the series regular. A regular time series has a well-defined frequency and has a value at every sample point in a contiguous time span\n", + "* Impute missing values in the target (via forward-fill) and feature columns (using median column values)\n", + "* Create grain-based features to enable fixed effects across different series\n", + "* Create time-based features to assist in learning seasonal patterns\n", + "* Encode categorical variables to numeric quantities\n", + "\n", + "In this notebook, AutoML will train a single, regression-type model across all time-series in a given training set. This allows the model to generalize across related series. To create a training job, we use AutoML Config object to define the settings and data. Here is a summary of the meanings of the AutoMLConfig parameters:\n", + "\n", + "|Property|Description|\n", + "|-|-|\n", + "|**task**|forecasting|\n", + "|**primary_metric**|This is the metric that you want to optimize.
Forecasting supports the following primary metrics
spearman_correlation
normalized_root_mean_squared_error
r2_score
normalized_mean_absolute_error\n", + "|**experiment_timeout_hours**|Experimentation timeout in hours.|\n", + "|**enable_early_stopping**|If early stopping is on, training will stop when the primary metric is no longer improving.|\n", + "|**training_data**|Input dataset, containing both features and label column.|\n", + "|**label_column_name**|The name of the label column.|\n", + "|**compute_target**|The remote compute for training.|\n", + "|**n_cross_validations**|Number of cross-validation folds to use for model/pipeline selection|\n", + "|**enable_voting_ensemble**|Allow AutoML to create a Voting ensemble of the best performing models|\n", + "|**enable_stack_ensemble**|Allow AutoML to create a Stack ensemble of the best performing models|\n", + "|**debug_log**|Log file path for writing debugging information|\n", + "|**time_column_name**|Name of the datetime column in the input data|\n", + "|**grain_column_names**|Name(s) of the columns defining individual series in the input data|\n", + "|**drop_column_names**|Name(s) of columns to drop prior to modeling|\n", + "|**max_horizon**|Maximum desired forecast horizon in units of time-series frequency|" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Model training" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "time_series_settings = {\n", + " \"time_column_name\": time_column_name,\n", + " \"grain_column_names\": grain_column_names,\n", + " \"drop_column_names\": [\"logmove\"], # 'logmove' is a leaky feature, so we remove it.\n", + " \"max_horizon\": NUM_TEST_PERIODS,\n", + "}\n", + "\n", + "automl_config = AutoMLConfig(\n", + " task=\"forecasting\",\n", + " debug_log=\"automl_oj_sales_errors.log\",\n", + " primary_metric=\"normalized_mean_absolute_error\",\n", + " experiment_timeout_hours=0.6, # You may increase this number to improve model accuracy\n", + " training_data=train_dataset,\n", + " label_column_name=target_column_name,\n", + " compute_target=cpu_cluster,\n", + " enable_early_stopping=True,\n", + " n_cross_validations=3,\n", + " verbosity=logging.INFO,\n", + " **time_series_settings\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "remote_run = experiment.submit(automl_config, show_output=False)\n", + "remote_run" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "remote_run.wait_for_completion()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Retrieve the best model\n", + "\n", + "Each run within an Experiment stores serialized (i.e. pickled) pipelines from the AutoML iterations. After the training job is done, we can retrieve the pipeline with the best performance on the validation dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "best_run, fitted_model = remote_run.get_output()\n", + "print(fitted_model.steps)\n", + "model_name = best_run.properties[\"model_name\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Forecasting\n", + "\n", + "Now that we have retrieved the best model pipeline, we can apply it to generate forecasts for the target weeks. To do this, we first remove the target values from the test set" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Generate forecasts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "X_test = test_df\n", + "y_test = X_test.pop(target_column_name).values" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "X_test.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# The featurized data, aligned to y, will also be returned. It contains the assumptions\n", + "# that were made in the forecast and helps align the forecast to the original data.\n", + "y_predictions, X_trans = fitted_model.forecast(X_test)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We need to align the output explicitly to the input, as the count and order of the rows may have changed during transformations that span multiple rows." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pred_automl = align_outputs(y_predictions, X_trans, X_test, y_test, target_column_name)\n", + "pred_automl.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Results evaluation & visualization" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Use automl metrics module\n", + "scores = metrics.compute_metrics_regression(\n", + " pred_automl[\"predicted\"],\n", + " pred_automl[target_column_name],\n", + " list(constants.Metric.SCALAR_REGRESSION_SET),\n", + " None,\n", + " None,\n", + " None,\n", + ")\n", + "\n", + "print(\"[Test data scores]\\n\")\n", + "for key, value in scores.items():\n", + " print(\"{}: {:.3f}\".format(key, value))\n", + "\n", + "# Plot outputs\n", + "%matplotlib inline\n", + "test_pred = plt.scatter(pred_automl[target_column_name], pred_automl[\"predicted\"], color=\"b\")\n", + "test_test = plt.scatter(pred_automl[target_column_name], pred_automl[target_column_name], color=\"g\")\n", + "plt.legend((test_pred, test_test), (\"prediction\", \"truth\"), loc=\"upper left\", fontsize=8)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We also compute MAPE of the forecasts in the last two weeks of the forecast period in order to be consistent with the evaluation period that is used in other quick start examples." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pred_automl_sub = pred_automl.loc[pred_automl.week >= max(test_df.week) - NUM_TEST_PERIODS + GAP]\n", + "mape_automl_sub = MAPE(pred_automl_sub[\"predicted\"], pred_automl_sub[\"move\"]) * 100\n", + "print(\"MAPE of forecasts obtained by AutoML in the last two weeks: \" + str(mape_automl_sub))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Combine AutoML Model with a Custom Model\n", + "\n", + "So far we have demonstrated how we can quickly build a forecasting model with AutoML in Azure. Next, we further show a simple way to achieve more robust and accurate forecasts by combining the forecasts from AutoML and a custom model that the user may have. Here we assume that the user have also constructed a series of linear regression models with each model forecasts the sales of a specfic store-brand using `scikit-learn` package." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Multiple linear regression models" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create price features\n", + "df_sub[\"price\"] = df_sub.apply(lambda x: x.loc[\"price\" + str(int(x.loc[\"brand\"]))], axis=1)\n", + "price_cols = [\n", + " \"price1\",\n", + " \"price2\",\n", + " \"price3\",\n", + " \"price4\",\n", + " \"price5\",\n", + " \"price6\",\n", + " \"price7\",\n", + " \"price8\",\n", + " \"price9\",\n", + " \"price10\",\n", + " \"price11\",\n", + "]\n", + "df_sub[\"avg_price\"] = df_sub[price_cols].sum(axis=1).apply(lambda x: x / len(price_cols))\n", + "df_sub[\"price_ratio\"] = df_sub.apply(lambda x: x[\"price\"] / x[\"avg_price\"], axis=1)\n", + "\n", + "# Create lag features on unit sales\n", + "df_sub[\"move_lag1\"] = df_sub[\"move\"].shift(1)\n", + "df_sub[\"move_lag2\"] = df_sub[\"move\"].shift(2)\n", + "\n", + "# Drop rows with NaN values\n", + "df_sub.dropna(inplace=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "After splitting the data, we use `fit()` and `predit()` functions from `fclib.models.multiple_linear_regression` to train separate linear regression model for each invididual time series and generate forecasts for the sales during the test period." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Split data into training and test sets\n", + "train_df, test_df = split_last_n_by_grain(df_sub, NUM_TEST_PERIODS)\n", + "train_df.reset_index(drop=True)\n", + "test_df.reset_index(drop=True)\n", + "\n", + "# Train multiple linear regression models\n", + "fea_column_names = [\"move_lag1\", \"move_lag2\", \"price\", \"price_ratio\"]\n", + "lr_models = fit(train_df, grain_column_names, fea_column_names, target_column_name)\n", + "\n", + "# Generate forecasts with the trained models\n", + "pred_all = predict(test_df, lr_models, time_column_name, grain_column_names, fea_column_names)\n", + "\n", + "pred_lr = pd.merge(pred_all, test_df, on=index_column_names)\n", + "pred_lr.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's check the accuracy of the predictions on the entire forecast period as well as in the last two weeks of the forecast period.\n", + "\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "mape_lr_entire = MAPE(pred_lr[\"prediction\"], pred_lr[\"move\"]) * 100\n", + "print(\"MAPE of forecasts obtained by multiple linear regression on entire test period: \" + str(mape_lr_entire))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pred_lr_sub = pred_lr.loc[pred_lr.week >= max(test_df.week) - NUM_TEST_PERIODS + GAP]\n", + "mape_lr_sub = MAPE(pred_lr_sub[\"prediction\"], pred_lr_sub[\"move\"]) * 100\n", + "print(\"MAPE of forecasts obtained by multiple linear regression in the last two weeks: \" + str(mape_lr_sub))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Combine forecasts from different methods\n", + "\n", + "We can combine the forecasts obtained by AutoML and multiple linear regression using weighted average and evaluate the final forecasts. Usually the combined forecasts will be more robust as a combination of two methods can reduce the chance of model overfitting. Here we use equal weights which can be further adjusted according to our confidence on each model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pred_final = pd.merge(\n", + " pred_automl[index_column_names + [\"predicted\", \"move\", \"week\"]],\n", + " pred_lr[index_column_names + [\"prediction\"]],\n", + " on=index_column_names,\n", + " how=\"left\",\n", + ")\n", + "pred_final[\"combined_prediction\"] = pred_final[\"predicted\"] * 0.5 + pred_final[\"prediction\"] * 0.5" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "mape_entire = MAPE(pred_final[\"combined_prediction\"], pred_final[\"move\"]) * 100\n", + "print(\"MAPE of forecasts obtained by the combined model on entire test period: \" + str(mape_entire))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pred_final_sub = pred_final.loc[pred_final.week >= max(test_df.week) - NUM_TEST_PERIODS + GAP]\n", + "mape_final_sub = MAPE(pred_final_sub[\"combined_prediction\"], pred_final_sub[\"move\"]) * 100\n", + "print(\"MAPE of forecasts obtained by the combined model in the last two weeks: \" + str(mape_final_sub))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Additional Reading\n", + "\n", + "\\[1\\] Nicolo Fusi, Rishit Sheth, and Melih Elibol. 2018. Probabilistic Matrix Factorization for Automated Machine Learning. In Advances in Neural Information Processing Systems. 3348-3357.
\n", + "\\[2\\] Azure AutoML Package Docs: https://docs.microsoft.com/en-us/python/api/azureml-train-automl/azureml.train.automl?view=azure-ml-py
\n", + "\\[3\\] Azure Automated Machine Learning Examples: https://github.com/Azure/MachineLearningNotebooks/tree/master/how-to-use-azureml/automated-machine-learning
\n", + "\n", + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "forecasting_env", + "language": "python", + "name": "forecasting_env" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/examples/grocery_sales/python/00_quick_start/lightgbm_single_round.ipynb b/examples/00_quick_start/lightgbm_point_forecast.ipynb similarity index 99% rename from examples/grocery_sales/python/00_quick_start/lightgbm_single_round.ipynb rename to examples/00_quick_start/lightgbm_point_forecast.ipynb index 72de6354..7a49baa7 100644 --- a/examples/grocery_sales/python/00_quick_start/lightgbm_single_round.ipynb +++ b/examples/00_quick_start/lightgbm_point_forecast.ipynb @@ -6,7 +6,7 @@ "source": [ "Copyright (c) Microsoft Corporation.\n", "\n", - "Licensed under the MIT License. " + "Licensed under the MIT License." ] }, { @@ -104,11 +104,7 @@ { "cell_type": "code", "execution_count": 3, - "metadata": { - "tags": [ - "parameters" - ] - }, + "metadata": {}, "outputs": [], "source": [ "# Use False if you've already downloaded and split the data\n", @@ -591,10 +587,6 @@ } ], "metadata": { - "author_info": { - "affiliation": "Microsoft", - "created_by": "Chenhui Hu" - }, "kernelspec": { "display_name": "forecasting_env", "language": "python", diff --git a/examples/grocery_sales/python/01_prepare_data/ojdata_exploration.ipynb b/examples/01_prepare_data/ojdata_exploration_retail.ipynb similarity index 100% rename from examples/grocery_sales/python/01_prepare_data/ojdata_exploration.ipynb rename to examples/01_prepare_data/ojdata_exploration_retail.ipynb diff --git a/examples/grocery_sales/python/01_prepare_data/ojdata_preparation.ipynb b/examples/01_prepare_data/ojdata_preparation_retail.ipynb similarity index 99% rename from examples/grocery_sales/python/01_prepare_data/ojdata_preparation.ipynb rename to examples/01_prepare_data/ojdata_preparation_retail.ipynb index 38508ec6..86a9afb1 100644 --- a/examples/grocery_sales/python/01_prepare_data/ojdata_preparation.ipynb +++ b/examples/01_prepare_data/ojdata_preparation_retail.ipynb @@ -160,7 +160,7 @@ "For demonstration, this is what the time series split on the Orange Juice dataset looks like, for the parameters listed above.\n", "For `HORIZON = 2` and `GAP = 2`, assuming the current week is week `153`, our goal is to forecast the sales in week `155` and `156` using the training data. As you can see, the first forecasting week is `two` weeks away from the current week, as we want to leave time for planning inventory in practice.\n", "\n", - "![Single split](../../../../assets/time_series_split_singleround.jpg)\n", + "![Single split](../../assets/time_series_split_singleround.jpg)\n", "\n", "We also refer to splits as rounds, so for `N_SPLITS = 1`, we have single-round forecasting, and for `N_SPLITS > 1`, we have multi-round forecasting." ] @@ -1120,7 +1120,7 @@ "\n", "For demonstration, this is what the time series splits would look like for `N_SPLITS = 5`, and using other settings as above:\n", "\n", - "![Multi split](../../../../assets/time_series_split_multiround.jpg)\n" + "![Multi split](../../assets/time_series_split_multiround.jpg)\n" ] }, { diff --git a/examples/grocery_sales/python/02_model/dilatedcnn_multi_round.ipynb b/examples/02_model/dilatedcnn_point_forecast_multiround.ipynb similarity index 99% rename from examples/grocery_sales/python/02_model/dilatedcnn_multi_round.ipynb rename to examples/02_model/dilatedcnn_point_forecast_multiround.ipynb index 48a41f47..2ca1bf60 100644 --- a/examples/grocery_sales/python/02_model/dilatedcnn_multi_round.ipynb +++ b/examples/02_model/dilatedcnn_point_forecast_multiround.ipynb @@ -38,7 +38,8 @@ "metadata": {}, "outputs": [], "source": [ - "%load_ext tensorboard" + "%load_ext tensorboard\n", + "%load_ext blackcellmagic" ] }, { @@ -104,15 +105,11 @@ { "cell_type": "code", "execution_count": 3, - "metadata": { - "tags": [ - "parameters" - ] - }, + "metadata": {}, "outputs": [], "source": [ "# Use False if you've already downloaded and split the data\n", - "DOWNLOAD_SPLIT_DATA = True\n", + "DOWNLOAD_SPLIT_DATA = False # True\n", "\n", "# Data directories\n", "DATA_DIR = os.path.join(git_repo_path(), \"ojdata\")\n", @@ -244,7 +241,7 @@ " data_filled = pd.merge(data_grid, train_df, how=\"left\", on=[\"store\", \"brand\", \"week\"])\n", "\n", " # Get future price, deal, and advertisement info\n", - " aux_df = pd.read_csv(os.path.join(TRAIN_DIR, \"auxi_\" + str(pred_round) + \".csv\"))\n", + " aux_df = pd.read_csv(os.path.join(TRAIN_DIR, \"aux_\" + str(pred_round) + \".csv\"))\n", " data_filled = pd.merge(data_filled, aux_df, how=\"left\", on=[\"store\", \"brand\", \"week\"])\n", "\n", " # Create relative price feature\n", @@ -941,10 +938,6 @@ } ], "metadata": { - "author_info": { - "affiliation": "Microsoft", - "created_by": "Chenhui Hu" - }, "kernelspec": { "display_name": "forecasting_env", "language": "python", diff --git a/examples/grocery_sales/python/02_model/lightgbm_multi_round.ipynb b/examples/02_model/lightgbm_point_forecast_multiround.ipynb similarity index 99% rename from examples/grocery_sales/python/02_model/lightgbm_multi_round.ipynb rename to examples/02_model/lightgbm_point_forecast_multiround.ipynb index 33cb1567..bcb2e620 100644 --- a/examples/grocery_sales/python/02_model/lightgbm_multi_round.ipynb +++ b/examples/02_model/lightgbm_point_forecast_multiround.ipynb @@ -103,11 +103,7 @@ { "cell_type": "code", "execution_count": 3, - "metadata": { - "tags": [ - "parameters" - ] - }, + "metadata": {}, "outputs": [], "source": [ "# Use False if you've already downloaded and split the data\n", @@ -248,7 +244,7 @@ " data_filled = pd.merge(data_grid, train_df, how=\"left\", on=[\"store\", \"brand\", \"week\"])\n", "\n", " # Get future price, deal, and advertisement info\n", - " aux_df = pd.read_csv(os.path.join(train_dir, \"auxi_\" + str(pred_round) + \".csv\"))\n", + " aux_df = pd.read_csv(os.path.join(train_dir, \"aux_\" + str(pred_round) + \".csv\"))\n", " data_filled = pd.merge(data_filled, aux_df, how=\"left\", on=[\"store\", \"brand\", \"week\"])\n", "\n", " # Create relative price feature\n", @@ -4133,10 +4129,6 @@ } ], "metadata": { - "author_info": { - "affiliation": "Microsoft", - "created_by": "Chenhui Hu" - }, "kernelspec": { "display_name": "forecasting_env", "language": "python", @@ -4152,7 +4144,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.10" + "version": "3.5.6-final" } }, "nbformat": 4, diff --git a/examples/README.md b/examples/README.md index 39b28c5b..69f5fa79 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,16 +1,16 @@ # Forecasting examples -This folder contains Python and R examples for building forecasting solutions presented in Python Jupyter notebooks and R Markdown files, respectively. The examples are organized according to forecasting scenarios in different use cases with each subdirectory under `examples/` named after the specific use case. - -At the moment, the repository contains a single retail sales forecasting scenario utilizing [Dominick's OrangeJuice data set](https://www.chicagobooth.edu/research/kilts/datasets/dominicks). The name of the directory is `grocery_sales`. +This folder contains Python examples for building forecasting solutions. To run the notebooks, please execute `jupyter notebook` and select the Jupyter kernel `forecasting_env` if you are using a local machine. Otherwise, if you use a remote VM, you can start the notebooks via `jupyter notebook --no-browser` and forward the port where the notebooks are running (e.g., 8888) to the local machine via `ssh @ -L 8888:localhost:8888`. ## Summary -The following table summarizes each forecasting scenario contained in the repository, and links available content within that scenario. - -| Directory | Content | Description | -|----------------------------------|----------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------| -| [grocery_sales](./grocery_sales) | [python/](./grocery_sales/python)
[R/](./grocery_sales/R) | Python and R examples for forecasting sales of orange juice in [Dominick's dataset](https://www.chicagobooth.edu/research/kilts/datasets/dominicks). | +The following summarizes each directory of the best practice notebooks. +| Directory | Content | Description | +| --- | --- | --- | +| [00_quick_start](./00_quick_start)| [auto_arima_forecasting.ipynb](./00_quick_start/auto_arima_forecasting.ipynb)
[azure_automl_forecast.ipynb](./00_quick_start/azure_automl_forecast.ipynb)
[lightgbm_point_forecast.ipynb](./00_quick_start/lightgbm_point_forecast.ipynb) | Quick start notebooks that demonstrate workflow of developing a forecasting model using one-round training and testing data| +| [01_prepare_data](./01_prepare_data) | [ojdata_exploration_retail.ipynb](./01_prepare_data/ojdata_exploration_retail.ipynb)
[ojdata_preparation_retail.ipynb](./01_prepare_data/ojdata_preparation_retail.ipynb) | Data exploration and preparation notebooks| +| [02_model](./02_model) | [dilatedcnn_point_forecast_multiround.ipynb](./02_model/dilatedcnn_point_forecast_multiround.ipynb)
[lightgbm_point_forecast_multiround.ipynb](./02_model/lightgbm_point_forecast_multiround.ipynb) | Deep dive notebooks that perform multi-round training and testing of various classical and deep learning forecast algorithms| +| [03_model_select_deploy](03_model_select_deploy) | Example notebook to be added soon | Best practice notebook for model selecting by using Azure Machine Learning Service and deploying the best model on Azure| diff --git a/examples/grocery_sales/R/01_dataprep.Rmd b/examples/grocery_sales/R/01_dataprep.Rmd deleted file mode 100644 index 05cd7d02..00000000 --- a/examples/grocery_sales/R/01_dataprep.Rmd +++ /dev/null @@ -1,96 +0,0 @@ ---- -title: Data preparation -output: html_notebook ---- - -_Copyright (c) Microsoft Corporation._
-_Licensed under the MIT License._ - -In this notebook, we generate the datasets that will be used for model training and validating. - -The orange juice dataset comes from the bayesm package, and gives pricing and sales figures over time for a variety of orange juice brands in several stores in Florida. Rather than installing the entire package (which is very complex), we download the dataset itself from the GitHub mirror of the CRAN repository. - -```{r, results="hide", message=FALSE} -# download the data from the GitHub mirror of the bayesm package source -ojfile <- tempfile(fileext=".rda") -download.file("https://github.com/cran/bayesm/raw/master/data/orangeJuice.rda", ojfile) -load(ojfile) -file.remove(ojfile) -``` - -The dataset generation parameters are obtained from the file `ojdata_forecast_settings.yaml`; you can modify that file to vary the experimental setup. The settings are - -| Parameter | Description | Default | -|-----------|-------------|---------| -| `N_SPLITS` | The number of splits to make. | 10 | -| `HORIZON` | The forecast horizon for the test dataset for each split. | 2 | -| `GAP` | The gap in weeks from the end of the training period to the start of the testing period; see below. | 2 | -| `FIRST_WEEK` | The first week of data to use. | 40 | -| `LAST_WEEK` | The last week of data to use. | 156 | -| `START_DATE` | The actual calendar date for the start of the first week in the data. | `1989-09-14` | - -A complicating factor is that the data does not include every possible combination of store, brand and date, so we have to pad out the missing rows with `complete`. In addition, one store/brand combination has no data beyond week 156; we therefore end the analysis at this week. We also do _not_ fill in the missing values in the data, as many of the modelling functions in the fable package can handle this innately. - -```{r, results="hide", message=FALSE} -library(tidyr) -library(dplyr) -library(tsibble) -library(feasts) -library(fable) - -settings <- yaml::read_yaml(here::here("examples/grocery_sales/R/forecast_settings.yaml")) -start_date <- as.Date(settings$START_DATE) -train_periods <- seq(to=settings$LAST_WEEK - settings$HORIZON - settings$GAP + 1, - by=settings$HORIZON, - length.out=settings$N_SPLITS) - -oj_data <- orangeJuice$yx %>% - complete(store, brand, week) %>% - mutate(week=yearweek(start_date + week*7)) %>% - as_tsibble(index=week, key=c(store, brand)) -``` - -Here are some glimpses of what the data looks like. The dependent variable is `logmove`, the logarithm of the total sales for a given brand and store, in a particular week. - -```{r} -head(oj_data) -``` - -The time series plots for a small subset of brands and stores are shown below. We can make the following observations: - -- There appears to be little seasonal variation in sales (probably because Florida is a state without very different seasons). In any case, with less than 2 years of observations, the time series is not long enough for many model-fitting functions in the fable package to automatically estimate seasonal parameters. -- While some store/brand combinations show weak trends over time, this is far from universal. -- Different brands can exhibit very different behaviour, especially in terms of variation about the mean. -- Many of the time series have missing values, indicating that the dataset is incomplete. - - -```{r, fig.height=10} -library(ggplot2) - -oj_data %>% - filter(store < 25, brand < 5) %>% - ggplot(aes(x=week, y=logmove)) + - geom_line() + - scale_x_date(labels=NULL) + - facet_grid(vars(store), vars(brand), labeller="label_both") -``` - -Finally, we split the dataset into separate samples for training and testing. The schema used is broadly time series cross-validation, whereby we train a model on data up to time $t$, test it on data for times $t+1$ to $t+k$, then train on data up to time $t+k$, test it on data for times $t+k+1$ to $t+2k$, and so on. In this specific case study, however, we introduce a small extra piece of complexity based on discussions with domain experts. We train a model on data up to week $t$, then test it on week $t+2$ to $t+3$. Then we train on data up to week $t+2$, and test it on weeks $t+4$ to $t+5$, and so on. There is thus always a gap of one week between the training and test samples. The reason for this is because in reality, inventory planning always takes some time; the gap allows store managers to prepare the stock based on the forecasted demand. - -```{r} -subset_oj_data <- function(start, end) -{ - start <- yearweek(start_date + start*7) - end <- yearweek(start_date + end*7) - filter(oj_data, week >= start, week <= end) -} - -oj_train <- lapply(train_periods, function(i) subset_oj_data(settings$FIRST_WEEK, i)) -oj_test <- lapply(train_periods, function(i) subset_oj_data(i + settings$GAP, i + settings$GAP + settings$HORIZON - 1)) - -save(oj_train, oj_test, file=here::here("examples/grocery_sales/R/data.Rdata")) - -head(oj_train[[1]]) - -head(oj_test[[1]]) -``` diff --git a/examples/grocery_sales/R/02_basic_models.Rmd b/examples/grocery_sales/R/02_basic_models.Rmd deleted file mode 100644 index 6dda9451..00000000 --- a/examples/grocery_sales/R/02_basic_models.Rmd +++ /dev/null @@ -1,87 +0,0 @@ ---- -title: Basic models -output: html_notebook ---- - -_Copyright (c) Microsoft Corporation._
-_Licensed under the MIT License._ - -```{r, echo=FALSE, results="hide", message=FALSE} -library(tidyr) -library(dplyr) -library(tsibble) -library(feasts) -library(fable) -``` - -We fit some simple models to the orange juice data for illustrative purposes. Here, each model is actually a _group_ of models, one for each combination of store and brand. This is the standard approach taken in statistical forecasting, and is supported out-of-the-box by the tidyverts framework. - -- `mean`: This is just a simple mean. -- `naive`: A random walk model without any other components. This amounts to setting all forecast values to the last observed value. -- `drift`: This adjusts the `naive` model to incorporate a straight-line trend. -- `arima`: An ARIMA model with the parameter values estimated from the data. - -Note that the model training process is embarrassingly parallel on 3 levels: - -- We have multiple independent training datasets; -- For which we fit multiple independent models; -- Within which we have independent sub-models for each store and brand. - -This lets us speed up the training significantly. While the `fable::model` function can fit multiple models in parallel, we will run it sequentially here and instead parallelise by dataset. This avoids contention for cores, and also results in the simplest code. As a guard against returning invalid results, we also specify the argument `.safely=FALSE`; this forces `model` to throw an error if a model algorithm fails. - -```{r} -srcdir <- here::here("R_utils") -for(src in dir(srcdir, full.names=TRUE)) source(src) - -load_objects("grocery_sales", "data.Rdata") - -cl <- make_cluster(libs=c("tidyr", "dplyr", "fable", "tsibble", "feasts")) - -oj_modelset_basic <- parallel::parLapply(cl, oj_train, function(df) -{ - model(df, - mean=MEAN(logmove), - naive=NAIVE(logmove), - drift=RW(logmove ~ drift()), - arima=ARIMA(logmove ~ pdq() + PDQ(0, 0, 0)), - .safely=FALSE - ) -}) -oj_fcast_basic <- parallel::clusterMap(cl, get_forecasts, oj_modelset_basic, oj_test) - -save_objects(oj_modelset_basic, oj_fcast_basic, - example="grocery_sales", file="model_basic.Rdata") - -do.call(rbind, oj_fcast_basic) %>% - mutate_at(-(1:3), exp) %>% - eval_forecasts() -``` - -The ARIMA model does the best of the simple models, but not any better than a simple mean. - -Having fit some basic models, we can also try an exponential smoothing model, fit using the `ETS` function. Unlike the others, `ETS` does not currently support time series with missing values; we therefore have to use one of the other models to impute missing values first via the `interpolate` function. - -```{r} -oj_modelset_ets <- parallel::clusterMap(cl, function(df, basicmod) -{ - df %>% - interpolate(object=select(basicmod, -c(mean, naive, drift))) %>% - model( - ets=ETS(logmove ~ error("A") + trend("A") + season("N")), - .safely=FALSE - ) -}, oj_train, oj_modelset_basic) - -oj_fcast_ets <- parallel::clusterMap(cl, get_forecasts, oj_modelset_ets, oj_test) - -destroy_cluster(cl) - -save_objects(oj_modelset_ets, oj_fcast_ets, - example="grocery_sales", file="model_ets.Rdata") - -do.call(rbind, oj_fcast_ets) %>% - mutate_at(-(1:3), exp) %>% - eval_forecasts() -``` - -The ETS model does _worse_ than the ARIMA model, something that should not be a surprise given the lack of strong seasonality and trend in this dataset. We conclude that any simple univariate approach is unlikely to do well. diff --git a/examples/grocery_sales/R/02a_reg_models.Rmd b/examples/grocery_sales/R/02a_reg_models.Rmd deleted file mode 100644 index 581f4e35..00000000 --- a/examples/grocery_sales/R/02a_reg_models.Rmd +++ /dev/null @@ -1,86 +0,0 @@ ---- -title: ARIMA-Regression models -output: html_notebook ---- - -_Copyright (c) Microsoft Corporation._
-_Licensed under the MIT License._ - -```{r, echo=FALSE, results="hide", message=FALSE} -library(tidyr) -library(dplyr) -library(tsibble) -library(feasts) -library(fable) -``` - -This notebook builds on the output from "Basic models" by including regressor variables in the ARIMA model(s). We fit the following model types: - -- `ar_trend` includes only a linear trend over time. -- `ar_reg` allows stepwise selection of independent regressors. -- `ar_reg_price`: rather than allowing the algorithm to select from the 11 price variables, we use only the price relevant to each brand. This is to guard against possible overfitting, something that classical stepwise procedures are wont to do. -- `ar_reg_price_trend` is the same as `ar_reg_price`, but including a linear trend. - -As part of the modelling, we also compute a new independent variable `maxpricediff`, the log-ratio of the price of this brand compared to the best competing price. A positive `maxpricediff` means this brand is cheaper than all the other brands, and a negative `maxpricediff` means it is more expensive. - -```{r} -srcdir <- here::here("R_utils") -for(src in dir(srcdir, full.names=TRUE)) source(src) - -load_objects("grocery_sales", "data.Rdata") - -cl <- make_cluster(libs=c("tidyr", "dplyr", "fable", "tsibble", "feasts")) - -# add extra regression variables to training and test datasets -add_regvars <- function(df) -{ - df %>% - group_by(store, brand) %>% - group_modify(~ { - pricevars <- grep("price", names(.x), value=TRUE) - thispricevar <- unique(paste0("price", .y$brand)) - best_other_price <- do.call(pmin, .x[setdiff(pricevars, thispricevar)]) - .x$price <- .x[[thispricevar]] - .x$maxpricediff <- log(best_other_price/.x$price) - .x - }) %>% - ungroup() %>% - mutate(week=yearweek(week)) %>% # need to recreate this variable because of tsibble/vctrs issues - as_tsibble(week, key=c(store, brand)) -} - -oj_trainreg <- parallel::parLapply(cl, oj_train, add_regvars) -oj_testreg <- parallel::parLapply(cl, oj_test, add_regvars) - -save_objects(oj_trainreg, oj_testreg, - example="grocery_sales", file="data_reg.Rdata") - -oj_modelset_reg <- parallel::parLapply(cl, oj_trainreg, function(df) -{ - model(df, - ar_trend=ARIMA(logmove ~ pdq() + PDQ(0, 0, 0) + trend()), - - ar_reg=ARIMA(logmove ~ pdq() + PDQ(0, 0, 0) + deal + feat + maxpricediff + - price1 + price2 + price3 + price4 + price5 + price6 + price7 + price8 + price9 + price10 + price11), - - ar_reg_price=ARIMA(logmove ~ pdq() + PDQ(0, 0, 0) + deal + feat + maxpricediff + price), - - ar_reg_price_trend=ARIMA(logmove ~ pdq() + PDQ(0, 0, 0) + trend() + deal + feat + maxpricediff + price), - - .safely=FALSE - ) -}) - -oj_fcast_reg <- parallel::clusterMap(cl, get_forecasts, oj_modelset_reg, oj_testreg) - -destroy_cluster(cl) - -save_objects(oj_modelset_reg, oj_fcast_reg, - example="grocery_sales", file="model_reg.Rdata") - -do.call(rbind, oj_fcast_reg) %>% - mutate_at(-(1:3), exp) %>% - eval_forecasts() -``` - -This shows that the models incorporating price are a significant improvement over the previous naive models. The model that uses stepwise selection to choose the best price variable does worse than the one where we choose the price beforehand, confirming the suspicion that stepwise leads to overfitting in this case. diff --git a/examples/grocery_sales/R/02b_prophet_models.Rmd b/examples/grocery_sales/R/02b_prophet_models.Rmd deleted file mode 100644 index 0d416e11..00000000 --- a/examples/grocery_sales/R/02b_prophet_models.Rmd +++ /dev/null @@ -1,67 +0,0 @@ ---- -title: Prophet models -output: html_notebook ---- - -_Copyright (c) Microsoft Corporation._
-_Licensed under the MIT License._ - -```{r, echo=FALSE, results="hide", message=FALSE} -library(tidyr) -library(dplyr) -library(tsibble) -library(feasts) -library(fable) -library(prophet) -library(fable.prophet) -``` - -This notebook builds a forecasting model using the [Prophet](https://facebook.github.io/prophet/) algorithm. Prophet is a time series model developed by Facebook that is designed to be simple for non-experts to use, yet flexible and powerful. - -> Prophet is a procedure for forecasting time series data based on an additive model where non-linear trends are fit with yearly, weekly, and daily seasonality, plus holiday effects. It works best with time series that have strong seasonal effects and several seasons of historical data. Prophet is robust to missing data and shifts in the trend, and typically handles outliers well. - -Here, we will use the fable.prophet package which provides a tidyverts frontend to the prophet package itself. As with ETS, prophet does not support time series with missing values, so we again impute them using the ARIMA model forecasts. - -```{r} -srcdir <- here::here("R_utils") -for(src in dir(srcdir, full.names=TRUE)) source(src) - -load_objects("grocery_sales", "data_reg.Rdata") -load_objects("grocery_sales", "model_basic.Rdata") - -cl <- make_cluster(libs=c("tidyr", "dplyr", "fable", "tsibble", "feasts", "prophet", "fable.prophet")) - -oj_modelset_pr <- parallel::clusterMap(cl, function(df, basicmod) -{ - df$logmove <- interpolate(select(basicmod, -c(mean, naive, drift)), df)$logmove - df %>% - group_by(store, brand) %>% - fill(deal:maxpricediff, .direction="downup") %>% - model( - pr=prophet(logmove ~ deal + feat + price + maxpricediff), - - pr_tune=prophet(logmove ~ deal + feat + price + maxpricediff + - growth(n_changepoints=2) + season(period=52, order=5, prior_scale=2)), - - .safely=FALSE - ) -}, oj_trainreg, oj_modelset_basic) - -oj_fcast_pr <- parallel::clusterMap(cl, function(mable, newdata, fcast_func) -{ - newdata <- newdata %>% - fill(deal:maxpricediff, .direction="downup") - fcast_func(mable, newdata) -}, oj_modelset_pr, oj_testreg, MoreArgs=list(fcast_func=get_forecasts)) - -destroy_cluster(cl) - -save_objects(oj_modelset_pr, oj_fcast_pr, - example="grocery_sales", file="model_pr.Rdata") - -do.call(rbind, oj_fcast_pr) %>% - mutate_at(-(1:3), exp) %>% - eval_forecasts() -``` - -It appears that Prophet does _not_ do better than the simple ARIMA model with regression variables. This is possibly because the dataset does not have a strong time series nature: there is no seasonality, and only weak or nonexistent trends. These are features which the Prophet algorithm is designed to detect, and their absence means that there would be little advantage in using it. diff --git a/examples/grocery_sales/R/README.md b/examples/grocery_sales/R/README.md deleted file mode 100644 index 6e3ca408..00000000 --- a/examples/grocery_sales/R/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# Forecasting examples in R: orange juice retail sales - -The Rmarkdown notebooks in this directory are as follows. Each notebook also has a corresponding HTML file, which is the rendered output from running the code. - -- [`01_dataprep.Rmd`](01_dataprep.Rmd) creates the training and test datasets -- [`02_basic_models.Rmd`](02_basic_models.Rmd) fits a range of simple time series models to the data, including ARIMA and ETS. -- [`02a_reg_models.Rmd`](02a_reg_models.Rmd) adds independent variables as regressors to the ARIMA model. -- [`02b_prophet_models.Rmd`](02b_prophet_models.Rmd) fits some simple models using the Prophet algorithm. - -If you want to run the code in the notebooks interactively, you must start from `01_dataprep.Rmd` and proceed in sequence, as the earlier notebooks will generate artifacts (datasets/model objects) that are used by later ones. - -## Package installation - -The following packages are needed to run the basic analysis notebooks in this directory: - -- rmarkdown -- dplyr -- tidyr -- ggplot2 -- tsibble -- fable -- feasts -- yaml -- here - -It's likely that you will already have many of these (particularly the [Tidyverse](https://tidyverse.org) packages) installed, if you use R for data science tasks. The main exceptions are the packages in the [Tidyverts](https://tidyverts.org) family, which is a modern framework for time series analysis building on the Tidyverse. - -```r -install.packages("tidyverse") # installs all tidyverse packages -install.packages("rmarkdown") -install.packages("here") -install.packages(c("tsibble", "fable", "feasts")) -``` - -The following packages are needed to run the Prophet analysis notebook: - -- prophet -- fable.prophet - -While prophet is available from CRAN, its frontend for the tidyverts framework, fable.prophet, is currently on GitHub only. You can install these packages with - -```r -install.packages("prophet") -install.packages("https://github.com/mitchelloharawild/fable.prophet/archive/master.tar.gz", repos=NULL) -``` diff --git a/examples/grocery_sales/R/forecast_settings.yaml b/examples/grocery_sales/R/forecast_settings.yaml deleted file mode 100644 index 638662ce..00000000 --- a/examples/grocery_sales/R/forecast_settings.yaml +++ /dev/null @@ -1,6 +0,0 @@ -N_SPLITS: 10 -HORIZON: 2 -GAP: 2 -FIRST_WEEK: 40 -LAST_WEEK: 156 -START_DATE: "1989-09-14" diff --git a/examples/grocery_sales/README.md b/examples/grocery_sales/README.md deleted file mode 100644 index 8bcfb30c..00000000 --- a/examples/grocery_sales/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# Forecasting examples - -This folder contains Python and R examples for building forecasting solutions on the Orange Juice dataset which is part of the [Dominick's dataset](https://www.chicagobooth.edu/research/kilts/datasets/dominicks). The examples are presented in Python Jupyter notebooks and R Markdown files, respectively. - - -## Orange Juice Dataset - -In this scenario, we will use the Orange Juice (OJ) dataset to forecast its sales. The OJ dataset is from R package [bayesm](https://cran.r-project.org/web/packages/bayesm/index.html) and is part of the [Dominick's dataset](https://www.chicagobooth.edu/research/kilts/datasets/dominicks). - -This dataset contains the following two tables: -- **yx.cs.** - Weekly sales of refrigerated orange juice at 83 stores. This table has 106139 rows and 19 columns. It includes weekly sales and prices of 11 orange juice brands as well as information about profit, deal, and advertisement for each brand. Note that the weekly sales is captured by a column named `logmove` which corresponds to the natural logarithm of the number of units sold. To get the number of units sold, you need to apply an exponential transform to this column. -- **storedemo.csv** - Demographic information on those stores. This table has 83 rows and 13 columns. For every store, the table describes demographic information of its consumers, distance to the nearest warehouse store, average distance to the nearest 5 supermarkets, ratio of its sales to the nearest warehouse store, and ratio of its sales to the average of the nearest 5 stores. - -Note that the week number starts from 40 in this dataset, while the full Dominick's dataset has data starting from week 1 to week 400. According to [Dominick's Data Manual](https://www.chicagobooth.edu/-/media/enterprise/centers/kilts/datasets/dominicks-dataset/dominicks-manual-and-codebook_kiltscenter.aspx), week 1 starts on 09/14/1989. Please see pages 40 and 41 of the [bayesm reference manual](https://cran.r-project.org/web/packages/bayesm/bayesm.pdf) and the [Dominick's Data Manual](https://www.chicagobooth.edu/-/media/enterprise/centers/kilts/datasets/dominicks-dataset/dominicks-manual-and-codebook_kiltscenter.aspx) for more details about the data. - - - -## Summary - -The following summarizes each directory of the forecasting examples. - -| Directory | Content | Description | -| --- | --- | --- | -| [python](./python)| [00_quick_start/](./python/00_quick_start)
[01_prepare_data/](./python/01_prepare_data)
[02_model/](./python/02_model)
[03_model_tune_deploy/](./python/03_model_tune_deploy/) |
  • Quick start examples for single-round training
  • Data exploration and preparation notebooks
  • Multi-round training examples
  • Model tuning and deployment example
| -| [R](./R) | [01_dataprep.Rmd](R/01_dataprep.Rmd)
[02_basic_models.Rmd](R/02_basic_models.Rmd)
[02a_reg_models.Rmd](R/02a_reg_models.Rmd)
[02b_prophet_models.Rmd](R/02b_prophet_models.Rmd) |
  • Data preparation
  • Basic time series models
  • ARIMA-regression models
  • Prophet models
| - diff --git a/examples/grocery_sales/python/00_quick_start/azure_automl_single_round.ipynb b/examples/grocery_sales/python/00_quick_start/azure_automl_single_round.ipynb deleted file mode 100644 index 027a3814..00000000 --- a/examples/grocery_sales/python/00_quick_start/azure_automl_single_round.ipynb +++ /dev/null @@ -1,1790 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Copyright (c) Microsoft Corporation.\n", - "\n", - "Licensed under the MIT License. " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Automated Machine Learning (AutoML) on Azure for Retail Sales Forecasting\n", - "\n", - "This notebook demonstrates how to apply [AutoML in Azure Machine Learning services](https://docs.microsoft.com/en-us/azure/machine-learning/concept-automated-ml) to train and tune machine learning models for forecasting product sales in retail. We will use the Orange Juice dataset to illustrate the steps of utilizing AutoML as well as how to combine an AutoML model with a custom model for better performance.\n", - "\n", - "AutoML is a process of automating the tasks of machine learning model development. It helps data scientists and other practioners build machine learning models with high scalability and quality in less amount of time. AutoML in Azure Machine Learning allows you to train and tune a model using a target metric that you specify. This service iterates through machine learning algorithms and feature selection approaches, producing a score that measures the quality of each machine learning pipeline. The best model will then be selected based on the scores. For more technical details about Azure AutoML, please check [this paper](https://papers.nips.cc/paper/7595-probabilistic-matrix-factorization-for-automated-machine-learning.pdf).\n", - "\n", - "This notebook uses [Azure ML SDK](https://docs.microsoft.com/en-us/python/api/overview/azureml-sdk/?view=azure-ml-py) which is included in the `forecasting_env` conda environment. If you are running in Azure Notebooks or another Microsoft managed environment, the SDK is already installed. On the other hand, if you are running this notebook in your own environment, please follow [SDK installation instructions](https://docs.microsoft.com/azure/machine-learning/service/how-to-configure-environment) to install the SDK." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Global Settings and Imports" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2\n", - "%matplotlib inline" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "System version: 3.6.10 |Anaconda, Inc.| (default, Jan 7 2020, 21:14:29) \n", - "[GCC 7.3.0]\n", - "This notebook was created using version 1.0.85 of the Azure ML SDK\n", - "You are currently using version 1.0.85 of the Azure ML SDK\n" - ] - } - ], - "source": [ - "import os\n", - "import sys\n", - "import math\n", - "import warnings\n", - "import datetime\n", - "import logging\n", - "import azureml.core\n", - "import azureml.automl\n", - "import pandas as pd\n", - "\n", - "from matplotlib import pyplot as plt\n", - "from fclib.common.utils import git_repo_path\n", - "from fclib.azureml.azureml_utils import (\n", - " get_or_create_workspace,\n", - " get_or_create_amlcompute,\n", - ")\n", - "from fclib.dataset.ojdata import download_ojdata, FIRST_WEEK_START\n", - "from fclib.common.utils import align_outputs\n", - "from fclib.evaluation.evaluation_utils import MAPE\n", - "from fclib.models.multiple_linear_regression import fit, predict\n", - "\n", - "from azureml.core import Workspace\n", - "from azureml.core.dataset import Dataset\n", - "from azureml.core.experiment import Experiment\n", - "from automl.client.core.common import constants\n", - "from azureml.train.automl import AutoMLConfig\n", - "from azureml.core.compute import ComputeTarget, AmlCompute\n", - "from azureml.automl.core._vendor.automl.client.core.common import metrics\n", - "\n", - "warnings.filterwarnings(\"ignore\")\n", - "\n", - "print(\"System version: {}\".format(sys.version))\n", - "print(\"This notebook was created using version 1.0.85 of the Azure ML SDK\")\n", - "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "# Use False if you've already downloaded and split the data\n", - "DOWNLOAD_SPLIT_DATA = True\n", - "\n", - "# Data directory\n", - "DATA_DIR = os.path.join(git_repo_path(), \"ojdata\")\n", - "\n", - "# Forecasting settings\n", - "GAP = 2\n", - "LAST_WEEK = 138\n", - "\n", - "# Number of test periods\n", - "NUM_TEST_PERIODS = 3\n", - "\n", - "# Column names\n", - "time_column_name = \"week_start\"\n", - "target_column_name = \"move\"\n", - "grain_column_names = [\"store\", \"brand\"]\n", - "index_column_names = [time_column_name] + grain_column_names\n", - "\n", - "# Subset of stores used in the notebook\n", - "USE_STORES = [2, 5, 8]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Set up Azure Machine Learning Workspace\n", - "\n", - "An Azure ML workspace is an Azure resource that organizes and coordinates the actions of many other Azure resources to assist in executing and sharing machine learning workflows. In particular, an Azure ML workspace coordinates storage, databases, and compute resources providing added functionality for machine learning experimentation, deployment, inference, and the monitoring of deployed models. To create an Azure ML workspace, first you need access to an Azure subscription. An Azure subscription allows you to manage storage, compute, and other assets in the Azure cloud. You can [create a new subscription](https://azure.microsoft.com/en-us/free/) or access existing subscription information from the [Azure portal](https://portal.azure.com/). Given that you have access to your Azure subscription, you can further create an Azure ML workspace by following the instructions [here](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-manage-workspace). You can also do so [using Azure CLI](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-manage-workspace-cli) or the `Workspace.create()` method in Azure SDK.\n", - "\n", - "Once you have created an Azure ML workspace, you can download its configuration file (`config.json`) from Azure Portal as follows\n", - "\n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Prepare Azure ML Workspace\n", - "\n", - "In the following cell, `get_or_create_workspace()` creates a workspace object from the details stored in `config.json` that you have downloaded. We assume that you store this config file to a directory `./.azureml`. In case the existing workspace cannot be loaded, the following cell will try to create a new workspace with the subscription ID, resource group, and workspace name as specified in the beginning of the cell.\n", - "\n", - "The cell can fail if you don't have permission to access the workspace. You may need to log into your Azure account and change the default subscription to the one which the workspace belongs to using Azure CLI `az account set --subscription `." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Workspace preparation succeeded.\n" - ] - } - ], - "source": [ - "# Please specify the AzureML workspace attributes below if you want to create a new one.\n", - "subscription_id = \"\"\n", - "resource_group = \"\"\n", - "workspace_name = \"\"\n", - "workspace_region = \"\"\n", - "\n", - "# Connect to a workspace\n", - "ws = get_or_create_workspace(\n", - " config_path=\"./.azureml\",\n", - " subscription_id=subscription_id,\n", - " resource_group=resource_group,\n", - " workspace_name=workspace_name,\n", - " workspace_region=workspace_region,\n", - ")\n", - "print(\n", - " \"Workspace name: \" + ws.name,\n", - " \"Azure region: \" + ws.location,\n", - " \"Resource group: \" + ws.resource_group,\n", - " sep=\"\\n\",\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Create compute resources for your experiments\n", - "\n", - "We run AutoML on a dynamically scalable compute cluster. In the next cell, we create an AmlCompute target with a specific cluster name, VM size, and maximum number of nodes if the cluster does not exist. Otherwise, we will reuse an existing one. For more options of VM sizes, please check the information in this [link](https://docs.microsoft.com/en-us/azure/virtual-machines/sizes-general)." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found existing cpu-cluster\n" - ] - } - ], - "source": [ - "# Choose a name for your cluster\n", - "cluster_name = \"cpu-cluster\"\n", - "# VM Size\n", - "vm_size = \"STANDARD_D2_V2\"\n", - "# Maximum number of nodes of the cluster\n", - "max_nodes = 4\n", - "\n", - "# Create a new AmlCompute if it does not exist or reuse an existing one\n", - "cpu_cluster = get_or_create_amlcompute(\n", - " workspace=ws,\n", - " compute_name=cluster_name,\n", - " vm_size=vm_size,\n", - " min_nodes=0,\n", - " max_nodes=max_nodes,\n", - " verbose=True,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Define Experiment\n", - "\n", - "To run AutoML, you need to create an Experiment. An Experiment corresponds to a prediction problem you are trying to solve, while a Run corresponds to a specific approach to the problem." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
SDK version1.0.85
Workspacechhamlws
SKUBasic
Resource Groupchhamlwsrg
Locationwestcentralus
Run History Nameautoml-ojforecasting
\n", - "
" - ], - "text/plain": [ - " \n", - "SDK version 1.0.85 \n", - "Workspace chhamlws \n", - "SKU Basic \n", - "Resource Group chhamlwsrg \n", - "Location westcentralus \n", - "Run History Name automl-ojforecasting" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# choose a name for the run history container in the workspace\n", - "experiment_name = \"automl-ojforecasting\"\n", - "\n", - "experiment = Experiment(ws, experiment_name)\n", - "\n", - "output = {}\n", - "output[\"SDK version\"] = azureml.core.VERSION\n", - "output[\"Workspace\"] = ws.name\n", - "output[\"SKU\"] = ws.sku\n", - "output[\"Resource Group\"] = ws.resource_group\n", - "output[\"Location\"] = ws.location\n", - "output[\"Run History Name\"] = experiment_name\n", - "pd.set_option(\"display.max_colwidth\", -1)\n", - "outputDf = pd.DataFrame(data=output, index=[\"\"])\n", - "outputDf.T" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Data Preparation\n", - "\n", - "We need to download the Orange Juice data and split it into training and test sets. By default, the following cell will download and spit the data. If you've already done so, you may skip this part by switching `DOWNLOAD_SPLIT_DATA` to `False`.\n", - "\n", - "We store the training data and test data using dataframes. The training data includes `train_df` and `aux_df` with `train_df` containing the historical sales up to week 135 (the time we make forecasts) and `aux_df` containing price/promotion information up until week 138. We assume that future price and promotion information up to a certain number of weeks ahead is predetermined and known. The test data is stored in `test_df` which contains the sales of each product in week 137 and 138. Assuming the current week is week 135, our goal is to forecast the sales in week 137 and 138 using the training data. There is a one-week gap between the current week and the first target week of forecasting as we want to leave time for planning inventory in practice." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Data download and split" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Data already exists at the specified location.\n" - ] - } - ], - "source": [ - "if DOWNLOAD_SPLIT_DATA:\n", - " download_ojdata(DATA_DIR)\n", - " df = pd.read_csv(os.path.join(DATA_DIR, \"yx.csv\"))\n", - " df = df.loc[df.week <= LAST_WEEK]" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "# Convert logarithm of the unit sales to unit sales\n", - "df[\"move\"] = df[\"logmove\"].apply(lambda x: round(math.exp(x)))\n", - "# Add timestamp column\n", - "df[\"week_start\"] = df[\"week\"].apply(lambda x: FIRST_WEEK_START + datetime.timedelta(days=(x - 1) * 7))\n", - "# Select a subset of stores for demo purpose\n", - "df_sub = df[df.store.isin(USE_STORES)]" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "# Split data into training and test sets\n", - "def split_last_n_by_grain(df, n):\n", - " \"\"\"Group df by grain and split on last n rows for each group.\"\"\"\n", - " df_grouped = df.sort_values(time_column_name).groupby( # Sort by ascending time\n", - " grain_column_names, group_keys=False\n", - " )\n", - " df_head = df_grouped.apply(lambda dfg: dfg.iloc[:-n])\n", - " df_tail = df_grouped.apply(lambda dfg: dfg.iloc[-n:])\n", - " return df_head, df_tail\n", - "\n", - "\n", - "train_df, test_df = split_last_n_by_grain(df_sub, NUM_TEST_PERIODS)\n", - "train_df.reset_index(drop=True)\n", - "test_df.reset_index(drop=True)\n", - "\n", - "# Save data locally\n", - "local_data_pathes = [\n", - " os.path.join(DATA_DIR, \"train.csv\"),\n", - " os.path.join(DATA_DIR, \"test.csv\"),\n", - "]\n", - "\n", - "train_df.to_csv(local_data_pathes[0], index=None, header=True)\n", - "test_df.to_csv(local_data_pathes[1], index=None, header=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Upload data to datastore\n", - "\n", - "The [Machine Learning service workspace](https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-workspace), is paired with the storage account, which contains the default data store. We will use it to upload the train and test data and create [tabular datasets](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.data.tabulardataset?view=azure-ml-py) for training and testing. A tabular dataset defines a series of lazily-evaluated, immutable operations to load data from the data source into tabular representation.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Uploading an estimated of 2 files\n", - "Uploading /data/home/chenhui/work/forecasting/ojdata/test.csv\n", - "Uploading /data/home/chenhui/work/forecasting/ojdata/train.csv\n", - "Uploaded /data/home/chenhui/work/forecasting/ojdata/test.csv, 1 files out of an estimated total of 2\n", - "Uploaded /data/home/chenhui/work/forecasting/ojdata/train.csv, 2 files out of an estimated total of 2\n", - "Uploaded 2 files\n" - ] - }, - { - "data": { - "text/plain": [ - "$AZUREML_DATAREFERENCE_1f003008a69b4030b4c6165a27ca7f24" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "datastore = ws.get_default_datastore()\n", - "datastore.upload_files(files=local_data_pathes, target_path=\"dataset/\", overwrite=True, show_progress=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Create dataset for training" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "train_dataset = Dataset.Tabular.from_delimited_files(path=datastore.path(\"dataset/train.csv\"))" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
storebrandweeklogmoveconstantprice1price2price3price4price5...price7price8price9price10price11dealfeatprofitmoveweek_start
297681113110.4010.030.040.040.030.03...0.040.030.020.020.0200.005.52330241992-03-12
297781113210.3910.030.040.040.040.03...0.030.030.020.020.0211.005.48323841992-03-19
29788111339.3710.050.040.040.030.04...0.030.030.020.020.0200.005.38117761992-03-26
29798111349.3410.040.040.040.030.03...0.040.030.020.020.0200.007.16113921992-04-02
298081113510.5110.040.040.040.040.03...0.040.030.030.020.0211.008.29368641992-04-09
\n", - "

5 rows × 21 columns

\n", - "
" - ], - "text/plain": [ - " store brand week logmove constant price1 price2 price3 price4 \\\n", - "2976 8 11 131 10.40 1 0.03 0.04 0.04 0.03 \n", - "2977 8 11 132 10.39 1 0.03 0.04 0.04 0.04 \n", - "2978 8 11 133 9.37 1 0.05 0.04 0.04 0.03 \n", - "2979 8 11 134 9.34 1 0.04 0.04 0.04 0.03 \n", - "2980 8 11 135 10.51 1 0.04 0.04 0.04 0.04 \n", - "\n", - " price5 ... price7 price8 price9 price10 price11 deal \\\n", - "2976 0.03 ... 0.04 0.03 0.02 0.02 0.02 0 \n", - "2977 0.03 ... 0.03 0.03 0.02 0.02 0.02 1 \n", - "2978 0.04 ... 0.03 0.03 0.02 0.02 0.02 0 \n", - "2979 0.03 ... 0.04 0.03 0.02 0.02 0.02 0 \n", - "2980 0.03 ... 0.04 0.03 0.03 0.02 0.02 1 \n", - "\n", - " feat profit move week_start \n", - "2976 0.00 5.52 33024 1992-03-12 \n", - "2977 1.00 5.48 32384 1992-03-19 \n", - "2978 0.00 5.38 11776 1992-03-26 \n", - "2979 0.00 7.16 11392 1992-04-02 \n", - "2980 1.00 8.29 36864 1992-04-09 \n", - "\n", - "[5 rows x 21 columns]" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "train_dataset.to_pandas_dataframe().tail()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Modeling\n", - "\n", - "For forecasting tasks, AutoML uses pre-processing and estimation steps that are specific to time-series. AutoML will undertake the following pre-processing steps:\n", - "* Detect time-series sample frequency (e.g. hourly, daily, weekly) and create new records for absent time points to make the series regular. A regular time series has a well-defined frequency and has a value at every sample point in a contiguous time span\n", - "* Impute missing values in the target (via forward-fill) and feature columns (using median column values)\n", - "* Create grain-based features to enable fixed effects across different series\n", - "* Create time-based features to assist in learning seasonal patterns\n", - "* Encode categorical variables to numeric quantities\n", - "\n", - "In this notebook, AutoML will train a single, regression-type model across all time-series in a given training set. This allows the model to generalize across related series. To create a training job, we use AutoML Config object to define the settings and data. Here is a summary of the meanings of the AutoMLConfig parameters:\n", - "\n", - "|Property|Description|\n", - "|-|-|\n", - "|**task**|forecasting|\n", - "|**primary_metric**|This is the metric that you want to optimize.
Forecasting supports the following primary metrics
spearman_correlation
normalized_root_mean_squared_error
r2_score
normalized_mean_absolute_error\n", - "|**experiment_timeout_hours**|Experimentation timeout in hours.|\n", - "|**enable_early_stopping**|If early stopping is on, training will stop when the primary metric is no longer improving.|\n", - "|**training_data**|Input dataset, containing both features and label column.|\n", - "|**label_column_name**|The name of the label column.|\n", - "|**compute_target**|The remote compute for training.|\n", - "|**n_cross_validations**|Number of cross-validation folds to use for model/pipeline selection|\n", - "|**enable_voting_ensemble**|Allow AutoML to create a Voting ensemble of the best performing models|\n", - "|**enable_stack_ensemble**|Allow AutoML to create a Stack ensemble of the best performing models|\n", - "|**debug_log**|Log file path for writing debugging information|\n", - "|**time_column_name**|Name of the datetime column in the input data|\n", - "|**grain_column_names**|Name(s) of the columns defining individual series in the input data|\n", - "|**drop_column_names**|Name(s) of columns to drop prior to modeling|\n", - "|**max_horizon**|Maximum desired forecast horizon in units of time-series frequency|" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Model training" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "time_series_settings = {\n", - " \"time_column_name\": time_column_name,\n", - " \"grain_column_names\": grain_column_names,\n", - " \"drop_column_names\": [\"logmove\"], # 'logmove' is a leaky feature, so we remove it.\n", - " \"max_horizon\": NUM_TEST_PERIODS,\n", - "}\n", - "\n", - "automl_config = AutoMLConfig(\n", - " task=\"forecasting\",\n", - " debug_log=\"automl_oj_sales_errors.log\",\n", - " primary_metric=\"normalized_mean_absolute_error\",\n", - " experiment_timeout_hours=1.0, # You may increase this number to improve model accuracy\n", - " training_data=train_dataset,\n", - " label_column_name=target_column_name,\n", - " compute_target=cpu_cluster,\n", - " enable_early_stopping=True,\n", - " n_cross_validations=3,\n", - " verbosity=logging.INFO,\n", - " **time_series_settings\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
ExperimentIdTypeStatusDetails PageDocs Page
automl-ojforecastingAutoML_45710381-d3fc-47c9-816d-9874c41b5355automlStartingLink to Azure Machine Learning studioLink to Documentation
" - ], - "text/plain": [ - "Run(Experiment: automl-ojforecasting,\n", - "Id: AutoML_45710381-d3fc-47c9-816d-9874c41b5355,\n", - "Type: automl,\n", - "Status: Starting)" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "remote_run = experiment.submit(automl_config, show_output=False)\n", - "remote_run" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "remote_run.wait_for_completion()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Retrieve the best model\n", - "\n", - "Each run within an Experiment stores serialized (i.e. pickled) pipelines from the AutoML iterations. After the training job is done, we can retrieve the pipeline with the best performance on the validation dataset." - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[('timeseriestransformer', TimeSeriesTransformer(logger=None,\n", - " pipeline_type=)), ('MinMaxScaler', MinMaxScaler(copy=True, feature_range=(0, 1))), ('GradientBoostingRegressor', GradientBoostingRegressor(alpha=0.9, criterion='mse', init=None,\n", - " learning_rate=0.1, loss='huber', max_depth=10,\n", - " max_features='sqrt', max_leaf_nodes=None,\n", - " min_impurity_decrease=0.0, min_impurity_split=None,\n", - " min_samples_leaf=0.15874989977926784,\n", - " min_samples_split=0.10734188827013527,\n", - " min_weight_fraction_leaf=0.0, n_estimators=50,\n", - " n_iter_no_change=None, presort='auto', random_state=None,\n", - " subsample=0.95, tol=0.0001, validation_fraction=0.1,\n", - " verbose=0, warm_start=False))]\n" - ] - } - ], - "source": [ - "best_run, fitted_model = remote_run.get_output()\n", - "print(fitted_model.steps)\n", - "model_name = best_run.properties[\"model_name\"]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Forecasting\n", - "\n", - "Now that we have retrieved the best model pipeline, we can apply it to generate forecasts for the target weeks. To do this, we first remove the target values from the test set" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Generate forecasts" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [], - "source": [ - "X_test = test_df\n", - "y_test = X_test.pop(target_column_name).values" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
storebrandweeklogmoveconstantprice1price2price3price4price5price6price7price8price9price10price11dealfeatprofitweek_start
85211368.5910.050.050.050.050.040.050.030.040.030.020.0300.0033.541992-04-16
86211379.1910.040.050.050.040.030.040.030.040.040.020.0300.0020.431992-04-23
87211389.7410.040.040.050.040.040.050.040.040.040.030.0311.0011.291992-04-30
195221369.1410.050.050.050.050.040.050.030.040.030.020.0310.0027.131992-04-16
196221378.7410.040.050.050.040.030.040.030.040.040.020.0300.0033.301992-04-23
\n", - "
" - ], - "text/plain": [ - " store brand week logmove constant price1 price2 price3 price4 \\\n", - "85 2 1 136 8.59 1 0.05 0.05 0.05 0.05 \n", - "86 2 1 137 9.19 1 0.04 0.05 0.05 0.04 \n", - "87 2 1 138 9.74 1 0.04 0.04 0.05 0.04 \n", - "195 2 2 136 9.14 1 0.05 0.05 0.05 0.05 \n", - "196 2 2 137 8.74 1 0.04 0.05 0.05 0.04 \n", - "\n", - " price5 price6 price7 price8 price9 price10 price11 deal feat \\\n", - "85 0.04 0.05 0.03 0.04 0.03 0.02 0.03 0 0.00 \n", - "86 0.03 0.04 0.03 0.04 0.04 0.02 0.03 0 0.00 \n", - "87 0.04 0.05 0.04 0.04 0.04 0.03 0.03 1 1.00 \n", - "195 0.04 0.05 0.03 0.04 0.03 0.02 0.03 1 0.00 \n", - "196 0.03 0.04 0.03 0.04 0.04 0.02 0.03 0 0.00 \n", - "\n", - " profit week_start \n", - "85 33.54 1992-04-16 \n", - "86 20.43 1992-04-23 \n", - "87 11.29 1992-04-30 \n", - "195 27.13 1992-04-16 \n", - "196 33.30 1992-04-23 " - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "X_test.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [], - "source": [ - "# The featurized data, aligned to y, will also be returned. It contains the assumptions\n", - "# that were made in the forecast and helps align the forecast to the original data.\n", - "y_predictions, X_trans = fitted_model.forecast(X_test)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We need to align the output explicitly to the input, as the count and order of the rows may have changed during transformations that span multiple rows." - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
week_startstorebrandpredictedweeklogmoveconstantprice1price2price3...price6price7price8price9price10price11dealfeatprofitmove
01992-04-16214075.571368.5910.050.050.05...0.050.030.040.030.020.0300.0033.545376
11992-04-16227212.491369.1410.050.050.05...0.050.030.040.030.020.0310.0027.139312
21992-04-16234075.571367.8510.050.050.05...0.050.030.040.030.020.0300.0032.552560
31992-04-16244011.431367.4210.050.050.05...0.050.030.040.030.020.0300.0034.981664
41992-04-16254336.831368.5910.050.050.05...0.050.030.040.030.020.0300.0028.805376
\n", - "

5 rows × 22 columns

\n", - "
" - ], - "text/plain": [ - " week_start store brand predicted week logmove constant price1 \\\n", - "0 1992-04-16 2 1 4075.57 136 8.59 1 0.05 \n", - "1 1992-04-16 2 2 7212.49 136 9.14 1 0.05 \n", - "2 1992-04-16 2 3 4075.57 136 7.85 1 0.05 \n", - "3 1992-04-16 2 4 4011.43 136 7.42 1 0.05 \n", - "4 1992-04-16 2 5 4336.83 136 8.59 1 0.05 \n", - "\n", - " price2 price3 ... price6 price7 price8 price9 price10 price11 \\\n", - "0 0.05 0.05 ... 0.05 0.03 0.04 0.03 0.02 0.03 \n", - "1 0.05 0.05 ... 0.05 0.03 0.04 0.03 0.02 0.03 \n", - "2 0.05 0.05 ... 0.05 0.03 0.04 0.03 0.02 0.03 \n", - "3 0.05 0.05 ... 0.05 0.03 0.04 0.03 0.02 0.03 \n", - "4 0.05 0.05 ... 0.05 0.03 0.04 0.03 0.02 0.03 \n", - "\n", - " deal feat profit move \n", - "0 0 0.00 33.54 5376 \n", - "1 1 0.00 27.13 9312 \n", - "2 0 0.00 32.55 2560 \n", - "3 0 0.00 34.98 1664 \n", - "4 0 0.00 28.80 5376 \n", - "\n", - "[5 rows x 22 columns]" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "pred_automl = align_outputs(y_predictions, X_trans, X_test, y_test, target_column_name)\n", - "pred_automl.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Results evaluation & visualization" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[Test data scores]\n", - "\n", - "explained_variance: 0.187\n", - "r2_score: 0.172\n", - "spearman_correlation: 0.703\n", - "mean_absolute_percentage_error: 117.345\n", - "mean_absolute_error: 6624.722\n", - "normalized_mean_absolute_error: 0.046\n", - "median_absolute_error: 3048.760\n", - "normalized_median_absolute_error: 0.021\n", - "root_mean_squared_error: 16663.119\n", - "normalized_root_mean_squared_error: 0.115\n", - "root_mean_squared_log_error: 0.890\n", - "normalized_root_mean_squared_log_error: 0.140\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAD4CAYAAAAZ1BptAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dfXQc1Znn8e8jWcaIF78nAcuWTOLNBvAEbK1DgEkEAtuQgD2zzDnOCDBvowDJLMzLBnO0B+NkdCYkcxYPE0yiBYOxewOEMIMheB1jYCYzJxhkCIiXEAuwhIDEwjaaDB7AL8/+UVdySeqSulstqSX9Puf06eqnblVdXUn9dN17u8rcHRERkXSKhrsCIiJSuJQkREQkkZKEiIgkUpIQEZFEShIiIpJo3HBXIN+mTZvmFRUVw10NEZERZfv27e+5+/Se8VGXJCoqKmhsbBzuaoiIjChm1pIuru4mERFJpCQhIiKJRl13U0/79++nra2NDz/8cLirUjAmTJhAWVkZJSUlw10VESlwoz5JtLW1ccwxx1BRUYGZDXd1hp27s3v3btra2pg9e/ZwV0dECtyo72768MMPmTp1qhJEYGZMnTpVZ1YikpFRnyQAJYge1B4ikqkxkSRGgqqqKgC++93v8vbbb6cts3bt2q7l66+/noMHDw5F1USkAKWaUlSsrqBoVREVqytINaUG5ThKEkAqBRUVUFQUPafy3NaHDh3KuOyKFSuYMWNG2nXxJLF69WqKi4sHXDcRGXlSTSlqH6mlpaMFx2npaKH2kdpBSRRjPkmkUlBbCy0t4B4919ZmnyieeuopLrjgAs477zzOPvtsnnvuOc466ywuuugi7rnnHrZt20ZVVRVnnHEGd999NwCPPvoo8+fP5+tf/zoHDhwA4LLLLqO5uZkPPviAiy66iC9/+ctcfvnlbNy4kaamJqqqqtiyZQtVVVUcOHCA1tZWzj77bM444wxuueUWAG6++WauuuoqzjnnHK666qq8tpeIDL+6rXXs27+vW2zf/n3Uba3L+7FG/eym/tTVwb7ubc2+fVG8pia7fX344Yds2bKF+++/n5///Ofs2rWLxx9/nOLiYhYtWsTGjRs55phjOPfcc6mpqeFv//Zv+ed//mf27t3LWWed1W1fDQ0NLFy4kNraWg4dOkRRURFz587lqaeeAqC+vh6AW265hVWrVvGHf/iHLFq0iEsuuQSAk046iTvvvJOFCxfy/vvvM2nSpJzaR0QKT2tHa1bxgRjzZxKtCW2aFO/LqaeeCsApp5zC448/zuc///muLqEXXniBCy+8kLPOOovf/va3tLe3U1RUxNFHH83MmTOZPr37JVN+85vfcPrppwNQVJT8a3r99deZN29e1/HffPNNAE4++WQAjj/+eDo6OrL/YUSkYM2aOCur+ECM+SQxK6FNk+J9eeGFF7qeq6uru725n3rqqfzsZz/jqaee4vnnn2fGjBkcOnSIDz74gLa2Ntrb27vt67Of/SxPP/00cHhMI92spBNOOIHt27cD8Pzzz9N5ccN4Wd2iVmR0qa+up7SktFustKSU+ur6vB+r3yRhZmvNbJeZvZRm3V+bmZvZtPDazOw2M2s2sxfNbF6s7HIz2xEey2Px+WbWFLa5zcK7m5lNMbMtofwWM5ucnx+5u/p6KO3e1pSWRvFslZSUsHjxYtasWcPChQu7rVu1alXXmcSyZcsAuOGGG/jSl77EqlWr+NSnPtWt/J/92Z+xadMmvvzlL3eNKyxYsIClS5fyi1/8oqvcDTfcwE033cTpp59OVVVV4qC3iIweNXNraLiggfKJ5RhG+cRyGi5ooGZuln3kmXD3Ph/Al4B5wEs94jOBzUALMC3Ezgc2AQacBmwL8SnAG+F5clieHNY9A3wxbLMJOC/EvwesCMsrgFv6q6u7M3/+fI975ZVXvD8bNriXl7ubRc8bNvS7SS9PPvmk19XVZb/hMMmkXURk7AAaPc17ar9nEu7+L8CeNKtuBb4FxPsylgD3hmM+DUwys+OARcAWd9/j7nuBLcDisO5Yd/9lqOS9wNLYvtaF5XWxeN7V1MDOnXDoUPSc7YC1iMholdPsJjO7EHjb3V/o0U8+A3gr9rotxPqKt6WJA3zS3d8FcPd3zewTfdSnFqgFmJXLYEIeVFVVdX0hTkRktMh64NrMSoE64KZ0q9PEPId4Vty9wd0r3b2y5ywhERHJXS6zmz4NzAZeMLOdQBnwnJl9iuhMYGasbBnwTj/xsjRxgN+F7ijC864c6ioiIgOQdZJw9yZ3/4S7V7h7BdEb/Tx3/y2wEbg0zHI6DegIXUabgYVmNjnMUloIbA7rfm9mp4VZTZcCD4dDbQQ6Z0Etj8VFRGSIZDIF9sfAL4HPmlmbmV3ZR/HHiGYuNQP/B7gWwN33AN8Bng2Pb4cYwDXAnWGb14lmOAF8FzjXzHYA54bXBWvnzp088cQTGZW95557ur770HkZDhGRQpTJ7Kavuftx7l7i7mXufleP9RXu/l5Ydnf/hrt/2t3nuntjrNxad/9MeNwdize6+8lhm2+GWU64+253r3b3OeE53QyrvMjH1RTTJYmkC/vFk4SISCEb89+4ztfVFBsaGli/fj3V1dXdLux35plndpWpqqrimWee4Ve/+hXV1dWsX78egH/4h3/gzDPPZNWqVXn92UREBmrMJ4l8XU2xtraWSy65hLvuuotdu3Zx//33c8UVV/Qqt2DBAk455RS2bt3adTG+qqoq/vVf/5XHHnss9x9ERGQQjPkkMRhXU4xf2K+T93H9pM6L8R155JE5H1NEZDCM+SSRr6splpSUdN0pLn5hP3fno48+oqmpKW1Z0O1ERaRwjfkkka+rKZ588sn827/9GzfccEO3+GWXXcaZZ57JT37yk67YV77yFZYuXcpPf/rT3CsuIjIErK9ukJGosrLSGxu7JlXx6quv8rnPfa7PbVJNKeq21tHa0cqsibOor64fnKspFpBM2kVExg4z2+7ulT3jY/7OdBBddne0JwURkVyM+e4mERFJNiaSxGjrUhsotYeIZGrUJ4kJEyawe/duvTEG7s7u3buZMGHCcFdFREaAUT8mUVZWlvYe0mPZhAkTKCsr67+giIx5oz5JlJSUMHv27OGuhojIiDTqu5tERCR3ShIiIpJISUJERBIpSYiISCIlCRERSaQkISIiiZQkREQkUb9JwszWmtkuM3spFvu+mf3azF40s380s0mxdTeaWbOZvWZmi2LxxSHWbGYrYvHZZrbNzHaY2f1mNj7Ejwivm8P6inz90CIikplMziTuARb3iG0BTnb3PwB+A9wIYGYnAsuAk8I2a8ys2MyKgduB84ATga+FsgC3ALe6+xxgL3BliF8J7HX3zwC3hnIiIjKE+k0S7v4vwJ4esZ+7+4Hw8mmg8xoPS4D73P0jd38TaAYWhEezu7/h7h8D9wFLLLol29nAg2H7dcDS2L7WheUHgWrTLdxERIZUPsYkrgA2heUZwFuxdW0hlhSfCrwfSzid8W77Cus7QvlezKzWzBrNrFHXaBIRyZ8BJQkzqwMOAKnOUJpinkO8r331Dro3uHulu1dOnz6970qLiEjGcr7An5ktB74KVPvh63C3ATNjxcqAd8Jyuvh7wCQzGxfOFuLlO/fVZmbjgIn06PYSEZHBldOZhJktBm4ALnT3fbFVG4FlYWbSbGAO8AzwLDAnzGQaTzS4vTEklyeBi8L2y4GHY/taHpYvAp5w3RRCRGRI9XsmYWY/BqqAaWbWBqwkms10BLAljCU/7e5Xu/vLZvYA8ApRN9Q33P1g2M83gc1AMbDW3V8Oh7gBuM/M/gZ4HrgrxO8C1ptZM9EZxLI8/LwiIpIFG20fzisrK72xsXG4qyEiMqKY2XZ3r+wZ1zeuRUQkkZKEiIgkUpIQEZFEShIiIpJISUJERBIpSYiISCIlCRERSaQkISIiiZQkREQkkZKEiIgkUpIQEZFEShIiIpJISUJERBIpSYiISCIlCRERSaQkISIiiZQkREQkkZKEiIgk6jdJmNlaM9tlZi/FYlPMbIuZ7QjPk0PczOw2M2s2sxfNbF5sm+Wh/A4zWx6LzzezprDNbRZump10DBERGTqZnEncAyzuEVsBbHX3OcDW8BrgPGBOeNQCd0D0hg+sBL4ALABWxt707whlO7db3M8xRERkiPSbJNz9X4A9PcJLgHVheR2wNBa/1yNPA5PM7DhgEbDF3fe4+15gC7A4rDvW3X/p7g7c22Nf6Y4hIiJDJNcxiU+6+7sA4fkTIT4DeCtWri3E+oq3pYn3dYxezKzWzBrNrLG9vT3HH0lERHrK98C1pYl5DvGsuHuDu1e6e+X06dOz3VxERBLkmiR+F7qKCM+7QrwNmBkrVwa800+8LE28r2OIiMgQyTVJbAQ6ZygtBx6OxS8Ns5xOAzpCV9FmYKGZTQ4D1guBzWHd783stDCr6dIe+0p3DBERGSLj+itgZj8GqoBpZtZGNEvpu8ADZnYl0Ar8SSj+GHA+0AzsAy4HcPc9ZvYd4NlQ7tvu3jkYfg3RDKojgU3hQR/HEBGRIWLRpKLRo7Ky0hsbG4e7GiIiI4qZbXf3yp5xfeNaREQSKUmIiEgiJQkREUmkJCEiIomUJEREJJGShIiIJFKSEBGRREoSIiKSSElCREQSKUmIiEgiJQkREUmkJCEiIomUJEREJJGShIiIJFKSEBGRREoSIiKSSElCREQSKUmIiEiiASUJM/sLM3vZzF4ysx+b2QQzm21m28xsh5ndb2bjQ9kjwuvmsL4itp8bQ/w1M1sUiy8OsWYzWzGQuoqISPZyThJmNgP4H0Clu58MFAPLgFuAW919DrAXuDJsciWw190/A9waymFmJ4btTgIWA2vMrNjMioHbgfOAE4GvhbIiIjJEBtrdNA440szGAaXAu8DZwINh/TpgaVheEl4T1lebmYX4fe7+kbu/CTQDC8Kj2d3fcPePgftCWRERGSI5Jwl3fxv4O6CVKDl0ANuB9939QCjWBswIyzOAt8K2B0L5qfF4j22S4r2YWa2ZNZpZY3t7e64/koiI9DCQ7qbJRJ/sZwPHA0cRdQ315J2bJKzLNt476N7g7pXuXjl9+vT+qi4iIhkaSHfTOcCb7t7u7vuBh4DTgUmh+wmgDHgnLLcBMwHC+onAnni8xzZJcRERGSIDSRKtwGlmVhrGFqqBV4AngYtCmeXAw2F5Y3hNWP+Eu3uILwuzn2YDc4BngGeBOWG21Hiiwe2NA6iviIhkaVz/RdJz921m9iDwHHAAeB5oAH4G3GdmfxNid4VN7gLWm1kz0RnEsrCfl83sAaIEcwD4hrsfBDCzbwKbiWZOrXX3l3Otr4iIZM+iD/OjR2VlpTc2Ng53NURERhQz2+7ulT3j+sa1iIgkUpIQEZFEShIiIpJISUJERBIpSYgMglRTiorVFRStKqJidQWpptRwV0kkJzlPgRWR9FJNKWofqWXf/n0AtHS0UPtILQA1c2uGs2oiWdOZhEie1W2t60oQnfbt30fd1rphqpFI7pQkRPKstaM1q7hIIVOSEMlR0rjDrImz0pZPiosUMiUJkRx0jju0dLTgeNe4Q6opRX11PaUlpd3Kl5aUUl9dP0y1FcmdkoRIDvoad6iZW0PDBQ2UTyzHMMonltNwQYMGrWVE0uwmkRz0N+5QM7dGSUFGBZ1JiORA4w4yVihJiORA4w4yVihJiORA4w4yVuh+EiIiovtJiIhI9pQkREQk0YCShJlNMrMHzezXZvaqmX3RzKaY2RYz2xGeJ4eyZma3mVmzmb1oZvNi+1keyu8ws+Wx+Hwzawrb3GZmNpD6iohIdgZ6JvH3wP9z9/8KfB54FVgBbHX3OcDW8BrgPGBOeNQCdwCY2RRgJfAFYAGwsjOxhDK1se0WD7C+IiKShZyThJkdC3wJuAvA3T929/eBJcC6UGwdsDQsLwHu9cjTwCQzOw5YBGxx9z3uvhfYAiwO64519196NLp+b2xfIiIyBAZyJnEC0A7cbWbPm9mdZnYU8El3fxcgPH8ilJ8BvBXbvi3E+oq3pYn3Yma1ZtZoZo3t7e0D+JFERCRuIEliHDAPuMPdTwU+4HDXUjrpxhM8h3jvoHuDu1e6e+X06dP7rrWIiGRsIEmiDWhz923h9YNESeN3oauI8LwrVn5mbPsy4J1+4mVp4iIiMkRyThLu/lvgLTP7bAhVA68AG4HOGUrLgYfD8kbg0jDL6TSgI3RHbQYWmtnkMGC9ENgc1v3ezE4Ls5ouje1LRESGwECvAvvnQMrMxgNvAJcTJZ4HzOxKoBX4k1D2MeB8oBnYF8ri7nvM7DvAs6Hct919T1i+BrgHOBLYFB4iIjJEdFkOERHRZTlERCR7ShIiIpJISUJERBIpSYiISCIlCRERSaQkISIiiZQkREQkkZKEiIgkUpKQgpVqSlGxuoKiVUVUrK4g1ZQa7iqJjDkDvSyHyKBINaWofaSWffv3AdDS0ULtI7UA1MytGc6qiYwpOpOQglS3ta4rQXTat38fdVvrhqlGImOTkoQUpNaO1qziIjI4lCSkIM2aOCuruIgMDiUJGRb9DUrXV9dTWlLaLVZaUkp9df1QVlNkzFOSkCHXOSjd0tGC412D0vFEUTO3hoYLGiifWI5hlE8sp+GCBg1aiwwx3U9ChlzF6gpaOlp6xcsnlrPz+p1DXyER0f0kpHBoUFpk5FCSkCGnQWmRkWPAScLMis3seTN7NLyebWbbzGyHmd0f7n+NmR0RXjeH9RWxfdwY4q+Z2aJYfHGINZvZioHWVQqDBqVFRo58nElcB7wae30LcKu7zwH2AleG+JXAXnf/DHBrKIeZnQgsA04CFgNrQuIpBm4HzgNOBL4WysoIp0FpkZFjQJflMLMy4CtAPfCXZmbA2cCfhiLrgJuBO4AlYRngQeAHofwS4D53/wh408yagQWhXLO7vxGOdV8o+8pA6iyFoWZujZKCyAgw0DOJ1cC3gEPh9VTgfXc/EF63ATPC8gzgLYCwviOU74r32CYpLiIiQyTnJGFmXwV2ufv2eDhNUe9nXbbxdHWpNbNGM2tsb2/vo9YiIpKNgZxJnAFcaGY7gfuIuplWA5PMrLMbqwx4Jyy3ATMBwvqJwJ54vMc2SfFe3L3B3SvdvXL69OkD+JFERCQu5yTh7je6e5m7VxANPD/h7jXAk8BFodhy4OGwvDG8Jqx/wqNv8m0EloXZT7OBOcAzwLPAnDBbanw4xsZc6ysiItkbjPtJ3ADcZ2Z/AzwP3BXidwHrw8D0HqI3fdz9ZTN7gGhA+gDwDXc/CGBm3wQ2A8XAWnd/eRDqKyIiCXRZDslIqilF3dY6WjtamTVxFvXV9ZqdJDKKJF2WQ3emk37pLnEiY5cuyyH90l3iRMYuJQnply7IJzJ2KUlILz1vCDTlyClpy+mCfCKjn8YkpEuqKcV1m65j93/u7oq1dLQwvng8JUUl7D+0vyuuC/KJjA06kxDg8OB0PEF0+vjgxxx7xLG6IJ/IGKQzCQHSD07H7fnPPbz3rfeGsEYiUgh0JiFA/4PQGn8QGZuUJAToOwlo/EFk7FKSECD93eIAph45VeMPImOYxiQEOPzNaV16Q0TidO0mERFJvHaTuptERCSRkoSIiCRSkhARkURKEiIikkhJQkREEilJjGA9r9aaakoNd5VEZJTR9yRGqGt/di0/bPwhTjSFWXeLE5HBkPOZhJnNNLMnzexVM3vZzK4L8SlmtsXMdoTnySFuZnabmTWb2YtmNi+2r+Wh/A4zWx6LzzezprDNbWZmA/lhR4tUU6pbguiku8WJSL4NpLvpAPBX7v454DTgG2Z2IrAC2Oruc4Ct4TXAecCc8KgF7oAoqQArgS8AC4CVnYkllKmNbbd4APUd0VJNKaZ9bxq2yrj4oYt7JYhOuluciORTzknC3d919+fC8u+BV4EZwBJgXSi2DlgalpcA93rkaWCSmR0HLAK2uPsed98LbAEWh3XHuvsvPfpa+L2xfY0p1/7sWi5+6OK093roSVdrFZF8ysvAtZlVAKcC24BPuvu7ECUS4BOh2AzgrdhmbSHWV7wtTTzd8WvNrNHMGtvb2wf64xSUVFOKOxrvyKisYbpaq4jk1YCThJkdDfwUuN7d/72vomlinkO8d9C9wd0r3b1y+vTp/VV5ROjsXrr4oYszKm8YV1derUFrEcmrASUJMyshShApd38ohH8XuooIz7tCvA2YGdu8DHinn3hZmviIkUpBRQUUFUXPqQxnqKaaUlz20BUZdS/hwPvlTHlqPWe8v2YAtT1cXzMYNy56zqbeIjL6DGR2kwF3Aa+6+/+OrdoIdM5QWg48HItfGmY5nQZ0hO6ozcBCM5scBqwXApvDut+b2WnhWJfG9lXwUimorYWWFnCPni+5JPmNN55QLr2njgN83P9BHHhoA6zeye6naqitzf0NPV5fgIMHo+eWlih+7bW5JbyByDXJikgeuXtOD+BMorepF4Ffhcf5wFSiWU07wvOUUN6A24HXgSagMravK4Dm8Lg8Fq8EXgrb/IBwafO+HvPnz/dCUF7uHqWH9I/SUvcNG6KyGza4l8zf4Fxf7qw0ZyXOzf08VuJcXN1rv+Xlg1Nfs+T6D4YNG6JjDOUxs7VhQ9RuZtFzIdVNRr98//0BjZ7mPVX3kxgkRUXRW1tfysth506YdlaK3afXwvh9/e/YAS+CZ78Om3p3L5nBoUODU9+eOus/GCoqDp/VDNUxs9F55rUv9isrLYWGBqipidbX1UFrK8yaBfX1UVwkH/r7+8tF0v0klCQGSdKbXJe5KVh8HRy1O3mYvqcDJfDw3dCU/FeQ65tov/VNI9eElImkpDWYx8xGX0msvj7//8AicYPxIUo3HRoEffWZ19dHbwxpzU3B0uVRgoDkBOGxxwdT+00QpaXRcXPRV32Tvuc+axC/kpG078E8ZjZaE76z2NoanUHs63FSuG9fFBfJh77+/vJNSSJH6Qam4wPHNTXRJ8fy8uh1tzfaxddB8cH+D9JRDqs8enz/vbQJorg42nd5+cA+qfasb3Fx9FxeDldf3TuBDCQhZSJd0hrsY2ajryQ2lP/AMjYN5YcoJYkcZfJpsaYmOvVzh6tvT1H81xWwsghK+5/aOt5Kuea/9P2OWFoK69ZF3S87dw68KyNe3wMHouedO2HNmsMJJB8JCfqfuRRPWvk6Zj71lcQK/SxIRr6h/BClMYkc9TfQO3Vq9Lx7wbXw334IRZm3c/nEcuqr66mZW9PnWME110Rv4CPNYAy6DYekwenR8vNJYcv35AgNXA9A5y8j24Fdrp8BE9/JbFAawGHDf9/Q7VvTqRRcnPCl66lT4b33sqxTASj0mUv5oNlNMtIoSWQplYLrroPdGXzpOa0cEgTPXMOGmjW93kz6ukD6SPz1FfrMJZGxSLObspBKwRVX5JggLj4HVlpmCeJgMbjB++XRN6c3rcl6BsxI/Day+uxFRg7dmS6Nujr4OIOrYvSSzdnDx6XwSEOvGUvpZsBMnZqcsOIzq2BkdGkkfY+gUGYuichhOpNII6upinNT8L9KMj97ADhkaRMEpP80/fd/DyUlfe9yJM3DL/SZSyJymJJEGhl3e1x8DvzxxTDuQJQcMkoQcNTP13PNmTW9xhqSPk3X1MDddx9+U00ykubhd063zdf0XREZHEoSaZx/fgaFrp8Bn96a3cD0QYN/3MCEHTWsWQPr12f+aTr+ptr5hbee1KcvIvmmMYkeUim4884+Clx9EnzylWg5mwTxejVseByAzuGFmprcPkGrT19EhoqSRA9f/zrs35+w8ibLvFsJouRwyOCf1ncbf+i85EWuOhOL5uGLyGBTkohJpeCDD9KsmJuCP7o48wTR+R2AZ65Jeznvgxlctqk/uZ6FiIhkQ0kiptfsoGyTA0QJ4qMj4bvJ94ZIGlMQESk0ShIx3WYHdY49ZJocIEoQHcfD6rcTi4wfr7EDERk5NLsppmt20P+cnF2CcOAQUffS6reprk5/b4ajjoK1a9VNJCIjR8EnCTNbbGavmVmzma0YzGPV1wPXnASl72c/c+nbztG/WMOGDfD4472/LLZhA/zHfyhBiMjIUtDdTWZWDNwOnAu0Ac+a2UZ3f2VQDvgHKWjOcNfhlqPVJ1Tz+M2Pw/ruqzWwLCKjQUEnCWAB0OzubwCY2X3AEmBQkkTd1syva1FSVMLHN+VygScRkZGj0LubZgBvxV63hVg3ZlZrZo1m1tje3p7zwVo7MruuxfFHH68EISJjQqEniXQjA73uRODuDe5e6e6V06dPz/lgsyb2fV2LEivBVzpv/1Xy7CURkdGk0JNEGzAz9roMeGewDlZfXU9pSe9pSeNsHBv+eIPOHkRkzCn0MYlngTlmNht4G1gG/OlgHazztqF1W+to7Whl1sRZXfeaFhEZiwo6Sbj7ATP7JrAZKAbWuvvLg3nMmrk1SgoiIkFBJwkAd38MeGy46yEiMhYV+piEiIgMIyUJERFJpCQhIiKJlCRERCSRuff6btqIZmbtQEsedjUNeC8P+xlMqmN+qI75MxLqqTqmV+7uvb6NPOqSRL6YWaO7Vw53PfqiOuaH6pg/I6GeqmN21N0kIiKJlCRERCSRkkSyhuGuQAZUx/xQHfNnJNRTdcyCxiRERCSRziRERCSRkoSIiCRSkujBzBab2Wtm1mxmK4bgeDPN7Ekze9XMXjaz60J8ipltMbMd4XlyiJuZ3Rbq96KZzYvta3kov8PMlsfi882sKWxzm5mlu5lTJnUtNrPnzezR8Hq2mW0Lx7vfzMaH+BHhdXNYXxHbx40h/pqZLYrFB9zuZjbJzB40s1+H9vxiobWjmf1F+D2/ZGY/NrMJhdCOZrbWzHaZ2Uux2KC3XdIxsqjj98Pv+0Uz+0czm5RrG+Xye8ikjrF1f21mbmbThrMds+bueoQH0eXIXwdOAMYDLwAnDvIxjwPmheVjgN8AJwLfA1aE+ArglrB8PrCJ6K59pwHbQnwK8EZ4nhyWJ4d1zwBfDNtsAs7Lsa5/Cfxf4NHw+gFgWVj+IXBNWL4W+GFYXgbcH5ZPDG16BDA7tHVxvtodWAdcFZbHA5MKqR2Jbr37JnBkrEFpGvwAAAPXSURBVP0uK4R2BL4EzANeisUGve2SjpFFHRcC48LyLbE6Zt1G2f4eMq1jiM8kuuVBCzBtONsx6/+rfO1oNDxC42+Ovb4RuHGI6/AwcC7wGnBciB0HvBaWfwR8LVb+tbD+a8CPYvEfhdhxwK9j8W7lsqhXGbAVOBt4NPyRvhf7B+1qu/DP8MWwPC6Us57t2VkuH+0OHEv0Bmw94gXTjhy+Z/uU0C6PAosKpR2BCrq/AQ962yUdI9M69lj3R0Aq3c/eXxvl8vecTR2BB4HPAzs5nCSGrR2zeai7qbvOf+JObSE2JMJp7KnANuCT7v4uQHj+RD917CveliaerdXAt4BD4fVU4H13P5Bmv111Ces7Qvls656NE4B24G6LusTuNLOjKKB2dPe3gb8DWoF3idplO4XVjnFD0XZJx8jFFUSfrnOpYy5/zxkxswuBt939hR6rCrUdu1GS6C5dH/OQzBE2s6OBnwLXu/u/91U0TcxziGdTt68Cu9x9ewb1GJY6En3Cmwfc4e6nAh8QnXYnGY52nAwsIer+OB44Cjivj/0ORztmouDqZWZ1wAEg1RnKsi65/D1nUq9SoA64Kd3qPNZx0ChJdNdG1HfYqQx4Z7APamYlRAki5e4PhfDvzOy4sP44YFc/dewrXpYmno0zgAvNbCdwH1GX02pgkpl13t0wvt+uuoT1E4E9OdQ9G21Am7tvC68fJEoahdSO5wBvunu7u+8HHgJOp7DaMW4o2i7pGBkLA7tfBWo89LfkUMf3yP73kIlPE30oeCH8/5QBz5nZp3Ko46C2Y6J89VuNhgfRp9E3wi+1c1DrpEE+pgH3Aqt7xL9P94Go74Xlr9B9sOuZEJ9C1Cc/OTzeBKaEdc+Gsp2DXecPoL5VHB64/gndB/quDcvfoPtA3wNh+SS6Dya+QTSQmJd2B34BfDYs3xzasGDaEfgC8DJQGvaxDvjzQmlHeo9JDHrbJR0jizouBl4Bpvcol3UbZft7yLSOPdbt5PCYxLC1Y1Z/t/na0Wh5EM04+A3RDIi6ITjemUSnjC8CvwqP84n6PLcCO8Jz5x+JAbeH+jUBlbF9XQE0h8flsXgl8FLY5gf0MeiWQX2rOJwkTiCabdEc/sGOCPEJ4XVzWH9CbPu6UI/XiM0Oyke7A6cAjaEt/yn8gxVUOwKrgF+H/awnehMb9nYEfkw0TrKf6BPrlUPRdknHyKKOzUT9953/Oz/MtY1y+T1kUsce63dyOEkMSztm+9BlOUREJJHGJEREJJGShIiIJFKSEBGRREoSIiKSSElCREQSKUmIiEgiJQkREUn0/wF4M7THnpv9oQAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "# Use automl metrics module\n", - "scores = metrics.compute_metrics_regression(\n", - " pred_automl[\"predicted\"],\n", - " pred_automl[target_column_name],\n", - " list(constants.Metric.SCALAR_REGRESSION_SET),\n", - " None,\n", - " None,\n", - " None,\n", - ")\n", - "\n", - "print(\"[Test data scores]\\n\")\n", - "for key, value in scores.items():\n", - " print(\"{}: {:.3f}\".format(key, value))\n", - "\n", - "# Plot outputs\n", - "test_pred = plt.scatter(pred_automl[target_column_name], pred_automl[\"predicted\"], color=\"b\")\n", - "test_test = plt.scatter(pred_automl[target_column_name], pred_automl[target_column_name], color=\"g\")\n", - "plt.legend((test_pred, test_test), (\"prediction\", \"truth\"), loc=\"upper left\", fontsize=8)\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We also compute MAPE of the forecasts in the last two weeks of the forecast period in order to be consistent with the evaluation period that is used in other quick start examples." - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "MAPE of forecasts obtained by AutoML in the last two weeks: 124.10334037331717\n" - ] - } - ], - "source": [ - "pred_automl_sub = pred_automl.loc[pred_automl.week >= max(test_df.week) - NUM_TEST_PERIODS + GAP]\n", - "mape_automl_sub = MAPE(pred_automl_sub[\"predicted\"], pred_automl_sub[\"move\"]) * 100\n", - "print(\"MAPE of forecasts obtained by AutoML in the last two weeks: \" + str(mape_automl_sub))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Combine AutoML Model with a Custom Model\n", - "\n", - "So far we have demonstrated how we can quickly build a forecasting model with AutoML in Azure. Next, we further show a simple way to achieve more robust and accurate forecasts by combining the forecasts from AutoML and a custom model that the user may have. Here we assume that the user have also constructed a series of linear regression models with each model forecasts the sales of a specfic store-brand using `scikit-learn` package." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Multiple linear regression models" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [], - "source": [ - "# Create price features\n", - "df_sub[\"price\"] = df_sub.apply(lambda x: x.loc[\"price\" + str(int(x.loc[\"brand\"]))], axis=1)\n", - "price_cols = [\n", - " \"price1\",\n", - " \"price2\",\n", - " \"price3\",\n", - " \"price4\",\n", - " \"price5\",\n", - " \"price6\",\n", - " \"price7\",\n", - " \"price8\",\n", - " \"price9\",\n", - " \"price10\",\n", - " \"price11\",\n", - "]\n", - "df_sub[\"avg_price\"] = df_sub[price_cols].sum(axis=1).apply(lambda x: x / len(price_cols))\n", - "df_sub[\"price_ratio\"] = df_sub.apply(lambda x: x[\"price\"] / x[\"avg_price\"], axis=1)\n", - "\n", - "# Create lag features on unit sales\n", - "df_sub[\"move_lag1\"] = df_sub[\"move\"].shift(1)\n", - "df_sub[\"move_lag2\"] = df_sub[\"move\"].shift(2)\n", - "\n", - "# Drop rows with NaN values\n", - "df_sub.dropna(inplace=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "After splitting the data, we use `fit()` and `predit()` functions from `fclib.models.multiple_linear_regression` to train separate linear regression model for each invididual time series and generate forecasts for the sales during the test period." - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
week_startpredictionstorebrandweeklogmoveconstantprice1price2price3...price11dealfeatprofitmovepriceavg_priceprice_ratiomove_lag1move_lag2
01992-04-1612507211368.5910.050.050.05...0.0300.0033.5453760.050.041.2712416.0028096.00
11992-04-2317664211379.1910.040.050.05...0.0300.0020.4397920.040.041.115376.0012416.00
21992-04-3021670211389.7410.040.040.05...0.0311.0011.29169600.040.040.949792.005376.00
31992-04-169551221369.1410.050.050.05...0.0310.0027.1393120.050.041.2111424.004992.00
41992-04-237452221378.7410.040.050.05...0.0300.0033.3062400.050.041.399312.0011424.00
\n", - "

5 rows × 27 columns

\n", - "
" - ], - "text/plain": [ - " week_start prediction store brand week logmove constant price1 \\\n", - "0 1992-04-16 12507 2 1 136 8.59 1 0.05 \n", - "1 1992-04-23 17664 2 1 137 9.19 1 0.04 \n", - "2 1992-04-30 21670 2 1 138 9.74 1 0.04 \n", - "3 1992-04-16 9551 2 2 136 9.14 1 0.05 \n", - "4 1992-04-23 7452 2 2 137 8.74 1 0.04 \n", - "\n", - " price2 price3 ... price11 deal feat profit move price \\\n", - "0 0.05 0.05 ... 0.03 0 0.00 33.54 5376 0.05 \n", - "1 0.05 0.05 ... 0.03 0 0.00 20.43 9792 0.04 \n", - "2 0.04 0.05 ... 0.03 1 1.00 11.29 16960 0.04 \n", - "3 0.05 0.05 ... 0.03 1 0.00 27.13 9312 0.05 \n", - "4 0.05 0.05 ... 0.03 0 0.00 33.30 6240 0.05 \n", - "\n", - " avg_price price_ratio move_lag1 move_lag2 \n", - "0 0.04 1.27 12416.00 28096.00 \n", - "1 0.04 1.11 5376.00 12416.00 \n", - "2 0.04 0.94 9792.00 5376.00 \n", - "3 0.04 1.21 11424.00 4992.00 \n", - "4 0.04 1.39 9312.00 11424.00 \n", - "\n", - "[5 rows x 27 columns]" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Split data into training and test sets\n", - "train_df, test_df = split_last_n_by_grain(df_sub, NUM_TEST_PERIODS)\n", - "train_df.reset_index(drop=True)\n", - "test_df.reset_index(drop=True)\n", - "\n", - "# Train multiple linear regression models\n", - "fea_column_names = [\"move_lag1\", \"move_lag2\", \"price\", \"price_ratio\"]\n", - "lr_models = fit(train_df, grain_column_names, fea_column_names, target_column_name)\n", - "\n", - "# Generate forecasts with the trained models\n", - "pred_all = predict(test_df, lr_models, time_column_name, grain_column_names, fea_column_names)\n", - "\n", - "pred_lr = pd.merge(pred_all, test_df, on=index_column_names)\n", - "pred_lr.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's check the accuracy of the predictions on the entire forecast period as well as in the last two weeks of the forecast period.\n", - "\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "MAPE of forecasts obtained by multiple linear regression on entire test period: 83.90865445283927\n" - ] - } - ], - "source": [ - "mape_lr_entire = MAPE(pred_lr[\"prediction\"], pred_lr[\"move\"]) * 100\n", - "print(\"MAPE of forecasts obtained by multiple linear regression on entire test period: \" + str(mape_lr_entire))" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "MAPE of forecasts obtained by multiple linear regression in the last two weeks: 72.11741385279376\n" - ] - } - ], - "source": [ - "pred_lr_sub = pred_lr.loc[pred_lr.week >= max(test_df.week) - NUM_TEST_PERIODS + GAP]\n", - "mape_lr_sub = MAPE(pred_lr_sub[\"prediction\"], pred_lr_sub[\"move\"]) * 100\n", - "print(\"MAPE of forecasts obtained by multiple linear regression in the last two weeks: \" + str(mape_lr_sub))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Combine forecasts from different methods\n", - "\n", - "We can combine the forecasts obtained by AutoML and multiple linear regression using weighted average and evaluate the final forecasts. Usually the combined forecasts will be more robust as a combination of two methods can reduce the chance of model overfitting. Here we use equal weights which can be further adjusted according to our confidence on each model." - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [], - "source": [ - "pred_final = pd.merge(\n", - " pred_automl[index_column_names + [\"predicted\", \"move\", \"week\"]],\n", - " pred_lr[index_column_names + [\"prediction\"]],\n", - " on=index_column_names,\n", - " how=\"left\",\n", - ")\n", - "pred_final[\"combined_prediction\"] = pred_final[\"predicted\"] * 0.5 + pred_final[\"prediction\"] * 0.5" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "MAPE of forecasts obtained by the combined model on entire test period: 87.2964359857758\n" - ] - } - ], - "source": [ - "mape_entire = MAPE(pred_final[\"combined_prediction\"], pred_final[\"move\"]) * 100\n", - "print(\"MAPE of forecasts obtained by the combined model on entire test period: \" + str(mape_entire))" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "MAPE of forecasts obtained by the combined model in the last two weeks: 84.39534839261313\n" - ] - } - ], - "source": [ - "pred_final_sub = pred_final.loc[pred_final.week >= max(test_df.week) - NUM_TEST_PERIODS + GAP]\n", - "mape_final_sub = MAPE(pred_final_sub[\"combined_prediction\"], pred_final_sub[\"move\"]) * 100\n", - "print(\"MAPE of forecasts obtained by the combined model in the last two weeks: \" + str(mape_final_sub))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Additional Reading\n", - "\n", - "\\[1\\] Nicolo Fusi, Rishit Sheth, and Melih Elibol. 2018. Probabilistic Matrix Factorization for Automated Machine Learning. In Advances in Neural Information Processing Systems. 3348-3357.
\n", - "\\[2\\] Azure AutoML Package Docs: https://docs.microsoft.com/en-us/python/api/azureml-train-automl/azureml.train.automl?view=azure-ml-py
\n", - "\\[3\\] Azure Automated Machine Learning Examples: https://github.com/Azure/MachineLearningNotebooks/tree/master/how-to-use-azureml/automated-machine-learning
\n", - "\n", - "\n" - ] - } - ], - "metadata": { - "author_info": { - "affiliation": "Microsoft", - "created_by": "Chenhui Hu" - }, - "kernelspec": { - "display_name": "forecasting_env", - "language": "python", - "name": "forecasting_env" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.10" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/grocery_sales/python/02_model/autoarima_multi_round.ipynb b/examples/grocery_sales/python/02_model/autoarima_multi_round.ipynb deleted file mode 100644 index d57f850b..00000000 --- a/examples/grocery_sales/python/02_model/autoarima_multi_round.ipynb +++ /dev/null @@ -1,588 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Copyright (c) Microsoft Corporation.\n", - "\n", - "Licensed under the MIT License." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# ARIMA: Autoregressive Integrated Moving Average\n", - "\n", - "This notebook provides an example of how to train an ARIMA model to generate point forecasts of product sales in retail. We will train an ARIMA based model on the Orange Juice dataset.\n", - "\n", - "An ARIMA, which stands for AutoRegressive Integrated Moving Average, model can be created using an `ARIMA(p,d,q)` model within `statsmodels` library. In this notebook, we will be using an alternative library `pmdarima`, which allows us to automatically search for optimal ARIMA parameters, within a specified range. More specifically, we will be using `auto_arima` function within `pmdarima` to automatically discover the optimal parameters for an ARIMA model. This function wraps `ARIMA` and `SARIMAX` models of `statsmodels` library, that correspond to non-seasonal and seasonal model space, respectively.\n", - "\n", - "In an ARIMA model there are 3 parameters that are used to help model the major aspects of a times series: seasonality, trend, and noise. These parameters are:\n", - "- **p** is the parameter associated with the auto-regressive aspect of the model, which incorporates past values.\n", - "- **d** is the parameter associated with the integrated part of the model, which effects the amount of differencing to apply to a time series.\n", - "- **q** is the parameter associated with the moving average part of the model.,\n", - "\n", - "If our data has a seasonal component, we use a seasonal ARIMA model or `ARIMA(p,d,q)(P,D,Q)m`. In that case, we have an additional set of parameters: `P`, `D`, and `Q` which describe the autoregressive, differencing, and moving average terms for the seasonal part of the ARIMA model, and `m` refers to the number of periods in each season.\n", - "\n", - "We provide a [quick-start ARIMA example](../00_quick_start/auto_arima_forecasting.ipynb), in which we explain the process of using ARIMA model to forecast a single time series, and analyze the model performance. Please take a look at this notebook for more information.\n", - "\n", - "In this notebook, we will train an ARIMA model on multiple splits (round) of the train/test data." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Global Settings and Imports" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "System version: 3.6.10 |Anaconda, Inc.| (default, Jan 7 2020, 21:14:29) \n", - "[GCC 7.3.0]\n" - ] - } - ], - "source": [ - "import os\n", - "import sys\n", - "import math\n", - "import warnings\n", - "import itertools\n", - "import numpy as np\n", - "import pandas as pd\n", - "import scrapbook as sb\n", - "\n", - "from tqdm import tqdm\n", - "from pmdarima.arima import auto_arima\n", - "\n", - "from fclib.common.utils import git_repo_path\n", - "from fclib.common.plot import plot_predictions_with_history\n", - "from fclib.evaluation.evaluation_utils import MAPE\n", - "from fclib.dataset.ojdata import download_ojdata, split_train_test, complete_and_fill_df\n", - "\n", - "pd.options.display.float_format = \"{:,.2f}\".format\n", - "np.set_printoptions(precision=2)\n", - "warnings.filterwarnings(\"ignore\")\n", - "\n", - "print(\"System version: {}\".format(sys.version))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Parameters\n", - "\n", - "Next, we define global settings related to the model. We will use historical weekly sales data only, without any covariate features to train the ARIMA model. The model parameter ranges are provided in params. These are later used by the `auto_arima()` function to search the space for the optimal set of parameters. To increase the space of models to search over, increase the `max_p` and `max_q` parameters.\n", - "\n", - "> NOTE: Our data does not show a strong seasonal component (as demonstrated in data exploration example notebook), so we will not be searching over the seasonal ARIMA models. To learn more about the seasonal ARIMA models, please take a look at the quick start ARIMA notebook, referenced above in the introduction." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# Use False if you've already downloaded and split the data\n", - "DOWNLOAD_SPLIT_DATA = True\n", - "\n", - "# Data directory\n", - "DATA_DIR = os.path.join(git_repo_path(), \"ojdata\")\n", - "\n", - "# Forecasting settings\n", - "N_SPLITS = 5\n", - "HORIZON = 2\n", - "GAP = 2\n", - "FIRST_WEEK = 40\n", - "LAST_WEEK = 156\n", - "\n", - "# Parameters of ARIMA model\n", - "params = {\n", - " \"seasonal\": False,\n", - " \"start_p\": 0,\n", - " \"start_q\": 0,\n", - " \"max_p\": 5,\n", - " \"max_q\": 5,\n", - "}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Data Preparation\n", - "\n", - "We need to download the Orange Juice data and split it into training and test sets. By default, the following cell will download and spit the data. If you've already done so, you may skip this part by switching `DOWNLOAD_SPLIT_DATA` to `False`.\n", - "\n", - "We store the training data and test data using dataframes. The training data includes `train_df` and `aux_df` with `train_df` containing the historical sales up to week 135 (the time we make forecasts) and `aux_df` containing price/promotion information up until week 138. Here we assume that future price and promotion information up to a certain number of weeks ahead is predetermined and known. In our example, we will be using historical sales only, and will not be using the `aux_df` data. The test data is stored in `test_df` which contains the sales of each product in week 137 and 138. Assuming the current week is week 135, our goal is to forecast the sales in week 137 and 138 using the training data. There is a one-week gap between the current week and the first target week of forecasting as we want to leave time for planning inventory in practice.\n", - "\n", - "The setting of the forecast problem are defined in `fclib.dataset.ojdata.split_train_test` function. We can change this setting (e.g., modify the horizon of the forecast or the range of the historical data) by passing different parameters to this functions. Below, we split the data into `n_splits=N_SPLITS` splits, using the forecasting settings listed above in the **Parameters** section." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Data already exists at the specified location.\n", - "Finished data downloading and splitting.\n" - ] - } - ], - "source": [ - "if DOWNLOAD_SPLIT_DATA:\n", - " download_ojdata(DATA_DIR)\n", - " train_df_list, test_df_list, _ = split_train_test(\n", - " DATA_DIR,\n", - " n_splits=N_SPLITS,\n", - " horizon=HORIZON,\n", - " gap=GAP,\n", - " first_week=FIRST_WEEK,\n", - " last_week=LAST_WEEK,\n", - " write_csv=True,\n", - " )\n", - "\n", - " print(\"Finished data downloading and splitting.\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To create training data and test data for multi-round forecasting, we pass a number greater than `1` to `n_splits` parameter in `split_train_test()` function. Note that the forecasting periods we generate in each test round are **non-overlapping**. This allows us to evaluate the forecasting model on multiple rounds of data, and get a more robust estimate of our model's performance.\n", - "\n", - "For visual demonstration, this is what the time series splits would look like for `N_SPLITS = 5`, and using other settings as above:\n", - "\n", - "![Multi split](../../../../assets/time_series_split_multiround.jpg)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Process training data" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Our time series data is not complete, since we have missing sales for some stores/products and weeks. We will fill in those missing values by propagating the last valid observation forward to next available value. We will define functions for data frame processing, then use these functions within a loop that loops over each forecasting rounds.\n", - "\n", - "Note that our time series are grouped by `store` and `brand`, while `week` represents a time step, and `logmove` represents the value to predict." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "def process_training_df(train_df):\n", - " \"\"\"Process training data frame.\"\"\"\n", - " train_df = train_df[[\"store\", \"brand\", \"week\", \"logmove\"]]\n", - " store_list = train_df[\"store\"].unique()\n", - " brand_list = train_df[\"brand\"].unique()\n", - " train_week_list = range(FIRST_WEEK, max(train_df.week))\n", - "\n", - " train_filled = complete_and_fill_df(train_df, stores=store_list, brands=brand_list, weeks=train_week_list)\n", - "\n", - " return train_filled" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Process test data" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's now process the test data. Note that, in addition to filling out missing values, we also convert unit sales from logarithmic scale to the counts. We will do model training on the log scale, due to improved performance, however, we will transfrom the test data back into the unit scale (counts) by applying `math.exp()`, so that we can evaluate the performance on the unit scale.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "def process_test_df(test_df):\n", - " \"\"\"Process test data frame.\"\"\"\n", - " test_df[\"actuals\"] = test_df.logmove.apply(lambda x: round(math.exp(x)))\n", - " test_df = test_df[[\"store\", \"brand\", \"week\", \"actuals\"]]\n", - " store_list = test_df[\"store\"].unique()\n", - " brand_list = test_df[\"brand\"].unique()\n", - "\n", - " test_week_list = range(min(test_df.week), max(test_df.week) + 1)\n", - " test_filled = complete_and_fill_df(test_df, stores=store_list, brands=brand_list, weeks=test_week_list)\n", - "\n", - " return test_filled" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Model training" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now let's run model training across all the stores and brands, and across all rounds. We will re-run the same code to automatically search for the best parameters, simply wrapped in a for loop iterating over stores and brands.\n", - "\n", - "> **NOTE**: Since we are building a model for each time series sequentially (900+ time series for each store and brand), it would take about 1 hour to run the following cell over all stores. To speed up the execution, we model only a subset of ten stores in each round (exacution time ~8 minutes). To change this behavior, and run ARIMA modeling over *all stores and brands*, switch the boolean indicator `subset_stores` to `False`." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - " 0%| | 0/5 [00:00 Note: Since `auto_arima` model makes consecutive forecasts from the last time point, we want to forecast the next `n_periods = GAP + HORIZON - 1` points, so that we can account for the GAP, as described in the data setup." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Model evaluation" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To evaluate the model, we will use *mean absolute percentage error* or [MAPE](https://en.wikipedia.org/wiki/Mean_absolute_percentage_error)." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "MAPE values for each forecasting round:\n", - "round\n", - "1 57.72\n", - "2 77.08\n", - "3 63.12\n", - "4 74.93\n", - "5 73.70\n", - "dtype: float64\n" - ] - }, - { - "data": { - "application/scrapbook.scrap.json+json": { - "data": 69.22142436904007, - "encoder": "json", - "name": "MAPE", - "version": 1 - } - }, - "metadata": { - "scrapbook": { - "data": true, - "display": false, - "name": "MAPE" - } - }, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Overall MAPE is 69.22 %\n" - ] - } - ], - "source": [ - "mape_r = result_df.groupby(\"round\").apply(lambda x: MAPE(x.predictions, x.actuals) * 100)\n", - "\n", - "print(\"MAPE values for each forecasting round:\")\n", - "print(mape_r)\n", - "\n", - "metric_value = MAPE(result_df.predictions, result_df.actuals) * 100\n", - "sb.glue(\"MAPE\", metric_value)\n", - "\n", - "print(f\"Overall MAPE is {metric_value:.2f} %\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The resulting MAPE value is relatively high. As `auto_arima` searches a restricted space of the models, defined by the range of `p` and `q` parameters, we often might not find an optimal model for each time series. In addition, when building a model for a large number of time series, it is often difficult to examine each model individually, which would usually help us improve an ARIMA model. Please refer to the [quick start ARIMA notebook](../00_quick_start/auto_arima_forecasting.ipynb) for a more comprehensive evaluation of a single ARIMA model.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now let's plot a few examples of forecasted results." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABDAAAARRCAYAAADQPHLpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdeZxcVZn/8c/TS9JJdychSXcCCZLQiQmJEMDABEEEA0lcyTAsMi4wKLiOjo4oODKg6KijP1QclckIAyqKimyOSFhCQHaDQSUk0WxAd/akO+mkl/Ty/P44t5JKpXpLV9Wt7vq+X69+Vde527m3blXd+9Q5zzF3R0REREREREQknxXFXQERERERERERkZ4ogCEiIiIiIiIieU8BDBERERERERHJewpgiIiIiIiIiEjeUwBDRERERERERPKeAhgiIiIiIiIikvcUwBARyRIzm2Rmbma3pZTfFpVPytJ2z4rWf3021l8IzOz66BieFcO2R5jZTWa2wczao3qcmOt6ZJKZjTez282s1sw6on0aFXe9CkVXn0U5roOb2dK4tj+Q6FiJiHStJO4KiIj0h5l5SlEnUA/8GbjF3e/Ifa2yKwp8rAdud/fLYq1MgYnOt8fd/awsbuY/gQ8D/wf8BOgANmdxe7lwGzAP+DmwBnCgJc4KSWaZ2QYAd58Ub01ERGQwUwBDRAaLL0WPpcA0YCFwtpm90d0/E1+10roG+DpQl6X1Pw8cB2zP0volu94J/NXd3xV3RTLBzIYA5wKPuPt7466PxOY4oCnuSgwQOlYiIl1QAENEBgV3vz75uZnNBR4G/sXMbnL3DXHUKx133wRsyuL6m4BV2Vq/ZN1RwBNxVyKDxhO6rG6MuyISH3fXZ1Iv6ViJiHRNOTBEZFBy90cJN/EGnAIH9wM3s9eb2S/MbKuZdSbnOjCz0Wb2NTNbaWbNZrbLzB41s3nptmVmlWZ2Y9S/v8XMVpnZZ+jiM7a7HBhmdmpUrzozazWzTWb2kJldFE2/ntB9BODSaD2Jv8uiebrMgWFmU83sx9H695nZxuj51DTz7s8DYWYXmNnzZtZkZjvN7E4zm9DV8U+zrssSdTSzBWa2NDqunjLf9Oj4vBbt/xYz+5mZTUuzznFm9i0zW21me82sIfr/NjM7Nt22u6hbj/3NE+uInr4l5bhfnzTfu6NzZVNU/41m9riZfawXx2hptA1L2cbSpHmKzOwjZvYHM9sT7fcfzOyjZnbI+ZZY3kIOih9Fr3tHV8ciabkhZvYJM3vAzF6J9mWnmT1iZm/raV+S1rMBeCV6mny+3pYy3yVm9piZ1UfvoZVm9kUzG5oy30Yzq02znVei9V6bUv72qPzLvairmdmlZva0mW2L6vGamS02s4tT5j3bzBaZ2ctmttvC58RLZnadmZWlWXfye+kSM3shei9ttPDZMTSa763R67U7OhY/MbMx6Y5r9DfSzP4rel1bovp80sysp/1NWtdwM7vGzF6Mzqc9ZvaMmV3Sy+XPis7bY4BjUt4btyXNd8j7LNPHJZp3YnRM1kXn7Q4zu9/MTuntMYnW0+v3svXhO8N68VmY7lhF5SVm9jEzezY6Fk1mttzCezXd+/+wP49ERPKVWmCIyGCWuIhPzZNRAzwH/BW4AxgG7AYws2OApcAk4PfAg0A5oVn/g2b2YXf/n/0bCBfYjxKCJH+K1jcKuBZ4S58qa3YF8ENCzoP7gb8B1cBs4GPAL6O6jQI+FW3v3qRVvNjD+k8BHgEqo/W/DEwH3gucZ2Zz3X1ZmkU/Brw7WuZx4O+Ai4FZZnaiu7f2YTcvABYAvwNuJhznRP0WAHcTugH9hpArYSJwPvAOMzvb3f8YzTsceIrwWj4czW+Em6jzgLuAdX2oV09eJHRTuo5wQ35b0rSlUZ2uBP6bkK/iN4QuPNXACcA/AT/oYRu3RetK3caGpHl+Avwj8BrwI8K5/ffRus8gvJapRgPPAnsIx7cT2NJDXUYD3wWeJhzfbcCRwLuAB8zsCnf/UQ/rAPgO4TVOPV/3n6tmdgtwOVAb1a8BmAPcAMw1s3PdvT2afQnwXjObnviV2symAK+Lps+Nlkt4a/T4aC/q+lVC9671hPfarmifTwEuBH6RNO/nCe+dp4HfAmXA6cD1wFlmdo67d6TZxj8Db4uOw1JCXpBPA6PN7D7gzmh9i4A3Ae8DxkbLpBpCeD+PipYbAvwD4XWbBny8px22kEh1CXAS8EfgVkLgdT7wMzOb6e5f7GE1GwjvjX+Jnn8naVq3n0lJMnJczOxk4CHC+buYcD6NJXQpfNLM/t7dH+ipMn15L/f1OyNJl5+FXdQp8bk4H1gN/IyQR+Zs4HuEz+X3H84+iIgMKO6uP/3pT38D9o9wA+dpys8h3Kh1AsdEZZMS8wP/0cX6lkbLvCelfBThYrwZGJdU/oVofb8GipLKJwM7o2m3pazrtqh8UlLZDKAtWmZmmnpNTPp/Urr1Jk0/K5p+fVKZASuj8vemzH9xVL4qZR+uj8p3A8enLPOzaNpFvXydLovm7wQWpJl+BCH56nZgRsq0mYSb7z8mlb0rWt+306xrCFCZZtuXdXMOLU0pS+z7WT3NmzTtBaAVqE4zbWwfz+lDtgFcEk37I1CRVF4OLIum/WO69wfwY6CkD3UYmnzOJZWPBF6KztNhvVxXl+dr0mtzd+r6kl6DTyWVXR6VfTyp7MNR2UPR8R+eNG05IZfAkF7UcwchiDI8zbSxKc+PBSzNfDdEdbm4i33ZBRyXcpxXEIKWO4C3JE0rIgSPHDgxZX0bovIngaFJ5aOBtdG0M3t6DTjwWfS5lPIywo14Z+q2uzl+G4ANfTmvM3lcCD/KrSHc1L8lZTtHEXIObUo+Xt3UtdfvZfr+nXEZ3XwW9uJYfQ8oTiovBm6Jpp13OPugP/3pT38D6U9dSERkUIiaIl9vZl81s7sIF98GfMfdX0mZfQsHkn4mr2MWodXEr939zuRp7t5A+GW8jPArZ8I/ES5EP+funUnzrwdu6sMufJRwAX6Du69InejuhzSb76M3EX4xfsZTRmZx918QboSmEX7FT3WTu/8lpSzxi+KpfazHfe7+YJryDxAu+K9z95dT6rci2t5JZjYjZbnm1BW5+z53b+xjvTKlnRCIOoi7ZyKh6uXR49Xuvidp3XsJLQIAPpRmuX3AZ/1AK4YeuXtrunPO3XcRfqU/gqhrVj99inDMLnf31NfyBsLNa3KrkkRLirlJZXOBrYT32xCiczjqYjALeNLd9/WyPm2Em+aDpL5+7r7O3VNbdsGB1gfzu1j/Te6+Mmk9rYSWHUXAb9398aRpncBPo6ezuljfNZ7UAsrdd3KgBco/dbEMsP/4vA9Y5u7/mTzN3VsI55QRWvxkWyaOyzsILbK+lzx/tMxGwug+4zn43OlOj+/lw/zOSOjqs/AQUfeQTxBaU3zak1r3RP//K1Fwuq/7ICIy0KgLiYgMFtdFj05ogv57wjCqP00z7588fbeH06LHkZYmfwRQFT0eByH3BTAFeM3d16aZf2lSvXoyJ3r8XS/n76uTo8clXUxfQrjxO4lDE0im61byWvR4RB/r8XwX5YljP6uLY//66PE4QteXxwm/qF4dNRt/gNCl5EVP33Q/F+4A/h+wwsx+EdXxKXfflqH1n0wIli1NM+1xwo33SWmmbXD3rX3dmJnNBK4CziR0pUjN7dDrHChdrH844QZ0OyHZbrrZWonebwDu/oqZrSOMMFRE1EqG0JXiccIN21xCa4yzCTfgXZ3zqe4gdGVYYWa/itb3TBS0Sa17OSH48veEc7OSA13WoOtjk+69lEhu+kKaaYmRiiammdZO6MKSamn0mO5cSHYK4df7tPlyCF25IOn4Z1EmjkviM+SYLvYnkefnOMLnRXd6+17u03dGiq4+C9N5PTCG0K3wi128V5pTtpPtzyMRkVgogCEig4K79zppHeFXrHQSSeHOjf66UhE9joweu8on0NV20hkVPWZraNVEXbsa/SRRPirNtIY0ZYlf84v7WI+ejv0VPSxfAeDuu81sDqElzbs58Iv3djP7AfAVdz/kl8dscvcbzWw7IWfIJwk5AdzMHgeu8vT5RfpiJLAzXWsCd2+Ptl2dZrm+nIcARMd2CeE64VFC/pPdRF0KCHlGhna5gt45gnDTX0XvA31E9bmCENBpi5Z/1N0bzewPHPiFfW7S/L3xaUL3i8uBq6O/djN7APhXd18D+3MRLCG0PnqJ0FJgGwd+6b6Oro/NIcEQDryXuptWmmba9i6CdYnXe2SaackS77lT6L41TUU30zIlE8clsT8X9rCtHvenD+/lvn5nJOvL+zKxnal0/17Zv50cfB6JiMRCAQwRKUTpmn7DgQvlT7l7b7p/JOYf18X08X2oUyJIMIHsDIGaqGtXdToyZb5s6enYz3L3P/dqRaGLwwct/Bw5g5Cw8ePAvxOanidGpEh07TnkOy9KYpgx7v5j4MfRet9E+IX+cmCxmR13OC0hkuwiJDUsTQ3OmFkJIVnh7nTVOoxtfZGQ3PZsd1+asq1rCAGM/kq85svd/eRu5zzYEkIA4xxC95hEWeLxGjMbTQhg7CLkDOlRFAz4LvBdM6smtEh6D+GGeGaU0LKVsO+nAre7+2XJ6zCzI+lbMKY/xppZcZogRuI93tN7OTH92+7+mcxWLRaJ/TnP3e/v78p6+V7u63fGQZvow7yJ7dzj7uf3egPZ/TwSEYmFcmCIiBzwbPT45t7MHOVZWANMMLOaNLOcdRjb7s0QlYkblr60flgePZ7VxfREea9u9rKgT8c+mQcr3P17HPgVdGHSLPXR49FpFp/dx8110ovj7u4N7v6Au19BSJQ4msPYtxTLCd/bZ6aZdmZUr0y9flMIrT2Wppn2lkxsIMrjsYIQHBjdh0WXEG7+5hKCVuuinDMQWlsUEUZjmEpIhNjnLkXuvtXd73b3i6Lt1QBviCZPiR5/nWbRjBybXioh3JSmOit6XJ5mWrLnCedzf8/LhA763iIrkw77M6Q7PbyXs7LNNFYRjc4TtQDqkyx9HomIxEIBDBGRSNSk9vfA+WZ2ebp5zOz46NfZhP8lfJZ+I+qTn5hvMqHZbm/9kNAs+to0iSoxs+S+3vWEG7jXpc7XjacIQ++dYWYXpKz7AsIN8F8JyTzj8L+EC/TrzOyQxKBmVmRmZyU9f4OZTUqznkRrmKaksmWEG7V/jPIuJNYxmpDYry92kD4QgpktiFpCpEqcL01ppvXFrdHj11L2Yzjw9ejpLf3cRsIGQmuPE5ILzeyDdJ2g8nDcSEi8eWu61jBmdkSU42S/6FfjFYRhS8/k4C4iTxNGofhC9LxX+S/MbKiZzbWU5ALRzWIiuJJ4/TZEj2elzHss8I3ebC+DvhYN5Zyow2hC6xkI76kuRcfxDmC2mV2b7tw1s5ros6w3dgBVZjasl/Nn2n2ELkAfN7O3p5vBzE5Lfu90pbfv5cP8zuizKAHv9wgt5W5Kd4zN7Mjk744cfB6JiMRCXUhERA72j4SbnlvM7JPAc4Qb64nACYRfYU8jjHoAIUnaQkKW+T+a2WJC3/OLCckw392bjbr7y2b2MeBmYLmZ3UdI2DaG0EqgkZCUEHffY2bPAW82szsIgYcO4P6uul+4u5vZpYThB38RrX8VYeSRhdH6P5A8kkouufuOKJByD/CsmT1KuEntJARqTiMci0QiyXOAG83sacJ+bCW8RudFy3wzad2bouP0fuBFM/stMAJ4O+E16inZYbJHgfeY2W8IyQXbgSfc/QngTqDFzJ4k3OQa4VfOU6J5H+nLMUnl7j8zs/OAiwiJ+e4lBLIWEobt/WXqCDP98B1CoOJJM/sloQn7bEK3iruAC7pZttfc/VYzeyOhn/7a6P3zKiFoMJkQoPhf4CMpiz7KgRYR+wMY7t5qZk/R9/wXwwivz4bovfUK4Vw7l5AY8f6kUTJ+Q2h59RkzO57Q0uF1wDuB39K3wGJ/bCLk2njJzO4n5IO4gHCT+4PonOzJJwgtVb4MvD86d7cQhh09jnDuXgKs73INBzwazf+gmT1BSMD6J3f/TZ/26jC5e5uZnQ8sBn4bfTa8SLhRPzqq27GE49PTzXtf3st9/c44XDcQkt5+BHiXmS0h5EyqJryGpwP/Rkhy3Nd9EBEZMBTAEBFJ4u610Q3VPxOCEu8lNIveTLgw/B7wl6T5W83sHOB6QtDiU4SLxa8QbsZ7FcCI1vU/ZvYS8FnCr7sLCSM0/Bn4Ucrs7we+DSwg3GAYUBvN29X6nzOzUwi/0J4DvCta/88Jw7eu7m1ds8HdH41+8f8s4eb5zYQcBxsJNwjJTfYXE26yzyQELUYQbugeBm5099TRGa4g3JhdQsiT8Sph2M1vEgICvfUpDnRfeDuh9c2XCIGQq6N6nxxNayHcCH8e+GGGkopeQhhN4HLgw1HZSkIg7YcZWD8A7v6gmb2LcK5cTAiQPU8Ioh1LhgIY0bY+bma/I9yYnUNIJLuT8Bp9kwNDZiZ7lAOvxWNpps0FtniaIYm7kBiK9mxCt4xEUG8tYYjjROsX3H2vmb2V0OrlLMJ5uo5wg3kj4Xjlwj7C8foPQq6OsVE9vk74nOpRlAz3LcCVhBvxfyAEbrYQAqifJryneuMrhNfuXYSb6WLgdkLAJyfc/c8Whjb9DCGglBjmehMh0HQd4TOvJ71+L/f1O6Mf+9ZmZgsJQ99eFu1fBSGB7HpCzp/kAGYuPo9ERHLOPO0w5iIiIiKSj8xsA4C7T4q3JiIiIrmlHBgiIiIiIiIikvcUwBARERERERGRvKcAhoiIiIiIiIjkPeXAEBEREREREZG8pxYYIiIiIiIiIpL3FMAQERERERERkbynAIaIiIiIiIiI5D0FMEREREREREQk7ymAISIiIiIiIiJ5TwEMEREREREREcl7CmCIiIiIiIiISN5TAENERERERERE8p4CGCIiIiIiIiKS9xTAEBEREREREZG8pwCGiIiIiIiIiOQ9BTBEREREREREJO8pgCEiIiIiIiIieU8BDBHJS2Y2yczczEoKcfsiIiKSWWa21Mw+VKjbFxkMFMAQGcTM7Hoz+2kOtzfDzJaZWX3094iZzUiafpWZvWRmjWa23syuylXdMs3MPhHta6uZ3ZZm+lwzW2VmTWb2mJkdE0M1RUREciaG6445Zvawme00s21m9iszOzJX288kC75iZnVmtisKdsyMu14i+UYBDBHp0mG0PtgIXACMBsYC9wN3Jq8S+ABwBLAA+ISZvScDVT2cuvbXRuArwK1p6jIWuBu4lnAslgG/yGntREREBpjD+C4/AlgETAKOARqB/42pLv11IXA58GbCtcMzwE9yXAeRvKcAhsggYGafjyL2jWa2Ovr1fwHwBeBiM9tjZn+K5j3KzO6Pfq1YY2ZXJK3nejO7y8x+ama7gcvMrMjMrjaztWa2w8x+aWaj09XD3RvcfYO7OyFY0QFMSZr+n+7+R3dvd/fVwH3A6T3s3uVmttHMNpnZv/ZQ11PN7Bkza4jm/y8zG5K0jJvZR8zsb1ELke+bmUXTis3sW2a23czWAe/orlLufre73wvsSDP5fGCFu//K3VuA64FZZja9h30VERHJe3l03fG76Lt2t7s3Af9Fz9cVNWb2fNTK4b7EupO6jn7QzF4FlkTlvzKzzdH8TyS3ijCz26Jrid9Gx+I5M6tJmn5u1Bpzl5n9F+HaqCuTgSfdfZ27dwA/BWZ0M79IQVIAQ2SAM7NpwCeAU9y9EpgPbHD3B4H/AH7h7hXuPita5OdALXAUobXEf5jZ3KRVngfcBYwC7gA+CSwE3hItUw98v4c6NQAtwPeiOqSbxwi/MqzoYRfPBqYC84CrzeycburaAXya0PrjNGAu8LGU9b0TOAWYBVxEOF4AV0TTTgJmE47N4ZoJ/CnxxN33AmujchERkQErH687kpxJz9cVHyC0dDgKaAduSpn+FuA4Dlwf/I5wHVIN/DGqY7JLgC8RWoOsAb4K+1tj/hr4IuG6ZC3dB1fuBKaY2evNrBS4FHiwh30RKTgKYIgMfB3AUGCGmZVGLSDWppvRzI4GzgA+7+4t7v4i8CPg/UmzPePu97p7p7s3Ax8G/s3da929ldCa4ILumla6+yhgJOECZ3kXs11P+Azqqannl9x9r7v/JZr3kq7q6u4vuPuzUQuPDcB/Ey5Ekn09ainyKvAYcGJUfhHwHXd/zd13Al/roV7dqQB2pZTtAir7sU4REZF8kHfXHdG2TgD+Hegpv9ZP3P2l6MeFa4GLzKw4afr10XVHM4C73+rujUl1mWVmI5Pmv9vdn3f3dkJwI3Fd8XbgZXe/y93bgO8Am7up1ybg98BqoJnQpeTTPeyLSMFRAENkgHP3NcC/EL5Ut5rZnWZ2VBezHwXsdPfGpLJXgAlJz19LWeYY4J6oW0YDsJJw8TKuh3rtBW4Gfmxm1cnTzOwThF9A3hFdEHQnuT6vRPuQtq7Rrxb/FzX13E34JWhsyvqSLx6aCMEGovWmbutw7QFGpJSNIPTNFRERGbDy8brDzKYQWkp8yt1/38MupH7Xl3LwtcL+6VH30q9H3Vl2AxuiScnz9+q6Iupem7qvya4jtBA9GigjtOpYYmbDe9gfkYKiAIbIIODuP3P3Mwhf+g58IzEpZdaNwGgzS24J8DqgLnl1Kcu8BrzN3Ucl/ZW5ex09KwKGk3ShYmaXA1cDc929thfrODqlrhu7qesPgVXAVHcfQeiL211/02Sb0mzrcK0gdFEBwMzKgRp6btYqIiKS9/LpusPCKF+PADe4e2+SXqZ+17cB27uozz8SuricQ2hZOimx2V5s56Driqjr7NFdz84sQveb2qgl6W2EbinKgyGSRAEMkQHOzKaZ2VvNbCgh70Qz4ZcKgC3AJDMrAnD314Cnga+ZWVnU3PKDHNqfM9nNwFejCwTMrMrMzuuiLuea2UnRLxYjgBsJfVdXRtPfS2gVca67r+vlLl5rZsOjpFn/RPejeVQCu4E9UcLMj/ZyGwC/BD5pZhPN7AhCkKVLZlZiZmVAMVAcHc9E89Z7gDeY2T9E8/w78Gd3X9WH+oiIiOSdPLvumEBItvl9d7+5l7vwPgvDvg8HvgzcFSXNTKcSaCUk7B5OF3m9uvBbYKaZnR9dH3wSGN/N/H8ALjSzcVEi0/cTWoes6cM2RQY9BTBEBr6hwNcJvx5sJiSZ+kI07VfR4w4z+2P0/yWEXxA2Em60r3P3h7tZ/3cJw6E+ZGaNwLPA33Ux7yhCsq5dhGRVU4AF0UgcEIYdHQP8wUKG8j1m1tMFx+OEL+9HgW+5+0PdzPtZwq8ljcD/0LehS/8HWExIvvlHwjCo3fki4aLtauB90f9fBHD3bcA/EBJ51ROOV0aGixUREYlZPl13fAg4Frgu6bpiTw/1/wlwW1T3MkJgoSs/JnQzqQNejurSK+6+nZDH4uuEAMhU4KluFvkG4RrkRaCBkP/iH9y9obfbFCkEFrpjiYiIiIiIiIjkL7XAEBEREREREZG8pwCGiIiIiIiIiOQ9BTBEREREREREJO8pgCEiIiIiIiIiea+k51kGl7Fjx/qkSZPiroaIiMig8cILL2x396q46xEHXVeIiIhkXlfXFgUXwJg0aRLLli2LuxoiIiKDhpm9Encd4qLrChERkczr6tpCXUhEREREREREJO8pgCEiIiIiIiIieU8BDBERERERERHJewWXA0NERApLW1sbtbW1tLS0xF2VAa+srIyJEydSWload1Xyms65zNE5JyIiyRTAEBGRQa22tpbKykomTZqEmcVdnQHL3dmxYwe1tbVMnjw57urkNZ1zmaFzTkREUqkLiYiIDGotLS2MGTNGN5L9ZGaMGTNGrQp6QedcZuicExGRVApgiIjIoKcbyczQcew9HavM0HEUEZFkCmCIiIiIiIiISN5TAENERCSPLF26lKeffrpf66ioqMhQbaQQ6JwTEZGBQkk85RD3Lq/jm4tXs7GhmaNGDeOq+dNYeNKEuKslIpITcX8GLl26lIqKCt70pjflbJsSL51zIiIy0MT13aUWGHKQe5fXcc3df6GuoRkH6hqauebuv3Dv8rq4qyYiknXZ/AxcuHAhb3zjG5k5cyaLFi0C4MEHH+Tkk09m1qxZzJ07lw0bNnDzzTfz7W9/mxNPPJHf//73XHbZZdx1113715P4pXvPnj3MnTuXk08+meOPP5777ruv33WU3NM5JyIiA02c94xqgSEH+ebi1TS3dRxU1tzWwTcXr1YrDBEZ8L70mxW8vHF3l9OXv9rAvo7Og8qa2zr43F1/5ufPv5p2mRlHjeC6d83scdu33noro0ePprm5mVNOOYXzzjuPK664gieeeILJkyezc+dORo8ezUc+8hEqKir47Gc/C8Att9ySdn1lZWXcc889jBgxgu3btzNnzhze/e53K+lhntE5JyIig02c94wKYMhBNjY096lcRGQwSb2R7Km8L2666SbuueceAF577TUWLVrEmWeeyeTJkwEYPXp0n9bn7nzhC1/giSeeoKioiLq6OrZs2cL48eP7XVfJHZ1zIiIy0MR5z6gAhhzkqFHDqEtz4h01algMtRERyayefrU+/etL0n4GThg1jF98+LTD3u7SpUt55JFHeOaZZxg+fDhnnXUWs2bNYvXq1T0uW1JSQmdnuJl1d/bt2wfAHXfcwbZt23jhhRcoLS1l0qRJtLS0HHYdJTt0zomIyGAT5z2jcmDIQa6aP42SooObgg4rLeaq+dNiqpGISO5cNX8aw0qLDyrLxGfgrl27OOKIIxg+fDirVq3i2WefpbW1lccff5z169cDsHPnTgAqKytpbGzcv+ykSZN44YUXALjvvvtoa2vbv87q6mpKS0t57LHHeOWVV/pVR4mHzjkRERlosvXd1RsKYMhBFp40gROPHrn/+YRRw/ja+ccr/4WIFISFJ03ga+cfz4RRwzAy9xm4YMEC2tvbOeGEE7j22muZM2cOVVVVLFq0iPPPP59Zs2Zx8cUXA/Cud72Le+65Z39CxSuuuILHH3+cU089leeee47y8nIA3vve97Js2TJmz57NHXfcwfTp0/u7+xIDnXMiIjLQLDxpAjecd6CFYS7vGZvGP8cAACAASURBVM3ds76RfDJ79mxftmxZ3NXIaxf99zM8vz78KrPiS/MpH6qeRiIycK1cuZLjjjsu7moMGumOp5m94O6zY6pSrNJdV+icyywdTxGR/LNh+17O+tZSvnXhLC5448SMr7+rawu1wJBD1NU3UxkFLdZv3xtzbURERLpnZmVm9ryZ/cnMVpjZl6LyyWb2nJn9zcx+YWZDovKh0fM10fRJSeu6JipfbWbz49kjERGR/FZbH3JgTMhxrkQFMOQgbR2dbNrVzOlTxgKwdtuemGskIiLSo1bgre4+CzgRWGBmc4BvAN9296lAPfDBaP4PAvXuPgX4djQfZjYDeA8wE1gA/MDMDu7kKyIiItQ1NAEw8QgFMCRGm3e10Olw+tSxFBcZa7YqgCEiIvnNg8QXVmn058Bbgbui8tuBhdH/50XPiabPNTOLyu9091Z3Xw+sAU7NwS6IiIgMKLX1zRQXGUeOLMvpdhXAkIMkmgJNHlPOMaOHqwWGiIgMCGZWbGYvAluBh4G1QIO7t0ez1AKJ7GITgNcAoum7gDHJ5WmWERERkUhdfTPjR5RRUpzbkIICGHKQ2voDTYGOrapQCwwRERkQ3L3D3U8EJhJaTaTL+pjIXG5dTOuq/CBmdqWZLTOzZdu2bTvcKouIiAxYtfXNTMhx9xFQAENS1DU0YwZHjipjSnUFG7Y30d7RGXe1REREesXdG4ClwBxglJklhtKaCGyM/q8FjgaIpo8EdiaXp1kmeRuL3H22u8+uqqrKxm6IiIjktdr6JibmOIEnKIAhKWrrm6muHMrQkmJqqsrZ19HJa1G3EhERyQ8VFRUAbNy4kQsuuKDbeb/zne/Q1NTUp/UvXbqUd77znYddv1wzsyozGxX9Pww4B1gJPAYkDtClwH3R//dHz4mmL/Ewrvz9wHuiUUomA1OB53OzF/lN55yIiCS0dXSyeXdLzhN4ggIYkqK2vomJRwwHoKY6XKysVTcSESk0mzbBW94CmzfnbJMdHR19Xuaoo47irrvu6naew7mZHICOBB4zsz8DfwAedvf/Az4PfMbM1hByXNwSzX8LMCYq/wxwNYC7rwB+CbwMPAh83N37/sIcDp1zIiIyQCQGflAXEoldbX3z/khaTVUUwFAiTxEpNDfcAE8+GR4zYMOGDUyfPp1LL72UE044gQsuuICmpiYmTZrEl7/8Zc444wx+9atfsXbtWhYsWMAb3/hG3vzmN7Nq1SoA1q9fz2mnncYpp5zCtddee9B63/CGNwDhZvSzn/0sxx9/PCeccALf+973uOmmm9i4cSNnn302Z599NgAPPfQQp512GieffDIXXnghe/aEz/gHH3yQ6dOnc8YZZ3D33XdnZL9zxd3/7O4nufsJ7v4Gd/9yVL7O3U919ynufqG7t0blLdHzKdH0dUnr+qq717j7NHf/Xc52QudcRvZbRESyLzHwQ+KH71wq6XkWKRTtHZ1s3nWgKdDIYaVUVQ5VIk8RGVzOOuvQsosugo99DJqaYO5ceP556OyEm2+G5cvhyivhsstg+3ZIbT6/dGmvNrt69WpuueUWTj/9dC6//HJ+8IMfAFBWVsaTTz4JwNy5c7n55puZOnUqzz33HB/72MdYsmQJn/rUp/joRz/KBz7wAb7//e+nXf+iRYtYv349y5cvp6SkhJ07dzJ69GhuvPFGHnvsMcaOHcv27dv5yle+wiOPPEJ5eTnf+MY3uPHGG/nc5z7HFVdcwZIlS5gyZQoXX3xxLw+m9IrOOZ1zIiKDSPLAD7mmAIbst6WxlfZOZ8KoA5G0KVUVaoEhIoXllVfAo4En3MPzDDj66KM5/fTTAXjf+97HTTfdBLD/xm3Pnj08/fTTXHjhhfuXaW1tBeCpp57i17/+NQDvf//7+fznP3/I+h955BE+8pGPUFISvtpHjx59yDzPPvssL7/88v567Nu3j9NOO41Vq1YxefJkpk6dur9+ixYtysh+Sy/onNM5JyIygOwf+GGkAhgSo9qdh0bSaqrLuf/Fjbg7ZulGlxMRGWC6+/V61y6orz/4ZrK+HhYsCM/Hju31r9+pUj9DE8/Ly8sB6OzsZNSoUbz44ou9Wj5Vbz6n3Z1zzz2Xn//85weVv/jii/qMzyadczrnREQGkdr6ZsZVljGkJPcZKZQDQ/ara0j0ZToQwJhSVcHulna279kXV7VERHLnhhtCM/5kHR0ZyUvw6quv8swzzwDw85//nDPOOOOg6SNGjGDy5Mn86le/AsKN35/+9CcATj/9dO68804A7rjjjrTrnzdvHjfffDPt7e0A7Ny5E4DKykoaGxsBmDNnDk899RRr1qwBoKmpib/+9a9Mnz6d9evXs3bt2v31kxzRObe/fiIiMjDU1jfFksATFMCQJIlkLEeNSm6BERJ5Kg+GiBSEZ56BfSkB23374Omn+73q4447jttvv50TTjiBnTt38tGPfvSQee644w5uueUWZs2axcyZM7nvvjDq53e/+12+//3vc8opp7Br16606//Qhz7E6173Ok444QRmzZrFz372MwCuvPJK3va2t3H22WdTVVXFbbfdxiWXXMIJJ5zAnDlzWLVqFWVlZSxatIh3vOMdnHHGGRxzzDH93l/pJZ1zOudERAaYuobmWPJfAJgnmiwWiNmzZ/uyZcvirkZe+txdf+Kx1dv4w7+ds79s065mTvvaEr6y8A28b44uLkRk4Fm5ciXHHXdcrHXYsGED73znO3nppZdirUcmpDueZvaCu8+OqUqxSnddoXMus/LheIqISNDR6Uz74u+48sxj+dyC6VnbTlfXFmqBIfslD6GaMH5EGcOHFKsFhoiIiIiISIHbsruF9k6PZQhVUABDkoSmQAefiGZGjUYiERHpl0mTJg2KX8Jl4NA5JyIi2ZBIOxBXFxIFMAQITYE2NjQzYdShJ+KU6grWbdsbQ61ERDKj0LpLZouOY+/pWGWGjqOISH6pawgjVyqJp8Rqa2MLbR2eNpJWU1VOXUMze1vbY6iZiEj/lJWVsWPHDt0I9ZO7s2PHDsrKyuKuSt7TOZcZOudERPJP7c7QAiPdD9+5UBLLViXv1HXTFGhKNBLJ+u17ecOEkTmtl4hIf02cOJHa2lq2bdsWd1UGvLKyMiZOnBh3NfKezrnM0TknIpJfauubGVsxlLLS4li2rwCGAN33ZaqpOjCUqgIYIjLQlJaWMnny5LirIQVE55yIiAxWcQ6hCupCIpHa+qgv06hDs8keM6ac4iJTIk8REREREZECVlvfFFv+C1AAQyJ1Dc2MrRjCsCGHNgUaUlLEMaOHayhVERERERGRAtXZ6WxsaFELDIlfbX0zE7oZy/dYDaUqIiIiIiJSsLbtaWVfRycTu7lvzDYFMAQIAYyJ3WSSnVJdwfrte2nv6MxhrURERERERCQf7M+bGNMIJKAAhhCaAtXVd5+MpaaqnLYO57XopBUREREREZHCkcibqC4kEqvt+5sCdd8CA2Ct8mCIiIiIiIgUnEQLDCXxlFi91osTsSYKYKxRHgwREREREZGCU9fQzOjyIQwfUhJbHRTAkKSmQF0nYxlRVkp15VC1wBARERERESlAtfXNTIgx/wUogCGESBrQ48lYU1WhFhgiIiIiIiIFqK6+Kdb8F6AAhhAiaaPLh1A+tPumQFOqK1i7dQ/unqOaiYiIiIiISNzcnbqG7gd+yAUFMKTXTYFqqsrZ3dLOtj2tOaiViIiIiIiI5IMde/fR0tapLiQSv9peNgWq2T8Syd5sV0lERERERETyRGIEku7yJuaCAhgFzt2pq+9dU6D9Q6kqD4aIiIiIiEjBSAz8EOcQqpDlAIaZfdrMVpjZS2b2czMrM7PJZvacmf3NzH5hZkOieYdGz9dE0yclreeaqHy1mc1PKl8Qla0xs6uzuS+D1fY9+2ht711ToPEjyigfUswajUQiIiIiIiJSMOqiFhiDNoBhZhOATwKz3f0NQDHwHuAbwLfdfSpQD3wwWuSDQL27TwG+Hc2Hmc2IlpsJLAB+YGbFZlYMfB94GzADuCSaV/qgN0OoJpgZNdUVaoEhIiIiIiJSQGrrmxlRVsKIstJY65HtLiQlwDAzKwGGA5uAtwJ3RdNvBxZG/58XPSeaPtfMLCq/091b3X09sAY4Nfpb4+7r3H0fcGc0r/RBYgjViaN7F0mrqQojkYiIiIiIiEhhCCOQxJv/ArIYwHD3OuBbwKuEwMUu4AWgwd3bo9lqgQnR/xOA16Jl26P5xySXpyzTVfkhzOxKM1tmZsu2bdvW/50bRBLJWHqbTXZKdQUbd7Wwt7W955lFRERERERkwOvtwA/Zls0uJEcQWkRMBo4CygndPVJ5YpEupvW1/NBC90XuPtvdZ1dVVfVU9YJSW9/EyGGlVPayKVBNVTkA67ZpJBIREREREZHBLjHwQ9z5LyC7XUjOAda7+zZ3bwPuBt4EjIq6lABMBDZG/9cCRwNE00cCO5PLU5bpqlz6oLaXI5Ak1FRpJBIREREREZFC0dDUxt59HYO7Cwmh68gcMxse5bKYC7wMPAZcEM1zKXBf9P/90XOi6Uvc3aPy90SjlEwGpgLPA38ApkajmgwhJPq8P4v7Myj1dgjVhGPGlFNcZBqJREREREREpAD0Ne1ANpX0PMvhcffnzOwu4I9AO7AcWAT8FrjTzL4Sld0SLXIL8BMzW0NoefGeaD0rzOyXhOBHO/Bxd+8AMLNPAIsJI5zc6u4rsrU/g5G7U1vfzJun9r5bzZCSIo4ZPVwtMERERERERApAXUNi5MpBHMAAcPfrgOtSitcRRhBJnbcFuLCL9XwV+Gqa8geAB/pf08K0c+8+mts6+nwiaihVERERERGRwpBogZEPAYxsD6MqeWz/EKp9DWBUVbB++17aOzqzUS0RERERERHJE7X1zVQMLWHksN4N/JBNCmAUsAORtL4lY5lSXUFbh/NatLyIiIiIiIgMTomBH0Jqy3gpgFHAautDX6a+DoeTGEpViTxFREREREQGt9r6prxI4AlZzoEh+a2uvpnKsr43BaqpPjCU6rmMy0bVRERECtK9y+v45uLVbGxo5qhRw7hq/jQWnjQh7mqJiEgBq2to5u8mj467GoACGAUtNAXq+1i+I8pKqa4cqhYYIiIiGXTv8jquufsvNLd1AOGC8Zq7/wKgIIaIiMRiV3MbjS3tfW61ny3qQlLAauubD7spUE2VRiIRERHJpG8uXr0/eJHQ3NbBNxevjqlGIiJS6OoOM29itiiAUaDcndr6psMeCmdKdQVrtu7B3TNcMxERkcK0sSF9cuyuykVERLJtf97EPMmBoQBGgdrV3MbefR2HHcCoqSqnsaWdbXtaM1wzERGRwnRUFxeHXZWLiIhkW11DogVGfnwXKYBRoA4MoXq4LTAqAVi7dW/G6iQiIlLIrpo/jWGlxQeVDSst5qr502KqkYiIFLra+maGlRYzunxI3FUBFMAoWImmQIfbl6mmOhpKVXkwREREMmLhSRP42vnHM27EUABGlJXwtfOPVwJPERGJTW19ExOOGIaZxV0VQAGMgtXfFhjjR5RRPqSYtRqJREREJGMWnjSB575wDsdPGElNdYWCFyIiEqu6hua86T4CCmAUrNr6ZiqGljByWOlhLW9m1FRrJBIREZFsmD9zHMtfbWDr7pa4qyIiIgWsPyNXZoMCGAUqcSL2pylQTVWFWmCIiIhkwbyZ4wF4eOWWmGsiIiKFak9rOw1NbXkzhCoogFGw+jOEasKU6go27mphb2t7hmolIiIiAFOrK5g0ZjiLVyiAISIi8aiL0g5MUBcSiVsm+jLVVIVEnuu2aSQSERGRTDIz5s8czzNrt7O7pS3u6oiISAGqa0gM/KAAhsRoV3MbjS3t/Y6kTamuAFAeDBERkSyYN3McbR3OY6u2xl0VEREpQP0d+CEbFMAoQP0dQjXhdaPLKS4y1igPhoiISMaddPQRjK0YykMvqxuJiIjkXm19M0NKihhbPjTuquynAEYBqstQJG1ISRHHjBmuFhgiIhIrMzvazB4zs5VmtsLMPhWVX29mdWb2YvT39qRlrjGzNWa22szmJ5UviMrWmNnVcexPQlGRce6McSxdtZXW9o44qyIiIgWorr6ZiaOGUVR0+AM/ZJoCGAXoQFOg/meTramqUAsMERGJWzvwr+5+HDAH+LiZzYimfdvdT4z+HgCIpr0HmAksAH5gZsVmVgx8H3gbMAO4JGk9sZg3cxx793Xw9JodcVZDREQKUG19U14l8AQFMApSbX0zw0qLOWJ4ab/XNaW6gg079tLe0ZmBmomIiPSdu29y9z9G/zcCK4EJ3SxyHnCnu7e6+3pgDXBq9LfG3de5+z7gzmje2LypZgwVQ0t46OXNcVZDREQKUCYGfsg0BTAKUGIIVbP+NwWqqaqgrcN5dWdTBmomIiLSP2Y2CTgJeC4q+oSZ/dnMbjWzI6KyCcBrSYvVRmVdlcdmaEkxZ02r4uGXt9DR6XFWRURECkjzvg6279mXkVb7maQARgHKZCQtMZTqWg2lKiIiMTOzCuDXwL+4+27gh0ANcCKwCfh/iVnTLO7dlKdu50ozW2Zmy7Zt25aRundn3szxbN+zj+Wv1md9WyIiIhDuGQEmjFILDIlZbX1zxvoy1URDqSoPhoiIxMnMSgnBizvc/W4Ad9/i7h3u3gn8D6GLCISWFUcnLT4R2NhN+UHcfZG7z3b32VVVVZnfmRRnT6uitNg0GomIiOTMgZErFcCQGDW2tLGruS1jTYFGlJVSXTlUI5GIiEhsLPSJvAVY6e43JpUfmTTb3wMvRf/fD7zHzIaa2WRgKvA88AdgqplNNrMhhESf9+diH7pTWVbKm2rGsnjFZtzVjURERLIvMfCDknhKrBJNgTIZSZtSXaEAhoiIxOl04P3AW1OGTP1PM/uLmf0ZOBv4NIC7rwB+CbwMPAh8PGqp0Q58AlhMSAT6y2je2M2bOY5XdjTx1y36vhURkeyra2imtNioriyLuyoHKYm7ApJbtTszN4RqQk1VBfe+WIe7ZyQxqIiISF+4+5Okz1/xQDfLfBX4apryB7pbLi7nzhjHF+99iYdWbGba+Mq4qyMiIoNcbX0zR44cRnFRft3fqQVGgUn0ZcpkMpYp1RU0trSzbU9rxtYpIiIiB1RXlnHS0aNYrOFURUQkB+qikSvzjQIYBaa2vpmhJUWMrRiSsXXWVCmRp4iISLbNmzmel+p27+8OKiIiki219ZkbuTKTFMAoMIkhVDPZ1aOmWkOpioiIZNv8meMBeHiFWmGIiEj2tLZ3sLWxlQmjMpd2IFMUwCgwYQjVzJ6I40eUUT6kmLVqgSEiIpI1k8eWM7W6gsUrNJyqiIhkz8aGFiD/hlAFBTAKTm0W+jKZGTUaiURERCTr5s8cz/MbdlK/d1/cVRERkUFqf95EBTAkTntb26lvastKJG1KVYVaYIiIiGTZvJnj6Oh0Hl21Ne6qiIjIIFVXnxi5UgEMiVEi6Vcmh1BNqKmuYOOuFva2tmd83SIiIhIcP2EkR44s4yHlwRARkSyprW+muMgYP6Is7qocQgGMApKNIVQTEiORrFMiTxERkawxM+bNGMcTf9tG876OuKsjIiKDUF1DM+NHlFFSnH/hgvyrkWRNoinQ0dnoQhKNRLJmW2PG1y0iIiIHzJs5npa2Tp7427a4qyIiIoNQNvImZooCGAWktr6ZISVFjK0YmvF1HzOmnJIiY+1WtcAQERHJplMnj2bksFIe0mgkIiKSBXX1zXmZwBMUwCgotfXNTBg1jKIiy/i6S4uLeN2Y4axRIk8REZGsKi0uYu70ah5dtYX2js64qyMiIoPIvvZONu9uyUrexExQAKOAZLspUE2VhlIVERHJhXkzx9HQ1MbzG3bGXRURERlENu9qodNhYhbyJmaCAhgFpK6hOasBjCnVFWzYsVe/BomIiGTZma+vYmhJkbqRiIhIRtU2hIEflANDYtW8r4Pte/ZltSlQTVUFbR3OqzubsrYNERERgeFDSnjz1CoefnkL7h53dUREZJCojQZ+UA4MiVVdQ/aGUE2YUh2GUl2roVRFRESybt7McdQ1NLNi4+64qyIiIoNEXX0zZnDkSAUwJEaJSFo2mwIdWxUNpapEniIiIll3znHjKDJ4aMXmuKsiIiKDRG19M+NHlDGkJD9DBflZK8m4AwGM7HUhGVFWyrgRQ5XIU0REJAdGlw/hlEmjWaw8GCIikiG19U1ZbbXfXwpgFIja+mZKi43qyqFZ3U5NVYVaYIiIiOTIvJnjWb2lkQ3b1X1TRET6L9sDP/SXAhgFora+iaNGDaOoyLK6ncRQqkooJiIikn3zZowD4OGX1QpDRET6p72jk027WvI2gScogFEwchVJm1JdQWNLO9saW7O+LRERkUJ39OjhzDhyBIuVB0NERPppS2MrHZ2e1bQD/aUARoGorW9m4qjsn4g1VWEkkjXKgyEiIpIT82eO54VX6/XjgYiI9EvtzuyPXNlfCmAUgJa2DrY1tuakKdD+oVSVB0NERCQn5s0chzs8ulLdSERE5PDVNWR/5Mr+UgCjAGzM4Yk4bsRQKoaWsHabkomJiIjkwvTxlRw9epi6kYiISL8kRq48Si0wJE65GEI1wcyoqSrXUKoiIiI5YmbMnzGep9bsYE9re9zVERGRAaq2vomqyqGUlRbHXZUuZTWAYWajzOwuM1tlZivN7DQzG21mD5vZ36LHI6J5zcxuMrM1ZvZnMzs5aT2XRvP/zcwuTSp/o5n9JVrmJjPL7hAbA1QigJGrbLIaSlUkXvcur+P0ry9h8tW/5fSvL+He5XVxV0lEsmzezPHs6+hk6eqtcVdFREQGqHwfQhWy3wLju8CD7j4dmAWsBK4GHnX3qcCj0XOAtwFTo78rgR8CmNlo4Drg74BTgesSQY9oniuTlluQ5f0ZkGrrmygpMsZVDs3J9mqqK9i0q0W/AonE4N7ldVxz91+oa2jGCV9E19z9FwUxRAa5Nx5zBGPKh/DQCuXBEBGRw1Nb35zXCTwhiwEMMxsBnAncAuDu+9y9ATgPuD2a7XZgYfT/ecCPPXgWGGVmRwLzgYfdfae71wMPAwuiaSPc/Rl3d+DHSeuSJHUNzRw5qoyS4tz0GEqMRLJO3UhEcu6bi1fT3NZxUFlzWwffXLw6phqJSC4UFxnnHDeOx1ZtZV97Z9zVERGRAaaz09nY0JzXQ6hCdltgHAtsA/7XzJab2Y/MrBwY5+6bAKLH6mj+CcBrScvXRmXdldemKZcUuRpCNWFKdTmA8mCIxCCRtLe35SIyeMybOY7G1naeWbcj7qqIiMgAs7WxlbYOz1nagcOVzQBGCXAy8EN3PwnYy4HuIumky1/hh1F+6IrNrjSzZWa2bNu2bd3XehCqrW/K6Yl4zJhySopMeTBEYtBV1uh8ziYtIplx+pSxDB9SzEMajURERPqorqEJyO8hVCG7AYxaoNbdn4ue30UIaGyJun8QPW5Nmv/opOUnAht7KJ+YpvwQ7r7I3We7++yqqqp+7dRA09rewdbG1pyeiKXFRbxuzHDWbtVQqiK5dtX8aRSn5DMeVlrMVfOnxVQjEcmVstJizppWxcMvb6GzM+1vOiIiImklBn44ulADGO6+GXjNzBJXzXOBl4H7gcRIIpcC90X/3w98IBqNZA6wK+pishiYZ2ZHRMk75wGLo2mNZjYnGn3kA0nrksimhhbcczOEarIpVRXqQiISg4UnTWBsxZD9z48aWcbXzj+ehSeph51IIZg3YzxbG1t5sbYh7qqIiMgAkghg5Hur3ZIsr/+fgTvMbAiwDvgnQtDkl2b2QeBV4MJo3geAtwNrgKZoXtx9p5ndAPwhmu/L7r4z+v+jwG3AMOB30Z8k2T+Eao5PxJrqCh5bvZX2js6cJQ8VkdDqavvefUypDsMZ//B9b2TW0aPirpaI5MjZ06spKTIeWrGFk193RM8LiIiIEO4bx5QPYfiQbIcI+iertXP3F4HZaSbNTTOvAx/vYj23AremKV8GvKGf1RzU4urLNKWqgrYO59WdTRwbjUoiItm3ZuseOjqdhScexbce+iurNzcqgCFSQEYOK+W0mjE8tGIzn18wDbN0KcNEREQOluu8iYdLP40PcrX1zRQXGUeOLMvpdmuqQ9BCiTxFcmv15kYA5s0cz7DSYlZFz0WkcMybMY512/eqK6eIiPRaXUNz3ifwBAUwBr3a+mbGjyjLeTeOY6sSQ6kqkadILq3a3MiQkiKOHVvO68dXsmrz7rirJCI5du6M8QAsXrEl5pqIiMhA4O7U1TfnPO3A4VAAY5CLqynQiLJSxo0YqhYYIjm2ctNuplZXUFJcxPRxlaza3EjooScihWL8yDJmHT1Kw6mKiEivbN+zj9b2zpwP/HA4FMAY5Orq42sKVKORSERybvXmRqaNrwRg2vhKdu7dx7Y9rTHXSkRybd6McfypdhebdjXHXRUREclztfXx5E08HApgDGL72jvZvLsltkjalOoK1m7do19/RXJk5959bG1s5bjxIwCYfmQIZKxWHgyRgjN/ZuhG8sjL6kYiIiLd2z9ypQIYEqfNu1rodJgYU1+mmqoKGlvb2daoX39FciGR7yIRuJgeBTIUwBApPFOqKzi2qlx5MEREpEd1DVEAYzDkwDCzC82sMvr/i2Z2t5mdnP2qSX/VxjSEasKUxEgk6kYikhOrNoVARaILyejyIVRXDmXlJgUwZGDQNUdmzZ85nmfX7WBXU1vcVRERkTxWW9/EyGGlVJaVxl2VHvWmBca17t5oZmcA84HbgR9mt1qSCYmmQHF1IampCgGMtUrkKZITqzc3MqZ8CFUVQ/eXTRtfyeotGolEBgxdc2TQvBnjaO90lqxWKwwREelanHkT+6o3AYyO6PEdwA/d/T5gSPaqJJlSW99MkYVs5HEYN2IoFUNLNJSqSI6s2rybaeMrMbP9ZdPHV/K3LXtou2yx4gAAIABJREFU7+iMsWYivaZrjgyaNXEU1ZVDeUjdSEREpBu1A2QIVehdAKPOzP4buAh4wMyG9nI5iVltfRPjRpQxpCSel8vMqKkq11CqIjnQ0ems3tK4P+9FwvTxI2ht72TDjqaYaibSJ7rmyKCiImPezHE8/tdttLR19LyAiIgUHHenrqF5QAyhCr27KLgIWAwscPcGYDRwVVZrJRmRD02BNJSqSG68urOJlrZOpkf5LxIS+TCUyFMGCF1zZNi8GeNp2tfBk3/bHndVREQkD9U3tdG0ryP2+8be6jGA4e5NwFbgjKioHfhbNislmVFbH38kraa6gk27WtjT2h5rPUQGu1WbDh6BJGFKdQXFRbZ/hBKRfKZrjsybc+wYKstKeOjlzXFXRURE8lBtfWilOxCGUIXejUJyHfB54JqoqBT4aTYrJf3X3tHJ5t0tsfdlSiTyXKdWGCJZtWpzI2YwtfrgAEZZaTGTx5azSi0wZADQNUfmDSkp4q3Tq3lk5VY6Oj3u6oiISJ6p2z/wwyAJYAB/D7wb2Avg7huBym6XkNht3v3/2bvz+KjK6/Hjn2cmk32SAEkmJEBYAtlEAXFFVLagtVZrtZs/tVq1i9auWvl2/Wr9SrW1dWurVsWtVWsVdwEBFQEVlE1IQsKeIRtkTybLzDy/P2YGIiRkm5k7Mznv12teJHfuzD3RLPeee55z2nG5teHfiEdGqUofDCECqqSqiQmjEoiLNh/3XG6GVZaQiHAh5xwBUFSQQV1rJxv31hkdihBCiBBzZHJlSuT0wOjUWmtAAyilEgIbkvAHo0eo+mSPiifKpKQPhhABVlrVfNzyEZ/8DCv769pkKZcIB3LOEQDn5aYRHWVi+Q6ZRiKEEOKL7A0OrDFRJMVFGR1Kv/QngfGityN4ilLqBuBd4LHAhiWGypfAMHotk8VsIntUPLtqZJSqEIHS1ulkX10bubakHp/P9U4m2VktVRgi5Mk5RwAkxkRxTk4qy7ZX4ckPCSGEEB4V9W1kjYhDKWV0KP3SnyaefwJeAv4L5AK/1Vo/GOjAxND4mrFkpsQaHImnD0a5VGAIETA7q1vQ+vgGnj6+ySQllZLAEKFNzjkCp6jARkW9g2L5PSCEEKKbihCYXDkQ/aoT0VqvAFYEOBbhR/Z6B7akGGKijl8PH2yT0hNZXVpDl8uNxdyfoh8hxEAcmUCS0XMCIysljsSYKEplEokIA3LOERjzC2yoV7axfEcVBZk9V2sJIYQYXrTW2OsdnDlxlNGh9FuvV5NKqWalVFMPj2allJwFh7hQGKHqk5OWSJdLs7+uzehQhIhIJVXNxEebGdvLz7zJpJhiS5RJJCJkyTlH4KUmxjAzewTLtksfDCGEEB5NDifNHU7DJ1cORK8JDK21VWud1MPDqrWW1H2Iq2hoC5lvxEneSSS7ZBKJEAFRUtXEFJsVk6n3tYt5o5MoqWqW9e8iJA31nEMpNVYptVopVayU2q6U+rF3+0il1AqlVJn33xHe7Uop9YBSqlwptVUpNaPbe13j3b9MKXVN4L7q4CsqyKC4sokDckNBCCEEnmtGCJ8RqtC/Jp4AKKXSlVLjfI9ABiWGxuXWVDa0h8w34qQ0TxN56YMhhP9prSmpaia/l/4XPnkZVhodXVQ3dQQpMiEGbxDnHE7g51rrfOBM4CalVAFwO7BSaz0ZWOn9HOBCYLL3cSPwd+9xRwK/A84ATgd+50t6RIKiQhuATCMRQggBhM7gh4HoM4GhlPqKUqoM2AO8D+wF3g5wXGIIqpvacbp1yCwhscZasCXFyCQSIQKgprmDhrYu8jJOfJM61+Zt5Cl9MEQIG+w5h9a6Umv9mffjZqAYyAIuAZ7y7vYUcKn340uAp7XHR3imnowGFgIrtNZ1Wut6PL04LvDX12e07FEJ5GVYWba9yuhQhBBChAC7N4ERKteN/dGfCow78dzN2Km1ngDMA9YGNCoxJKGYSctJT2SXVGAI4XfF3gaeub008PTxJTikD4YIcUM+51BKjQemAx8DNq11JXiSHEC6d7cs4EC3l1V4t/W2PWIUFdjYuLeOwy1SjSWEEMGydJOdWYtXMeH2N5m1eBVLN9mNDgnwXDfGWcyMiLcYHUq/9SeB0aW1PgyYlFImrfVqYFqA4xJD4BuhGipLSMAzSnVXTYusvxfCz0q9CYneJpD4JMdbGJ0ce2R/IULUkM45lFKJeEaw/kRrfaJyo54axugTbD/2ODcqpTYqpTbW1tb2N7yQUFSYgVvDyuIao0MRQohhYekmO4te3oa9wYEG7A0OFr28LSSSGBX1bYwZEYdSvfdRCzX9SWA0eE8IPgCeU0rdj2etqQhRvlKgUGniCZ4ERnOHk9pmueMjhD+VVDWTkRRLSnx0n/vmZliPVGwIEaIGfc6hlLLgSV48p7V+2bu52rs0BO+/vqv2CmBst5ePAQ6eYPsXaK0f1VrP1FrPTEtL6/cXFwoKM5PISolj+Q5ZRiKEEMFw77JSHF2uL2xzdLm4d1mpQREdZW9whNRN7/7oTwLjEqAN+CnwDrALuDiQQYmhqah3kGaNIdZiNjqUI3K8k0jKZRKJEH5VUtVMXh8NPH3yMpLYVdtCl8sd4KiEGLRBnXMoz62jx4FirfV93Z56DfBNErkGeLXb9qu900jOBBq9S0yWAUVKqRHe5p1F3m0RQylFUaGND8oO0doh96OEECLQDjY4BrQ9mCrqHSHVdqA/+j2FRGvtBNbjaaglt/BCWCiNUPWZlOYdpSp9MITwmy6Xm/Ka5j77X/jkZVjpcmn2HJKGuiK0DeKcYxZwFTBXKbXZ+/gSsBhY4G0MusD7OcBbwG6gHHgM+KH3uHV4+nBs8D7u8G6LKEUFGXQ63XywM7yWvwghRDganRzb4/ZMg6/Xmtu7aHR0hVUDT4CofuzzATDbeydiJbAR+AZwZSADE4Nnr3dwUlay0WF8gS0phsSYKKnAEMKPdte20uXS5PcxgcTHV6lRXNnEFFv/kh5CBNmgzjm01h/Sc/8K8DQCPXZ/DdzUy3s9ATwxgJjDzmnjRzAi3sLyHdVcOHW00eEIIUREOz83jX99cuAL2+IsZm5dmGtQRB72htBrO9Af/anAUFrrNuAy4EGt9VeBgsCGJQbL7dbetUyhlUlTSjEpLYFdtXLnVwh/8Y1E7W8FxsTURKJMShp5ilAm5xxBEGU2MS/fxsriallSJoQQAaS1Zpu9iXRrzJFKjMQYM3dfNpVLpxs75OroCNUITGAopc7Cc/fjTe+2/lRuCAPUNHfQ5dIhuZZpUnqiVGAI4UclVc1EmdSRJVp9iY4yMSktUUapilAm5xxBUlRgo6ndyce7I26FjBBChIxP99Wzzd7ILfMms37RPM7PTWNkQgyXTMs0OjQqfIMfQvC68UT6k8D4MbAIeEVrvV0pNRFYHdiwxGDZG0JvhKrPpLREqpraaZGmYUL4RWlVMznpiURH9budEXmjrVKBIUKZnHMESUNbFwD/7/GPmbV4VUiM8xNCiEjz5Lq9JMVGcdkMT7XFwsIM9te1UVpt/LlYRX0bMVEm0hJjjA5lQPo869Vaf6C1/orW+o/ez3drrW8JfGhiMHyZtLEhmMDwTSLZLY08hfCLksqmfi8f8cnNsGJvcNDU3hWgqIQYPDnnCI6lm+z87rXtRz63NzhY9PI2SWIIIYQfHWxw8M7nVXzz9HHER3uKCeflp6MULN9ebXB0nt/9WSPi8AzyCh/9v20nwsKRUqCU0OqBAUcnkcgyEiGGrtHRxcHGdvL62cDTJ8+b8JAqDCGGr3uXleLocn1hm6PLxb3LSg2KSAghIs+zH+1Da81VZ2Yf2ZZujWXGuBEs215lYGQeFfWOsGvgCZLAiDgV9W2MSogmLtpsdCjHyR4VT5RJyShVIfzAl4DIG2AFhi/hIX0whBi+Dno7z/d3uxBCiIFp73Lx70/2s6DAxtiRX7yxvLDQxvaDTVTUtxkUnYe9PvQGP/RHnwkMpdSs/mwToaGi3hGS/S8ALGYT2aPipQJDCD/wTSDxjUbtr9HJsVhjoyj1vl6IUCLnHMGR2csdt962CyGEGJhXN9upb+viO2dPOO65ooIMwNhlJG2dTg63dobsdeOJ9KcC48F+bhMhINQzaZPSEmWUqhB+UFLVTFJsFBlJsQN6nVKK/IwkSiqlAkOEJDnnCIJbF+YSZzm+UvOmOZMMiEYIISKL1pon1+4lL8PKmRNHHvf8+NQEcm1Wlu8wbhmJr+IuHBMYvY4m844xOxtIU0r9rNtTSUDorU8QuN2aigYH8wtsRofSq5z0RFaV1NDlcmMxywomIQarpLKJvNFJg2q8lJthZekmO1rrsGvcJCKTnHME16XTPd3w711WysEGB2nWGGqaO9hZLRWSQggxVB/trqOkqpk/fm1qr+dZRYU2Hl5dTn1rJyMSooMcIRyoD98ExomuIKOBRDxJDmu3RxNweeBDEwN1qLWDTqc7pL8RJ6Ul4nRr9tcZu+ZLiHDmdmt2VrcMuP+FT26GleYOJ3ZZ7y5Ch5xzBNml07NYe/tc9iy+iE9+NZ8rzxjHMx/toywERvsJIUQ4e3LtHkbEW7hkWlav+xQVZODW8G6xMctIQnnwQ196rcDQWr8PvK+UWqK13hfEmMQgVYRBJs03SnVXTcuRqSRCiIGxNzho6XAOeAKJT/7oo5NIQnnJmRg+5JzDeD9bMIXXthzkjjd28PR1p0t1lhBCDMKBujbeLa7m++dNIraHpXo+J2UlkZkcy/Id1Vwxc2wQI/Sw1zuwmBXp1pigH3uoeq3AUEr91fvhQ0qp1459BCk+MQBHExihe0EyMS0BgHKZRCLEoPkmiAy0gafPFJv1C+8jhNHknMN4oxJj+Mn8KawpO8SqkhqjwxFCiLD09Pq9KKW46qzsE+6nlKKoMIM1ZbU4Ol0n3DcQKurbyEyJw2QKv2R1rxUYwDPef/8UjEDE0PlG8YTyPF9rrAVbUgy7aqSRpxCDVVLpmSDiS0QMlDXWwpgRcZLAEKFEzjlCwNVnZfPcx/v4w5vFzJ6cRnSU9KoSQoj+au1w8vyGA1xwUgajk/u+HisqsLFk3V7e31nLBSdlBCHCo+wNoTu5si8nWkLyqfff94MXjhgKe72DEfEWEmJOlJcyXk56olRgCDEEJdXNjBsZT+IQftbzMqwySlWEDDnnCA0Ws4nfXFTAtUs28PT6vVw/e6LRIQkhRNh4eZOd5nYn180a36/9T58wkuQ4C8t3VAU9gVFR72BOblpQj+kvfabWlVKzlFIrlFI7lVK7lVJ7lFK7gxGcGJiKEB+h6jMpLZHdNS1orY0ORYiwVFLZRO4gG3j65GZY2VXbSocz+GWLQvRGzjmMNycvnfNz07j/3TIOtXQYHY4QQoQFrTVL1u5halYyM8aN6Ndroswm5uWns7K4BqfLHeAIj2rvclHb3BEW14096U9t4OPAfcA5wGnATO+/IsRU1LeF9PIRn5z0RJo7nNQ0y4mREAPV3uViz6FW8oeYwMjLSMLl1rKcS4QaOecIAb++qABHl4s/L99pdChCCBEW1pQdYldtK9fOGj+gJshFBRk0Orr4ZE9dAKP7ooMNoT/44UT6k8Bo1Fq/rbWu0Vof9j0CHpkYEK112Kxl8k0f2VUjy0iEGKjymhbcGnIHOYHExzeCtbRalpGIkCLnHCEgJz2Rq87K5vkN+9l+sNHocIQQIuQtWbeX1MQYLjp59IBed96UNGItJpbvCN441aMjVEP/urEn/UlgrFZK3auUOkspNcP3CHhkYkAOt3bS3uUOiwSGb5Sq9MEQYuCKvQ08BzuBxGd8agLRZhMlldLIU4QUOecIET+ZN4WUOAt3vL5DlnwKIcQJ7DnUyqqSGq48YxwxUb2PTu1JXLSZ2ZPTWL69Kmi/a+2+CoyR4bmEpD8d4M7w/juz2zYNzPV/OGKwwmGEqk+6NYbEmCipwBBiEEqrmomJMjF+VMKQ3sdiNpGTniiTSESokXOOEJEcb+FnRbn8ZunnvPN5FRdOHdhdRSGEGC6eWrcXi1lx5ZnjBvX6hYUZrNhRzef2JqaOSfZzdMerqG/DbFLYrDEBP1Yg9JnA0FrPCUYgYmiOjFANgwoMpRST0hPZVStr74UYqJKqZqbYrJj9MLc7L8PKul1SnS9Ch5xzhJZvnTaWZ9fv4663ipmTl06sZWB3FoUQItI1t3fx0qcVfPnkTNKtsYN6j3l56ZgULN9RFZQEhr3ewejkWKLM4Tkqu88EhlLqtz1t11rf4f9wxGDZfWuZwiCBATApLYF15XLhJMRAlVQ1+23sVd5oKy9vstPQ1klKfLRf3lOIoZBzjtASZTbx24sLuPKfH/P4h3u4aU6O0SEJIURIeenTClo6nHzn7PGDfo8RCdGcPmEky7ZX8fOiXP8F14uKekfY9r+A/vXAaO32cAEXAuMDGJMYhIp6B8lxFpJiLUaH0i+T0hKpamqnpcNpdChChI1DLR0caukY8ghVH18jUFlGIkKInHOEmFk5qRQV2Hh4dTnVTe1GhyOEECHD7dY8tW4vM8alcMrYlCG918LCDHZWt7DnUOAr1D2DH0K/7UBv+kxgaK3/3O1xF3A+kBXwyMSAhMsIVR9fI0/pgyFE/5V6Ew35o4c2gcTHN4mkpFImkYjQIOccoelXF+XjdGnueafU6FCEECJkvLezhr2H2/jOrAlDfq8FBTYAVuyoGvJ7nUin001VU3tYDH7ozWAWvsQDE/u7s1LKrJTapJR6w/v5BKXUx0qpMqXUC0qpaO/2GO/n5d7nx3d7j0Xe7aVKqYXdtl/g3VaulLp9EF9LxAiXEao+R0apyiQSIfrNN4HEXxUY6dYYRsRbKK2WCgwRsgZ0ziECI3tUAtedM4H/flbB5gMNRocjhBAh4cm1e7ElxXDhSRlDfq8xI+IpzExi2fbAjlOtbHSgdfi0HehJnwkMpdQ2pdRW72M7UArcP4Bj/Bgo7vb5H4G/aK0nA/XAd73bvwvUa61zgL9490MpVQB8EygELgD+5k2KmIGH8ZSXFgDf8u477GitqagPr1Kg7FHxRJkU5VKBIUS/lVQ1k5oYQ2qif7pGK6XIzbDKEhIRMvxwziEC5Oa5OaQmxnDH69tlrKoQYtgrq25mTdkhrjozG4ufmmEuLMzgs/311DQHbrme/cjkyghOYABfBi72PoqATK31Q/15c6XUGOAi4J/ezxWeUWgveXd5CrjU+/El3s/xPj/Pu/8lwPNa6w6t9R6gHDjd+yjXWu/WWncCz3v3HXbq27po63SF1TeixWwie1S8VGAIMQClVc1Hln34S15GEqVVzbjdckEiQsKgzzlEYCXGRHHbBbl8tr+B17YcNDocIYQw1JJ1e4mOMvGt0wc3OrUnRYU2tIaVxTV+e89jVfgSGCnhc+P7WP3pgbGv28OutR5I18W/ArcBbu/no4CGbu9RwdG1rVnAAe8xnUCjd/8j2495TW/bh51wm0Dik5OeKBUYQvSTy63ZWR2IBIaVtk7XkT9oQhhpiOccIsAunzGGqVnJ3P1WCW2d8r9GCDE8NbZ18fJndi45JZNRfqqKBci1WRk3Mp5l2wPXB6OiwYFJQUby4Ea+hoKADX9VSn0ZqNFaf9p9cw+76j6eG+j2nmK5USm1USm1sba29gRRh6eK+jYg/EqBJqUlsu9wG10ud987CzHM7T3cSofTTZ6fGnj6+PppFFdJI08hxImZTIrfXlxAVVM7/3h/t9HhCCGEIV7YuB9Hl4vvzBrv1/dVSrGw0Ma68sM0t3f59b19KurbsCXFEh0VsDRAwAUy8lnAV5RSe/Es75iLpyIjRSkV5d1nDOCrQ6wAxgJ4n08G6rpvP+Y1vW0/jtb6Ua31TK31zLS0tKF/ZSHmSClQGPXAAE8Cw+nW7K9rMzoUIUJeSaWnT4W/KzCm2KwodXTCiRBCnMhp40dy8SmZPPL+LuwNUrklhBheXG7NU+v2cfqEkRRmJvv9/YsKM+h0uXmvNDA33T19E8PrpvexApbA0Fov0lqP0VqPx9OEc5XW+kpgNXC5d7drgFe9H7/m/Rzv86u0p0vUa8A3vVNKJgCTgU+ADcBk71STaO8xXgvU1xPKKurbsMZEkRxnMTqUAfGNUpVlJEL0rbSqCZM6+nPjLwkxUYwbGS8JDCFEv91+YR5Kwd1vFfe9sxBCRJAVO6qxNzi4zs/VFz4zxo0gNTGa5TsCM43EHmaDH3piRO3IL4GfKaXK8fS4eNy7/XFglHf7z4DbAbTW24EXgR3AO8BNWmuXd13szcAyPFNOXvTuO+zYGxxh1/8CYGJaAiCjVIXoj+KqZiakJhBrMfv9vXNtVllCIoTot6yUOG48dxJvbK1kw946o8MRQoigWbJuD1kpcczPtwXk/c0mxfx8G6tLauhwuvz63k6Xm6qmdrJSwu+6sbugJDC01u9prb/s/Xi31vp0rXWO1voKrXWHd3u79/Mc7/O7u73+Lq31JK11rtb67W7b39JaT/E+d1cwvpZQFG4jVH2ssRYykmKlAkOIfiipavJ7/wufvNFJ7D3USnuXf/9QCiEi1/fPm8jo5Fj+9/XtMsVICDEsFFc28dHuOq4+K5soP41O7UlRoY2WDifrdx326/tWNbXjcmtZQiKMpbUO27VMSzfZqW/r5OXP7MxavIqlm+xGhyRESGrpcHKgzkGezb/9L3zyMqy4tSznEkL0X3x0FLdfmMfn9iZe+rTC6HCEECLglqzdS6zFxDdOG9v3zkNw9qRUEqLNfl9GUhGmkyuPJQmMMNfkcNLS4Qy7BMbSTXYWvbyNDqdnAom9wcGil7dJEkOIHvj6UwSsAsM3iaRSlpEIIfrvK6dkMmNcCvcsKw1Yx3whhAgFda2dLN1s57IZY0iJjw7osWItZs7PTWfFjmq/VrjZw3Tww7EkgRHmDoTpCNV7l5XiOKZc3dHl4t5lpQZFJEToOpLA8PMEEp/sUQnEWkzSyFMIMSBKKX53cSGHWjp4ePUuo8MRQoiA+fcn++lwuvnO2eODcryiQhu1zR1sOtDgt/f0VWCMTo7123saQRIYYS5cR6ge7GX0Wm/bhRjOSqqaSIyJCljTJbNJMTndSokkMIQQA3TK2BS+NmMMT3y4h32HW40ORwgh/K7L5ebZj/ZxTk4qUwK0nPdYc/LSsZgVy3dU+e09K+rbSLfGBKQhfDBJAiPMVXgrMMKtm2xmL/H2tl2I4aykqpncDCsmkwrYMfIyJIEhhBic2y7IJcqsuOtNGasqhIg8y7ZXUdnYHrTqC4CkWAtnThzF8u3VaO2fZST2hvDsm3gsSWCEOXuDg4RoMynxFqNDGZBbF+YSd0z2L9Zi4taFuQZFJERo0lpTUtlEboCWj/jkZlg51NLBoZaOgB5HCBF5bEmx3DQnh+U7qllbfsjocIQQwq+WrN1L9qh45ualB/W4RYUZ7DnU6rcm6xX1DrLCrGq/J5LACHO+EapKBe7ObCBcOj2Luy+bSlZKHL7Irzh1DJdOzzI0LiFCTVVTO03tTvIDnMDIy/A0CJU+GEKIwfjuORMYOzKOO17fgdPlNjocIYTwi20VjWzcV8/VZ40PaCVsT4oKbAB+mUbicmsqG6UCQ4SAcB2hCp4kxtrb57Lr/75EamIMdW3SwVyIY5VUehIKuRmBmUDikzfakyCRZSRCiMGItZj5nwvzKa1u5t8bDhgdjhBC+MWT6/aQEG3mipljgn5sW1Is08amsGz70Ptg1DS30+XSYdd2oCeSwAhz9vq28J/la1LMz0/n/dJaOpyuvl8gxDBSXOUZbRroJSSpiTGkJkZTIqNUhRCDdMFJGZw5cST3LS+lUW5KCCHCXG1zB29sqeTyU8eQFGvMcv2iQhtbKxqHPOjg6AjV8L5uBElghLVGRxdN7c6I+EZcUGCjpcPJx7vrjA5FiJBSWtVMZnIsyXGB/8OZl5FEabVUYAghBkcpxW+/XEijo4u/rtxpdDhCCDEk//p4P50uN9cEsXnnsRYWZgDwbvHQlpFUSAJDhAJ7mI5Q7cmsnFTiLGZW+GGNlxCRpKSymbzRgV0+4pObYWVndTMut3+6XQshhp+CzCS+cdo4nlm/z2+N54QQItg6nW6e/Xgf5+emMTEt0bA4JqUlMiktYcjLSI5Orgz/60ZJYISxcB2h2pNYi5nZk1N5t9h/o4KECHedTje7alsCvnzEJzfDSnuXm32HW4NyPCFEZPpF0RTios384c0dRocihBCD8ta2SmqbO7h21gSjQ2FhYQYf7a4b0tI8e4OD1MRo4qLNfe8c4iSBEcbsDZFTCgSeZSSVje1sPyhr8IUA2FXbgtOtyQtSAiNfJpEIIfxgVGIMP543mfdKa1ldUmN0OEIIMSBaa55cu4eJaQnMzkk1OhyKCjNwuTWrSgdfqV5R74iIm94gCYywVlHvIM5iZmRCtNGh+MXcvHRMCllGIoSXL5GQH6QlJJNtiZiUTCIRQgzd1WeNZ2JqAne+uYMuGasqhAgjmw40sKWikWvPDv7o1J6cnJWMLSmGZZ8P/hrJXu+IiLYDIAmMsFbhnUCilPE/WP4wKjGGU7NHSAJDCK/iqiYsZsWE1ISgHC/WYmZ8agIlVVIFJYQYmugoE7/+cj67a1t5ev0+o8MRQoh+e3LtXqyxUVw2I/ijU3tiMimKCjJ4f2ct7V0Dn9jodmsqGhxhP7nSRxIYYcze4IiY5SM+8/Nt7KhsOtLfQ4jhrLSqmZx0KxZz8H5V52VYZQmJEMIv5uSmc+6UNP767k4Ot3QYHY4QQvSpqrGdt7dV8o2ZY0mIiTI6nCOKCm04ulysKTs04Nceau2g0+mOmOtGSWCEsYr6yEtgLCiwAbCyWNbMClFS2Ry0/hc+ubYk9tW10dbpDOpxhRCRRynFby7Kp6Xdyex7VjPh9jeZtXgVSzfZjQ5NCDEASzfZmbV4VVB+hoN5rJ48+9E+XFrAYgPLAAAgAElEQVRz9Vnjg3rcvpwxYRTW2CiWD2IaiW+EqvTAEIZqbu+ioa0rYtYy+Uz0jgqSZSRiuGto66SqqT3oCYy80Va0hp3VMv5QCDF02w82YTIp2jpdaDzVo4te3iZJDCHCxNJNdha9vA17gyPgP8PBPFZP2rtc/OuT/czPtzFuVGhdY0VHmZiXl867xdU4B9hXyJfAiJTrxtCpixED4ptAEimZtO7mF9h4fM0emtq7SIq1GB2OEIbwNdIM1ghVH1/CpLSqiWljU4J6bCFE5Ll3WSku9xfHozu6XNy7rJRLp2cZFJUQor/uXVaK45i+C44uF4te3sb7O2v9eqx3Pq/q8VjB+n3x2paD1LV2cu3Z4wN+rMEoKsxg6eaDbNxXz5kTR/X7dXZfBUaEVO5LAiNM2esja4Rqd0UFNh55fzfvldbylVMyjQ5HCEOUVHoaaQZrAonP2BHxxEebKa6UPhhCiKE76L3h0t/tQojQ0tvPqqPLxaf76v16rGOTFz72BgcvbjzA+blppFtj/XpMH601S9buJddm5axJ/U8OBNN5U9KIjjKxfHv1gBIYFfVtpMRbSAyhnh5DERlfxTAUaaVA3U0bO4JRCdGs2FEtCQwxbJVWNzMi3kK6NSaoxzWZFFNs0shThBel1BPAl4EarfVJ3m2/B24AfLcI/0dr/Zb3uUXAdwEXcIvWepl3+wXA/YAZ+KfWenEwv45IlJkSd6Rq9NjtQojQ19vPcFZKHB/cNsevx5q1eFWPxzIpuO2lrQBMzUpmTm4a5+elc8qYFMx+GnP6yZ46dlQ2cfdlU0N2wmNCTBSzc1JZvqOK33w5v99xRtrgB+mBEaYq6tuIiTKRmhhtdCh+ZzYp5uWn815pDZ1OmR0vhqfiymZyM6yG/BHNy7BSUtWE1rrvnYUIDUuAC3rY/het9TTvw5e8KAC+CRR6X/M3pZRZKWUGHgYuBAqAb3n3FUNw68Jc4izmL2yLjjJx68JcgyISQgzErQtzOTZHEGcxB+RnuKffF3EWM3++4hTeumU2ty7MJdZi4qHV5Vz2t3XM/MMKfvL8Jl7dbKe+tXNIx16ybi8p8RYunRbaS9uKCm1U1DvYUdn/kfcV9Y6IajsgFRhhyu6d5RuqGcKhmp9v48WNFXyyp45zJqcaHY4QQeV2a3ZWN/P1mWMNOX5ehpXnNxygtrmD9KTAlGoK4U9a6w+UUuP7ufslwPNa6w5gj1KqHDjd+1y51no3gFLqee++O/wc7rDiW7d+77JSDjY4UAqyR8ZJ/wshwkRGcixuDclxUTQ5nGSmxHHrwtyA/Awf+/vi2GMVZCZx05wcGto6WVN2iNWlNbxfWsvSzQcxKZg2NoU5uenMyUunMDOp39dJFfVtLNtexY3nTiIu2tz3Cww0P9+GSW1j+fZqCjOT+9xfa4293sF5U9KCEF1wSAIjTHlGqEbe8hGf2ZPTiLWYeLe4WhIYYtg5UN9GW6cr6BNIfHIzPH03SqqaJYEhwt3NSqmrgY3Az7XW9UAW8FG3fSq82wAOHLP9jJ7eVCl1I3AjwLhx4/wdc8S5dHrWkQuQf67ZzR/eLGbj3jpmjh9pcGRCiL48sLKMNGsMa26bQ6wl8Bf33X9f9CYlPpqLT8nk4lMycbs1W+2NrC6p4b3SGv68Yid/XrGTdGsM5+emMSc3nVmTU084GOCZj/ahlOKqs7L9/eX43ajEGGZmj2T5jmp+umBKn/vXtXbi6HJFVAWGLCEJU54ERuR8Ix4rLtrMOTlprNhRLWXsYtjxNdDMC3IDTx9f4qSkqv/liUKEoL8Dk4BpQCXwZ+/2nm7J6RNsP36j1o9qrWdqrWempUXOXa1g+PYZ4xiVEM39K8uMDkUI0YcNe+tYt+sw3zt3YlCSF4NhMimmjU3hpwum8OrN57DhV/P58xWncPqEkbzzeRU/eO4zZtyxgm8+up5H3t/FzurmI9cWSzfZOevulTzy/m4sZsWGPXUGfzX9U1Roo7iyiQN1bX3uWxGBgx+kAiMMtXU6qWvtjKhMWk8WFHhmHRdXNlOQacyFnBBGKK1qRimYYks05PgjEqKxJcUcGeUqRDjSWlf7PlZKPQa84f20Aui+PmsMcND7cW/bhZ/ER0dxw7kTWfx2CZ/tr2fGuBFGhySE6MUDK8tITYzmyjNCvzLBJ80aw9dOHcPXTh2D0+Xms/0NrC6tYXVJDXe/XcLdb5eQlRLH+FFxbNjbQKfL02+vvcvNope3AYT8Ereiggz+8GYxy7ZXcf3siSfc19cUNZIq96UCIwxF8gjV7ubm2VAKVuyo7ntnISJISVUT2SPjiY82Lsecm5Ekk0hEWFNKje726VeBz70fvwZ8UykVo5SaAEwGPgE2AJOVUhOUUtF4Gn2+FsyYh4urzsxmRLyFB6UKQ4iQ9dn+etaUHeKG2RNDvi9Eb6LMJk6fMJJfXpDHOz85l/WL5nL3ZVMpzExi3a66I8kLH0eXi3uXlRoUbf+NGxVPXoaV5dv7vkaqqPdUaWRF0HWjJDDCUCSPUO0uzRrD9LEprCiuMjoUIYKqtMozgcRI+RlWympacLpkEpAIfUqpfwPrgVylVIVS6rvAPUqpbUqprcAc4KcAWuvtwIt4mnO+A9yktXZprZ3AzcAyoBh40buv8LOEmCiunz2R1aW1bK1oMDocIUQPHlhZxsiEaP7fmeFTfdGX0clxfOv0cTx69cxe9znYwxjXUFRUmMHGfXUcauk44X72egfW2CiS43rvARJuJIERhnyZtEivwABYUJDB5/YmKhvD45eJEEPl6HSx53AreRnGLpvKzbDS6XSz93CroXEI0R9a629prUdrrS1a6zFa68e11ldpradqrU/WWn9Fa13Zbf+7tNaTtNa5Wuu3u21/S2s9xfvcXcZ8NcPD1Wdlkxxn4QGpwhAi5Gw50MB7pbVcP3sCCTGR2XEgs5el+L1tDzULC224NawqrjnhfpE2QhUkgRGWKhocRJtNpCXGGB1KwC0osAHwriwjEcNEWU0zWkP+aGMrMHwVIL6GokII4U/WWAvXnzOBd4tr+NzeaHQ4QohuHlxVRkq8havPGm90KAFz68Jc4o5pTBpnMXPrwlyDIhqYgtFJZKXEsWz7iSvV7Q2RN7lSEhhhqKLeQdaIOEym/s02DmeT0hKYkJrAij6yi0JEihJvwiDX4AqMnPREzCYlfTCEEAFzzazxWGOjeHCVVGEIESo+tzfybnEN158zgcQIrb4AT6POuy+bSlZKHArISonj7sumhnwDTx+lFEWFNtaUH6K1w9njPlrriJxcGbnflREsEr8Re6OUYkGBjSfX7qG5vQvrCWY4CxEJSqqaibOYGTfS2Gx5TJSZiakJMolECBEwSbEWrps1gftXllFc2US+QaOjhRBHPbCyjKTYKK4+e7zRoQTcpdOzwiZh0ZOFhRk8uXYvH+ys5cKpo497vtHRRUuHM+KuG6UCIwzZI3At04nMz7fR5dK8v7PW6FCECLiSqiam2DzVD0bLzbBSUtVkdBhCiAh23awJWGOkCkNEjqWb7MxavIoJt7/JrMWrWLrJbnRI/bbjYBPLd1Rz3TkTSJKbhiFvZvYIRsRbel1GUhGhkyslgRFm2rtcHGrpiLhvxBM51fvDKX0wRKTTWlNS1Wx4A0+f/NFJVNQ7aOmlNFEIIYYqOd7Cd2aN561tVbJkTYS9pZvsLHp5G/YGBxpP/4FFL28LmyTGg6vKsMZEce3ZE4wORfRDlNnEvHwbK0tq6OphapwvgZGVIj0whIGGywjV7swmxdw8G6t6+eEUIlLUtnRQ19pp+AhVn1ybJw65qBBCBNJ1syaQEG3modXlRocixJDcu6wUR5frC9scXS7uXVZqUET9V1rVzNufV3HtrPEkx0v1RbhYWJhBc7uTj3fXHfecvUEqMEQI8I1QzYqwb8S+LCiw0dTuZMPe4384hYgUvgaeeQZPIPHxxSHLSIQQgTQiIZqrzx7PG1sPUl4jCVMRvg56Lxj7uz2UPLiqjIRoM9edI9UX4WT25FTiLOYel5FU1LcRH20mJcISUpLACDORmknry7lTUomOMrFClpGICOardAiVJSRZKXFYY6KkAkMIEXA3zJ5InMXMQ6ukCkOEp4a2zl77V2WGeO+68ppm3txWyTVnjyclPtrocMQAxFrMnDcljRU7qnG7tWdjZSWcdx5New4wZkQcShnfV82fJIERZirqHVjMinRrrNGhBFV8dBTn5KTybnE1WmujwxEiIIqrmki3xjAyITROHpRSTMmwHqkMEUKIQBmZEM1VZ2bz2paD7K5tMTocIQakw+nixmc+RWtNdNQXL6/MJsWtC3MNiqx/HlxVTpzFzPWzJxodihiEokIbVU3tbLU3ejbccQesWcOcFx+JyMEPksAIMxX1DjJT4kJiQkGwLSiwcaDOQWm1XEyJyFRa1UxeiI0RzPNOIpHEoRAi0K6fPZHoKBMPr95ldChC9JvWmtv/u41P9tRx3zemcc/XTiYrJQ4FJMZE4XJrkuNCt4R/V20Lr285yFVnZYfMDRQxMHPz0jGbFMu3V3mqLx57DLRm4fo3yKPN6PD8ThIYYcZe3xaRmbT+mJeXDsCK7bKMREQep8tNWU0LeSHSwNMnL8NKU7uTqqZ2o0MRQkS4NGsMV56RzdLNdvYdbjU6nJASzqM5I91f3i3jlU12flE0hUumZXHp9CzW3j6XPYsv4tPfzCcvw8pt/91KXWun0aH26OHV5URHmbhBqi/CVkp8NGdOHMnyHdVw3XXg8jSSdQMXvfpPY4MLAElghJmKesew63/hk54Uy7SxKbxbLAkMEXn2Hm6l0+kOuQRGrrcfhywjEUIEw/fOnUiUSfGwTCQ5ItxHc0aylz6t4IGVZXx95hhumpNz3PMxUWbu+/o0Gto6+fXSbSFXzbj3UCuvbj7I/zsjm9TEGKPDEUNQVJBB4qaN6HfeObItxu0k/53/wi9/CV1dBkbnX5LACCPtXS5qmjuG1QjVYy0osLGlopFquRssIkyxN0EQKiNUfXzxlEgjTyFEEKQnxfKt08fx8md2DtRFXunzYITzaM5Itm7XIRa9vJVZOaO466tTe22UWJCZxE8XTOGtbVUs3RxaSaeHV5cTZVLceJ5UX4S7BQU2Hnr1j8dtNzm74J574NJLoS0yfqdKAiOM+EYwDdclJOD54QSkCsMI3o7GVB0/pimsjxUiSqqaMJsUOemJRofyBclxFjKTY2WUqhAiaL5/3iRMSvG396QXBoT3aM5IVV7TzPee+ZTxoxL425WnYjGf+JLqe+dOYmb2CH776vaQ+f92oK6NlzfZ+fYZ44bdcIBIlJkShysxkWPTaMrlgjFj4O23oagIGhoMic+fJIERRobrCNXuJqcnkj0qXsapGuHOO+HDDz3/BtoddwTvWCGitKqZiakJxESZjQ7lOHmjk4bVKFVZay6EsTKSY/nGaWN56dMDR859hrPRKT1fXIb6aM5IVdvcwXee3EBMlJknrz2tXw06zSbFn79+Ci635hf/2XJ03KWBHl5djtmk+P55k4wORQzVtm3gcvH6s8sY/8s3qG508IfXt5P7q7fQbjccOAAvvgiffOK5QVhZaXTEQyIJjDBSUe9NYIwcvktIlFLMz7exrvwwLR1Oo8MZPior4cknwe2Gf/4T7r8fHn0UHnwQ/vxnz3aAV16Bn/0MbroJrr8erroKrr766PvcdRfMmgUzZ8LUqZCbC6eddvT5r38doqLgH//wvOcTTwybKoziytCbQOKTm2FlV20LnU630aEEnKw1FyI0/OB8z0XV39+TXhgXFGYct81iDv3RnJHI0eni+qc3cqilg8evmTmgZd3ZoxL4zZcLWLfrME+t3xuwGPujor6Nlz6t4FunjcWWJNUXYW3nTs+59S9/SZH3d8WKHdXYGxxkjYg7urTp8svhzTehpcXzCGOSwAgjFfVtRJkUNuvwbrKzoMBGp8vNmp21RocyfNx559EkRWcn/OQn8L3vwS23wC9+cbQx0Jo1ntFNL7zgKVVbuxY2bz76PiYTxMWBzQY5OTB9+hcTGF/+Mpx8Mpi9VQjt7fCrXwXnazRQU3sX9gZHyDXw9MnLsNLl0uw+FN5/8PpD1poLERoyU+K4YuZYXtxQQWXj8K3C0Frz8Z46UhMsZKbEooCYKBNOl2aEjLwMKrdb89MXNrO1ooEHvjmdU8amDPg9vnnaWOblpbP47RLKa4yrbPz7e7swKcX3z5fqi7DmcMAVV0B0NPz4x0xOT2RCagLLtld5Bz8ck2BbsABKSmDyZNAa9u83Ju4hkgRGmFi6yc4TH+7F6dacd+97w/pu4MzsEaTEW2QZSbD4qi86u43/io2Fzz6D2lpobPT84gS47z5oboZDh8Buh927YevWo69btAjefdeTAX7lFXj+efjb344+v2ABFBcfGf8EwHPPRXwVxk7v8ozQTWB4KkOGwzISWWsuROj4wXmTcGvNP4ZxL4yVxTVsP9jELy/MZ93t87yjOReQNzqJm577jOJK6U8ULHe/Xcw726v49UUFR+50D5RSiru/NpX4aDM/fWELXa7gVzYebHDw4sYDXDFzDKOTZRlSWLvlFs959jPPwNixKKUoKrCxftdh9hxq7bntgMW75GnxYs9Nw7VrgxuzH0gCIwz4Spp9dwWHe0lzlNnE3Nx0VpXW4DTgF/+w0736wse3lCQ1FZKSoJfO2345ltYR3wvDN+EjVJeQTExLwGJWw2ISSW9rymWtuRDBN3ZkPJefOoZ/bzgwLKePaa15YFUZY0fGcen0rCPbE2OieOI7M0mIMXPdkg3D8r9NsD3z0T4eW7OHa87K5rpZ44f0XunWWP7vq1PZZm/kwVXBXyL1j/c9CcEfSPVFeHvmGc+5+KJFcOGFRzbHWsw43ZqWDidvbDnY+/Xit78N6emem4dvvRWkoP1DEhhhQEqaj7egwEZDWxcb99UbHUrkW7/+i9UX4Pl83brgHeuVV+D3v/f/8UJESVUT1tgoMpNDcx2qxWxiUloiJcPgTl9PJ3SxUSZZay6EQX54fg4ut+aR93cbHUrQvVday9aKRm46P+e4KRejk+N44jun0eTo4rolG2iVvmABs7qkht+9+jnz8tL57cWFvY5LHYgLp47msulZPLy6nM0HgjcVoqqxnec/OcDlp44ZUP8OEYKmTIErr/Q0vvdausnOox8crVhranf2ftM7O9vTMD8vDy65BP71r2BE7ReSwAgDUtJ8vNlT0og2m3hXlpEE3qZNniqIYx+bNgXnWG63J7P8v/8LTz3l/2OGgNKqZvIyrH45KQqUvAzrsFhC4ruTmW6NOTKKbNrYlC/c/RRCBM+4UfF8dXoWz328j5rm4VNpoLXm/pVlZKXEcdmMMT3uU5iZzEPfnkFxZRO3/HsTrhCYbBFpth9s5OZ/fUb+6CQe+NZ0zCb//Z3+/SWF2Kwx/OyFzTg6XX2/wA8e+WAXLq354fk5QTmeCABfpfIZZ8Czz3qa33t5bnp/sZL5hDe909Phvfc8TUCvvRYqKgIUtH8FLIGhlBqrlFqtlCpWSm1XSv3Yu32kUmqFUqrM++8I73allHpAKVWulNqqlJrR7b2u8e5fppS6ptv2U5VS27yveUCF8tn/EIzu5a7scC5pToyJ4uycUaworkZr+YMd0ZTyTCWZNw9uuAFWrzY6Ir/SWlNS2UxuiPa/8MnNSOJgYzuNbV1GhxIwjY4ulqzdy4UnZfDJr+azZ/FFXDtrPJ/sraO8JvIbmAoRqm6ak0OXy81jHwyfKow1ZYfYfKCBH86ZRHRU76frc/LS+d9LTmJlSQ13vL5dzon8qLLRwXVLNpAcZ+GJ75xGQkxU3y8agKRYC3+64hR2H2pl8dvFfn3vntQ0tfOvj/dz2fQsxg7jiYZhTWtPouEXv/B8fIxB3fROSoJ33oHly2FMz8nSUBPICgwn8HOtdT5wJnCTUqoAuB1YqbWeDKz0fg5wITDZ+7gR+Dt4Eh7A74AzgNOB3/mSHt59buz2ugsC+PUY5vQJI4/bFmcxD/uS5vn5NvYdbqNMLiwin8UCL73k6Zr81a96Gn1GCHuDg+YO55FGmaEqb7QnwVJaHblVGM+s30tzh5Ob5hy9M3XznBzio6P40zBesieE0SakJnDptCye/Wg/h1o6jA4n4HzVF6OTY7n81L4vKK46M5sbZk/gqfX7eGLt3sAHOAy0dDi5bslGWjtcPHHtaQEbNXp2TirXzfL8v1tTFtjpeo9+sBunW3PzXKm+CFtPPAFPPw1Wa4/95wbdxys2Fs47z/PxCy/Aj398fE+6EBKwBIbWulJr/Zn342agGMgCLgF8deBPAZd6P74EeFp7fASkKKVGAwuBFVrrOq11PbACuMD7XJLWer32pJuf7vZeEaOmuZ13i2vIz7CSlRKHArJS4rj7sqnDvqR5fr4NQKaRDBcpKZ4mQ4mJsGGD0dH4TWmITyDx8cVXWhWZfTDaOp08/uEe5uSmcVJW8pHtoxJjuGH2RN7ZXsVn+6XnjhBGuWluDu1OF/9cs8foUAJu3a7DfLqvnh+cP4mYKHO/XrPownwuKMzgD2/uYNn2yJ7cFWhOl5ubnvuMndXN/O3KGQG/wXDbBbnkpCdy63+2BqzK8VBLB89+vI9LpmWSPSohIMcQAbZlC9x8M8yfD7/+dY+73LowlzjLF39nDPim9+bN8MADcNVV0BWaVbdB6YGhlBoPTAc+Bmxa60rwJDmAdO9uWcCBbi+r8G470faKHrZHlHvfKaXD6eLhK2ew9va57Fl8EWtvnzvskxcAGcmxnDwmWRIYw0l2tmd+9dVXGx2J3/gme0wJ8QRGRlIsSbFRFEdoH4x/fbyf+rYubp47+bjnrp89gdTEaBa/XSLl2UIYZFJaIhefnMnT6/dS19rZ5/7h7P6VZdiSYvj6zLH9fo3JpPjLN6Zx8pgUfvz8JrYEsTFkJNFa87vXtvP+zlr+cOlJnDslbXBvVFnpuaPdjzHwsRYzf/n6NA61dPCbVz8f3PH68NgHu+l0ur9QYSjCSFMTXHEFjBgBzz0H5p4Tm5dOz+Luy6YO7ab3//0f3H23p6nnpZdCW5t/vgY/CngCQymVCPwX+InW+kS37nrqX6EHsb2nGG5USm1USm2srQ1seZY/bT7QwH8+reC6cyYwMS3R6HBC0oJ8G5sPNAyrxl7DXqL3Z+HNNz2JDFdwGl8FSklVM2NGxJEUazE6lBNSSpE3OikiG3m2d7l45IPdnD1pFKdmjzju+YSYKG6ZN5lP9tTxXmn4/A0RItL8aG4Oji4Xj38Yub0wPtp9mE/21PH98yYRa+lf9YVPXLSZf149k9TEGL771EYO1IXehUeoe2zNbp77eD/fP28S3zp93ODf6M47PRMe+jkGfuqYZG6ZN5nXthzk9S0HB3/cHhxu6eDp9fu4+JRMJsn1RHj69FNPMuz55z2NN0/g0ulZQ7vprRTcfjs88gi8/TYUFUF7aF1nBTSBoZSy4ElePKe1ftm7udq7/APvvzXe7RVA91TzGOBgH9vH9LD9OFrrR7XWM7XWM9PSBplJDTK325MBTrPG8KMe7ggKjwWFnmUkK4tr+thTRJzycs8M7NtuMzqSISmpbAr55SM+vkkkkVaF8J9PK6ht7uDmE9yZ+uZp48geFc8f3ynBLZ3+hTDEZJuVL00dzVPr9tHQFplVGA+sLCM1MWbQF89p1hiWXHsanU4X1y3ZQKMjNEvAQ9Hb2yr5v7dKuOjk0dw2lD5zr77qaT7udsOTT/arCgPgh+dPYtrYFH699PMjE7H84fEP99DudPEj6X0RvubMgb174dxzg3fMG2+EF1/0VBLFBqYHzGAFcgqJAh4HirXW93V76jXAN0nkGuDVbtuv9k4jORNo9C4xWQYUKaVGeJt3FgHLvM81K6XO9B7r6m7vFfb++1kFWw40sOjCPBL93PU4kuTarIwZESfLSIajH/8YfvQjuO8++NvfjI5mUDqcLnYfag35Bp4+eRlJtHQ4qaiPnBHOXS43/3hvFzPGpXDWpFG97hcdZeLnRbmUVDXz6pYe5qkLIYLiR3NzaOlwRmSzyg1761i36zDfP2/igKsvustJt/KPq05l7+FWfvjcp3Q6Q7cZX6j4bH89P3lhMzPGpfDnK07BNJhxqW43LF7sKbv3Jfpdrn5XYUSZTdz39VPocLq49aWtfrlZ0NDWyVPr9nLR1NHkpIfHzRLRzaefwlPe1pEjjx/qEHCXXw533eX5eMsW2LUr+DH0IJAVGLOAq4C5SqnN3seXgMXAAqVUGbDA+znAW8BuoBx4DPghgNa6DrgT2OB93OHdBvAD4J/e1+wC3g7g1xM0Te1d/PGdUmaMS+HSadLr4kSUUiwosPFh+SHaOp1GhyOC7S9/gYsv9iQy3nzT6GgGrLymBZdbh/wIVZ/cI408I2cZySub7NgbHPxo7mT6msT95amjKcxM4k/LdtLhDO+lS0KEq7yMJC4ozODJtXsirrrAU30RzZVnZA/5vc6elMrdl53M2vLD/OqVbRFXOedP+w+3ccNTG7ElxfLY1TMHlzyqrYWLLoJFi8DU7fKqsxMefRT+/vd+vc3EtET+50v5fLCzlmc/3j/wOI7x+Id7aO10STV3OGpo8PS9+M1voMXgiYtuN3z723DOObB1q7GxENgpJB9qrZXW+mSt9TTv4y2t9WGt9Tyt9WTvv3Xe/bXW+iat9SSt9VSt9cZu7/WE1jrH+3iy2/aNWuuTvK+5WUfIb+cHV5ZxuLWD33+lcHAZ4GFmQb6NTqebD3YeMjoUEWxms6fJ0LRpsGKF0dEMWEmlJxGQPzq8EhglETKJxOXW/P29XRRmJnF+bt/LC00mxe0X5mFvcPDcR0M/sRRCDM6P5uXQ3O5kSQRVYXy6r541ZYe4YfZE4qIHX33R3eWnjuGWeZP5z6cVPLy63C/vGWka2zru1t4AACAASURBVLq4dsknON2aJ689jVGJMYN7o9JS+OADmD0boo6pnHa54Ic/hGuv7deF6FVnZjN7cir/92Yxew61Di4ePF/bkrV7+dLUjLC5UWKIATRcDRqtPd8vBw54lnEkGty7xGSC//zHc9597rmwdq2x4Rh6dHGc8poWnly7l2/MHMvJY1KMDicsnDZhJEmxUbxbLMtIhqXERHjvPU81RpgprW4mOsrE+DAZaZYYE8XYkXFHJqeEuze3VbLnUCs3z8nps/rCZ/bkNGbljOKh1eU0t0fW3V8hwkVhZjILCmw8/uHuiPk5fGBlGSMTovl/Zw69+qK7n86fzFenZ/Gn5Tt5dbMsf+uu0+nme89uZH9dG49ederAG1y63bB6tefjc86BffugudlTddGd1mCzeZYCzJjhWRZwAkop7r38FKKjTPz0hc04XYNbAvTkuj00dzi5eY5UX5zQ//zPgBquBsVf/wpLl8K998KZZxodjUdBgSdxYbPBggXw1luGhSIJjBCiteZ/X99OXLSZXwyledAwYzGbmJuXzqqSGlzSXG94slo9XZOLiz1rT5vCo0KguLKJyemJRJnD51dxri0yJpG43ZqHV5WTk57IwsKMAb32lxfkUdfayWMfRO4kBCFC3S1zJ9PU7uTp9fuMDmXINh9o4P2dtVw/ewIJfu57ppRi8demcvqEkdz6n618sqeu7xcNA1prbv/vVj7aXcc9l5/MGRN774HUo5oauOACmDsXPvvMsy01FTZt8iQsjn1UVcGqVZ6RlHPnepYHnEBGcix3XnoSmw808Pf3Bt53oKm9iyc+3ENRgY2CzPDosxV0XV2eZT9LlniSUY89BrtD4O/6vn3wy1/CV7/q6fcWSrKzYc0ayM/3LIvS2pAKlvA5ax4G3i2uYU3ZIX46fwqpgy1hG6bmF9ioa+3ks/31RocijFRRAW+8Ad/4BjhDvydKaVVz2JV15o+2svtQa9j3gHi3uJrS6mZumjNpwEv1Th6TwkUnj+afH+6REc5CGGTqmGTm5qXz2JrdtHSE/u/7E3lwZRkp8RauPmt8QN4/JsrMo1edypiRcdz4zEZ21xq8nj4E3L+yjJc32fnp/Cl8dfqYvl/Q3XvveZaurlnj6W8xfXr/Xnf++Z5GiC+8ACneKuvGxl53/8opmVx8Sib3ryzjc3vv+/XkqbV7aWp3css8qb74ApcLPvrI87HFAjt3epZFgCehUVh4tKrGKNnZ8NJL8MQTnptzoSY93fPf6PnnPfH9/vdBr2CRBEaIaO9ycecbO5icnshVZ/m3fHA4OG9KGhazkmkkw92CBZ7RZe+8AzfffLQLeAiqa+2kprmD/DCZQOKTm2HF5daU14TvCbDWmodWlzNuZDwXn5w5qPf4RVEunU43D66UdeVCGOWWeZNpaOvimTCuwvjc3sjKkhq+O2tCQKfOpcRH8+R3TsOkFNct2UBda2SOoe2Plz+r4K/vlvG1GWO4Zd4AR4suXgzz5kFSEnz8Mdxww8AuMkeN8lRugKeH15QpnnOWXtx5SSGjEqP56Qubae/q342Dlg4n//xwD/Pz0zkpK7n/sUWy5ma4/36YPPnocp/KSs+NL1e3/66dnZ7/R+CpxmgdfA+SAXO7YccOz8df+crRJFcoSkqChAQoL/dUrgxwZPBQSQIjRDz+4R7217Xxu4sLsYRROXmosMZaOHPiKFbsqJZO28Pd9dfD7bfDI4/An/5kdDS98jXCzAuTBp4+eb5GnpXhu4zkg7JDbK1o5IfnTxr08p0JqQl88/Sx/PuT/ewdQpM1IcTgTRubwnlT0nhsze6wnUR2/8oykmKjuGbW+IAfK3tUAo9dPZODje3c8PTGfl8QR4Klm+zMWryK8be/yc9e3EJOWgJ3Xza13/2PjkhK8kxj2LgRTj55aEGdcoqnn8CFF8LPfw4dHcftkhIfzT2Xn0JZTQv3Livt19s+vX4vjY4umTwCnukwt90GY8fCT34CmZmeZpRjxngqBtzH9BeJivKcPwJccw2MHw9//GNwpoDcc4+nsmfbtsAfy1/+9KejU3cGMDJ4qORKOQRUNjp4aFU5CwttnDM51ehwwlZRgY09h1rZVSsXE8PeXXd5lpG8/nrILiXxJQDCbQnJ+FEJREeZKK0O3wTGw6vKGZ0cy2UzBlg2fIxb5k3GYjbx5xU7/RSZEGKgbpk3mbrWzrCcDLT9YCMrdlRz3TkTSIq1BOWYp2aP4C9fn8an++r5xX+24B4GvcOWbrKz6OVt2BscR7ZVNDh4a1tl/95g5Up45RXPxz/4ATz9tH+mQhQWeqo4broJ7rsPzjrLM83kGOdNSeOqM7N5/MM9rNt14ol7rR1OHvtgN+fnpnHK2BC+gx9oDu//644OePBBWLjQs3Tkww89vSXMZli//viGq52dsG6d5+N77oFTT/XcFBs/3lN90xygc5/334df/Qq+9jU46aTAHMPfKis9jWl9FSydnUGrwpAERghY/HYJLq359UUFRocS1ubl2wBkGYnwZIOXLIHly48fZxYiSquaGZUQTVqY9buJMpuYnJ4YtpNIPt59mE/21vG9cycSHTW0P4Hp1li+e84EXt9ykG0VA1ufLITwj1OzRzB7ciqPfLALR2d4VRQ8tKoca0wU1549IajHvejk0dx+YR5vbK3kT8v7d1c/nN27rBTHMdUm7V3uvisaXC747W89y1P/+EfPslSl/NuXIC4OHnoIXn0V9u+HrVt73G3Rl/KYkJrAL17cQtMJJu88+9E+6tu6hmfvC7fb0wdtzhy4+GLPtjFjwG739B0544wv7t9bw9VNmzzPn3WWZ3nP+vVw2mmwaJHnAv3/s3ff8VFV6R/HPyeFFAiEQOgl9N47SBMVxV1FVOyKumLX3bW336q49lVXRREbtrWAgA1BESnSQ1Gkd0gBEnpIT87vjzuBAAmQZDIt3/frlRfJvTP3njMzzDzz3HOe4267d8NVV0Hz5k5NFV+se1GUokaweGgUhhIYXrZ02z6+WZnEbQOa0jAm0tvN8Wv1oiNoX7+qllMVR3i487N/P4wc6RuVpQtZt+sQrepElXz4qg9oVSeKdcn+sdLLid78dRM1q1Tiyp6N3HK80QObUj0ylBdnrHPL8USk5O4Z0oLUtGz+t8R/RmGs23WIH//cxah+cVSL9Mzoi8JuHdCUq3o24q3Zm/nCjx630kgqNPLiTLY7O5OcWhdjxsD11zujMMrz8/qii5x6Apdf7vz944/HrVQSWSmEV0Z2YvfhLJ76dk2Rh8jIzmP83C30b1GTro2ql19bfU1GhvPFv107J3GxaZMzLadgSnlMTNmO37u383wsWuRMUwb46it49tmyr3qXlwfXXOPEqhMnOqvq+YvTjWApR0pgeFFevuVf36ymXrVwbh9UwiJCUqRz29Rh+Y79pBw+eR6hVFCpqU7gMWwY7PON5ePy8i3rdx+mtZ8V8CzQpk5V9hzO8rsicCt3HmDexlRu6d+U8NBgtxyzangodw5uzryNqfy28dRDe0WkfPSIi6FP0xqMm7PZb+o6vDFrE5UrBXPzWZ4dfVHAGMOYi9sxoGUsj039k3kbU7zSDk+IqVypyO31oiOKvsOePU4tgqVLndGcEyY4BQvLW0HRxr17nURG587HfRns0qg6dw5qxtfLE5j+58nD9D9bvJ29R7K5t6KNvnjnHbj1VoiMhM8+cy5Y3Xef+xNOvXo554BjUz7i4pxpy2VJZPTrB2PHlr2miqedbgRLOVICw4u+WLqDNcmHePTCNkRUck8wXdGd07YW1sKv6/Z4uyniK1q0gKlTYetWZ95jEUWyPG3HvnQyc/KPFsT0NwV1OwoKkfqLN2dtolpEKNf0du9KT9f2bkz96AhemL6uQswnF/FF9wxpQcrhLL5cutPbTTmtjbsPM21VMjf0jSM6sugv154QEhzE2Ku70KJWFe74dDnr/XRq4KkcSM8mOzePE7/KRoQG88DQVkXfqVYtp+Dj0qVOIUdPq1HDufASFAQDBjijQFx1Bu4e0oL29avy6JRVxy3jnZmTx7g5W+jbrAbd48o44qA4yckwcKDHVpoo1po1zuovn3/u/H3jjc7StvHxToHVUA+MaBo7FpYscZIPjz/uJDI++KBkx8jPd2pxPPUU3HRTuTQzUCmB4SUH0rN5ecZ6ejWJ4cIOdb3dnIDRtm5V6kdH8JPqYEhh/fs78xbnznWG/3l5pZqC6Rf+tgJJgYJ2+1OwuybpEDPX7uamcliqMDw0mH+e25JViQeZ9ucZFoUTEbfq06wGPZvE8PbszWTl+vYojDd/3UREaDB/69/U200hKjyUD0b1IDIsmBs/XMKeQ5mnv5MfeXzqn2Tk5PPP81pSPzoCA9SPjuC5ER0Y3qX+sRsmJMB558HKlc7fjz4Kbb1Ym65XL+dK9siRx+pw5OYSGhzEqyM7k5aVyyNfrzq68t7nS3aQmpZVvqMvxoxximB6aKWJ41gLM2c6o2nbtXNGWux0JSurVXMSK56ektujh1MsfulSZ2nWqq5RtWlpcPA0dbGSkqBLF5g/v/zbGYB8s7pdBfDqzxs4mJHDkxe188s58L7KGMM5bWrxZfxOMrLzNLJFjrn6amdY4fjxztWDut5LHK7bdRhjoEUt/0xgxFYJI6ZyJb9KYIydvYkqYSGM6htXLscf3qU+4+du4eUZ6xnaro6WwxbxgnuHtOCa9xbT89+/cCgjh3rRETwwtNXxX1S9bHNKGt/9nsQt/ZsWO7XB0+pFR/D+DT0Y+c5CRry1gDxr2XUw0zOPX3IyXHmlU2SxTh23HvqblYl8/0cy95/XkrvOblH8sqI//gjXXQeZmbB9uzN1wxdUq+Z8UR861Cnw6SpK3qJ2FA+d35ox36+hy9M/czAjB2OgWc3K9Gpao3zasnUrvPeeM2pg/HioX9/56dzZWQ42Lw/WrnVqOERFOau0VCrD6/vE18U11zgjLmrXdhIot90GNX1k5cbu3eHbb4/9/eqrzqoyf/873HvvsalBBXJznaKdmzaVvT5HBaUIywvW7TrEJ4u2c23vxrSp659z4H3ZuW3rkJmTz2+bNB/dXQrWT2/y8A/0e34WU1ckertJpfPYY/D7715NXoDzHtCkRmW/TbAZY2hVO4q1fpLA2JySxrRVyVzXp3G5FcsLDjI8dEErtu1N5ws/GMIuEoj2HMrEGDiYkYMFEg9k8MjkVT71mTV21iYqhQRxywDvj74orH39alzTqxEJBzJIPpjpucevnK7qJx/M4Impf9KlUTS3DWxW9I1ycpwlMocNc76ML1sGF1/s1naUmTHONJYnnnD+nj0b7riDmiaXIAMHXK/1fAs792eU7blasMAZrfrIIzBihLOc5z//6ex74QXn8QLnC/hjj8GoUU6CAZwilB06OFMpatSAsDDn5z//cfYnJ0Pfvk4y5tJLnfvefbfz3INTo2zCBJg0CWbMcJaWnTfPmZ4Bzhf+99+Hbducbb6SvCjKX/7ijAh58knn8XjyyWMFWZOToWlTZ0TwO+9AmzZebKj/0ggMD7PW8uS3q6kaEco/z23p7eYEpF5NY4gKD+HnNbs4t21tbzfH7xWsn16wBFlBQAP41FWtM2IMVK/ufPjed58TqJx9tsebsX7XYdrW8+/kZeu6UXy5dCf5+ZagIN8eRfbWr5sJCwkq92J5g1vVomdcDP+duZERXepT2c1TVUTk1F7+acNJMwQzcvJ4acZ6n/i82pZ6hKkrE7mpXxNq+uAS2tNWnVzboFwfv+Rk5wtzfj689ZbzhbVZM2jUyBmSX7AiR0qK86U46Myuu+bnWx6c9Ac5eZZXR3YmpLgRcePHO1/Mb73VuWoeUUxRT1+yeDG8/TYdJv9IiwvvZ19EVd789kXuuughUqpUP/VzlZ4OGzfChg3Oz/r1zuiGF1909l95pTMtIzTUeR5atnS+YCcnw0cfHX+s8HAnmdLE9blaubKTzEhLg8OHj/3btauzPzfXuc3Bg850nYL9Xbo4z/XmzU4tixN98gk888yxZVH9QZcuTu21FSvg6aedGherVsHXXzvTmHfudB7Xa6/1dkv9lkZgeNi0VbtYtGUf95/XyquFmwJZaHAQg1rV4pe1e8hTQb0yK2r9dCeg8eNlI9PTnbmUI0Y4WXBPFaVKTiav/wCO7EikVe1yTmCUc7Gt1nWiSM/OI2ntFo8+fiU918596UxdmcjVPRuX7AtDKc5ljOGhC1qTmpbFB79tLddzlZqvFGETKQelWi7Tg8b+uonQ4CBGD/St0RcFPP74jRnjJC/ASU4cPOgMq//wQ/j+e2e7tc5V7PBw50v14MHOiISvvz62f906OHLk6GE/XbydeRtTeezCNsTVLLR6SMH734YNzt+jR8O0aTBunH8kLwAeeghmzCAq7QDffvQP3vzmeXrsXM3dC5yClrv3pTmP4bRp8NprzgoZBc4915nyMXKkM4rh11+dldoKTJzoJDjS053pIN984xTLLPw8FcjPh48/dgqegvP4jRzpFKO8915nhMbzzzvPF0DDhvDzz85SpKtXO1N19u07VryyUydnmu/vv8Pw4UenywDeqbnhDl26wJQpTiLjqaec19+MGc6+bdv0OVwGSmB4UEZ2Hv/+YQ1t6lblqp6NvN2cgHZu29rsPZLNyp37vd0Uv1dc4JJ4IJMXpq9jVcLBo0Wk/EbVqvDDD05AdOGFzlWfp5929lnrfDCf+FPQx7LsHzOGoPnzufu3z2ldp0r59rGci20VLAGb/7QHi3qVok/j5mwm2BhGl3S4dikfv26Nq3Ne29q8M3dL8cvM5uc7q+EcOeIE7E8+6Zzrqaec5fNO/ClYOScnp+j9BeuwZ2cXvb9g2G9WllMMzltF2ETKWXHLYha7XKYH7dibzuQViVzdqxG1osK93ZwiefTxKxh9UfD+lZ/vjLT4+WfnffHdd53teXnw0kvOVIaePZ3bz5oFf/zh7N+/37maXaUK1KxJVsfO1L3xau7JWM81vRo5dS0WL3bO9+STzkWLrl2dL+6hoXDBBe7vW3k77zxG/f09ltZvQ6+ENQRhuXzVTJ6d/gZrX7nUWX3twgvhH/9wpikUxCcPPwxffeUUKk1Lc0YCFF45o1cvaN78+OQBwMKFx56nAtnZxy3xWmaVKjmjOWJjYfp0Z8RGwXk+/NC/v+x37uxMxxkzxll1BJzXtT6HS8343RePMurevbuNj4/3yrlf+XkDr/+yka9u7UPPJiraUp4OZuTQbczP3Ny/CY9coPllZdHnuV9IPnhyVfKwkCBy8y15+ZYG1SMY1qEu57evQ+cG0T4/peCoH3905r6CE8js2AG7dxddwOujj+D6650vf/37n7x/8mRnmdbCxyzsiy+cOZ+ZxVR4X7gQevd2golbbnG2FRT4Nca5KtG2Lbz5pjP95cT9GzY4VzhefNEJ0jIKJZ4iI50hm9WrO/NoX3vt5PPv3+8ELf/857HAsUB4uBNYgnPF6vPPsUB6Vi6ROZnO8nQREc7Vk4cfdoKP4OBjP3FxzpWegvsvWeJcbSvY36rVseGpd9/t9KXw/du1g7vucuaNZmY62y6++NgVs44d4cEHnd/vuce5qoMzUuin1bsJ6t2Lv7733LHzp6cf37+BA4895tdf7yQWvvnGCTCCg+Hll51iXJmZztzW3Nzjf26+GW6/3XmMevcmJzuHvQeOUCXEUMXkO8/HPfc4V8Vatjx5FZzQUCfJEBZW9DK/n3ziDDWdN89ZUu9EU6Y4V6ymTXOC1hPNnAlDhjhXGe++22lzwfPlpqJ5xphl1trubjmYn/FmXCHHO3HKY4GBLWvywaieBHvxs+nhr/9g8opE5j04mNpVfTOBUdzjd9+5Lbnb3atb3HGHU9Og8BfjSpWcIfZjx575cY4ccYbrb99O/vbtLP/tD6JTk4l96nGq3fY357PzxM/04GDn/Tguzi1d8YapKxLJ+NtoLlsxg1CbR1ZwCEsbdaDWoL607N/N+axp1cqZeuNPiwW463XhawpqXxSOAd38ORyIiostNEHXQ3buS2fcnM1c1KmekhceUC0ilN5NazBzzW4lMMogKzePyCIKTUaEBvPciA4MbBnLz2t2M+3PZD6cv5Xxc7dQt1o457evw7AOdenWqLpvJzO+++7Yl0dwsuFPPOFcCT9Rp07Ov40aFb2/oBBTixZF7//uu6NDMHNNEMHdumIuvPDYl9kGDY6d59FHjx/RAccKVnXp4iQZTtwf5VrRpGtXJ3D5889jX8BbtHC+HINzhWX06JPbVxDg9Ot3crBT+GrMoEEQFYUBkj6fStNd2wi2+ceuJvTu7ZwrL+/YT+FiW3XrQuPGzmNRsL/w0N2MDOfqW+H7V616/BDWvDznS3lsrPN34Urny5cfvVKTcSSbzpm5xJhWx/bHx8OhQ8f3r2HDY78vXuzcPy/v2Lm++MJJYBjjBB8hIU5SJyTE+Sl47MPCoG9fQkNC2LHzEJv2ZXJR90ZUadfO2R8T4wzbLbhfSIiTKFm61NlvrTMXeOTI49vX3fXZ3awZvP46J+nQwfm3bdui97d01VuaO/fYc1vwfPlzQChygoK5/y/NWE/SgQzqRofTPLYKczakcsdny3jtii5eKZ6csD+dScsSuLpXI59NXsDJj1/tquFk5uTx8aLtXNK1Pg2qR7rvZO66ql+5srNCBfDGzI28Wn0Db17dheYd6zn74+Kcz98XX3SOXfC5+NJLfv3+N7xOEHmrfyHYOp9VYXm59ElaS/Cz0/z7C7EnRnt4Q1HTcPQ5XGoageEht32yjDkbUph1/0DqVvP+UMaKYML8rTz53Rpm3TeQprHlPFw/AOXnW/7x1Uq+WZnEtb0b8eu6FJIOZBS7rNrBjBx+Wbubaat2MXdjCtm5+cRGhXF+uzpc0KEOPeNiii+k5Q2ezIYH4rmSk8lu3IRKOYVGDPhQn/amZdHvhVlc2KEe/xnZqVzPVeRhDmYw6KXZXNixLq+MLGZJvgB6XWgEhkZg+LIPftvKmB/W0KlBNO/d0N3jBTQfnbKKifE7mfPAYJ+YzlISG3cfZsTbC6hbLZxJt/elanj5rORUVn8kHOCStxbwl451+e+VXY7fGYhXvwN1pEKg6tLFmbpzos6dnRoZUqTiYgsf+jYRuH7bmMr01bu46+zmSl540DmuFUhmrt3t5Zb4p5d+Ws83K5N4YGgrnhnegfkPn83W5y9k/sNnF1nhulpEKCO6NuC9G7qz/Ilzef2qLnRvXJ2Jy3Zy9buL6fXsLzwyeRVzN6SQk5dfxBk97FTZcJ3rjM4TZH23T+//tpWs3HzuGFzM8nluPFdR6laLYFTfOKasSGTdrkNF3ygQXxciPuims5ow7tpurNt1iEvems/mlDSPnTvpQAYT43cysntDv0teALSoHcW4a7uxJeUId3y63Dc+v0+QmZPHP75cSWyVMJ6+qP3JNwjE979AHakQqFascEZZnvij5EWpKIFRznLy8nnqu9U0ioks9yX85HgNqkfStm5Vfl6jBEZJfbpoO2/P3szVvRpxx6ASfgEEqoSFcFGnerx9bTeWP3Eub13Tlb7Na/LtykSu/2AJ3Z+ZyQMTf2fWut1k5ead/oDlwZMf/oF4roULCcnNKf/zuM5Vkj4dTM/h44XbGdahLs1KOvrKjY/f7YOaERUWwovT15f7uU5Lwa5UcEPb1eGL0X3IyM5jxFsLWLxlr0fOO27OZqx13g/8Vb/mNXluRAd+25TK41P+9LnC3S9MX8fmlCO8dHlHqkUWMUIkEN//9IVYKjDVwChnnyzczsY9abx7fXfCQz0/77KiO6dtbd6ctZG9aVnU8ME1133RzDW7+b9v/mRI61o8fVE7TBmLP0VWCmFYh7oM61CXzJw85m5I4cc/dzH9z11MXJZAVFgI57StzQXt6zCgZSzT/9x1dP5tcdNV3MKTH/Kuc81ev4dRHy7li9G96d20Rrmeq9ytWMHW1CMMfnk2L17WkZHdG57+PmU4V0l8tHAbaVm53DW4ebmf61SiIytx+6DmvDB9HYu37KXXic+5F16DgcoY8wHwF2CPtba9a1sM8CUQB2wDRlpr9xvnTe2/wDAgHRhlrV3uus8NwOOuwz5jrf3Ik/2Q8tW5YTRT7ujHDR8u4br3l/DS5R25uHM5fL647DqYyRdLdnJZtwburR/hBZd3b8iOfem8MWsTjWpEcmdp3l/LwfxNqXw4fxs39GlM/xaxRd8owN//RCoajcAoR6lpWbw6cwMDWsZyTpta3m5OhXRe29rkW5i1bo+3m+IXft95gLs/X0H7+tV44+oubq9ZER4azHnt6vDqFZ2Jf+IcPhzVg/Pb12HWuj2M/mQZHZ+cwX1frSTxQAYWSDyQwSOTVzF1RaJb2+ENU1ckcs/nThD1jy9XBkSfGsVEEh4axLrkw95uylFpWbl8MH8r57SpRZu6Vb3dHEb1jaN21TCen77O565aBpgJwPknbHsY+MVa2wL4xfU3wAVAC9fPaOBtOJrw+BfQC+gJ/MsYU73cWy4e1TAmksm396VLo2ju/WIlY3/dVG7/N8fN2UyetdwxyDe+7JfVP89tycWd6/HSjPV8+3uSt5vDwYwc7p/4O01jK/OwCraLVBhKYJSjl2esJyM7j//7S9syX8WW0mlXryp1q4VrGskZ2LE3nZs/WkrNqEq8f0MPIiuV7wCtsJBgBreuxUuXdyL+8XP45OaehAQHkXdCHJmRk8dLM4oZgu8nCpamO5TprGuefDAzIBIz3/2eRF6+5YP5W+n3/Cyf6M9ni7ZzID3HZ64ORlQK5h/ntGTFjgP8pPehcmOtnQvsO2HzxUDBCIqPgOGFtn9sHYuAaGNMXWAo8LO1dp+1dj/wMycnRSQAREdW4uObezLc9WX8kcmr3F7bYc+hTD5fsoMRXerTqIZ/j74oYIzhxcs60jMuhvsn/k78thP/y3nWk9+uFUd9uQAAIABJREFUZs/hLF4d2dkrq8uIiHcogVFO/kg4wJfxO7mxXxzNa2kFDG8xxnBOm9rM25hKZo6Xai34gf1Hshn14RJy8y0TbuxJbJRnp9uEBgfRv0UsGdlFP0eJBzL4Ze1u8vP98wr2SzPWk3HC68/fEzMFSZkcV8bJF0bLZObk8e68rfRvUZMujXznwvll3RrQLLYyL81YT64PFsALYLWttckArn8LhkLWB3YWul2Ca1tx2yUAhYUE8+oVnbn77OZ8sXQnN38Uz+HMnNPf8QyNn7uFnLx8n0mmuktYSDDvXNeN+tER3PJxPNtSj3ilHdNWJTNlRSJ3DW5Op4bRXmmDiHiHEhjlID/f8uS3q6lROYx7hrTwdnMqvHPb1iYjJ4/5m1K93RSflJmTx98+jifhQAbvXd+95EUP3ai4Cu1BBm7+KJ5BL8/mvXlbOJjhviCzPGXm5PHd70kkHsgocn9SMdv9gS8mZb5cupPUtKzS1b4oRyHBQTwwtDWb9qTx9fIEbzdHoKghkfYU208+gDGjjTHxxpj4lJQUtzZOPMcYw33nteKFSzswf1Mql49bSPLBsr8vpxzO4tPF2xneuT5xNSu7oaW+pXrlSnw4qgfGGG6csJT9R7JPfyc32nMok8emrKJjg2rcdbZvvd+LSPlTAqMcTF2ZyPIdB3jw/FZE+eh62RVJr6YxVAkL0XKqRcjLt/zjy5Us37Gf167oTPe4GK+254GhrYg4odhtRGgwL13WkTeu6kKtqDCe+WEtvZ/9hcemrGLjbt+pvVDAWsvSbft4ZPIf9Pj3TO7+fAXBxcwg88cl9QoUl3zxVlImOzefcXM20yOu+snFMn3A0Ha16dIomld/3qjRYJ6z2zU1BNe/BcWQEoDCVWcbAEmn2H4Sa+14a213a2332NhiCgeK37iiRyM+HNWDhP0ZXDJ2AWuSiln6+Ay9N28L2bn53BnAX67jalbm3eu7kXggg9GfxHvsfc1ay0Nf/0F6dh6vjOxMqJtrdYmI79P/ejdLy8rluR/X0alBNS7r2sDbzRGc4Y4DW8Uyc+0ev52CUF7+/cNafvxzF48Na8OwDnW93RyGd6nPcyM6UD86AgPUj47guREduLRbQ/7aqR6Tbu/L93efxYUd6zJxWQLnvjqXq99dxE+rd5Hn5ed25750/jtzI4Nens3l4xYydUUS57apzWd/68XLl3UqMjHzwNBWXmpt2RWXfLHAKz9v8PiX9MnLE0g+mMldZ/vmqDdjDA+d35pdhzL5aME2bzenovgWuMH1+w3AN4W2X28cvYGDrikmM4DzjDHVXcU7z3NtkwpgQMtYJt3eB2Pg8nELmLOhdCNr9qZl8fHC7fy1Uz2vjmj0hG6NY3hlZCeWbtvPg5P+8EiM9fmSnfy6PoWHL2itKdoiFZSWUXWzN2ZtJOVwFuOv60ZQkAp3+orqkaGkHM6i2aPTyndpTj/y/m9b+WD+Vm7sF8ff+jf1dnOOGt6l/imfm/b1q/Hy5Z145ILWfLF0J58u2s7oT5bRoHoE1/VuzBU9GhIdWckjbT2cmcOPq3YxaXkCS7Y6xcz6NK3B3We34IL2dagcduwt1gQZzywP6yEPDG3FI5NXHTeNJDwkiHb1qvL6Lxv5/vcknrmkPX2b1Sz3tuTm5fP2nM10bFCNAS3K/3yl1btpDQa3imXsr5u4skcjqkVqhJ67GGM+BwYBNY0xCTiriTwPfGWMuRnYAVzuuvk0nCVUN+Eso3ojgLV2nzFmDLDUdbunrbXerVIoHtW6TlWm3tmPGz9cyk0TlvLM8PZc1bNRiY7x/m9byczN87mpbOXlLx3rsWNfOi9OX0+jmEjuL8fE/Pa9R3jmhzX0a16DG/rEldt5RMS3mYq2rFv37t1tfHx8uRx7S0oaQ1+by8Wd6/Py5Z3K5RxSclNXJPLw5D/IzDlWPC8iNJjnRnTw6y+QZfHjqmTu+N9yzmtbm7eu6UawHyfbcvPy+WnNbiYs2MaSrfsIDw3iki71uaFvHK3ruH8Zzbx8y/xNqXy9PIEZq3eRmZNPk5qVubSrk3hpUD0wqs2fiakrEotMyszbmMLjU/9k+950Lu3agMcubENM5fJLKk1dkcjfv1zJO9d1Y2i7OuV2HndYm3yIYa/P49YBzXj4gtbebo7bGGOWWWu7e7sd3lCecYV4R1pWLnf9bzmz16dwx6Bm3H9eqzO6KLX/SDZnvTCLQa1rMfbqrh5oqW+w1vLI5FV8sXQnL17akZE9Gp7+TiWUl28Z+c5CNuw+zIy/D/DrKZgicmaKiy00AsONxny/hrCQYB4833+HhQeil2asPy55AU6xwed/XFchExjLtu/j3i9X0qVhNP+9sotfJy/AKZA4rENdhnWoy5qkQ3y8cBuTlyfy+ZKd9GoSw4394jinTW1CyjhPduPuw0xansDUFYnsPpRF1fAQLu3agEu7NaBLw+gKuVRycaNl+reIZcbfB/DGrI28M2cLs9bt5tFhbbisWwO3P075+Zaxv26iVe0ozm1T263HLg9t6lZleOf6fDh/Kzf0bUzdagrCRXxNlbAQ3ru+O//37Wremr2ZhP0ZvHR5R8JCTr1U5wfzt3IkO497fHQqW3kxxjBmeHsSD2Tw6JRV1IuO4Cw3j4Z7Z+5mlm136nUpeSFSsWkEhpvMWrebmybE89iwNtwywHeG4ws0efiHosvIA61qRzGgZU36t4ilZ5MYwkMDex3xLSlpXPr2AqIjK/H17X3L9aq4N+0/ks2X8Tv5ZOF2Eg9kUK9aONf2acyVPRqVqM/7jmTz7cpEJq9I5I+EgwQHGQa1jOXSbg04u3WtgH+9uMOG3Yd5ZPIqlm3fT++mMfz7kg5unRc+/c9kbvt0Oa9f1YWLOtVz23HL08596Qz5zxxGdK3P85d29HZz3EIjMDQCIxBZa3ln7hae/3EdPeNieOe6blQv5jPkYHoOZ70wi7Na1OTta7t5uKW+4VBmDpe/vZCkAxl8fUdfWtaOcstxVycdZPjY+ZzXtg5vXt2lQl4wEKmIiostlMBwg6zcPIa+OpegIMP0ewdQKUS1UX1Jv+dnFbmMZdXwEDo0qMbSrfvJzsunUkgQvZrE0L+Fk9BoXScqoD4kU9OyGPHWAo5k5TL5jr40rhF4S7udKDcvn5lr9/DRgm0s3LKXsJAgLu5cjxv6xtGuXrUip0AM61CXWev2MHl5Ar+u30NOnqVt3apc2q0BF3WqR2xUmLe75Xfy8y1fLN3J8z+uJTMnnzsGN+P2Qc1OezXzdKy1/OWN30jPzmPmPwf61Wiip75bzUcLtvHTPwYGRCE6JTCUwAhk3/2exH0Tf6dBdAQTbuxJoxonTxV8beYGXpu5kWn39KdtPfdPX/QXiQcyGD52PpWCg5hyZ19qRYWX6XiZOXlc/OZ89qVn89PfBxSbQBKRwKMEhos7A42CLz8FX45HD2jCo8PauuXY4j5TVySeVGywcA2MjOw8Fm/dy7yNqczdkMLGPWkAxEaF0b95Tfq3rMlZzWP9+otrenYuV41fxPrdh/lidB86N4z2dpM8bv2uw3y0cBtTlieSkZNHk5qRJOzPICfv2HtgcJAhLNiQnpNPzSphXNKlHiO6NqBN3YobjLrTnsOZPPP9Wr79PYmmsZX59/AO9GlW+iVPf123hxsnLOXFyzoysrv751yXp71pWQx8aTb9mtfgnev8/3u/EhhKYAS6pdv2ccvH8QQbw7s3dKdro+pH9x3KzOGs52fRu2kNxl9fIf8bHGdVwkFGvrOQFrWr8MXo3kRWKv2M9WenrWX83C18eGMPBreq5cZWioivUwLDxV2Bxum+FItvKa7YYFGSD2Ywb2Mq8zam8tvGFPan5wDQtm5V+resyYAWsXRrXN1vpg/k5uVz26fLmLVuD+9c151z2/p+nYDydDA9h6/id/L89HVFLr0aERrEW9d0o3+LmmWumyFFm7MhhcenrmLnvgwu79aAR4e1KfFVNWstl769gN2Hspj9wCBC/fC5ev2Xjbzy8wZiq4SRmpbl16vTKIGhBEZFsDX1CKM+XMKug5n898rOnN/eWX78jV828p+fN/D93WfRvn41L7fSN8xcs5vRn8QzpE1txl1bumLhi7fs5cp3F3FVz0Y8e0mHcmiliPgyJTBc3BVoFDctoX50BPMfPrvMxxffkJ9vWZ10iLkbU5i3MYVl2/eTk2cJDw2iV5Ma9G9RkwEtY2lRq4pPTjex1vLEN3/y6aIdjLm4Hddp2bGjiquNYoCtz1/o6eZUOBnZebw+ayPvzt1C1YhQHhvWhhFd65/x/6MFm1O5+t3FjBnenut6Ny7n1paPL5fs4KHJq47b5q+JcCUwlMCoKPamZXHLx/Gs2HmAizvVY/HWfSQfzCQ8JIjnL+3od/93y9OE+Vt58rs13NSvCf/315KNUD6cmcMF/51HcJBh2j39j1uWXEQqBq1C4mZJRSQvTrVd/FNQkKFDg2p0aFCNOwc350hWLou37mXuhlTmbkzhmR/Wwg9rqVM13Kmd0TKWs5rX9JnimOPmbOHTRTu4dWBTJS9OUC86osgkpKqbe0ZEpWAeOr81F3eux6OTV3HfxN/5enkCzwxvT9MzKPL55qxN1IoK4/JuDTzQ2vLx+qxNJ23LyMnj6e/X0DS2MnWqhlOjSphf1fYQCXQ1qoTxv1t6c8U7C5i6Muno9szcfB5xJSSVxHCM6teE7fvS+WD+VhrXiOSGvnFnfN8x368h6UAGE2/ro+SFiBxH7wilpC8/FVPlsBDObl2bs1s70zAS9qfzm2u6yU9rdjNxWQLGQPt61Y4WA+3WuLpXCrt+szKRF6av46+d6vHQ0NYeP7+ve2BoqyKngT0wVMsge1LrOlWZdFtf/rdkBy9MX8f5/53HXYObc+vApsUW+Vy2fT8LNu/l8Qvb+M1UrqIUl/DedySbi96cDzh1WWpHhVG7Wjh1qoZTp4h/a1cNP6PHoSRT6USkeOGhwaSkZZ+0PSMnj5dmrNf/q0Iev7AtCfszeOq71TSoHsGQM1ju+qfVu/gqPoE7BjWjW+MYD7RSRPyJEhilpC8/AtCgeiRX9mzElT0bkZdvWZV4kHkbUpi3MZXxc7fw1uzNRFYKpnfTY9NNmtasXO7TTRZu3sv9E3+nV5MYXr68I0G6gnuSggBTX+i8LyjIcG3vxpzXtjZPf7+GV37ewDcrE3n2kg70anpykc+xv26iemQoV/dq5IXWuk9xifDYqDD+Pbw9uw9lknwwk12HMtl9KJP1uw8zd0MKR7LzTrpP9chQalcNp261Y0mNuq5/61QLZ/n2/Yz5fg0ZOfmAs1KArhaLlF7ygcwit2sk7vGCgwz/vbIzV45fxF3/W8HE2/qcsk5IaloWj0xeRdu6Vfn7OS092FIR8ReqgVEGupolp3I4M4dFW/Yxb2MKczeksG1vOuDUSSkYndGveQ2iI9073WTD7sNc+vYC6lQNZ9JtfakWGerW44uUt1/X7+GJqX+SsD+Dkd0b8MgFx4p8/pl4kL+88Rv3n9eSu85u4eWWlk1pi0Efzsw5ltw4mHn0992HnGTHroNZpKZlnVEb3FW3STUwVAOjolEttJLZcziTS8YuICcvn6l39ityxLK1ltGfLGPO+hS+u/ssWtWJ8kJLRcRXqIiniwIN8ZYde9OZtymFeRtSmb85lcOZuRgDHRtEM8CV0OjSKLpMqynsPpTJJWPnk5NvmXJHXxpUP3mtehF/kJGdx2u/bOC9eVupFhHKBe1rM3t9CokHMjHAsyPac1VP/yzeWVh5JcKzc/PZc/hYcuOu/60o8nbuKlqrBIbiiopGq9GV3Ibdh7n0rQXUrx7BxNv6EBV+/AWWifE7eWDSHzw2rA23DGjqpVaKiK9QAsNFgYb4gty8fH5POMi8jc50kxU79pNvoUpYCH2a1Tia0GhcI/KMp5ukZeUyctxCtu89wpe3nnqIpoi/WJt8iFs/iWfHvuOvdOqLQsmU99ViJTAUV1REGolbcr9tTGXUh0vo06wGH4zqcfSizc596Vzw33m0q1eVz2/pramvIqIERgEFGuKLDmbksHBzKnM3pjJ3QwoJ+50vGg1jIujfIpYBLWrSp1lNqkUcf7WicPBUKSSI7Nx8JtzUk4EtY73RDZFy0ff5X0gqYr65hmqfufK+WqwEhuIKkTP11dKdPPj1H/RpGsOOfekkHcgkNDgIg2XmfYNoGKPRoyKiZVRFfFq1iFDOb1+X89vXxVrL9r3pTu2Mjal8uzKJ/y3eQZCBzg2jGdAylv4tYtmemsZjU1cf/UKSlZtPaLBh/5GTK6OL+DMVyys7Fa0VEV8xskdDfl6zi5/X7jm6LTvPiWGWbd+vBIaInJISGCI+xhhDXM3KxNWszHV94sjJy2flzgPM25DCnI2p/PeXjbw2cyMGOHH8VE6e1RJuEnC0bLV7DO9SX+8NIuITVicfOmmbYhgROROlrxYoIh4RGhxEj7gY/nleK765sx8rnjiXsVd3PSl5UUBXpSXQPDC0FRGhwcdt07LVIiL+SyPrRKS0lMAQ8TPRkZW4sGNd6hdz9VlXpSXQDO9Sn+dGdKB+dAQGp/aFCniKiPiv4mIVxTAicjqaQiLipx4Y2qrIony6Ki2BSNMfREQCh2IYESktJTBE/JSK8omIiIg/UgwjIqWlBIaIH9NVaREREfFHimFEpDRUA0NEREREREREfJ7fJzCMMecbY9YbYzYZYx72dntERERERERExP38OoFhjAkGxgIXAG2Bq4wxbb3bKhERERERERFxN79OYAA9gU3W2i3W2mzgC+BiL7dJRERERERERNzM3xMY9YGdhf5OcG0TERERERERkQDi7wkMU8Q2e9KNjBltjIk3xsSnpKR4oFkiIiIiIiIi4k7+nsBIABoW+rsBkHTijay146213a213WNjYz3WOBERERERERFxD39PYCwFWhhjmhhjKgFXAt96uU0iIiIiIiIi4mYh3m5AWVhrc40xdwEzgGDgA2vtai83S0RERERERETczFh7UsmIgGaMSQG2u/mwNYFUNx/TFwRivwKxTxCY/QrEPkFg9isQ+wTqV0k0ttZWyDma5RRXQGC+/gKxTxCY/QrEPkFg9isQ+wTqlz8prz4VGVtUuARGeTDGxFtru3u7He4WiP0KxD5BYPYrEPsEgdmvQOwTqF/iXYH4PAVinyAw+xWIfYLA7Fcg9gnUL3/i6T75ew0MEREREREREakAlMAQEREREREREZ+nBIZ7jPd2A8pJIPYrEPsEgdmvQOwTBGa/ArFPoH6JdwXi8xSIfYLA7Fcg9gkCs1+B2CdQv/yJR/ukGhgiIiIiIiIi4vM0AkNEREREREREfJ4SGGfAGPOBMWaPMebPIvbdb4yxxpiarr+NMeZ1Y8wmY8wfxpiunm/x6ZWwT9e4+vKHMWaBMaaT51t8ZkrSr0Lbexhj8owxl3mupWeupH0yxgwyxqw0xqw2xszxbGvPXAlfg9WMMd8ZY3539etGz7f49IrqkzHmSWNMous5WWmMGVZo3yOu94r1xpih3mn16ZWkX8aYc40xy4wxq1z/nu29lp9aSZ8v1/5Gxpg0Y8z9nm/x6ZXiNdjRGLPQ9f9qlTEm3Dstr1gCMa6AwIwtAjGuAMUWrr8VW3hRIMYWgRhXgO/FFkpgnJkJwPknbjTGNATOBXYU2nwB0ML1Mxp42wPtK40JnHmftgIDrbUdgTH49tytCZx5vzDGBAMvADM80bhSmsAZ9skYEw28BVxkrW0HXO6hNpbGBM78uboTWGOt7QQMAv5jjKnkgTaW1ASK6BPwqrW2s+tnGoAxpi1wJdDOdZ+3XK9HXzSBM+wXzjrgf7XWdgBuAD7xUBtLYwJn3q+j+4Afy71lpTeBM38NhgCfAre53i8GATmeamgFN4HAiysgMGOLCQReXAGKLUCxhbdNIPBiiwkEXlwBPhZbKIFxBqy1c4F9Rex6FXgQKFxI5GLgY+tYBEQbY+p6oJklUpI+WWsXWGv3u/5cBDQo/xaWTgmfK4C7ga+BPeXctFIrYZ+uBiZba3e47hso/bJAlDHGAFVc98st90aW0Cn6VJSLgS+stVnW2q3AJqBnuTWuDErSL2vtCmttkuvP1UC4MSas3BpXBiV8vjDGDAe24PTLJ5WwT+cBf1hrf3fdd6+1Nq/cGidHBWJcAYEZWwRiXAGKLQpujmILrwnE2CIQ4wrwvdhCCYxSMsZcBCQWPDmF1Ad2Fvo7wbXN552iT4XdjO9nCY9TXL+MMfWBS4BxXmlYGZziuWoJVDfGzHYNsbveC80rtVP0602gDZAErALutdbme7p9ZXCXa5j0B8aY6q5tfvteUUhR/SrsUmCFtTbL0w0ro5P6ZYypDDwEPOXdppVaUc9VS8AaY2YYY5YbYx70ZgMrukCMKyAwY4tAjCtAsQWKLXxFIMYWgRhXgJdiCyUwSsEYEwk8BvxfUbuL2ObzS72cpk8FtxmME2Q85Kl2ldVp+vUa8JC/XXE8TZ9CgG7AhcBQ4AljTEsPNq/UTtOvocBKoB7QGXjTGFPVg80ri7eBZjjtTgb+49rul+8VhRTXLwCMMe1whlHf6vmmlUlx/XoKZ6hkmrcaVgbF9SkEOAu4xvXvJcaYIV5pYQUXiHEFBGZsEYhxBSi2QLGFrwjE2CIQ4wrwYmwR4s6DVSDNgCbA786oMxoAy40xPXEynQ0L3bYBTmbX1xXbJ2vtLmNMR+A94AJr7V4vtrOkTvVcdQe+cG2vCQwzxuRaa6d6q7Fn6HSvv1Rr7RHgiDFmLtAJ2OCtxpbAqfp1I/C8ddZ93mSM2Qq0BpZ4q7Fnylq7u+B3Y8y7wPeuP/31vQI4Zb8wxjQApgDXW2s3e6F5pXaKfvUCLjPGvAhEA/nGmExr7ZteaGaJnOY1OMdam+raNw3oCvzi8UZKIMYVEJixRSDGFaDYQrGFDwjE2CIQ4wrwbmyhERilYK1dZa2tZa2Ns9bG4TxRXa21u4BvgeuNozdw0Fqb7M32nolT9ckY0wiYDFxnrfWHD6ujTtUva22TQtsnAXf4Q5BxmtffN0B/Y0yI66pDL2CtF5t7xk7Trx3AEABjTG2gFc6cQZ9njp+rfglQUMH5W+BKY0yYMaYJToE+nw+aChTXL+MUe/sBeMRaO98bbSuL4vplre1f6LX5GvCsvwQZp3gNzgA6GmMijVN0ayCwxtPtk8CMKyAwY4tAjCtAsYViC98QiLFFIMYV4N3YQiMwzoAx5nOcCqo1jTEJwL+ste8Xc/NpwDCcojnpONldn1PCPv0fUAOnkjFArrW2u0caWkIl7JdfKEmfrLVrjTHTgT+AfOA9a+1JS4n5ghI+V2OACcaYVTjDIx8qyOz6kqL6BAwyxnTGGcK5DdewR2vtamPMVzhv6rnAnb467Lgk/QLuAprjDDF+wrXtPOuDRd9K2C+/UMLX4H5jzCvAUte+adbaH7zR7oomEOMKCMzYIhDjClBs4aLYwosCMbYIxLgCfC+2MM6oKRERERERERER36UpJCIiIiIiIiLi85TAEBERERERERGfpwSGiIiIiIiIiPg8JTBERERERERExOcpgSEiIiIiIiIiPk8JDBHxS8aY2cYYn1tyT0RERPyP4goR/6AEhoiIiIiIiIj4PCUwRMQjjDEPGmPucf3+qjFmluv3IcaYT40x5xljFhpjlhtjJhpjqrj2dzPGzDHGLDPGzDDG1D3huEHGmI+MMc94vlciIiLiDYorRComJTBExFPmAv1dv3cHqhhjQoGzgFXA48A51tquQDzwT9f+N4DLrLXdgA+Afxc6ZgjwGbDBWvu4Z7ohIiIiPkBxhUgFFOLtBohIhbEM6GaMiQKygOU4AUd/4FugLTDfGANQCVgItALaAz+7tgcDyYWO+Q7wlbW2cPAhIiIigU9xhUgFpASGiHiEtTbHGLMNuBFYAPwBDAaaAVuBn621VxW+jzGmA7DaWtunmMMuAAYbY/5jrc0st8aLiIiIT1FcIVIxaQqJiHjSXOB+17/zgNuAlcAioJ8xpjmAMSbSGNMSWA/EGmP6uLaHGmPaFTre+8A0YKIxRglZERGRikVxhUgFowSGiHjSPKAusNBauxvIBOZZa1OAUcDnxpg/cAKP1tbabOAy4AVjzO84QUnfwge01r6CM2z0E2OM3tNEREQqDsUVIhWMsdZ6uw0iIiIiIiIiIqekrKKIiIiIiIiI+DwlMERERERERETE5ymBISIiIiIiIiI+TwkMEREREREREfF5SmCIiIiIiIiIiM9TAkNEREREREREfJ4SGCIiIiIiIiLi85TAEBERERERERGfpwSGiIiIiIiIiPg8JTBERERERERExOcpgSEiIiIiIiIiPk8JDBERERERERHxeUpgiIjPMsZsM8acU1HPLyIiImVjjJltjPlbRT2/SKBRAkOkAjPGPGmM+dSD54szxlhjTFqhnyc8dX53MsYMNsb8aow5aIzZ5u32iIiIeIMXYonexpifjTH7jDEpxpiJxpi6njq/Oxlj2htjZhhjUo0xtoj9dxlj4o0xWcaYCV5ooojPUQJDRErNGBNSyrtGW2uruH7GeLktpXUE+AB4wMPnFRERCRil+PyuDowH4oDGwGHgQy+1paxygK+Am4vZnwQ8gxNviAhKYIhUCMaYh4wxicaYw8aY9caYIcaY84FHgStcIyF+d922njHmW9eVjU3GmFsKHedJY8wkY8ynxphDwChjTJAx5mFjzGZjzF5jzFfGmBg3Nr+HMWaNMWa/MeZDY0y4qy2DjDEJrr7tAj40xlQ3xnzvuiKz3/V7g0Ltn22MGWOMme96LH4yxtQstP86Y8x2Vz8eO1WjrLVLrLWfAFvc2FcRERGf5CuxhLX2R2vtRGvtIWttOvAm0O80zW9mjFniGjX5TcGxC40MvdkYswOY5do+0Rizy3X7ucaYdoXaP8EYM9YY84PrsVgUmFk6AAAgAElEQVRsjGlWaP+5xph1rvu+CZjiGmWtXW+tfR9YXcz+ydbaqcDe0/RPpMJQAkMkwBljWgF3AT2stVHAUGCbtXY68CzwpWskRCfXXT4HEoB6wGXAs8aYIYUOeTEwCYgGPgPuAYYDA1332Q+MPU2ztruSDx8WTiAU4xpXm5sBLYHHC+2rA8TgXIEZjfOe9qHr70ZABk5gU9jVwI1ALaAScD+AMaYt8DZwnasfNYAGiIiIVHA+GksUGEAxCYBCrgduch07F3j9hP0DgTaufgH8CLTAiRWWu9pY2FXAUzijQTYB/wZwxTRf48QqNYHNnD65IiIloASGSODLA8KAtsaYUGvtNmvt5qJuaIxpCJwFPGStzbTWrgTew/lSX2ChtXaqtTbfWpsB3Ao8Zq1NsNZmAU8ClxUzDDMV6IGTYOgGRHFyUHCiN621O621+3AChKsK7csH/mWtzbLWZlhr91prv7bWpltrD7tuP/CE431ord3gavtXQGfX9suA7621c139eMJ1fBERkYrOl2KJwufqCPwfp5/O+Ym19k9r7RGcz/eRxpjgQvuftNYecbUFa+0H1trDhdrSyRhTrdDtJ7tGYubixDEFscQwYI21dpK1Ngd4Ddh1mraJSAkogSES4Ky1m4C/43wA7zHGfGGMqVfMzesB+1xf/gtsB+oX+nvnCfdpDEwxxhwwxhwA1uIEOrWLaEuatTbeWptrrd2NczXnPGNM1VN0ofD5trvaWCDFWptZ8IcxJtIY845rGsghYC4QfUKQUjiQSAequH6vV/hcriBHQzZFRKTC86VYooAxpjnOSIl7rbXzTtOFE2OJUJwREiftN8YEG2Oed01nOQRsc+0qfPszjSUsJ/dVRMpACQyRCsBa+z9r7Vk4AYIFXijYdcJNk4AYY0xUoW2NgMTChzvhPjuBC6y10YV+wq21iZxewbGKnR8KNDyhLUmnaMt9QCugl7W2Ks6w0tMdv0By4XMZYyJxppGIiIhUeL4USxhjGgMzgTGuelSnc2IskYMzKrSo9lyNM8XlHKAaTrFQKF0sYU44t4iUkRIYIgHOGNPKGHO2MSYMyMSpC5Hn2r0biDPGBAFYa3cCC4DnjDHhrqGZN3PqaR7jgH+7ggmMMbHGmIuLaUsvV3uCjDE1cOagzrbWHjzF8e80xjRwFdx6FPjyFLeNcvXvgOv2/zrFbU80CfiLMeYsY0wl4GlO8R7p6kM4zlUc43q8KpXgfCIiIn7Bx2KJ+jjFNsdaa8edYReuNca0dV2ceBqYZK3NK+a2UUAWzijMSJwaH2fqB6CdMWaEa/rLPTj1uopkHOE4NblwPV5hhfaHuPYHA8Gu/Z5eKUXEpyiBIRL4woDnca407MIpSPWoa99E1797jTHLXb9fhXO1IQmYglNj4udTHP+/wLfAT8aYw8AioFcxt20KTMdZ8uxPnADhqmJuW+B/wE84q31swVlOrDivARE4fV3kOtcZsdauBu50nS8Zp4BYwinuMgAngJvGsYKhP53p+URERPyIL8USf8OJJ/5lnJVP0owxaadp/yfABFfbw3ESC8X5GGeaSSKwxtWWM2KtTQUux3ms9uIUAp1/irs0xokfCoqQZgDrC+1/3LXtYeBa1++Fi5mLVDjGmZolIiIiIiIiIuK7NAJDRERERERERHyeEhgiIiIiIiIi4vOUwBARERERERERn6cEhoiIiIiIiIj4PCUwRERERERERMTnVbh1hGvWrGnj4uK83QwREZGAsWzZslRrbay32+ENiitERETcr7jYosIlMOLi4oiPj/d2M0RERAKGMWa7t9vgLYorRERE3K+42EJTSERERERERETE5ymBISIiIiIiIiI+TwkMEREREREREfF5Fa4GhoiIVCw5OTkkJCSQmZnp7ab4vfDwcBo0aEBoaKi3myIiIuI1ii3cp6SxhRIYIiIS0BISEoiKiiIuLg5jjLeb47estezdu5eEhASaNGni7eaIiIh4jWIL9yhNbKEpJCIiEtAyMzOpUaOGAowyMsZQo0YNXW0SEZEKT7GFe5QmtlACQ0REAp4CDPfQ4ygiIuLQZ6J7lPRxVAJDRETEh8yePZsFCxaU6RhVqlRxU2tERETE3wVSbKEaGOI1U1ck8tKM9SQdyKBedAQPDG3F8C71vd0sEangvP3eNHv2bKpUqULfvn09dk7xHd5+/YmIiPt5+709kGILjcAQr5i6IpFHJq8i8UAGFkg8kMEjk1cxdUWit5smIhVYeb43DR8+nG7dutGuXTvGjx8PwPTp0+natSudOnViyJAhbNu2jXHjxvHqq6/SuXNn5s2bx6hRo5g0adLR4xRcAUlLS2PIkCF07dqVDh068M0335S5jeJd+mwUEQk8ii3cSyMwxCtemrGejJy847Zl5OTx0oz1utIkIuXmqe9WsybpULH7V+w4QHZe/nHbMnLyeHDSH3y+ZEeR92lbryr/+mu70577gw8+ICYmhoyMDHr06MHFF1/MLbfcwty5c2nSpAn79u0jJiaG2267jSpVqnD//fcD8P777xd5vPDwcKZMmULVqlVJTU2ld+/eXHTRRZqT68f02Sgi4n8UW3iWEhjiFUkHMkq0XUTEE04MME63vSRef/11pkyZAsDOnTsZP348AwYMOLpsWExMTImOZ63l0UcfZe7cuQQFBZGYmMju3bupU6dOmdsq3qHPRhGRwKPYwr2UwBCvqBcdQWIRAVm96AgvtEZEKorTXc3o9/ysIt+b6kdH8OWtfUp93tmzZzNz5kwWLlxIZGQkgwYNolOnTqxfv/609w0JCSE/3wlyrLVkZ2cD8Nlnn5GSksKyZcsIDQ0lLi5OS5z6OX02ioj4H8UWnqUaGOIVDwxtRURo8HHbIkKDeWBoKy+1SESk/N6bDh48SPXq1YmMjGTdunUsWrSIrKws5syZw9atWwHYt28fAFFRURw+fPjofePi4li2bBkA33zzDTk5OUePWatWLUJDQ/n111/Zvn17mdoo3qfPRhGRwKPYwr2UwBCvGN6lPs+N6EBYiPMSDA8N4rkRHTTHV0S8quC9qX50BAbn6og73pvOP/98cnNz6dixI0888QS9e/cmNjaW8ePHM2LECDp16sQVV1wBwF//+lemTJlytNDWLbfcwpw5c+jZsyeLFy+mcuXKAFxzzTXEx8fTvXt3PvvsM1q3bl3W7ouXFbz+YiJDAYitEqbPRhERP6fYwr2MtdbbbfCo7t272/j4eG83Q1zO/s9stqQcoWnNysy6f5C3myMiAWjt2rW0adPG280IGEU9nsaYZdba7l5qkleVR1yxNy2Lbs/M5IGhrbhzcHO3HltERMpOsYV7lSS20AgM8ao9h7IwBrbuPUJGdt7p7yAiIhLgalQJo1XtKBZt2evtpoiIiPgUJTDEa9KycknLyqVzw2ishQ27D5/+TiIiIhVA76YxxG/bT44bqtSLiIgECiUwxGt2HXQq2g5sGQvAul3Fr58sIiJSkfRpVoOMnDz+SDjg7aaIiIj4DCUwxGt2H3ISGD3jYogIDWbdLo3AEBERAejZpAYAi7bs83JLREREfIcSGOI1BQmMOtXCaVUninXJSmCIiIgAxFSuROs6qoMhIiJSmBIY4jW7CiUw2tSNYt2uQ1S0VXFERESK07tpDeK37Sc7V3UwREREQAkM8aLdBzOJCg8hslIIretUZX96DnsOZ3m7WSIiPq9KlSoAJCUlcdlll53ytq+99hrp6eklOv7s2bP5y1/+Uur2iXv0bqo6GCIi4hn+ElsogSFes+tQJnWqhgPQuk4UAGuTVchTRHxAcjIMHAi7dnnslHl5JV9Kul69ekyaNOmUtylNkCG+oVeTGABNIxERCQSKLdxCCQzxml2HsqhTrSCBURVAhTxFxDeMGQO//eb86wbbtm2jdevW3HDDDXTs2JHLLruM9PR04uLiePrppznrrLOYOHEimzdv5vzzz6dbt27079+fdevWAbB161b69OlDjx49eOKJJ447bvv27QEnSLn//vvp0KEDHTt25I033uD1118nKSmJwYMHM3jwYAB++ukn+vTpQ9euXbn88stJS0sDYPr06bRu3ZqzzjqLyZMnu6XfUjbVj9bBUCFPERG/p9jCLf0OcctRREphz6FMmsfWBKBaZCj1qoWzTiMwRKS8DRp08raRI+GOOyA9HYYMgSVLID8fxo2DFStg9GgYNQpSU+HEYZWzZ5/RadevX8/7779Pv379uOmmm3jrrbcACA8P57fffgNgyJAhjBs3jhYtWrB48WLuuOMOZs2axb333svtt9/O9ddfz9ixY4s8/vjx49m6dSsrVqwgJCSEffv2ERMTwyuvvMKvv/5KzZo1SU1N5ZlnnmHmzJlUrlyZF154gVdeeYUHH3yQW265hVmzZtG8eXOuuOKKM3wwpbz1aVaDz5fsICs3j7CQYG83R0REiqLYwmOxhUZgiFfk5Vv2HM6iTrWwo9ta1YnSCAwR8b7t26GgoLC1zt9u0LBhQ/r16wfAtddeezSwKPhAT0tLY8GCBVz+/+zdeXjc5Xnv//etfRtJ1jpCNtiyNgMBQwwIrIQGynaSBtJf0oYuoT20JGna9IRenMKv5zS/k/zS0tKeNE0TUhJISEpIkxxKaLNQAgkJjk1iMITF1uINbzOWN4220TbP+WO+Y2RbsraZ+c5In9d1zaWZZ77LPbYv+/E9z3Pf73sf69ev54Mf/CCHDh0CYNOmTdx6660A/O7v/u601//hD3/Ihz70IfLy4t9NVFVVnXHMli1beP3119m4cSPr16/n4YcfZu/evezYsYM1a9bQ0tKCmfE7v/M7SfnMsngdTdVEx2P8cn+/36GIiMhCaW6RtLmFVmCIL44OjjIZcydrYAC0N5Tz054jjE3EKMhTbk1EUuRs32r098Px46dOMo4fhxtvjL+uqZnztyKnM7NpX5eWlgIQi8WorKzkpZdemtP5p3POzemY6667jkcfffSU8ZdeemnWc8UfV6ypwgy27DzKZavPnDiKiEgG0NwibXML/S9RfJFooVo/NYERDDARc+w6MuhXWCKy3H3yk/HlnVNNTiZlv+obb7zB5s2bAXj00Ufp7Ow85f3y8nLWrFnDt771LSA+IXj55ZcB2LhxI9/4xjcAeOSRR6a9/vXXX88XvvAFJiYmADh2LF43IRAIMDAQX93W0dHBpk2b6O3tBWB4eJju7m7a29vZvXs3O3fuPBmfZIbKkgLag+Vs2a1CniIiWUlzi5PxJYMSGOKLcCTeLnVqAmNdg1fI85C2kYiITzZvhrGxU8fGxuBnP1v0pdetW8fDDz/MRRddxLFjx/jwhz98xjGPPPIIDz74IBdffDEXXHAB3/nOdwD4zGc+w+c+9zkuu+wy+vun30rwB3/wB5x77rlcdNFFXHzxxXz9618H4I477uCmm27iHe94B7W1tXzlK1/h1ltv5aKLLqKjo4MdO3ZQVFTEAw88wDvf+U46Ozs577zzFv15JXmubKpm657jjE7Mv5q8iIj4THOLpM4tzCWWsiwTGzZscFu3bvU7jGXva1v28j8ff5Xn/99rTyYxxidjXPCXT/L7nau556Z1PkcoIkvF9u3bWbfO379T9uzZw7ve9S5effVVX+NIhul+Pc3sBefcBp9C8lU65hX/+VqIO772At/84JVcvkbbSERE/Ka5RXLNZ26hFRjii3B/lNwco6bszSKe+bk5NNeVaQWGiIhMy8weMrPDZvbqlLH7zGyHmf3SzP7NzCqnvHePmfWaWZeZ3TBl/EZvrNfM7p4yvsbMnjezHjP7VzMrSN+nm9kVa6rjdTB2aRuJiIgsb0pgiC9CkSi1ZYXk5pxa2KW9IcCOkFqpisjSsnr16iXxDUkG+Apw42ljTwEXOucuArqBewDM7Hzg/cAF3jmfN7NcM8sFPgfcBJwP3OodC/A3wKedcy3AceD21H6cuakoyef8hnI271QCQ0RE4pbr3EIJDPFFOBKlvqLojPF1wXLCkVGODY1Nc5aIiCxnzrmfAMdOG/tP59yE93ILsNJ7fjPwDefcqHNuN9ALXO49ep1zu5xzY8A3gJstXir9GuDb3vkPA7ek9APNQ0dTNS++cZzouOpgiIjI8qUEhvgiHIlSHyg8Y7y9IQCgVRgiklTLrd5TqmTBr+N/Bb7vPW8E9k15b783NtN4NXBiSjIkMZ4ROpqqGZ2I8fK+E36HIiIiZMW/iVlhvr+OSmCIL0L9UYLTrMBoC3oJDNXBEJEkKSoq4ujRo5poLJJzjqNHj1JUdObf3ZnAzP4CmAASveCmaz7vFjA+3b3uMLOtZra1r69vIeHO2+Wrq7w6GMdmP1hERFJKc4vkWMjcIi+F8YhMa2Rskkh04pQWqgm1ZYVUlxbQFVICQ0SSY+XKlezfv590/UdzKSsqKmLlypWzH5hmZnYb8C7gWvfmbHI/sGrKYSuBg97z6caPAJVmluetwph6/Cmccw8AD0C8C0myPsfZVJTkc8E55WzedYQ/pSUdtxQRkRlobpE8851bKIEhaReKRAEITpPAMDMV8hSRpMrPz2fNmjV+hyEpYmY3An8OXO2cG57y1hPA183sfwPnAC3Az4mvtGgxszXAAeKFPn/LOefM7EfAe4nXxbgN+E76PsnsOtZU89Ute4mOT1KUn+t3OCIiy5bmFv7RFhJJu7CXwJhuBQZAe7CcrvAAkzEtyRIRkTeZ2aPAZqDNzPab2e3APwEB4Ckze8nMvgDgnHsN+CbwOvAD4CPOuUlvdcUfA08C24FvesdCPBFyp5n1Eq+J8WAaP96sOpqqGZuI8ZLqYIiIyDKlFRiSdokERrDizCKeAO3BANHxGHuPDtFUW5bO0EREJIM5526dZnjGJINz7lPAp6YZ/x7wvWnGdxHvUpKRLltTRY7B5p1H6Wiq9jscERGRtNMKDEm7UP/ZV2CsaygHYIfqYIiIiJxUUZzPBedUsGXXUb9DERER8YUSGJJ2oUiU0oJcAkX5077fXFdGjsGOQ6qDISIiMlVHUxXb9p0gOj7pdygiIiJppwSGpN3hyCj107RQTSjKz6WptoztWoEhIiJyikQdjBffOO53KCIiImmnBIakXSgSpT5w9l6/7UF1IhERETldog7Gll3H/A5FREQk7ZTAkLQL9UcJnmUFBsQTGPuOjTA4OpGmqERERDJfeVE+FzaqDoaIiCxPKU1gmNnHzOw1M3vVzB41syIzW2Nmz5tZj5n9q5kVeMcWeq97vfdXT7nOPd54l5ndMGX8Rm+s18zuTuVnkeSIxRyHB6IzFvBMaA/GC3l2aRuJiIjIKTqaqnnpDdXBEBGR5SdlCQwzawQ+Cmxwzl0I5ALvB/4G+LRzrgU4DtzunXI7cNw51wx82jsOMzvfO+8C4Ebg82aWa2a5wOeAm4DzgVu9YyWDHRseY3zSESyfvoVqQntDAEDbSERERE5zZVM1Y5MxXtyrOhgiIrK8pHoLSR5QbGZ5QAlwCLgG+Lb3/sPALd7zm73XeO9fa2bmjX/DOTfqnNsN9BLv0X450Ouc2+WcGwO+4R0rGSwcOXsL1YTGymIChXnsOKQVGCIiIlNtWL3Cq4OhbSQiIrK8pCyB4Zw7APwd8AbxxEU/8AJwwjmXKGywH2j0njcC+7xzJ7zjq6eOn3bOTOOSwU4mMGapgWFmtDeokKeIiMjpAkX5vKWxQoU8RURk2UnlFpIVxFdErAHOAUqJb/c4nUucMsN78x2fLpY7zGyrmW3t6+ubLXRJoVD/KADBWVZgQLwOxo5DAzg37W+riIjIstWxtppt+44zMqY6GCIisnykcgvJrwK7nXN9zrlx4DHgKqDS21ICsBI46D3fD6wC8N6vAI5NHT/tnJnGz+Cce8A5t8E5t6G2tjYZn00WKBSJYga1gbPXwIB4HYyB0QkOnBhJQ2QiIiLZo6OpmvFJx4tvqA6GiIgsH6lMYLwBdJhZiVfL4lrgdeBHwHu9Y24DvuM9f8J7jff+My7+1fsTwPu9LiVrgBbg58AvgBavq0kB8UKfT6Tw80gSHI5EqSkrJD939j96iU4kqoMhIiJyqg3nrSA3x1QHQ0RElpVU1sB4nngxzheBV7x7PQD8OXCnmfUSr3HxoHfKg0C1N34ncLd3ndeAbxJPfvwA+IhzbtKrk/HHwJPAduCb3rGSwUKRKPWzdCBJaAuqE4mIiMh0AkX5XNhYoQSGiIgsK3mzH7JwzrmPAx8/bXgX8Q4ipx8bBd43w3U+BXxqmvHvAd9bfKSSLqH+KCtXFM/p2LLCPFZVFbM9pBUYIiIip7uyqZoHn9vFyNgkxQW5focjIiKScqluoypyinAkOmsL1anag+V0KYEhIiJyho6mKsYnHS/sVR0MERFZHpTAkLSJjk9yfHh8Th1IEtYFA+zqGyQ6rirrIiIiU21YXaU6GCIisqwogSFp0zcQb6E6rxUYDeXEHPQeHkxVWCIiIlmprDCPi1ZWsFkJDBERWSaUwJC0CUWiANRXzGcLSbyQ5/ZDKuQpIiJyuo6mal7ed4LhsQm/QxEREUk5JTAkbUL98QTGfLaQnFddSlF+DjtUB0NEROQMHU3VTMRUB0NERJYHJTAkbcKR+ScwcnOMtvqAWqmKiIhMY8N5K8jLMTbv1DYSERFZ+pTAkLQJR6IU5edQXjy/7r3twXK2HxrAOZeiyERERLJTqVcHQ4U8RURkOVACQ9ImFBmlvrwIM5vXee0NAY4NjdE3OJqiyERERLJXR1M1v9zfz9Co6mCIiMjSpgSGpE24PzqvDiQJbV4hzy7VwRARETmD6mCIiMhyoQSGpE0oEp1X/YuE9mA5ADsOKYEhIiJyug2rvToY2kYiIiJLnBIYkhbOuXgCYx4tVBOqSguoLy9kuwp5ioiInKGkII+LV1WqDoaIiCx5SmBIWvSPjDM2EaMuULig89uD5VqBISIiMoOOpirVwRARkSVPCQxJi1CiheoCVmBAvJBn7+FBxidjyQxLRERkSehoqmYy5vjFnmN+hyIiIpIySmBIWoT6vQTGAmpgAKwLljM2GWP3kaFkhiUiIrIkvPW8FeTnGlt2KYEhIiJLlxIYkhZhbwXGQrqQQHwFBsD2Q6qDISIicrqSgjwuXqk6GCIisrQpgSFpEY6MAlBXvrAaGE01ZeTnGjvUSlVERGRaHU3VvHKgn0HVwRARkSVKCQxJi1AkSlVpAYV5uQs6vyAvh7W1ZezQCgwREZFpXblWdTBERGRpUwJD0iLcH13w9pGE9mCALq3AEBERmdal5ybqYGgbiYiILE1KYEhahCJRggvcPpLQ3lDOwf4o/cPjSYpKRERk6SguyGX9qkoV8hQRkSVLCQxJi3AkuuAWqgntwXghzx0hbSMRERGZTkdTNa8e6GcgqmS/iIgsPUpgSMqNT8Y4MjhGXWBxCYx1DeUAKuQpIiIygyub4nUwtu457ncoIiIiSacEhqTc4YF4B5LFrsCoCxSyoiRfKzBERERmcMm5KyjIzVEdDBERWZKUwJCUC/VHAQgusoinmdEeLGf7Ia3AEBERmU6iDsZmJTBERGQJUgJDUi4ciScwFtuFBKC9Id6JJBZzi76WiIjIUtSxNl4HI6I6GCIissQogSEpl0hgLHYLCcC6YDkj45O8cWx40dcSERFZijqaqog52LpH3UhERGRpUQJDUi4UiVKQm8OKkvxFX6u9IdGJRNtIREREpnPpyToYSmCk0uPbDrDx3mdYc/d32XjvMzy+7YDfIYmILHlKYEjKhfuj1JUXYmaLvlZLXQAztVIVERGZSVF+LuvPrWTzTtXBSJXHtx3gnsde4cCJERxw4MQI9zz2ipIYIiIppgSGpFwoEl10Ac+E4oJc1lSXskOFPEVElh0ze8jMDpvZq1PGqszsKTPr8X6u8MbNzP7RzHrN7JdmdumUc27zju8xs9umjL/VzF7xzvlHS0bm3SdXNlXz2sF++kdUByMV7nuyi5HxyVPGRsYnue/JLp8iEhFZHpTAkJQLR0apT0L9i4T2hoBWYIiILE9fAW48bexu4GnnXAvwtPca4CagxXvcAdwP8YQH8HHgCuBy4OOJpId3zB1Tzjv9Xlmjo6ladTBS6OCJkXmNi4hIciiBISnlnCMciVIfSGICI1jO3mPDDI1OJO2aIiKS+ZxzPwFO/x/5zcDD3vOHgVumjH/VxW0BKs2sAbgBeMo5d8w5dxx4CrjRe6/cObfZOeeAr065Vta55NxKCvJytI0kRc6pLJ7XuIiIJIcSGJJSA6MTDI9NEqwoTNo124MBnIPusLaRiIgI9c65QwDezzpvvBHYN+W4/d7Y2cb3TzOelYryc7n03Eq27FYCIxXuuqGNgtxTdxgV5+dy1w1tPkUkIrI8KIEhKRXuj7dQrU9SDQyAdQ3lgDqRiIjIWU1Xv8ItYPzMC5vdYWZbzWxrX1/fIkJMrY6mal47GFEdjBS45ZJGrl1Xf/J1Q0URf/3rb+GWS7I25yUikhWUwJCUCkXiCYxkFfEEaKwspqwwjx2HVAdDREQIe9s/8H4e9sb3A6umHLcSODjL+Mppxs/gnHvAObfBObehtrY2KR8iFTqaqnEOfrFbdTBSITfnzZzXp39zvZIXIiJpoASGpFQ4MgpAMIlFPHNyjLZggO1agSEiIvAEkOgkchvwnSnjH/C6kXQA/d4WkyeB681shVe883rgSe+9ATPr8LqPfGDKtbLS+lVeHYxd2kaSCj3hQS44J74qtEtzEhGRtFACQ1IqHEn+FhKI18HoCg0Qr7MmIiLLgZk9CmwG2sxsv5ndDtwLXGdmPcB13muA7wG7gF7gi8AfATjnjgGfBH7hPT7hjQF8GPiSd85O4Pvp+FypUpSfy1vPXcEWJTCSbnwyxq4jg7ytpZbyojy6VJdLRCQt8vwOQJa2UH+UiuJ8ivJzk3rd9mCAR55/g1AkSkOFKn6LiCwHzrlbZ3jr2mmOdcBHZrjOQ6v3HQoAACAASURBVMBD04xvBS5cTIyZpqOpmn94upv+4XEqSvL9DmfJ2HNkiPFJR1uwjLZggG6twBARSQutwJCUCkWiSa1/kdCeKOR5SBMGERGRmXQ0VeEcPK9uJEnVHR4EoKUuQFswQFdYq0JFRNJBCQxJqcORKPVJrH+R0BYMALA9pEKeIiIiM1l/biWFeTls2aVCnsnUHR4gx6C5roy2+gAD0YmThctFRCR1lMCQlApFotQHCpN+3fKifBori7UCQ0RE5CwK83J563mqg5Fs3eEBzqsupSg/l9b6+Jcqau8uIpJ6SmBIykxMxugbGE1qB5Kp1jUE2KEVGCIiImfV0VTN9lCEE8NjfoeyZHSHB2ipKwM4mcBQHQwRkdRTAkNS5sjgGDGX/A4kCe3Bcnb2DTE6MZmS64uIiCwFV66t9upgaBtJMoxOTLLn6PDJxMWK0gLqAoXqRCIikgZKYEjKJPaCpqKIJ0B7Q4DJmKP38GBKri8iIrIUXLSygqL8HG0jSZLdR4aYjDlavXpcEK/N1a0EhohIyimBISkTTiQwUrSFpD0Y70TSpSWbIiIiM3qzDoZWYCRDYt7RWl92cqytPkBPeJDJmDqRiIikkhIYkjKJBEZdefKLeAKsri6hMC9HRbNERERm0bGmmu2HIhwfUh2MxeoJD5KbY6ypKT051hoMMDoRY+/RIR8jExFZ+pTAkJQJ9UfJyzFqSlOTwMjLzaGlvozth1TIU0RE5GyuXFsNqA5GMnSHB7wvUXJPjrUlCnlqG4mISEopgSEpE4pEqQsUkpNjKbtHe7BcKzBERERmcdHKStXBSJLu8ABtU+pfALTUl2EGXSHV5RIRSSUlMCRlDkdGqU9R/YuE9mCAvoFRjgyOpvQ+IiIi2awgL4cN51UpgbFI0fFJ9h4bpqXu1ARGSUEe51aVaAWGiEiKKYEhKROKRKkPpDaBsa5BhTxFRETm4sq11ewIDXBMdTAWrPfwIM5xsoXqVK31AbVSFRFJMSUwJGXC/dGUdSBJaPeWcKoOhoiIyNl1NFUB8PPdWoWxUIkVFm3BsjPea6sPsPvIEKMTk+kOS0Rk2VACQ1JiaHSCgdEJ6stTm8CoLiukNlCoOhgiIiKzeEtjJcX5uWqnugjd4UHyc43zqkvPeK81GGAy5th5WJ1IRERSRQkMSYmQ10I1WJGaDiRTtQcD7AhpBYaIiMjZFOTlsGH1Cjbv1AqMheoJD9BUU0Z+7plTaHUiERFJPSUwJCXCXgIj1SswIF4Hoyc8yMRkLOX3EhERyWYdTdV0hQc4quLXC9J9eICW+jO3jwCsqSklP9dUB0NEJIWUwJCUSGcCo60+wOhEjD1Hh1N+LxERkWzW0VQNwM93axvJfA2NTrDv2MjJlRanK8jLoammjG5taxURSZmUJjDMrNLMvm1mO8xsu5ldaWZVZvaUmfV4P1d4x5qZ/aOZ9ZrZL83s0inXuc07vsfMbpsy/lYze8U75x/NzFL5eWTuQv3xb3aCaUhgtDfEJxLaRiIiInJ2F62soKQgl81qpzpvvYcHAWiZIYEB8ToYqsslIpI6qV6B8RngB865duBiYDtwN/C0c64FeNp7DXAT0OI97gDuBzCzKuDjwBXA5cDHE0kP75g7ppx3Y4o/j8xROBIlUJhHaWFeyu/VXFdGbo6x45AmDCIiImeTn5vDhtVVbFECY94StS1aZ9hCAtBWX8aBEyMMRMfTFZaIyLKSsgSGmZUDbwceBHDOjTnnTgA3Aw97hz0M3OI9vxn4qovbAlSaWQNwA/CUc+6Yc+448BRwo/deuXNus3POAV+dci3xWTgSpT7FLVQTCvNyWVtbqhUYIiIic9DRVEV3eJAjqoMxL93hAQrycqbtQJLQ6q3O6PFWa4iISHKlcgVGE9AHfNnMtpnZl8ysFKh3zh0C8H7Wecc3AvumnL/fGzvb+P5pxiUDhCJR6stT34EkoT1YznatwBAREZmV6mAsTHd4kOba+KrPmbQHy+PHahuJiEhKpDKBkQdcCtzvnLsEGOLN7SLTme5fA7eA8TMvbHaHmW01s619fX1nj1qSItwfTUsBz4T2hgAHTowQ0ZJNERGRs3pLo1cHQ+1U56UnPHDW7SMAK1cUU5yfq04kIiIpksoExn5gv3Puee/1t4knNMLe9g+8n4enHL9qyvkrgYOzjK+cZvwMzrkHnHMbnHMbamtrF/WhZHaxmOPwwGhaCngmrPO+8ejSNx4iIiJnlZ+bw2WqgzEvkeg4B/ujtAZnLuAJkJNjtNaXaT4iIpIiKUtgOOdCwD4za/OGrgVeB54AEp1EbgO+4z1/AviA142kA+j3tpg8CVxvZiu84p3XA0967w2YWYfXfeQDU64lPjoyNMpEzBFMUw0MmNKJ5JDqYIiIiMymo6mansOqgzFXPeF4TYvWurMnMCBeB6NbKzBERFIi1V1I/gR4xMx+CawH/gq4F7jOzHqA67zXAN8DdgG9wBeBPwJwzh0DPgn8wnt8whsD+DDwJe+cncD3U/x5ZA4OR+KToXRuIQmWF1FRnK/WZSIiInPQ0VQFoFUYc9RzsgPJ7AmMtmCAI4NjSg6JiKRASntcOudeAjZM89a10xzrgI/McJ2HgIemGd8KXLjIMCXJQv1RIL0JDDOjTb3XRURE5mR33yAG/PHXt/HX39vBXTe0ccslqoU+k+7wIMX5uaxcUTzrsW3eNpPu8AA1ZekraC4ishykegWGLEOhSDyBkc4aGADrggG6QgPEYtPWchURERHg8W0H+IvHXztZ+fzAiRHueewVHt92wNe4Mll3eICW+jJyztKBJKHNW6WhTiQiIsmnBIYkXTgSJcegpqwgrfdtbyhncHSCAydG0npfERGRbHLfk12MjE+eMjYyPsl9T3b5FFHm6w4P0DKH+hcAtYFCKkvy1YlERCQFlMCQpAtHotQGCsnLTe8fr3ZvyeZ2FfIUERGZ0cEZEv0zjS93J4bHODwwOmsL1QQzo7U+oE4kIiIpoASGJF0oMprW+hcJrfUBzFAdDBERkbM4p3L6Og4zjS933YkOJLO0UJ2qrT5Ad3iQeIk3ERFJFiUwJOnC/VFfEhilhXmcV1XCjpBWYIiIiMzkrhvaKM7PPWWsOD+Hu25o8ymizNY9jw4kCW3BAIOjExz0CpuLiEhyKIEhSReKRNNewDOhPVjOjkNagSEiIjKTWy5p5K9//S00Tllx8d+ua1UXkhn0hAcoK8zjnIq5z21OdiLRqlARkaRSAkOSKjo+Sf/IOMF5/COfTO0NAfYcHWJkbHL2g0VERJapWy5pZNPd1/DsXb8CcMaKDHlTV3iA5royzGbvQJLQ6hX81LZWEZHkUgJDkirstVD1YwsJxFdgxBz0HNaEQUREZDbnVZeyqqqYn/Yc8TuUjNUTHjzZGnWuKkryCZYXndx+IiIiyTFrAsPM3mdmAe/5/zCzx8zs0tSHJtko1J9IYBT6cv9EJxJtIxERyVyaW2SWzuYatuw8ysRkzO9QMs7RwVGODo3RMscOJFO1BtWJREQk2eayAuN/OucGzKwTuAF4GLg/tWFJtgp5KzD8qoFxblUJxfm5bFchTxGRTKa5RQbpbK5lYHSCl/f3+x1KxjnZgWSeKzAg/qVKb9+gEkMiIkk0lwRGopjAO4H7nXPfAQpSF5Jks5NbSHyqgZGTY7QFA1qBISKS2TS3yCBXra3GDDb1ahvJ6RJbQNrm0UI1obU+wNhEjL3HhpMdlojIsjWXBMYBM/tn4DeA75lZ4RzPk2UoHBmlpCCXQGGebzGsawiwIxRR73URkcyluUUGWVFawIXnVPCc6mCcoTs8QHlRHnWB+W+NTdTN0DYSEZHkmctk4TeAJ4EbnXMngCrgrpRGJVkrFIlSX140r0rdydYeLOf48DiHB0Z9i0FERM5Kc4sMs7G5hhffOM7Q6ITfoWSUnvAgrfWBBc1r4p1LlMAQEUmmWRMYzrlh4DDQ6Q1NAD2pDEqyV7g/6lsBz4REIc/th1QHQ0QkE2lukXk6m2uYiDme333U71AyhnOOrvAALQuofwFQXJDLeVUl6kQiIpJEc+lC8nHgz4F7vKF84F9SGZRkr1Ak6lsBz4T2YDmg3usiIplKc4vMs2H1CgrzcniuRwmMhL6BUfpHxmlbQAeShLZggC4lMEREkmYuW0jeA7wbGAJwzh0EFpaKliXNOcfhyKhvBTwTKkryOaeiSEs2RUQyl+YWGaYoP5fLVlepkOcUi+lAktBWH2DPkSGi45OzHywiIrOaSwJjzMWrIToAMytNbUiSrY4PjzM2GfN9BQZAe0O5tpCIiGSupM8tzOxjZvaamb1qZo+aWZGZrTGz582sx8z+1cwKvGMLvde93vurp1znHm+8y8xuWGxc2aSzpYau8ACHvY5iy11i5cRCt5AAtAYDxBz0Hh5MVlgiIsvaXBIY3/QqhVea2R8CPwS+mNqwJBuF+r0WqhmQwGgLBtjZN8jYhHqvi4hkoKTOLcysEfgosME5dyGQC7wf+Bvg0865FuA4cLt3yu3AcedcM/Bp7zjM7HzvvAuAG4HPm1nuQuPKNp3NNQBs2qlVGAA94QGqSguoKVt4h99EJxLVwRARSY65FPH8O+DbwP8B2oC/dM59NtWBSfYJRzIngdEeDDA+6dh1RN94iIhkmhTNLfKAYjPLA0qAQ8A13n0AHgZu8Z7f7L3Ge/9ai7eZuBn4hnNu1Dm3G+gFLl9kXFnj/IZyVpTk81O1UwXiSYeWurJFdVZbXVNKfq6pDoaISJLkzeUg59xTwFMpjkWyXMhLYAR9roEBsK7BK+R5aOBkUU8REckcyZxbOOcOmNnfAW8AI8B/Ai8AJ5xzib6g+4FG73kjsM87d8LM+oFqb3zLlEtPPWfJy8kxrmquYVPvEZxzvrZE95tzjp7wILdcsrjf/vzcHNbWltGtulwiIkkx4woMMxsws8g0jwEzU3EBOUNiBUZdwN82qgBrakopyM1he0h/VEVEMkWq5hZmtoL46ok1wDlAKXDTNIe6xCkzvDfT+On3u8PMtprZ1r6+voUFnaHe1lxDODK67Gs2HOqPMjA6QWtw8bVl24KBkwVBRURkcWZMYDjnAs658mkeAeecvtKWM4QjUWrKCsjPnUtpldTKz82hua6MHYf0jYeISKZI4dziV4Hdzrk+59w48BhwFfEaG4nVpiuBg97z/cAqAO/9CuDY1PFpzpn6OR5wzm1wzm2ora1dRNiZZ6NXB+O5Zd6NJFGzorVu4S1UE1rrAxw4MUIkOr7oa4mILHdz/p+mmdWZ2bmJRyqDkuwU6o9mRP2LhPaGADu0AkNEJGMlcW7xBtBhZiVeLYtrgdeBHwHv9Y65DfiO9/wJ7zXe+894XVGeAN7vdSlZA7QAP19EXFlnVVUJ51WXLPt2qj1JaKGakCjk2aM6GCIiizZrAsPM3m1mPcBu4FlgD/D9FMclWSgUGc2IFqoJ64LlhCOjHB8a8zsUERGZItlzC+fc88SLcb4IvEJ8fvMA8OfAnWbWS7zGxYPeKQ8C1d74ncDd3nVeA75JPPnxA+AjzrnJhcaVrTqba9iy6xjjk8u3k1dXeICaskJWlC68A0lCm7cNpSukbSQiIos1lxUYnwQ6gG7n3Bri32psSmlUkpXCkSj1GVDAM6G9IT5h2KHCWSIimSbpcwvn3Medc+3OuQudc7/rdRLZ5Zy73DnX7Jx7n3Nu1Ds26r1u9t7fNeU6n3LOrXXOtTnnluUXNp3NNQyOTvDyvhN+h+KbnvAAbcHFbx8BaKwsprQgV61URUSSYC4JjHHn3FEgx8xynHM/AtanOC7JMqMTkxwbGsuoFRiJ7iPaRiIiknE0t8hgV62twYxl2041FnP0HB6kpW7x20cg3t2lpT5Al75QERFZtLkkME6YWRnwE+ARM/sMMDHLObLMHI6MAlBf7n8HkoTaQCHVpQUq5Ckiknk0t8hgFSX5XNRYsWzrYBw4McLw2GRS6l8ktNUH6AoPEC+1IiIiCzWXBMbNwDDwMeL7QXcCv5bKoCT7JFqoZlIRT1AhTxGRDKW5RYbrbKlh274TDCzDzhmJrR7J2kIC0BoMcGxojCODqsslIrIYc+5C4pybADYTL7Sl/xHKKUJeAiOYQTUwIL6NpCs8wGRM33iIiGQazS0y18bmGiZjjud3HfM7lLTr9jqQNCdpCwm82YlEdTBERBZnLgmMnwBFZtYIPA38PvCVVAYl2SfsbSHJpBoYAO3BANHxGHuPDvkdioiIvElziwz31vNWUJSfw3PLcBtJT3iAYHkRFcX5Sbvmm51IlMAQEVmMuSQwzDk3DPw68Fnn3HuA81MblmSbcCRKQV5OUv+xT4Z1DYlCnpowiIhkEM0tMlxhXi6Xr6lelgmMrvAALfXJ2z4CUFNWQFVpgVZgiIgs0pwSGGZ2JfDbwHe9sbzUhSTZKNQfJVhehJn5HcopmuvKyDHYcUgrk0VEMojmFlmgs7ma3sODhPqjfoeSNpMxR+/hwZNbPpLFzGitL9MXKiIiizSXBMafAvcA/+ace83MmoAfpTYsyTahSDTjto8AFOXn0lSrCYOISIbR3CILdDbXAiyrVRj7jg0zOhFLageShLb6AD3hAWKqyyUismCzftvhnPsJ8b2qide7gI+mMijJPocjUd6ystLvMKbVHgzwy/39fochIiIezS2yQ3swQHVpAZt6j/Det670O5y06PK2eCR7CwnEO5EMjU1y4MQIq6pKkn59EZHlYM5dSERm4pzzVmAU+h3KtNY1lPPGsWEGRyf8DkVERCRr5OQYG5treK73CM4tj1UDPScTGMlfgdEeVCcSEZHFUgJDFi0yMkF0PEZ9Bm4hgTdbl6nyt4iIyPx0NtfQNzB6srXoUtcdHqSxspiywuSXZEkkRbqUwBARWbBZExhmtnEuY7J8hSLx4l6ZmsBob4hPGHaEVMhTRCQTaG6RPTa21ADw054+nyNJj+7wAK0p2D4CUF6UzzkVRfpCRURkEeayAuOzcxyTZSqRwAhWZGYCo7GymEBhHjsOacIgIpIhNLfIEo2VxTTVlLJpGRTynJiMsatviNZg8rePJLQGA0pgiIgswozr47z2ZlcBtWZ255S3yoHcVAcm2SOcSGBk6AoMM6O9IaAVGCIiPtPcIjttbK7h/7y4n7GJGAV5S3f38Z6jw4xNxmitS10Co60+wM96jzI+GSM/d+n+WoqIpMrZ/uYsAMqIJzkCUx4R4L2pD02yRdjrD18byMwingDtwXJ2HBpYNkXIREQylOYWWaizpYbhsUm2vXHc71BSKlHAMxUtVBPaggHGJmPsPTqUsnuIiCxlM67AcM49CzxrZl9xzu1NY0ySZUKRKCtK8inKz9wvz9obAgxsmeDAiRFWrlDrMhERP2hukZ06mqrJMdjUe4Qrmqr9DidlusIDmEFzXWpqYMCbyZGu0CDNKVzpISKyVM24AsPM/sF7+k9m9sTpjzTFJ1kgHIlmbAHPhPZgOaBOJCIiftLcIjtVFOdz8apKfrrE62D0hAc5t6qE4oLUfSHTXFdGjkGXtrWKiCzI2XpEfc37+XfpCESyVzgymrEFPBMSy0Jvf3grjZXF3HVDG7dc0uhzVCIiy47mFlmqs7mGz/2ol0h0nPKifL/DSYnu8AAtKV4VUZSfy+rqUrVSFRFZoLNtIXnB+/ls+sKRbBSKRLngnHK/w5jR49sO8L/+/fWTrw+cGOGex14BUBJDRCSNNLfIXp3NNXz2mV427zzKDRcE/Q4n6cYmYuw+MsR159en/F6t9QElMEREFmjW8sdmttHMnjKzbjPbZWa7zWxXOoKTzDc+GePI4Ch1GbyF5L4nuxgZnzxlbGR8kvue7PIpIhGR5U1zi+xzybkrKCnIXbLtVHcfGWIi5mhLYQvVhLZggD1Hh4ieNjcREZHZnW0LScKDwMeAFwD9TSun6BsYxbnMbaEKcPDEyLzGRUQk5TS3yDIFeTlcsaaK53qWZgKj21sRkeotJBBPYDgHvYcHubCxIuX3ExFZSubSgLrfOfd959xh59zRxCPlkUlWCEXiLVSDFZnbQvWcyuJ5jYuISMppbpGFNjbXsOvIEAeW4BcAPeEBcgyaaktTfq9EJ5IdKiwuIjJvc0lg/MjM7jOzK83s0sQj5ZFJVjjsJTAyuQvJXTe0UXxai9fCvBzuuqHNp4hERJY9zS2yUGdLDcCS3EbSFR5gdXVpWlrCr64uoSA35+SqDxERmbu5bCG5wvu5YcqYA65JfjiSbUL9mZ/ASBTqvO/Jrvi2EYOWujIV8BQR8Y/mFlmorT5ATVkhz/Uc4Tc2rPI7nKTqCQ+eXBmRanm5OaytK1NrdxGRBZg1geGce0c6ApHsFIqMkp9rVJUU+B3KWd1ySePJhMWnn+rmM0/30Ht4gOY07HUVEZFTaW6RncyMzuZqftpzhFjMkZNjfoeUFNHxSfYcHeJdFzWk7Z7twQBbdmnXlIjIfM2awDCzv5xu3Dn3ieSHI9kmHIlSFyjKqknMbVet5p9/spMvPLuLv3vfxX6HIyKy7Ghukb06W2p5/KWD7AgNcH4Gt1Cfj519g8QctKRpBQbE62D827YD9I+MU1Gcn7b7iohku7nUwBia8pgEbgJWpzAmySLhSJRgReZuH5lOVWkB77/sXL7z0gEO9S+9QmQiIllAc4ss1dm89Opg9IQHAdLSQjWhLVgGoDoYIiLzNGsCwzn391MenwJ+BVDxAAHiXUgyuYXqTP7gbWuIOXjwp7v9DkVEZNnR3CJ7BSuKaK4r46dLKIHRHR4gL8dYXZ36DiQJiXobqoMhIjI/c1mBcboSoCnZgUh2CvdHqSvP3BaqM1m5ooR3X3wOX//5G5wYHvM7HBGR5U5ziyzS2VzDz3cfZXRi0u9QkqI7PMCamlIK8hYyLV6YxspiygrztAJDRGSeZv2b2sxeMbNfeo/XgC7gM3O9gZnlmtk2M/sP7/UaM3vezHrM7F/NrMAbL/Re93rvr55yjXu88S4zu2HK+I3eWK+Z3T33jy3JMBAdZ2hsMitXYAB88Oomhscm+drmvX6HIiKyrCx2biH+6myuIToe48W9J/wOJSm6w4O0pnH7CMQLorbWqxOJiMh8zSXV/C7g17zH9cA5zrl/msc9/hTYPuX13wCfds61AMeB273x24Hjzrlm4NPecZjZ+cD7gQuAG4HPe0mRXOBzxPfNng/c6h0raRKOxFuoZlsNjIT2YDnXtNfx5Z/tYWRsaXyLJCKSJRY7txAfXdFURW6O8Vxvn9+hLNrI2CT7jg/T6kNXsrZggO7wAM65tN9bRCRbzaUGxt4pjwPOuYm5XtzMVgLvBL7kvTbiPd6/7R3yMHCL9/xm7zXe+9d6x98MfMM5N+qc2w30Apd7j17n3C7n3BjwDe9YSZNwZBSA+ixdgQHwoavXcmxojG+9sM/vUERElo3FzC3Ef4GifNavquS53uxvA9p7eBDnoLW+LO33bq0PcHx4nL6B0bTfW0QkW6V6s98/AP8diHmvq4ETUyYq+3mzaFcjsA/Ae7/fO/7k+GnnzDQuaRLqj6/AyOYExmWrV/DW81bwwE92MTEZm/0EERERobO5hlf2n6B/eNzvUBaly6tBkc4WqgltiUKeqoMhIjJnKUtgmNm7gMPOuRemDk9zqJvlvfmOTxfLHWa21cy29vVl/3LHTBFKbCHJ4gSGmfGhq9ey//gI333lkN/hiIiIZIXOlhpiDjbvyu5uJD3hAQpyc1hdXZL2eyfqbqgOhojI3KVyBcZG4N1mtof49o5riK/IqDSzPO+YlcBB7/l+YBWA934FcGzq+GnnzDR+BufcA865Dc65DbW1tYv/ZALEa2CUF+VRXJDrdyiLcm17HS11ZXzh2V3ahyoiIjIH61dVUlqQy097sjuB0R0eoKm2lLzc9HUgSagpK6SmrECdSERE5iFlf1s75+5xzq10zq0mXoTzGefcbwM/At7rHXYb8B3v+RPea7z3n3Hx/00+Abzf61KyBmgBfg78AmjxupoUePd4IlWfR84UjkSztoDnVDk5xgevXsv2QxGe7dYKHRERkdnk5+bQ0VTNpt5sT2AM0urD9pGE1voAXeFB3+4vIpJt0p9uhj8H7jSzXuI1Lh70xh8Eqr3xO4G7AZxzrwHfBF4HfgB8xDk36dXJ+GPgSeJdTr7pHStpEoqMZnX9i6neffE5NFQUcf+Pd/odioiISFbobKlhz9Fh9h0b9juUBRkcneDAiRHa0txCdarW+gA94QFiMa0AFRGZi7QkMJxzP3bOvct7vss5d7lzrtk59z7n3Kg3HvVeN3vv75py/qecc2udc23Oue9PGf+ec67Ve+9T6fgs8qZwf3TJJDAK8nL4g7c18fzuY7z4xnG/wxEREcl4nc01AFm7CqMnUcCzLv0dSBLaggGGxybZf3zEtxhERLKJHyswZAmYjDn6BkezuoDn6d5/2SoqivP5glZhiIiIzKq5roz68kKey9IERqL2hN9bSECdSERE5koJDFmQI4OjTMYc9UugBkZCaWEet121mqe2h+k9rP2oIiIiZ2NmbGyu4Wc7j2blFoju8CCFeTmsqkp/B5KE1voyLxYlMERE5kIJDFmQ8BJooTqd37tqNYV5OTzwE63CEBERmU1ncw3HhsZ4/VDE71DmrTs8QEt9Gbk55lsMgaJ8GiuL1UpVRGSOlMCQBQn1xxMY9eWFPkeSXFWlBfzmhlX827YDHOrXflQRkWxhZpVm9m0z22Fm283sSjOrMrOnzKzH+7nCO9bM7B/NrNfMfmlml065zm3e8T1mdtvMdxR4sw5GNm4j6QkP0lrn3/aRhLZgQCswRETmSAkMWZClugID4A/e1kTMwUPP7fY7FBERmbvPAD9wzrUDFxPvUHY38LRzrgV42nsNcBPxtuwtwB3A/QBmVgV8HLgCuBz4eCLpIdOrKy+itb4s6wp59o+ME4pEafGx/kVClSjjZAAAIABJREFUa32AnX2DjE/G/A5FRCTjKYEhCxKKRMnNMarLltYKDIBVVSX82kUNfP35N+gfHvc7HBERmYWZlQNvx2vN7pwbc86dAG4GHvYOexi4xXt+M/BVF7cFqDSzBuAG4Cnn3DHn3HHgKeDGNH6UrNTZXMvPdx8jOj7pdyhzluhA0hb0rwNJQluwjPFJx+4jQ36HIiKS8ZTAkAUJR0apCxT6um80lT549VqGxib52pY9fociIiKzawL6gC+b2TYz+5KZlQL1zrlDAN7POu/4RmDflPP3e2MzjctZdLZUMzoR44W92dOGvDscL9bdkglbSOrLAVQHQ0RkDpTAkAUJR6LUL8HtIwnrGsp5R1stX960J6u+URIRWabygEuB+51zlwBDvLldZDrTZd/dWcZPPdnsDjPbamZb+/r6FhLvknLFmmryciyr6mB0hwcoKcilsbLY71Boqi0lN8cyog7G49sOsPHeZ1hz93fZeO8zPL7tgN8hiYicQgkMWZBQf3TJFfA83YeuXsvRoTG+tXXf7AeLiIif9gP7nXPPe6+/TTyhEfa2huD9PDzl+FVTzl8JHDzL+Cmccw845zY45zbU1tYm9YNko9LCPC49dwXP9WRXAqOlroycDFhJWpSfy+rqEt9XYDy+7QD3PPYKB06M4IADJ0a457FXlMQQkYyiBIYsSCgSXZIFPKe6fE0Vl55byQM/3cWECmuJiGQs51wI2Gdmbd7QtcDrwBNAopPIbcB3vOdPAB/wupF0AP3eFpMngevNbIVXvPN6b0xm0dlSw6sH+zk+NOZ3KHPSHR6kNQMKeCa0BQN0+bwC474nuxg5bdXpyPgk9z3Z5VNEIiJnUgJD5m14bIKB6AT1FUs7gWFmfOjqtew7NsJ3XznkdzgiInJ2fwI8Yma/BNYDfwXcC1xnZj3Add5rgO8Bu4Be4IvAHwE4544BnwR+4T0+4Y3JLDY21+Ac/GznUb9DmdWxoTGODI5mVAKjtT7AG8eGGR6b8C2Ggyembx8/07iIiB+UwJB5C0dGgaXZQvV0v7qunua6Mr7w7C6cO2MbtIiIZAjn3Eveto6LnHO3OOeOO+eOOueudc61eD+Pecc659xHnHNrnXNvcc5tnXKdh5xzzd7jy/59ouxy8coKAoV5WVEHI1FroqXe/w4kCe3BAM5B7+FB32I4Z4Z6ILk5llXbg0RkaVMCQ+Yt1B8FWNJFPBNycowPvr2J7Yci/ET/eIuIiEwrLzeHjrXVPNeb+UVN32yhmlkrMMDfTiR/dl3rGVVsC3JzqCjO43cefJ6PPPIih/q1GkNE/KUEhsxbOLJ8EhgAN69vpKGiiPt/3Ot3KCIiIhmrs7mGfcdGeOPosN+hnFV3eJBAYV5GrSQ9r7qUgrwcXzuRTDqHA1aU5GNAY2Uxf/vei9h097XceV0rP9we5tq/f5Z/fnYn46oNJiI+yfM7AMk+IS+BEVziNTASCvJyuL1zDf//d7fz0r4TrF9V6XdIIiIiGaezpQaAn/b28dvV5/kczcy6wgO01Jdh5n8HkoTcHKOlrowdPq3AGBmb5O/+s4v1qyr5tz+66oxfm49e28It6xv5xH+8xl9/fwfffmE//+vmC7hqbY0v8YrI8qUVGDJv4UiUssI8ygqXT/7r1svPpaI4ny/8eKffoYiIiGSkpppSGiqK2JTBdTCcc/SEBzJq+0hCW33AtxUYDz63i3BklL9457oZEzvnVpfwpdsu40sf2MDI+CS/9cXn+eij2zjsfbElIpIOSmDIvIUjUerLC/0OI61KC/P4wJXn8eTrIXb2+VdgS0REJFOZGZ3NNfxs51EmY5lZ+PrI4BjHh8dpqcvABEYwQDgyyonh9Lai7RsY5f4f7+SGC+q5bHXVrMf/6vn1/PDOq/noNc384NUQ1/z9szz43G61nBeRtFACQ+Yt1B9dNvUvpvq9q1ZTmJfDA8/u8jsUERGRjNTZUsOJ4XFeO9jvdyjTSqxwyKQWqgmt3qqQ7nB6vyj5zNPdjE7E+PMb2+d8TlF+Lnde38Z/fuztvPW8FXzyP17nXZ99jp/vVtdhEUktJTBk3sKR0YwqfJUu1WWF/MaGVTy2bf/JTiwiIiLypkRNhExtp/pmAiNzWqgmtCU6kaRxG0nv4UEe/fk+fvuKc2mqnf+vyeqaUr7y+5fxhd95KwPRCX7jnzdz5zdfom9gNAXRiogogSHzFIu5+BaSZVLA83R/+LYmYg4e2rTb71BEREQyTm2gkPZggOcytPV4d3iQypJ8agOZtxW2oaKIQGEeXaFI2u557/d3UJKfy0evbVnwNcyMGy8M8tSdb+cj71jLv798kGv+/sc8/LM92lYiIkmnBIbMy7HhMSZiblmuwABYVVXCuy5q4JEte+kfHvc7HBERkYzT2VzD1j3HGRmb9DuUM/SEB2itC2RUB5IEM6M1GKA7lJ4tJFt2HeWH28N8+B1rqS5bfEKnpCCPu25o5wf/7e1cvLKSjz/xGu/+p028sPd4EqIVEYlTAkPmJbF1YrkV8Zzqg29fy9DYJP/y/F6/QxEREck4nS01jE3G+MWezKqH4Jw72UI1U7UFA3SFB3AutUVQYzHHX31vO+dUFPFfN65J6rXX1pbxtdsv53O/dSnHhsb4f+7/Gf/92y9zdFDbSkRk8ZTAkHkJRxIJjOW5AgPg/HPK+ZW2Wr68aTfR8cz7dklERMRPl6+poiA3J+PaqYYjowxEJzKyhWpCW32A/pFxDqe4hsS///Igv9zfz59d30ZRfm7Sr29mvPOiBp7+s6v54NubeOzFA1zz98/yL1v2ZmyHGhHJDkpgyLyEvARGcJnWwEj40NVrOTI4xrde2O93KCIiIhmlpCCPS8+rzLhCnokCnpnYQjUh0R2lK5S6Qp7R8Un+9gddnN9QznsuaUzZfSDehv6e/7KO7//p21jXEOB/PP4q7/n8Jl7ed4LHtx1g473PsObu77Lx3md4fNuBlMYiIkuDEhgyL+HIKDkGtUnYK5nNrlhTxfpVlXzxJ7tUoEpEROQ0nc01vHYwklHbBjK5A0lCIrZUJjC+unkPB06M8BfvXEdOTnpqgbTUB3j0Dzv4zPvXE+qPcvPnNvFn33qZAydGcMCBEyPc89grSmKIyKyUwJB5CfdHqSkrJC93ef/RMTM+/CtreePYMN9/NeR3OCIiIhmls6UWgJ/tPOpzJG/qDg9QXVqQlIKVqVJdVkhNWWHKWqkeHxrjs8/08o62WjY216TkHjMxM25e38jTf3Y1pYW5Z2wlGRmf5L4nu9Iak4hkn+X9v1CZt1AkuqzrX0x13bp61taWcv+Pd6a82JaIiEg2eUtjBYGivIxqp9odHjy5RSOTtQcDJ1eLJNtnn+llaHSCe/7LupRcfy4CRfkMj05fQ+zgiZE0RyMi2UYJDJmXsBIYJ+XkGB+8ei2vH4rw0wyaoImIiPgtN8e4am01z/UeyYgkv3Mu3kI1g7ePJLTWxxMYsSQXu9x7dIivbdnDb162yvdEzjmVxfMaF5HM41cdGyUwZF7CkSjBisxdeplut6xvJFhexP0/3ul3KCIiIhmls6WWAydG2HN02O9QOHBihKGxSVqyYAVGW7CM6HiMfceT++v2tz/oIj83h4/9amtSr7sQd93QRvFp3U9yDP7suhafIhKR+Xh82wHueewVX+rYKIEhcxYdn+T48DhBrcA4qSAvh9s717B511Fe3nfC73BEREQyRqdXYyETupH0hAcBMrqFakJidcSOJBbyfGHvcb77yiH+8G1N1GXAPO6WSxr5619/C42VxRhQWZJPzJERyS4Rmd19T3YxMn7qVrB01bFRAkPm7HAkXkk8E/7hyyS3XnEu5UV5fOFZrcIQERFJWF1dQmNlMc/19PkdypsdSDK4hWpCYpVId5ISGM45/up726kNFHLH25uScs1kuOWSRjbdfQ27730nL/3l9bzvrSv57I96+Um3/39eROTsZqpXk446NkpgyJyFIlEArcA4TVlhHh+4cjU/eC3Ezr5Bv8MRERHJCGZGZ3MNP9t59IyOE+nWFR6gLlBIRUm+r3HMRVlhHquqipPWieTJ10K8sPc4d17XSmlhXlKumQqfuPlCWurK+Ni/vkTYm3OKSGbys46NEhgyZycTGBVKYJzu9zaupiA3hy/+ZJffoYiIiGSMzpYaBqITvHKg39c4esKDWbF9JKGtPjmdSMYmYtz7/R201pfxvreuTEJkqVNckMvnf/tSRsYn+ZOvb2NiMuZ3SCIyg7tuaCPHTh0rzs/lrhvaUn5vJTBkzg57CQx1ITlTTVkhv7FhFY+9eEDfGoiIiHiuWlsN4Os2kljM0XN4gJYs2D6S0FofYFffEGMTi/tP/Nef38ueo8Pcc9M68nIzf9rfXBfgU++5kJ/vOcb/fqrb73BEZAadLTU4F18xZkBjZTF//etv4ZZLGlN+78z/m0wyRqg/SnF+LuVFmbv80E9/+LYmJmIxHnput9+hiIiIZITqskIuOKfc10Ke+44PEx2PZUUL1YS2YICJmGPXkYVvTY1Ex/nM0z1sbK7mV9pqkxhdEh06BFdfDaHQyaH3XLKS91+2is//eCc/6jrsY3AiMpP/ePkgDnjsj65i973vZNPd16QleQFKYMg8hCJR6ssLMbPZD16Gzq0u4V0XncMjz79B/8i43+GIiIhkhM7mGl7Ye5zhsQlf7t/tdSDJhhaqCYlOJF2LKOT5+R/t5MTIOPfctC5z526f/CQ891z85xT/37svoD0Y4M5/fSktRQFFZH7+bdsBzm8oP/l3VTopgSFzFo5EtX1kFh+8uonB0Qn+Zctev0MRETnF49sOsPHeZ1hz93fZeO8zaenVLgJgBuOTjvP/8klf/uyd7ECSRSsw1taWkZdjC66DceDECA9t2s171jdyYWNFkqNLkkOH4MtfhlgMHnrolFUYRfnxehhjEzH+5NFtjKsehkjG2Nk3yMv7+/n1S9Oz4uJ0SmDInIUjoyrgOYsLzqng7a21fHnT/2XvvsOiurYGDv82vYgUAVGwYO+KYostpmlMMzGmaJqJURPTc1NM7ndzU26aqaZoNCYm0RRj1DQTY9TYu6jYQMWKFAEp0hn298eZERBQQJgZxvU+zzwD58zM2XtmxH3WWXvtI+SdszayEELYyuKoeKYsjCY+PReNcXIzZWG0BDFEnVscFc+c9UfO/m6L715sUhZNfT3w8bD/FUgs3FycCA/0JiaxZlNI3lkagwKetkJBvRp79VUjeAGQlwddusDjj8PChZCSQqugBrwxqhvbjp7mnb9ibNtWIcRZP0fF46Tghu5NbXJ8CWCIKtFak5iZJ0uoVsFDQ1qTciafn7afsHVThBACgKlLY8g9J6iaW2hi6lI5KRB1a+rSGPIKy149t/Z3LzbpTL2aPmLRLqRmK5Hsjs9gUVQ89w8MJ9QKSxrWiCX7oqCgZNvp0zBzJowaBS+/DMCNnYN53bSfn5ZsY/m+JBs1VghhobVm0Y54BrQJtFlmvgQwRJWk5xRSUFRMsAQwLqhfqwC6N/Nj5uo4m697L4QQQKVzyGVuuahrtv7umYo1h07VryVULdo39uFYWg7Z+VWvHaK15n+/7yPA242HLm9dh627SKWzLyxcXODee2HdOnj4YWPbjh2MeedfbP34bsIH9yZ73Hj4/ntIS7N+m4UQbDt6muNpuYzsYZvpIyABDFFFiealQSUD48KUUjw0pDVHU3P4Y3eCrZsjhBCVTv9raq9XZ4XDqOw7FtzQ3SrHP5pqLEXaNrj+1L+wsBTHO5Bc9WkkK2OS2RCXyuNXtqWhPU+Z2bChbPYFGL9v2gSXXQYdOxrbuneHjRtJ+8+rnPALQX3/Hdx5J2zbZuzfuxfmzoXjx63bfiEuUYui4vFwdWJYlxCbtUECGKJKzgYwfK0z4KjvrunUmFZB3kz/5xBaSxaGEMK2uoY2LLfN09WZZ+x5frxwCM8Ma4+nq3O57dn5RUQdO13nxy8p4Fn/MjA6mLNGYqu4EkmRqZjXl+wnPNCbMX2b12XTLl5UFGhd/hYVVfZxrq7Qty8BL/+brJ9+puuj3zH7vR9g4EBj/4IFcPfd0Lw5tG4N998PX38N+fnlj1nBkq11xprHEsJKCoqK+W1XAsM6h9DA3cVm7ZAAhqiSZHMAQ1YhqRonJ8XEwa3YczKTtQdTbN0cIcQlLDkrjzUHUunRzJdQP08UEOrnyRu3dLXamu3i0jUyIpQ3bula5rv3/PD2+Hu7ccfMjfy682SdHt+yhGqbepiB0SzACw9XJ2KqWAdj/tYTHEw+w3PDO+Dq7HhD/Ou6NWHsgFa8muTNX3EZxsYXX4Tt2+H996FrV1i8GCZPBmdz0Ozbb40VTuLi4JVXKlyytU5UsjysEPXZPzHJZOQW2nzsYLvQiahXEjOMSHawjwQwqmpkRCjvLYtlxqpDDGobZOvmCCEuUR8tP0ihqZj3b48gPNDb1s0Rl6CREaHlBryjI5sxae42Hv0uirhT2Tx2ZRuUUrV+7NikLJoFeOJtw6uFNeXspGgbXLVCntn5Rby3LJbIFv4M69zYCq2zjRev60jUsXT+9eNOfm/SkGYBXhARYdyeeMKoq3H0qFFPA2DWLPjnn7IvMnMm/N//QUgIjB1rZEsoVXIbOBBeesl47J13QkZG2f1XXmkcC+D226GwsOz+fv1KloedORMCA6FVK+O+USMID4fGtfQZJSTAHXfADz8Y/RGiDi2KiqeRtxuD2gTatB2OF54VdSIxM49G3m64uchXpqrcXZx5YGA46w6msutEuq2bI4S4BB1Jyea7zce4vXczCV4Iu9KogTtzx/fllohQ3v87lid+2FEny4/HJmXRLrj+TR+xaNfYh/1VmELy2eo4Us7k8+J1HeskEGQv3F2c+WRMTzTwyLfbKSg6pxCok5MRILBYsQJ274ZBg4x9YExVsWRGFBUZAYj8fMjNhexsY0lXi9RUOHUKkpKMYEF8fNkCoocOwYEDEBMD+/bBnj1GMMFSoLSoyMj8uO8+uP566N8fpk0ree3gYKPex6BBMHIkjB8Pf/9t7M/Ohl9+gfXrjddPTQXTOf9GJNNDWElGbiHL9yVzQ/emuNg4w0vORkWVJGXmyfSRGrizT3N8PFyYseqQrZsihLgEvbcsFldnJx6/sq2tmyJEOe4uzrx7W3eeGdaen3ecZMysjaScqaB2QQ0Vmoo5nJJdL5dQtWgf0oBTWfmkZRdU+pikzDxmrY7jum5NiGjub8XW2UbzRl5MvbU7O09k8PqSfed/sFIQEABbtpQEFUwmI0MiMdEINqxZYwQB1q0zggVvvFHy/L/+Mp67datROHT7diMgYbF1K0RHG0GSPXtg+XLj99IFSj08YONGo0DpkiVw110l+0aNgi5djFofcXHG/rg4Y9+hQ3DTTTBgAHToYGRwuLoaNT4AVq+Gzz4ryfR45RX47jsj0GLpZ32tw+ao9Uqsdaw6OM4f0QkUmIq5pec500dsUO9FAhiiSpIy8yqtYi8q5+Phyj39W/DH7kTiTlW9irgQQlys3fEZ/LLzJPcPbClLYAu7pZRi8tA2fDq2J3tOZjLyk3XEVLFo5YUcScmm0KRpH1L/6l9YtA8xCvCebxrJe3/FUlRczHPDOlirWTY3vEsI4wa0ZM76I/x5oRXfKlqy1WSqm6yFio5VXGwEHfr0gWuvLVlhpVEjmD4dfvzRyBTZtQtOnoQJE4z9bdsawZM//zRWWvnwQ/j3v6FbN2P/tGllMz1eegnGjIEdO4xtv/0G7u4QFga9esGIETBuHBw8aOw/ftwIuOzZAykp5dtdyuKoeG584Uc2Ne/KDS8uYHFUfC29YWYmE5w5A2lpLI6KZ+HNEylevYY/h9/F0j+2QHKyMZWnouKsF8uaWSxVPZbWJdlBBQVGv/PyjG1Q8n6dOQNZWZCZabw/lsDZyy8bx3nhBSOD6NQp4z20vH95eUbg4eRJI6voxAnjZsk+ys6GY8eM6VhHjsDhw6xdtoX2/q50DfU1jnfgAMTGwtNPWz0LqP5NCBQ2kZSZR7cwP1s3o16677JwZq05zKw1cbxxSzdbN0cIcYl4e2kMvp6uTBjc2tZNEeKCRnRtQqifJ+O/3sqo6ev5eEwEl7cPvqjXtBS/bFuPp5C0N2ePxCZl0a9Vo3L79ydm8uO244wbEE7zRl7Wbp5NTbm2I9uPnuaZBbvo1MS38v5XtmTr+vW136jaPJanJ0RGVrwvIQF+/73sNg8PI9gREWH83qqVcXKZlFRyi442tgH8+qtR8NTCxQWCgoxMlPBwWLYM/v6b6CJP1sbl8mj0P/Q+vofRS75gilMDfGP2MFSnGlNvcnKM+8JC46QZYPZsIzCTm1vyGA8PWLrU2P/AA7BwobHd/J5lN23Ge6Pf5K9tS3FCM3znchjRp6SNnTsb2S4AV11lrFrj7g5ubsZ9ZCTMm2fsf/BB46S89P4ePUr6//bbRtbArFklWSzdusHEicb+f/3LCA5YAgmFhUamwaRJxv5hw4x2W/YVFho1U555xuhTx45ln5ufb9wXFxuFZWfMMF7HsgIPwFtvwbPPGtk3bSvInPz0U3joISPY1bNn+f3ffGPUaLHUYPnyS+NmsXixkdWzYgVcd1355//9t/H8334zaquU8jEw/+MfjSlq8+cb729pX35ZUlumjkkAQ1xQQVExKWcKaGylNdsdTZCPO7dFhjF/ywmevKqdXAkVQtS59YdSWB17ihdGdMDX09XWzRGiSro38+PnyQN44Kut3D9nCy/d0Jl7L2tZ49eLTTqDk6qfK5BYNG7oTkMPl0qzUt5Ysp8G7i48ekUbK7fM9txcnPh4TE+um7aGyd9uZ8FD/XF3Kb9kb7mlWeuStY5VWabH/PnGSTYYq7KUng5zrltvhU6djJP40kGOgABj/7Zt6A8+oGtBAe+Uetptu5fz0YAxHH5vAUO3/FzuZYe49AOlmPT7MgZHrybfzYN8VzfyXdzJ9G7I/01dCcB1Gf606nyFsc/VeMwRPHlw1VyUNvpWpJzZHNaJDRGX8/SQluBX6mLq9dcbQYL8fONWUFC2/klurlE3xJLBkJ9vTCmymDEDDh8u+b2oCKZOLQlgLF5sZDi4uBhTd1xdjaCQRU6O8Xru7tCggbHf3zyFy9UVhg4t+9xVq2Dv3pLPKiIChg8vWwD2ssuM/QEBxpSg0vuUgr59jf2hoUYAxtIfy/5evcpmQjg7G3VXLMEIS/ZOly4lAZTSr9/BnMXVu7cRgDLvX74/mSW7k3jq6t7GtssvN4Ilc+YY/SoqKslq+uSTct+J2qZ0fZ0bVUORkZF669attm5GvXLidA4D31rJm7d05Y4+dr6uuJ06lprD5e+s5MHBrZhybUdbN0cI4cC01oz8dD3JmXms/NfleLhWMKCvZUqpbVrrSi4VOjYZV9S+7PwiHv8+ir/3JXNP/xb85/pONSoa99DcbexPzGLlvy6v/UZa0egZ69EaFjx0WZntaw6c4u7Zm3lxREceHNyqkmc7vr/2JDLhm23c078Fr9zUxdbNsY6IiJKpIqX16HFRQZTcAhNRx0+z5fBpNh9JZfuR07ieyeTVpZ8yInYdrsUm8p1d+KHbNXzc/3ZGtmpAoZs7Be6eFLq5U+jmjnaq+f8569buZs1n4/EoKsliyXVxY/DE2Wz5+K7zPLMGEhKMgETpoq2enkYNktrOIrDWsergOFprrn5/Nf5ervw4qdTfICv0qbKxhWRgiAtKyjS+mI2lBkaNNW/kxYiuTfh24zEmD21DQw+5IiqEqBtL9ySy83g6b4/qZpXghRC1zdvdhc/ujuTNP/Yxa81hjqTm8PGYiGr/3xmblEXbepx9YdGusQ+/7DyJ1vrsCiOmYs3rS/YT5u/JPZe1sHELbeuaziGMHxjO52sP0yc8gOu7NbV1k+peLWV6ZOYVsu3oaTYfTmPz4TR2nUin0KRRCjqGNOT2Ps1Zv2YXww5uxLXYWAHF3VTE6Oi/+fHacbzw9C210g6LhR+8dDb7wsJJF/P81h+BWg5gnK82Sm1nEVjrWHVwnD0nMzmYfIb/3XxOcNCa7985JIAhLigp0yj4EiJTHy7KpCGt+W1XAvM2HuOhy2VOuhCi9hWZinl7aQytg7zLVwoXoh5xdlK8eF0nWgU14P8W72bUp+v54r7eNAuoWp2H/CITR1JzuLZLkzpuad3rEOLDvE1FJGbm0cTXE4BFUfHsS8hk2p0RFU+buMQ8d20Hth07zfM/RdO5qa8sG12J1DP5bDmSxiZzwGJfQibFGlycFN3CfLl/YDh9wwPo1SLg7PTDuLlvVRhU+PDAr8Cttdq+K9LjcDcVldnmbipiaNrBWj0OUH9ro1j5OIui4nF1VlzX9Zy/pdZ8/84hAQxxQYkZ5gwMCWBclC6hvgxqG8jstYcZN6ClXBkVQtS6BdtOEHcqmxl39bL5Ou1C1IY7+zSnRYAXk+ZuY+Qn65h5Ty96tQi44PPiTmVjKta0bewYGRgAMYlZNPH1JLfAxLt/xdA9zJcbutX/AE1tcHUuVQ9j3nYWPnyZw4+zFkfFM3VpDCfTc2nq58kzw9ozMqJs4Ppkei6bD1sCFqkcOpUNgIerExHN/Hn0irb0DQ+gR3M/vNwqPi1sdSAaKggqtDqwq9b75Ld/d5l+Bfq4k3omn8vbB/N5scbJSV34RarKEWuj1PJxikzF/LLzJEPbB+Pn5Vanx6qOOgtgKKWaAV8DIUAxMFNr/aFSKgD4AWgJHAFu01qfVkZO3IfACCAHuE9rvd38WvcC/za/9Gta66/M23sBcwBPYAnwuL7UinpYQVJmHm4uTvh7ybSHi/XQ5a0ZM2sTC7fHM6av1BMRQtSevEITH/x9gIjmfgzr3NjWzRGi1lzWJpBFkwfwwJwt3DlzE2/f2q3cidq5LMuOtg+pvyuQWLQrtRLJ5e2D+WLdYRIy8vjg9h5FKUmNAAAgAElEQVRnp5QICPXz5L3bunP/nK288tteXr+5q62bVGcWR8UzZWE0uYXGtI749FymLNxFclYeDT1cjSkhR9I4cToXAB93FyJb+nNrr2b0CQ+ga6gvbi5VDHJb+UR1ZERomX/fc9Yd5r+/7mXWmjgmDpEMZmtafyiVU1n5dpfRWZcZGEXA01rr7UopH2CbUmoZcB+wXGv9plLqeeB54DngWqCt+dYXmA70NQc8XgIiAW1+nV+01qfNj5kAbMQIYAwH/qjDPl2SEjPzaNzQXf6TrAX9WzWie5gvM1cf4vbezXCuzUiyEOKS9tX6IyRm5vHBHZfuSY1SyhnYCsRrra9XSoUD3wMBwHbgbq11gVLKHeMiSy8gFbhda33E/BpTgAcAE/CY1nqp9XsiztU6qAGLHh7ApLnbeOKHHcSdOsMTV7Wr9IpsbFIWzk7KIaYS+Hu7Eezjzv7ELFLO5DP9n0Nc3akxfStYVvVSd0WHxkwc0orPVsXRNzyAm3rY14lXbZm6NOZs8MIit7CY15fsB6CRtxt9wgN4YGA4fcID6BDSsN6OOe+9rCWbj6Tx9tIYerXwJ7LlhTOwRO1YHBVPQw+Xi17SurbVWX6p1jrBkkGhtc4C9gGhwE3AV+aHfQWMNP98E/C1NmwE/JRSTYBhwDKtdZo5aLEMGG7e11BrvcGcdfF1qdcStSgpM0/qX9QSpRSThrTmSGoOf+5OtHVzhBAOIiO3kE//OcSQdkH0u7RPah7HGG9YvAW8r7VuC5zGCExgvj+ttW4DvG9+HEqpTsAdQGeMiyKfmoMiwg74e7vxzQN9uS0yjGkrDvLo91HknXMSZxGbdIaWjbwcpj5E+xAfYpOy+PDvA+QWmnj+2g62bpLd+tc17enVwp8XFkZz6NQZWzenTpxMz6103/Knh7D131cx/a5ejBsQTuemvvU2eAHG2PnNUd0I8/fkkW+jSMsuuPCTxEXLKSjizz2JXNetid1Nx7LKBFmlVEsgAtgENNZaJ4AR5AAsIZ1Q4Hipp50wbzvf9hMVbBe1LCkzX+pf1KJrOofQKtCbGasOITOehBC1YcaqQ2TkFvLs8Pa2borNKKXCgOuAz82/K+AKYIH5IedeNLFcTFkAXGl+/E3A91rrfK31YeAg0Mc6PRBV4ebixFujujHl2g4siU7g9pkbSc7KK/e4A0lZDjF9xMJZwe74TL7ZeBQPFyeiT2TYukl2y6iHEYGbixOT522vNMhVn4VUsjJgqJ8nrYMaOFwWXkMPVz4Z05O07AKe/GEHxcUyfq5rf+1JIqfAxEg7zGKq8wCGUqoB8BPwhNY683wPrWCbrsH2itowQSm1VSm19dSpUxdqsihFa01iRp4EMGqRs5NiwuBWRMdnsO5gqq2bI4So55Iy8/hy3WFu6tGUzk19bd0cW/oAeBaj7hZAIyBda22pPlf6QsfZiyPm/Rnmx1d20UTYEaUUE4e0ZsZdvYhNzGLkx+vYe7JkiJlbYOJoWg5tgx0jgLE4Kp51h0rGC9kFJqYsjGZxVLwNW2Xfmvh68t7tPdifmMV/f9lj6+bUqsy8Qtycy58Gebo688wwxw1idwn15f9u6MSq2FNMX3XI1s1xeIui4gn186S3HU7ZqdMAhlLKFSN4MU9rvdC8Ock8/QPzfbJ5+wmgWamnhwEnL7A9rILt5WitZ2qtI7XWkUFBQRfXqUtMZl4RuYUmmUJSy27uGUqwjzsz5A+wEOIifbj8AEUmzdNXO+7A9UKUUtcDyVrrbaU3V/BQfYF9Vbo4IhdG7MOwziH8OKk/Jq0ZPWM9y/clAXDo1Bm0Lil+Wd9NXRpDoans1zC30MTUpTE2alH9MLR9MA9f3prvtxxnUdSJCz+hHkjPKeCuzzdxMiOPcZe1INTPE4WRefHGLV0vWNy2vrurb3Ou79aEd/+KYVOcXASsK6ey8llz4BQjI5rW7sovtaQuVyFRwGxgn9b6vVK7fgHuBd403/9cavsjSqnvMYp4ZmitE5RSS4HXlVL+5sddA0zRWqcppbKUUv0wpqbcA3xUV/25VCVlmpdQrSRVTdSMu4szDwwM540/9hN9IoOuYZf0VVMhRA3FnTrDD1uOM7Zvc5o38rJ1c2xpAHCjUmoE4AE0xMjI8FNKuZizLEpf6LBcHDmhlHIBfIE0Kr9oUobWeiYwEyAyMlJymW2oS6gvP08eyPivtzD+663c1L0pq2KNoNLLv+6h0FRc70/qKqt3cL46CMLw1NXt2Hr0NM/+uIs3luznVFZ+pUuO2ruUM/nc9fmms0tlX9mxMS/daOtWWZdSijdu6cqek5k8+l0USx4fRGADd1s3y+H8svMkxRq7nD4CdZuBMQC4G7hCKbXDfBuBEbi4Wil1ALja/DsYq4jEYcw3nQU8DKC1TgNeBbaYb6+YtwE8hDHX9SBwCFmBpNZZAhiSgVH7xvRtjo+Hi2RhCCFq7N1lsbi7OPHoFW1t3RSb0lpP0VqHaa1bYhThXKG1HgusBG41P+zciyb3mn++1fx4bd5+h1LK3byCSVtgs5W6IWooxNeD+RP707VpQxbvOMnpnEIAkrPyHWKqRVM/z2ptFyVcnJ24rmsIhcWa5Kx8NJYlR+vX9yIpM487Zm7kSGo2s++L5MqOl+5S2T7mehjpuYU8+cMOTFIPo9YtjoqnS2hD2tppFltdrkKyVmuttNbdtNY9zLclWutUrfWVWuu25vs08+O11nqy1rq11rqr1nprqdf6Qmvdxnz7stT2rVrrLubnPKKlImKtS8wwZ2A0lOhmbfPxcOXufi34Y3cCh1Oybd0cIYTZ4qh4Bry5gvDnf2fAmyvsdpAbfSKD33clMH5gOEE+8je6Es8BTymlDmLUuJht3j4baGTe/hTGku5orfcA84G9wJ/AZK2141UAdEBebi6kVLA6gSNMtXhmWHs8z1kFwNHrHdSmmasPl9tWn74X8em53P7ZBhLSc5kzrg+D2sp0+E5NG/LfGzqz5kAKn648aOvmOJSDyVlEx2dwc0TYhR9sI1ZZhUTUX2enkEgGRp0YNyAcF2cnZq6Os3VThBAYwYspC6OJT8+1+yt1b/25H38vVx4c3MrWTbErWut/tNbXm3+O01r3MV8AGa21zjdvzzP/3sa8P67U8/9nvjDSXmstmZ31SEJ6+dVIoP5PtRgZEcobt3S95Ood1Jb6PAXnWGoOt83YQOqZAr5+oO+lvkx2GXf2acZNPZry/t+xrD+UYuvmOIzFUSdxUnBD9ya2bkqlJIAhzisxMw8/L1e7W//XUQT5uHNrrzB+2naC5MyKB16ijiUkwJAhkJjoWMcSNTJ1aQy55yy5Z49X6tYeSGHtwRQmD22Dj4errZsjhF1w5KkWIyNCWff8FRx+8zrWPX+FBC+qobLP39vdhUJTcYX77MGhU2cY/dl6sguK+PbBfvRq4X/hJ11ClFK8fnNXWgZ68/j3OziVlW/rJtV7xcWaxTviGdg2iGAf+714LQEMcV5JmflS/6KOTRjUiqLiYr5Yd8TWTbk0vfoqrF1r3DvSsazFwYIy9eFKndaat/7cT6ifJ3f1a2Hr5ghhN2SqhahIRd8LZyfFmfwiRs/YwPG0HBu1rHIxiVnc/tlGTMWa7yf0k2LvlfB2d+HTsT3Jyivk8e+jpB7GRdp69DQnTudyc0RTWzflvOpsFRLhGJIy82T6SB1rGejNtV2bMG/jUR4e2pqGcjXVOCm+4w744QcICam9183KgtOnIT3duMXFwezZUFwMX34JTZuCiwsUFkJRkXHfsyeMGmU8/6GHoKCgZF9REVx7LYwbB7m5cMMNxvbSzx8/HiZPht27Yfp043WmT4f9+42+3XWX8RpZWfDjj+DvX/YWHAweNfg3WFfv4blKB2U++aTujmMlPh4uZOYVldve1M9+/g4uiU4kOj6Dd0Z3l+w4IUqxZCVMXRrDyfTcervahKhdlX0vXJwVUxZGM+LDNbx2cxduspMVF3bHZ3D37E24uTgxb3x/2gQ3sHWT7FqHkIa8cmMXnv1pF9OWH+DJq9vZukn11qKoeLzcnBnWuQ7HjbVAAhjivBIz8ugQYp8VaB3JQ0Na8/uuBL7ddIxJQ1rbujkVs9YJMVR+UpyXVxKAsNy7ucFVVxn7334bYmNLAhTp6RARAbNmGfs7dYIT56wF72RORDOZ4JVXjACFhbMz3H9/SQDjl19AKSPI4epq3EdElLxOXp6xzdu7ZL+fn7H/gw+MxxSb01V37YIjR+Dyy43fjx2DBx4o/1589hlMmGA8/tZbywc4JkyAHj0gKQnWrzeO5+8P774La9bASy8Zr3H6tHG8oqKyAZjevcHHBw4fhqiosvuKioxj+vrC1q2wcmXJ9qIi4/398kujT7NnG8Gerl0hLAwaNzbev3rki7WHycwrwlkpTOfUhG7RyIviYm3z9dALTcW881cM7Ro34GY5KROinJERoRKwEOVU9r3oHubH499H8fj3O1hzIIWXb+yMt7vtTo+ijp3mni8209DDlW8f7EuLRt42a0t9MjoyjI2HU5m24gC9WwYwsG2grZtU7+QXmfh910mGdQ7By82+QwT23TphU0WmYlLOyBQSa+gS6sugtoHMXnuY+y5raZ9XVV991TghnjjRyCgofZJ7883Gyfn69bBnT9l9AE89ZdwvWGCcCJc+Sfb0hHfeMfa/+65xkvzHH8ZJ8fTpEBMDf/9t7L/mGqMNpUVEwPbtxs+//QYHD5acxDdubJxMl+6DyWTsN5ngnnsg3zxnsqDAyHSIizOe4+JiBCtKiz9PIUd3dyPoUpGEBJg3ryR4oTVkZ0N0dEkwqF0749inT5e9DRpk7HdzMwIEp09DWhocOmT8fP31RgBj82a45Zbyx/7qK3j5ZVixAsaOLb9/61bo1QuWLTM+23MNHGgEMFatgmefLbtPKSNQA8ZnOX58yT4XFyOjZd8+8PKC3383gkthYcatWTOj7y7V+G+oDoNo3246xiu/7eXaLiFc3TGYOT9t5MW5r/Da3f8huG1Llu9P5ukfdzL11m64ONtu9uX8rcc5nJLNrHsicbZxMEUIIeq7ZgFezJ/Yn2nLD/DxyoNsO3qaaXdE2GTKxubDaYz7cjOBPu7MG9+XMH8vq7ehvlJK8drILkSfyOCJH6JY8tggguX8pVpW7j9FZl5RvQgAq0tt5dHIyEi9devWCz9QkJCRS/83VvC/m7swtq/Ms65r6w6mMPbzTbxxS1fu7NPc1s0pKyEBWrUyMgwqkp9vnGA/8kj5aQRubiVBgvvvN07kLdkJrq4QFAR79xr7H30U5s41ruyDcYLcsaMRFAEjAJKSYgQgLLegIGhdg6yVhx82sgZKZ1y4uRkn4bU9FcIax8rKKglqvPWWEfQxmUqOM2UKbNtmvO+lbz17GhkYKSlw8mTJdstnFBJi/JyfbwSdLPuSkoz3vfR3wt3dyHbJzjYyXZKSSrJf7rvPCKaU5u9vBGMApk0zskAsAY6wMGje3Ah0lH4fP/sMJk2q1c/op20n+NeCnQxtH8yMu3rh5uJU7lifrDzI1KUxDOvcmGl3RuDuYv0gY26BiSFTV9I8wIsfJ/VHnRtgsyGl1DatdaSt22ELMq4QwjFsjEvlyR92kHImn2eGtWf8wFZWy7pbeyCFB7/eSlM/D+aN70eIr5x818SBpCxu/Hgd3cJ8mTe+r00vONQ3E7/Zyraj6WyccoXdvG+VjS0kgCEqteN4OiM/WcfseyO5smNjWzfH4WmtuemTdWTlFfH3U0Ps6+pq6RNwV1fjqv/zz5ec5HbubGRgpKUZtSBKnwC7uBhX4KuiokCJp6eRmVDb01YiImDHjvLbe/QwplLU12NZ6z2sblBGayO4cuJEyS03Fx5/3Nh/992waJER/LDo3NmoHWLZ/+23RhaLs7OR9dOzpxGYAWMqzcmTRiaIyWTcDxgAr71m7L/ySiNIY9lXVAQjR/Lr3U/x+PdRbJ0xDn9VhDKZjEBNbm7Je3fwIDz6KDFF7vx1qhi/Fk25bURP3HtHQocORt+KikqyUaqripkln/5zkLf/jOHHSf3p3TKgTo9VXRLAkHGFEI4gPaeA537axdI9SQxqG8i7t3Wv89UYVuxPYtLc7bQK9Gbu+L4ENnCv0+M5up+2neDpH3fy6BVtePoaKeBbFek5BfT533Lu6teC/9zQydbNOauysYVMIRGVSswwToCkiKd1KKWYNKQ1D8/bztI9iYzoaifrLyckGHUOLCeqhYXw55/w6aflT4ACanhSZfHqqyXTLCxMpropEFnbgQN7OZa13sMNG8oGL8D4ff36ih+vlPH9CAiAbt3K7//mGyMQkJkJx48bAY7SAfYtW0r6ZTLB0qVli5smJho3FxcjwHHu1JSmTY1ME0tQzdmZvV5BPPHDDiJbBOBz1x0otLFv5UojcGIyGbeXXoLYWNonJ9M2JQWnDcXwPeS/8CLu/3sNkpONfwuWoqtBQcb9+PFGgdbMTGNalGV7UBA0alTSxioUQk3PKWD6P4e4skNw9YMXJpORQWPpiwMVXRVCiNrk5+XGjLt68e3mY7zy615GfLiGqaO7M7R9cJ0c78/diTz63XY6hDTk6/v74O/tVifHuZSM6hXGxrhUPl55kN4tAxjcLsjWTbJ7S6ITKTAVc0tP+58+ApKBIc7jq/VHeOmXPWx58SqCfCQabA2mYs1V763Cx8OFnycPsI8UcWtOtbBmpoKjcsT3sA6ySv6JSWbC19vo1LQhc8f3pYGlaNuFjlVczF9r9vL+vLUENw/h/SdHEJB/Bj76yAhknDpVcj9lirHKzPbtRp2R0pQypksNHQrh4UaAwcnJeJyTkxFs+OADI4vk779JGTeRrOxcQn3ccEMb+xcuhD59jGyKCRNKAi6WLJPt243P/ZNPjOldYARNLLVnajErRzIwZFwhhKOJTcrise+i2J+YxQMDw3l2ePtanT748454npq/k+5hvnw5rg++nrIKXW3JLTAx8pN1nDqTz5LHBsmUnAu4bcYG0nIKWPbkYPs49zCTDAxRbYmZebg6KxpJNNhqnJ0UEwa3YsrCaNYfSmVAGzuoolzdK+0Xo76eYNsTR3wPazmrZP3BFCZ+s422jRvw1f19SoIXVTmWkxPXDOmCa+NgJs3dxu2fbWDe+L4Ev/RS5Qfs1MnI6Cgd3EhONlZsKX08rY1pMJ07G1kk5ikpKc7ubPFpSuPmDQhvFWjsc3YuWeGmdWtjKV/LdksWSrD5imH//vDmm/Dzz0bR1ot8/4QQ4lLQrrEPiycP4I0l+5i99jAb41KZdmcErYMuflnTH7ce59mfdtG7ZQBf3Ne77P9D4qJ5ujnzydie3PjxWh77LopvH7R9PYzFUfF2ucTz8bQcNh9J45lh7e0qeHE+koEhKvXU/B1siktj3fNX2Lopl5S8QhOD3l5JhxAfvnmgr62bI4Tt1WJWydYjadw9ezPNA7z4bkI/As4N0FbjWBsOpTL+qy0E+rgz94G+NAuoZsX4KmaWPP/TLhZuj2f500Oqf4xqHqumJANDxhVCOLJle5N4dsFO8gqLefnGzoyODKvxyd7cjUf59+LdDGobyMy7I/F0s8OV5xzE4qh4nvhhBw9d3prnhnewaTumLIwmt9B0dpunqzNv3NLV5kGMj1cc4J2/Ylnz7NCajzHqSGVjC/soMSrsUlJmHo0bytQRa/NwdeaBgeGsOZDC7vgMWzdHCNuLijKyE869VTN4sfN4Ovd9uYUmvh7MHd+3fPCimsfq37oRc8f3JT2nkNEzNnAw+Uz1+nW+bA+zg8lnmL/1OGP7Nb+4gUUVjiWEEKJiV3dqzB+PD6ZHMz+e/WkXj3wXRUZuYbVfZ/baw/x78W6u7BDMrHskeFHXRkaEcmefZkz/5xAr9yfbpA2FpmJe/W1vmeAFQG6hialLY2zSJgutNYui4ukTHmB3wYvzkQCGqFRiRp4U8LSRMX2b4+PuwvRVh2zdFCEcwt6TmdzzxWb8vV2Z92DfWqvrE9Hcn+8n9KOoWHP7ZxvYezKz6k+uwvSsd5bG4OnqzOShbS6uodacCiaEEA4oxBz8fmZYe/7cnciID9ew7WhalZ//ycqDvPrbXq7tEsL0u3rh4SrBC2t46YbOdGzSkCfn7+Bkeq5Vjqm1ZufxdP77yx76vb6c1OyCCh9nrfZUZnd8JodOZXOzHUxlqQ4JYIhKJWXmSwDDRhp6uDK2Xwv+iE7gSEr2hZ9QxxZHxTPgzRWEP/87A95cweKoeFs3SYgqO5CUxV2zN+Ht5sy34/vRxNezVl+/Y5OGzJ/YD3cXJ+6YuYHtx05X7YkXyPbYcTydP/ck8uDgVhe/rF4tZbEIIcSlzNlJMXloGxZM6o+TE9z22UamLT+AqbjyKflaa977K4apS2MY2aMpH90ZgZuLnIJZi4erM5+MiaCwqJhHvt1Ooan4wk+qoeNpOXy0/ABXvreKmz5Zx7ebj9GvVaOKMz6Bpn61Ox6proVRJ3BzdmJEFztZ+bCK5F+PqNCZ/CLO5BdJ1V4bun9AS1ycnZi5Js6m7bDM24tPz0UD8em5TFkYLUEMUS8cTslmzOebcHFSzHuwX52lSLYKasD8Sf0J8Hbjrs83sf5gykW9ntaat/7YTyNvN8YPalVLrRRCCFEbIpr7s+SxQVzfrQnvLYtlzKyNFV5N11rzxh/7mbbiILdHNuPd23rYvJjkpahVUAPeGNWN7cfSeaeWp21k5BTy7aZjjJ6xnkFvr+TdZbEENXDnzVu6suXFq/hkbE/+c30nPCvIuBkdGVarbamOIlMxv+48yRUdgvH1ql8r4EjJW1GhpEyj0FuIZGDYTHBDD0b1DGPBthM8cVVbgn1s81lMXRpT6bw9WxceEuJ8jqflMHbWRkzFmh8m9CM80LtOjxfm78X8Sf25+/PN3DdnC9PH9uTKjo1r9FqrD6SwIS6V/97QSarTCyGEHfLxcOWD23swuG0Q//l5N9d+uIa3RnUjzzxGOpmei5ebM9kFJu7u14KXb+yMk1P9WOXBEd3YvSmb4lL5bHUcfcIDavz/M0BBUTErY5JZtD2eFfuTKTAV0zrIm2eGteemHk0J8y97scQyXrZ8Lxo39KDQZGLO+iOM6NqEdo19LqpvNbH2YAopZwq4uWf9G8vLqEhUKCnDCGAESxFPm5o4uBU/bDnGnHVHeNZG1ZMrm59n63l7QpxPQkYuYz7fSHaBie8e7EdbKw0Ogn08+GFiP+79YjMTv9nG+7f34IbuTav1GsXFRvZFmL8nd/ZtXkctFUIIcbGUUozqFUavFv489n0Uk+Zuw9lJnZ1Skl1gwsVJ0bO5nwQv7MD/Xd+JHcfTeWr+Tn5/bGC5QMP5aK3Zfuw0C7fH83t0Auk5hQQ2cGNsv+bcEhFGl9CG512ZZmREaJkLf8fTchg1fT33zN7Mgof6V6sttWFxVDy+nq5c3j7IqsetDZLDJCqUKBkYdqFloDfXdmnCNxuPkpVX/WrXF2tTXGql/+G6uTiRnJVX4T4hbCk5K4+xszaRnl3I1/f3oVPThlY9vp+XG3PH96WneUD7w5Zj1Xr+b9EJ7E3I5Olr2uHuIkXehBDC3rUM9GbBpMto4O5Srh5GUbHmnb9ibdQyUZpRD6MnpmLNI99GUVB04XoYh1OyeW9ZLEOm/sOo6Rv4afsJBrcN4stxvdk45UpeuqEzXcN8q72sbrMAL75+oA85BUXcM3szKWfya9qtasvOL2LpniSu69akXo4zJIAhKnQ2gCE1MGxu0pDWZOUV8e2m6p0EXYzs/CL+8/Nubp+5EV9Pl3LFplydFUWmYoZ/sIa/9yZZrV1CXEhadgF3fb6JxMw8vhzXm+7N/GzSDh8PV74a14fBbYN47qdoZq89XKXnFRQV8+5fMXQI8eGm7vUvrVMIIS5Vbi5OZOcXVbhPslbtR8tAb96+tRs7jqfz4NdbKixSn5ZdwNcbjjDyk3UMfecfPlpxgGYBnrwzujtb/3010+6MYGj74IuuZ9IhpCFf3Nebkxm53PflZqtdrFy6J5HcQlO9W33EQgIYokLJmfn4eLjg5SazjGyta5gvA9sEMnvtYfKLTBd+wkVaeyCFa95fzTcbj3L/gHDWPncFb4/qRqifJwoI9fNk6q3dWfrkYEIaejD+6628sCianIKK/9MWwloycgq5e/Ymjqbm8Pk9kUS2DLBpezzdnJl1TyTXdgnh1d/2Mm35AbSuvFI9wA9bjnE0NYfnhneQdGMhhKhnKltVwtarTYiyRnRtwqA2jVgVm1KmSP0zC3Zy3bTV9Pnf3/zn5z3kFZqYcm0HNjx/JfPG9+PWXmG1XpcqsmUA08f2Yn9CFg9+vZW8wrof6y+KiifM35PIFv51fqy6IGenokKJGXkyfcSOTBrSmrtmb2LR9nju6FM3c+Iz8wp5/fd9fL/lOK2CvFkwqT+9WhgngOfO27NYNPky3vsrlplr4tgYl8qHt0fQNcy3TtonxPmcyS/i3i83E5uUxax7IrmsTaCtmwQYV+Q+ujOC536K5r1lsWTnF/H8tR0qTDXNzi/iw+UH6dMyoF7OSRVCiEvdM8PaM2VhdJni556uzjwzrL0NWyUqcuhUdrlthSbNvoQsxg9qxcgeoVabgjq0QzDvjO7OEz/s4LHvovh0bM86W60mOTOPdQdTmDy0TbWnvdgLycAQFUrMzKOxBDDsxoA2jegS2pCZq+POu9Z4TS3fl8Q1761m/tbjPHR5a5Y8Nuhs8OJ83F2cmTKiI/Me6EtOvombP13Hp/8crJM2ClGZnIIi7v9yC9HxGXw8pieXtw+2dZPKcHF2Yuqt3binfws+Wx3HvxfvpriCfyNfrjtMypl8nqskwCGEEMK+jYwI5Y1bupbJWn3jlq6yapsdSsiouI6b1vDCiI5Wr581MiKUl27oxF97k3hhUfQFMzZr6pedJynWcFOP+vudlAwMUc7iqHh2nUinWIqWsYUAACAASURBVMOAN1fwzLD28ofXxpRSPDSkDZO/3c5fexK5tmuTWnnd09kFvPLbXhZFxdMhxIeZ9/SiW1j1awZc1iaQP58YxIuLdvP2nzGsijnFe7f3IFRSJkUdyys0MeHrbWw9msaHd0QwrHOIrZtUIScnxcs3dqaBuwuf/nOI7Pwi3hnd/ewVltPZBXy2Ko6rOzWmVz1N6RRCCFF51qqwL039PImvoDaJLaf7jBsQzumcQqYtP4C/txtTru1Y68dYFBVPtzBf2gQ3qPXXthYJYIgyFkfFM2XhLiwXB+PTc5myMBpA/hjb2PAuIbRs5MWMVYcY3iXkoq/Q/hGdwP/9vJv0nEIev7Itk4e2KVesszr8vNz4eEwEQ7cH89LPuxn+wWpev7lrtZeQFKKqCoqKeXjedtYeTOGd0d3t/rumlOLZ4R3wdndh6tIYcgpMXNOpMe//feDsIKpnc9sUHRVCCCEuJfY63efJq9qevagR4OXGxCGta+21Y5Oy2HMyk5du6FRrr2kLMoVElPH2n/vJLSy7pFBuoYmpS2Ns1CJh4eykmDC4NTtPZLAhLrXGr3MqK5+H523joXnbCfH14NdHB/Lk1e0uKnhhoZTi1l5hLHl8EG2CG/Dod1E89cMOmywBKxxbkamYx76LYsX+ZP53cxdu7RVm6yZV2eShbXj5xs78tTeJZ3/aVeYK0LTlB89WQRdCCCFE3bDX6T5KKf57Y2eu79aEN/7Yz/wtx2vttRdHxePspLi+m31f8LkQycC4BOUVmjiamsOR1GyOpmZzOCWHo6nZHE3N4WQl88Fk+Sf7cEvPUN5bFsv0fw5xWevqFSnUWvPzjpP899c95BSYeHZ4eyYMalUnRYJaNPLmx4n9+WjFQT5acYDNR9L44PYeNl8VQjgGU7Hmqfk7+XNPIv+5vhNj+7awdZOq7d7LWvL+37Gk55QN7lkCxrYeQAkhhBCOzl6n+zg7Kd67rQcZuYU8v3AXvl6uFz1FtrjYOA8Y1DaQIB/3WmqpbUgAo55YHBXP1KUxnEzPpamf5wXrUuQUFHE01QhMHEnN4UhKtjlgkVOuaE2AtxstGnnRJzyAv/clkZVXfjlMWf7JPni4OnP/wJa8/WcMu+Mz6BJatRU/EjPyeHFRNMv3J9OzuR9v39qNNsE+ddpWF2cnnry6HYPbBfHED1Hc9tkGHhnahkevbItrHVVWFo6r9N9ATzfns0G4+weG27ppNZaRU3FmkgSMhRBCiEubm4sTn93di7Gfb+LR76L4alwf+rduVOPX23wkjfj0XJ4dXv9XxJEARj1g1KUomaNlqUuRX2iiS5jv2WwKI0hhBC2SMvPLvEZgAzdaNPKmf+tGhDfypkWgNy0bedGikTe+nq6VHgvsYz6YKHFXvxZMX3mIGasO8fGYnud9rNaa+VuP89pv+ygsLub/ru/EfZe1xNnJeisc9Grhz5LHBvHfX/YybcVBVh9I4YPbe9Ay0NtqbRB1o7qB1Ys5Tum/SzkFJlycFE1963dg1R4LiAkhhBDCPni5ufDlfb0ZPWMDD369le8n9KvyxctzLY6Kx9vNmWs62Wex8+qQAIYdKzQVk5iRx2u/7y0TUAAjzfg5c3FNiyAfd1o28mJQ2yDCA71p0ciLlo2Mex8PV6rCcvJhjZMSUTMNPVwZ0685s1bHcTQ1mxaNKg4EHE/LYcrCaNYeTKFfqwDeGtWt0sfWNR8PV969rTtDOwTxwsJoRkxbw39v6MzoyDCbLhdprRNwR1RZYBUqLvirtSa30MSZ/CKy802cySviTH6R+fcKfs4rIrugiDP5JjYcSqHQVHY5saJiXe+nWthrATEhhBBC2Ac/Lze+eaAvo6av594vNvPjpP60CqreCiJ5hSZ+j05gWJcQPN2c66il1iMBjItwsSc/Z/KLOJmeS/zpXOLTzbfTuca29FySMvPOrgZSmU/H9jwbqPB2r52P017ng4kSDwwI58u1R5i1Jo7XRnYts6+4WPPNxqO89ed+FPDayC6M6dMcJytmXVTm+m5N6dncn6fm7+DZn3axMiaZ12/uir+3m9XbUt0TcFEiK6+Q15fsqzCw+vzCXSzYdqLCwMSF/p4BKAUN3Fxo4OGCt7sLDdxdygUvLOr7VAsJGAshhBDiQkJ8PfjmgT6MnrGBu2dvZsFD/WlSjSzUlfuTycor4mYHGV9IAKOGLnTyU1ysSTmTfzYwURKoyDv7e0Zu2fnPrs6KJr6eNPXz4LLWgYT6eRDq78nbf8aQml1Qrg2hfp6M6Nqk7jsr7E5wQw9G9Qpl/tYTPH5lu7PFeA6nZPPcgl1sPpLG4HZBZ6sr25Omfp58O74fs9bE8c5fMWw/dpr3buvBgDbVK0p6MfIKTZWegNf3q/pwccHV4mLNKcvfrlIB1ZPpuZwwB1srqpNjkVdYTHZBET4eLjTx9aCBe0kgoiQo4UwDd1e83Z3xMd9b9nu6OpfLyhnw5gqHnWohAWMhhBBCXEiroAZ8dX8f7pi5kXtmb2b+xP5VvgC4MCqeYB/3ai8AYK8kgFFDU5fGVDyt46ddvP93LAnpeRSYyi5H6uPuQqi/J6F+nkS28CfU35OmfsbvoX6eBPm4V1ibwN3FWdKMRTkPDmrFd5uPM/Sdf8jON04Ys/OL8HZ3Yeqt3bi1l22nZ5yPk5Ni4pDWDGgTyGPfRzH28008OCicfw1rj7tL7aS25RWaOJaWw+GU7DLFbI3VdnLRlWQDxKfnkl9kqrV2WNuFgqt5hSYSMvLOBidOnBOoSMjILZfx4OPhcvbvVJ/wAEL9PJmx6hCnKyhCGernyaKHB9Rqn2SqhRBCCCEudV1CfZl1TyT3frmZcXO2MG983wtm4J/OLuCfmGSr18CrSxLAqKHKUpfzi4rpGurL8M4hZ4MVTf08CfX3pGEV61CcS9KMRUV2ncjASRlTkQAy84pwUvD0Ne0YHdnMxq2rmi6hvvz+6CD+t2Qvs9YcZu3BVG7s3oS5G49V6bueW2A6uxywpYDt4ZSKV9vx93KlRSNv+oQH0KKRF1+tP1LhCThA79f+5rpuTRjZI5TeLQPsYvpNVVUWXH1mwU5e+30fKWfKFvhVChr7eNDUz4PuzfwY0bXJ2eyvpua/XxX97Wrc0MNqQQX5GyiEEEIIAf1bN+KjOyN4aO42Js3dxux7e+PmUvnqfr9HJ1Bo0g41ZlK6ssuQDioyMlJv3br1ol+nspTmUD9P1j1/xUW/vhAX4mjfweX7knjsuyiyC8qefHu4OvHIFW1oFdjACFak5HDYHLQ4d7WdRuYlgVsGep8tYNuykfGzr1fZk/CKVtzxcHXivstakpyZz597EskpMBHq58mNPZpyc0Qo7RrX7dKzNWUq1kTHZ7A69hTvLYut9HF39G52NuurqZ8nYf6eNG7ocd7/+M5HiqAKC6XUNq11pK3bYQu1Na4QQgghqmr+luM8+9Muru/WhA/viKg0u+LW6evJzCtk6ROD7TYzuzKVjS0kA6OGJKVZ2FplWUD1tbDhlR0b4+PhWi6AkVdYzDtLS07KAxuUrLZjWQq4ZSNvWgR6VSvL6UJX9V8rKGLZ3iQWR8Uzc3Uc0/85RMcmDbk5oik3dg8lxNejFnpdcwkZuayJTWHVgVOsO5hCek4hShm1dCoqehnq58mbo7rVahukfoMQQgghhPXd1rsZp3MKeOOP/fh7ufHKTZ3LBSiOpeaw9ehpnh3evt4FL85HAhg1JCnNwtaa+nk6XGHDpMy8Svf99uhAWgZ606CWVtuB85+Ae7m5cFOPUG7qEUrKmXx+35XAoqh4Xl+ynzf+2E//Vo0YGRHK8C4hNZ4eVh25BSY2HU5lzYEUVsee4kDyGQCCfdy5qmNjBrUNZGCbQNYcSJHgqhBCCCGEg5s4pDVpOQV8tioOf283nrq6XZn9i3fEA3BTD8c6P5UAxkWQq4/ClhwxC6iyoEyonyddQn1t0CJDYAN37r2sJfde1pLDKdn8vCOexVHxPLtgF/+3eDdXdWzMyIhQhrQLqvF0jHNprYlJymJ17CnWHEhh0+E0CoqKcXNxom94ALdFNmNQu0DaN/YpE1WX4KoQQgghxKXh+eEdSM8uZNryA/h7uTJuQDhgjCMXR8XTr1WA3a1IeLEkgCFEPeWIJ6r1ISgTHujNE1e14/Er27LjeDo/7zjJrztP8nt0An5erlzXtQk3R4TSq4V/tdP1Us/ks/ZgCqtjU1hz4BTJWUaNj3aNG3B3vxYMbhdE3/AAPFzPv0KKBFeFEEIIIRyfUor/3dyF0zkFvPzrXvy93BgZEcquExnEpWQzcUgrWzex1kkRTyGEXamPhSELTcWsPZDC4h3xLN2TSF5hMWH+nozsEcrIiKa0CfapsF8jujZh+7HTrDlwitWxKew+mYHW4OflysA2gQxuG8SgdoE08XWsyLlwPFLEU8YVQgghbCev0MR9X25mU1waAd5upGYXAPD6LV0Y06eFjVtXM5WNLSSAIYQQtehMfhF/7Ulk8Y6TrD1wimINYX4eJGXllymu6aTAxUlRYNI4Oyl6NvczByyC6Brq6zBrdYtLgwQwZFwhhBDCtr7ffJQpi3ZT+vTe09WZN27pavcXAysiq5AIIYQVNHB34ZaeYdzSM4zkrDx+3ZnAm3/sK7cySLEGF2cnPhrTg/6tG1mlEKgQQgghhHBMH604xLm5CbmFJqYujamXAYzK1E61OSGEEOUE+3jwwMBwiipY1hSMlUWGdbbOKiZCODKlVDOl1Eql1D6l1B6l1OPm7QFKqWVKqQPme3/zdqWUmqaUOqiU2qWU6lnqte41P/6AUupeW/VJCCGEqI6TFRTCP9/2+koCGEIIUccqW9q2Pi95K4SdKQKe1lp3BPoBk5VSnYDngeVa67bAcvPvANcCbc23CcB0MAIewEtAX6AP8JIl6CGEEELYs0tlvCkBDCGEqGPPDGuP5zkrh9jb6ipC1Gda6wSt9Xbzz1nAPiAUuAn4yvywr4CR5p9vAr7Who2An1KqCTAMWKa1TtNanwaWAcOt2BUhhBCiRi6V8abUwBBCiDrmiEveCmGvlFItgQhgE9BYa50ARpBDKRVsflgocLzU006Yt1W2XQghhLBrl8p4UwIYQghhBSMjQh3uPxAh7I1SqgHwE/CE1jpTqUpX86lohz7P9nOPMwFj6gnNmzevWWOFEEKIWnYpjDdlCokQQggh6j2llCtG8GKe1nqheXOSeWoI5vtk8/YTQLNSTw8DTp5nexla65la60itdWRQUFDtdkQIIYQQlZIAhhBCCCHqNWWkWswG9mmt3yu16xfAspLIvcDPpbbfY16NpB+QYZ5qshS4Rinlby7eeY15mxBCCCHsgEwhEUIIIUR9NwC4G4hWSu0wb3sBeBOYr5R6ADgGjDbvWwKMAA4COcA4AK11mlLqVWCL+XGvaK3TrNMFIYQQQlyIBDCEEEIIUa9prddScf0KgCsreLwGJlfyWl8AX9Re64QQQghRW2QKiRBCCCGEEEIIIeyeBDCEEEIIIYQQQghh9ySAIYQQQgghhBBCCLsnAQwhhBBCCCGEEELYPWXUsbp0KKVOAUdr+WUDgZRafk174Ij9csQ+gWP2yxH7BI7ZL0fsE0i/qqOF1jqoll+zXqijcQU45vfPEfsEjtkvR+wTOGa/HLFPIP2qT+qqTxWOLS65AEZdUEpt1VpH2rodtc0R++WIfQLH7Jcj9gkcs1+O2CeQfgnbcsTPyRH7BI7ZL0fsEzhmvxyxTyD9qk+s3SeZQiKEEEIIIYQQQgi7JwEMIYQQQgghhBBC2D0JYNSOmbZuQB1xxH45Yp/AMfvliH0Cx+yXI/YJpF/Cthzxc3LEPoFj9ssR+wSO2S9H7BNIv+oTq/ZJamAIIYQQQgghhBDC7kkGhhBCCCGEEEIIIeyeBDCEEEIIIYQQQghh9ySAUQVKqS+UUslKqd0V7PuXUkorpQLNvyul1DSl1EGl1C6lVE/rt/jCqtmnsea+7FJKrVdKdbd+i6umOv0qtb23UsqklLrVei2tuur2SSl1uVJqh1Jqj1JqlXVbW3XV/A76KqV+VUrtNPdrnPVbfGEV9Ukp9V+lVLz5M9mhlBpRat8U89+KGKXUMNu0+sKq0y+l1NVKqW1KqWjz/RW2a/n5VffzMu9vrpQ6o5T6l/VbfGE1+A52U0ptMP+7ilZKedim5ZcWRxxXgGOOLRxxXAEytjD/LmMLG3LEsYUjjivA/sYWEsComjnA8HM3KqWaAVcDx0ptvhZoa75NAKZboX01MYeq9+kwMERr3Q14FfsuPjOHqvcLpZQz8Baw1BqNq6E5VLFPSik/4FPgRq11Z2C0ldpYE3Oo+mc1Gdirte4OXA68q5Rys0Ibq2sOFfQJeF9r3cN8WwKglOoE3AF0Nj/nU/P30R7NoYr9AlKAG7TWXYF7gW+s1MaamEPV+3V2H/BHnbes5uZQ9e+gCzAXmGT+e3E5UGithl7i5uB44wpwzLHFHBxvXAEytgAZW9jaHBxvbDEHxxtXgJ2NLSSAUQVa69VAWgW73geeBUpXQr0J+FobNgJ+SqkmVmhmtVSnT1rr9Vrr0+ZfNwJhdd/CmqnmZwXwKPATkFzHTauxavZpDLBQa33M/FxH6ZcGfJRSCmhgfl5RnTeyms7Tp4rcBHyvtc7XWh8GDgJ96qxxF6E6/dJaR2mtT5p/3QN4KKXc66xxF6GanxdKqZFAHEa/7FI1+3QNsEtrvdP83FSttanOGifOcsRxBTjm2MIRxxUgYwvLw5Gxhc044tjCEccVYH9jCwlg1JBS6kYg3vLhlBIKHC/1+wnzNrt3nj6V9gD2HyUso7J+KaVCgZuBGTZp2EU4z2fVDvBXSv1jTrG7xwbNq7Hz9OtjoCNwEogGHtdaF1u7fRfhEXOa9BdKKX/ztnr7t6KUivpV2iggSmudb+2GXaRy/VJKeQPPAS/btmk1VtFn1Q7QSqmlSqntSqlnbdnAS50jjivAMccWjjiuABlbIGMLe+GIYwtHHFeAjcYWEsCoAaWUF/Ai8J+Kdlewze7Xqr1AnyyPGYoxyHjOWu26WBfo1wfAc/XtiuMF+vT/7N15fJT1uf//15WdbIRkwhK2DPsiBBERC9atFayitse2Wtvac9p6Wttf+/31V0/1nPbrOV09p/3W1p7TWlvt8qtLq3VB61KtouKCgshiAFmCQBIgELIRsn++f8w9NED2zMw9mbyfj8c8yHzmnnuuG8LkyjWfz/VJAc4CLgOWA98ysxkxDG/Aermu5cDbQBGwAPhvM8uNYXiD8QtgKqG4K4H/440PyfeKTrq7LgDMbC6hadT/HPvQBqW76/oPQlMlG/wKbBC6u6YUYBlwnffnh83sYl8iHOYSMa+AxMwtEjGvAOUWKLeIF4mYWyRiXgE+5hYpkTzZMDIVCAIbQ7POmAC8ZWaLCVU6J3Y6dgKhym686/aanHMHzGw+8GvgUufcER/j7K+e/q0WAQ944wHgQ2bW5px71K9g+6i377/DzrljwDEzewkoAd71K9h+6Om6/hG4zTnngJ1mVgbMAt7wK9i+cs4dDH9tZr8CnvDuDtX3CqDH68LMJgCPAJ92zu3yIbwB6+G6zgGuNrP/AvKADjNrcs79tw9h9ksv34MvOucOe489CSwE/hbzICUR8wpIzNwiEfMKUG6h3CIOJGJukYh5BfibW2gGxgA45zY750Y754qdc8WE/qEWOucOAKuAT1vIEqDWOVfpZ7x90dM1mdkk4GHgU865ofDD6oSerss5F+w0/hBw41BIMnr5/nsMOM/MUrxPHc4BtvoYbp/1cl17gYsBzGwMMJPQmsG4ZyevVf8wEO7gvAq4xszSzSxIqEFf3CdNYd1dl4Wavf0FuMU594ofsQ1Gd9flnDuv0/fmT4DvD5Uko4fvwWeA+WaWaaGmW+cDpbGOTxIzr4DEzC0SMa8A5RbKLeJDIuYWiZhXgL+5hWZg9IGZ3U+og2rAzPYDtzrn7u7m8CeBDxFqmtNIqLobd/p5Tf8bKCDUyRigzTm3KCaB9lM/r2tI6M81Oee2mtnTwCagA/i1c+60rcTiQT//rb4D/NbMNhOaHvmNcGU3nnR1TcAFZraA0BTOPXjTHp1z75jZnwi9qbcBX4rXacf9uS7gy8A0QlOMv+WNXeLisOlbP69rSOjn9+BRM/sx8Kb32JPOub/4Efdwk4h5BSRmbpGIeQUot/Aot/BRIuYWiZhXQPzlFhaaNSUiIiIiIiIiEr+0hERERERERERE4p4KGCIiIiIiIiIS91TAEBEREREREZG4pwKGiIiIiIiIiMQ9FTBEREREREREJO6pgCEiQ5KZrTazuNtyT0RERIYe5RUiQ4MKGCIiIiIiIiIS91TAEJGYMLN/MbOveF/fbmbPe19fbGZ/MLNLzOw1M3vLzB40s2zv8bPM7EUzW29mz5jZuFPOm2RmvzOz78b+qkRERMQPyitEhicVMEQkVl4CzvO+XgRkm1kqsAzYDHwT+IBzbiGwDvia9/jPgKudc2cB9wDf63TOFOBe4F3n3DdjcxkiIiISB5RXiAxDKX4HICLDxnrgLDPLAZqBtwglHOcBq4A5wCtmBpAGvAbMBM4AnvXGk4HKTuf8JfAn51zn5ENEREQSn/IKkWFIBQwRiQnnXKuZ7QH+EXgV2ARcCEwFyoBnnXPXdn6Omc0D3nHOndvNaV8FLjSz/+Oca4pa8CIiIhJXlFeIDE9aQiIisfQS8HXvz5eBLwBvA68DS81sGoCZZZrZDGA7UGhm53rjqWY2t9P57gaeBB40MxVkRUREhhflFSLDjAoYIhJLLwPjgNeccweBJuBl51wV8BngfjPbRCjxmOWcawGuBv7TzDYSSkre1/mEzrkfE5o2+v+bmd7TREREhg/lFSLDjDnn/I5BRERERERERKRHqiqKiIiIiIiISNxTAUNERERERERE4p4KGCIiIiIiIiIS91TAEBEREREREZG4pwKGiIiIiIiIiMQ9FTBEREREREREJO6pgCEiIiIiIiIicU8FDBERERERERGJeypgiIiIiIiIiEjcUwFDREREREREROKeChgiIiIiIiIiEvdUwBCRIcHMis3MmVnKcHx9ERERiSy/f7b7/foiQ5EKGCLDiJn9u5n9IYavl2ZmD5nZHu8H9AWnPH6TmW0xs3ozKzOzm2IVW6SZ2R/MrNLM6szsXTP7nN8xiYiIRJsPucUcM1tnZke923NmNqfT44mUWxSb2ZPedR4ws/9WsUOGOxUwRKTPBvhDcw3wSeBAV6cEPg2MAlYAXzazawYeYacTx/4H/A+AYudcLnAF8F0zOyvGMYiIiAwpA/h5XQFcDeQDAWAV8EDnU5I4ucXPgUPAOGABcD5wY4xjEIkrKmCIJCAz+4aZlXufPmw3s4vNbAXwr8DHzazBzDZ6xxaZ2SozqzaznWb2+U7n+XdvBsUfzKwO+IyZJZnZzWa2y8yOmNmfzCy/qziccy3OuZ8459YA7V08/l/Oubecc23Oue3AY8DSXi7vn8yswpvt8P/1EutiM3vNzGq84//bzNI6PceZ2RfMbIf36cb/mJl5jyWb2Y/M7LCZ7QYu6yko59w7zrnm8F3vNrWXaxERERkS4ii3qHHO7XHOOULFinZgWqfHEya3AILAn5xzTc65A8DTwNxeniOS0FTAEEkwZjYT+DJwtnMuB1gO7HHOPQ18H/ijcy7bOVfiPeV+YD9QROgTje+b2cWdTnkl8BCQB9wLfAW4itCnAEXAUeB/IhC3AecB7/Ry6IXAdOAS4GYz+0APsbYD/y+hT2jOBS7m9E8uLgfOBkqAjxH6+wL4vPfYmcAiQn83vV3Dz82sEdgGVAJP9vYcERGReBePuYWZ1QBNwM+8GLo6ZqjnFj8FrjGzTDMbD1xKqIghMmypgCGSeNqBdGCOmaV6n1Ls6upAM5sILAO+4VX33wZ+DXyq02GvOecedc51OOeOA/8M/Jtzbr834+DfgasjMK3y3wm9J/2ml+P+wzl3zDm32Tv22u5idc6td8697n0Kswf4JaHkqLPbvE9z9gIvEJqiCaGE4yfOuX3OuWpCS0R65Jy7EcghlCw9DDT3/AwREZEhIe5yC+dcHjCSUGFlQzeH/TtDO7d4kdCMizpCBaF1wKO9PEckoamAIZJgnHM7gf9F6If2ITN7wMyKujm8CKh2ztV3GnsPGN/p/r5TnjMZeMSbOlkDbCWU2IwZaMxm9mVC61Uv67QMozud43mP0DV0GauZzTCzJyzU+KqO0Cc0gVPO17k3RyOQ7X1d1MVr9co51+4tmZkAfLEvzxEREYln8ZpbOOeOAXcCvzez0Z0fG+q5hZklAc8Q+kAky3uNUcB/9nItIglNBQyRBOScu885t4xQQuD4+w87d8qhFUC+meV0GpsElHc+3SnP2Qdc6pzL63TLcM6VMwBm9k/AzcDFzrn9fXjKxFNiregh1l8QWs4x3Wuu+a+E1sv2RWUXr9UfKagHhoiIJIg4zi2SgEw6FUgSJLfI9479b+dcs3PuCKHZIR/q42uJJCQVMEQSjJnNNLOLzCyd0NrQ4/y9geZBoNir6uOc2we8CvzAzDLMbD7wWUJrPLtzJ/A9M5vsvV6hmV3ZQzzpZpbh3U3zXifczOo6Qp9cfNA5t7uPl/gtby3oXOAfgT/2cGwOoWmXDWY2i/7NiPgT8BUzm2BmowglQl0ys9Fmdo2ZZXsNupYTmn76fD9eT0REJC7FU25hZh80szO9n7e5wI8J9czY6j2eELmFc+4wUAZ80cxSzCwPuB7Y2I/XE0k4KmCIJJ504DbgMKEpjKMJfToA8KD35xEze8v7+lqgmNCnDY8Atzrnnu3h/D8ltGXZX82sHngdOKeH47cTSnTGE5oKeZzQpzcA3wUKgDct1L28wczu7OX6XgR2An8DfuSc+2sPx34d+ARQD/yKnhOSU/3Ki3cj8BahKZzdE8Qo3gAAIABJREFUcYQSmP2EkqgfAf/LOfdYP15PREQkXsVTbpFHqEloLbCL0A4kK5xzTd7jiZJbAHyE0FawVV58bYQaiIoMWxbagUhEREREREREJH5pBoaIiIiIiIiIxD0VMEREREREREQk7qmAISIiIiIiIiJxTwUMEREREREREYl7KmCIiIiIiIiISNxL8TuAWAsEAq64uNjvMERERBLG+vXrDzvnCv2Oww/KK0RERCKvu9xi2BUwiouLWbdund9hiIiIJAwze8/vGPyivEJERCTyussttIREREREREREROKeChgiIiIiIiIiEvdUwBARERERERGRuDfsemCIiMjw0trayv79+2lqavI7lCEvIyODCRMmkJqa6ncoIiIivlFuETn9zS1UwBARkYS2f/9+cnJyKC4uxsz8DmfIcs5x5MgR9u/fTzAY9DscERER3yi3iIyB5BZaQiIiIgmtqamJgoICJRiDZGYUFBTo0yYRERn2lFtExkByCxUwREQk4SnBiAz9PYqIiIToZ2Jk9PfvUQUMERGROLJ69WpeffXVQZ0jOzs7QtGIiIjIUJdIuYUKGCIiInEkEkmGiIiISFgi5RYqYIiIxMCjG8pZetvzBG/+C0tve55HN5T7HZJ0I1r/VldddRVnnXUWc+fO5a677gLg6aefZuHChZSUlHDxxRezZ88e7rzzTm6//XYWLFjAyy+/zGc+8xkeeuihE+cJfwLS0NDAxRdfzMKFC5k3bx6PPfZYROIUf+m9QkQk8Si3iBztQiIiEmWPbijnloc3c7y1HYDymuPc8vBmAK46c7yfockpovlvdc8995Cfn8/x48c5++yzufLKK/n85z/PSy+9RDAYpLq6mvz8fL7whS+QnZ3N17/+dQDuvvvuLs+XkZHBI488Qm5uLocPH2bJkiVcccUVWpM7hOm9QkQk8Si3iCwVMEREouyHz2w/8UMr7HhrOz98Zrt+KYmx/3j8HUor6rp9fMPeGlraO04aO97azr88tIn739jb5XPmFOVy68q5vb72HXfcwSOPPALAvn37uOuuu3j/+99/Ytuw/Pz8vl4GENp67F//9V956aWXSEpKory8nIMHDzJ27Nh+nUfih94rRESGHuUWsaUChohIlFXUHO/XuPjn1ASjt/G+Wr16Nc899xyvvfYamZmZXHDBBZSUlLB9+/Zen5uSkkJHR+j1nXO0tLQAcO+991JVVcX69etJTU2luLhYW5wOcXqvEBFJPMotIsuXAoaZ5QG/Bs4AHPBPwHbgj0AxsAf4mHPuqIXmq/wU+BDQCHzGOfeWd57rgW96p/2uc+53MbwMEZE+KcobQXkXv4AU5Y3wIZrhrbdPM5be9nyX/1bj80bwx38+d8CvW1tby6hRo8jMzGTbtm28/vrrNDc38+KLL1JWVnbSNM+cnBzq6v7+SU5xcTHr16/nYx/7GI899hitra0nzjl69GhSU1N54YUXeO+99wYcn8QHvVeIiAw9yi1iy68mnj8FnnbOzQJKgK3AzcDfnHPTgb959wEuBaZ7txuAXwCYWT5wK3AOsBi41cxGxfIiRET64qblM0lJOnnt4IjUZG5aPtOniKQ7Ny2fyYjU5JPGIvFvtWLFCtra2pg/fz7f+ta3WLJkCYWFhdx111185CMfoaSkhI9//OMArFy5kkceeeREo63Pf/7zvPjiiyxevJi1a9eSlZUFwHXXXce6detYtGgR9957L7NmzRpUjOK/aH3/iYiIf5RbRJY552L7gma5wEZgiuv04ma2HbjAOVdpZuOA1c65mWb2S+/r+zsfF7455/7ZGz/puO4sWrTIrVu3LgpXJiLSvX/4+Sus31sDwJjcdG65dLbWtMfI1q1bmT17dp+Pf3RDOT98ZjsVNccpyhvBTctn6t+qk67+Ps1svXNukU8h+SrSeUXo+28b5TVNpKUk8V//MF/ffyIicUa5RWT1J7fwYwnJFKAK+I2ZlQDrga8CY5xzlQBeEWO0d/x4YF+n5+/3xrobP42Z3UBo9gaTJk2K3JWIiPRRfXMbeZmp1DS28l9Xl3D+jEK/Q5JuXHXmeCUV4pvw99/tz77LHc/v4NypBX6HJCIig6TcInL8WEKSAiwEfuGcOxM4xt+Xi3Slqz1bXA/jpw86d5dzbpFzblFhoX5pEJHYamptZ1fVMVbMDXVwLqtq8DkiEYl3K0uKcA7+sqnS71BERETihh8FjP3AfufcWu/+Q4QKGge9pSN4fx7qdPzETs+fAFT0MC4iElfePVhPe4fj/TMKyU5PoezwMb9DEpE4N210NnPG5bJqo1IbERGRsJgXMJxzB4B9ZhbuWnIxUAqsAq73xq4HHvO+XgV82kKWALXeUpNngEvMbJTXvPMSb0xEJK6E9wafW5RLMJBF2ZFGnyMSkaFgZUkRb++rYV+13jNERETAv11I/h/gXjPbBCwAvg/cBnzQzHYAH/TuAzwJ7AZ2Ar8CbgRwzlUD3wHe9G7f9sZEROJKaWUd2ekpTByVGSpgHNYSEhHp3eXzxwHw+CbNwhAREQF/mnjinHsb6Kpb+cVdHOuAL3VznnuAeyIbnYhIZJVW1DFrbA5JSUYwkMXjmypobmsnPSW59yeLyLA1MT+ThZPyWPV2BTdeMM3vcERERHzn1wwMEZFhoaPDse1APXOKcgGYUpiFc7BXy0hkELKzswGoqKjg6quv7vHYn/zkJzQ29u/7bfXq1Vx++eUDjk8i54qSIrYdqGfHwXq/QxERkQQ2VHILFTBERKJo39FGGprbmDMuVMAIBrIA2K1GnnKK9vb2fj+nqKiIhx56qMdjBpJkSPz40PxxJBk8rmaeIiLST4mYW6iAISISReEGnuEZGMVeAUM7kcS5yko4/3w4cCAip9uzZw+zZs3i+uuvZ/78+Vx99dU0NjZSXFzMt7/9bZYtW8aDDz7Irl27WLFiBWeddRbnnXce27ZtA6CsrIxzzz2Xs88+m29961snnfeMM84AQknK17/+debNm8f8+fP52c9+xh133EFFRQUXXnghF154IQB//etfOffcc1m4cCEf/ehHaWgI9WR5+umnmTVrFsuWLePhhx+OyHXL4I3OyWDJlAIe31RJaFWtiIgMScotInLdKmCIiERRaWUdyUnGjDE5AORmpBLITqOsSgWMuPad78CaNaE/I2T79u3ccMMNbNq0idzcXH7+858DkJGRwZo1a7jmmmu44YYb+NnPfsb69ev50Y9+xI033gjAV7/6Vb74xS/y5ptvMnbs2C7Pf9ddd1FWVsaGDRvYtGkT1113HV/5ylcoKirihRde4IUXXuDw4cN897vf5bnnnuOtt95i0aJF/PjHP6apqYnPf/7zPP7447z88ssciFByJZFxRUkRZYeP8Y5XEBURkSFIuUVErtmXJp4iIsNFaUUdUwuzyEj9e8PO0E4kKmD45oILTh/72MfgxhuhsREuvhjeeAM6OuDOO2HDBrjhBvjMZ+DwYTh1Xejq1X162YkTJ7J06VIAPvnJT3LHHXcA8PGPfxyAhoYGXn31VT760Y+eeE5zczMAr7zyCn/+858B+NSnPsU3vvGN087/3HPP8YUvfIGUlNCP9vz8/NOOef311yktLT0RR0tLC+eeey7btm0jGAwyffr0E/Hdddddfbouib4VZ4zlm49uYdXGCs4YP9LvcERE5FTKLWKWW6iAISISRaWVdZwTPPnNPhjI4vltVT5FJL167z0IT9V3LnQ/Asysy/tZWaFlRR0dHeTl5fH222/36fmncs716ZgPfvCD3H///SeNv/32270+V/yTl5nG+2cU8sTGCm5eMYukJP1biYgMKcotIkZLSEREouTosRYqa5uY7TXwDAsGsjnc0Ex9U6tPkQ1zq1effvOmU1JbC0ePnpxkHD0KK1aE7gcCpz+3j/bu3ctrr70GwP3338+yZctOejw3N5dgMMiDDz7ovbRj48aNACxdupQHHngAgHvvvbfL819yySXceeedtLW1AVBdXQ1ATk4O9fWhHSyWLFnCK6+8ws6dOwFobGzk3XffZdasWZSVlbFr164T8Q1FZpZsZhvM7Anv/m/NrMzM3vZuC7xxM7M7zGynmW0ys4WdznG9me3wbtf7dS2nuqKkiIraJtbvPep3KCIicirlFjHLLVTAEBGJkq2VJzfwDAvvRLLnsHaGiDvf+U5oemdn7e0RWa86e/Zsfve73zF//nyqq6v54he/eNox9957L3fffTclJSXMnTuXxx57DICf/vSn/M///A9nn302tbW1XZ7/c5/7HJMmTWL+/PmUlJRw3333AXDDDTdw6aWXcuGFF1JYWMhvf/tbrr32WubPn8+SJUvYtm0bGRkZ3HXXXVx22WUsW7aMyZMnD/p6ffJVYOspYzc55xZ4t/BHUJcC073bDcAvAMwsH7gVOAdYDNxqZqNiEnkvPjBnDOkpSdqNRERkqFFuEdHcwoZbR+tFixa5devW+R2GiAwDv355N9/9y1bWffMDBLLTT4y/e7CeS25/iZ9es4ArF4z3McLhYevWrcyePbtvB595JnQ1zXLBgtB61QHas2cPl19+OVu2bBnwOeJFV3+fZrbeObfIp5DCMUwAfgd8D/iac+5yM/st8IRz7qFTjv0lsNo5d793fztwQfjmnPvnro7rSizzii/d+xZry47w+i0Xk5Ksz6BERPyi3CKy+pNb6KefiEiUlFbUMSY3/aTiBcCk/EzMtJVqXNqwITS189TbIBIMiZmfAP8CnPIxF9/zloncbmbh/4zjgX2djtnvjXU3HhdWlozjcEMLr+0+4ncoIiLSV8otIkoFDBGRKCmtrGPOKf0vADJSkxmfN0IFjGGkuLg4IT4hiVdmdjlwyDm3/pSHbgFmAWcD+UC4xXpXXcVcD+Onvt4NZrbOzNZVVcWuIe8FM0eTnZ6iZSQiIjJscwsVMEREoqCptZ2dhxpO638Rpq1URSJqKXCFme0BHgAuMrM/OOcqXUgz8BtCfS0gNLNiYqfnTwAqehg/iXPuLufcIufcosLCwshfTTcyUpO5ZO4YntpygOa29pi9roiISLxQAUNEJAp2HmqgrcMxZ9zILh8PBrIoqzrGcOtD5Bf9PUdGvP49Ouducc5NcM4VA9cAzzvnPmlm4yC06whwFRD+qGoV8GlvN5IlQK1zrhJ4BrjEzEZ5zTsv8cbixsqSIuqb2njp3cN+hyIiMqzF68/Eoaa/f48qYIiIREFpRWgHktnjcrp8PBjIor65jcMNLbEMa1jKyMjgyJEjSjQGyTnHkSNHyMjI8DuU/rjXzDYDm4EA8F1v/ElgN7AT+BVwI4Bzrhr4DvCmd/u2NxY3lk0LMCozVctIRER8pNwiMgaSW6REMR4RkWGrtLKOzLRkJhdkdfl4eCvVssPHKMxJ7/IYiYwJEyawf/9+YtmrIFFlZGQwYcIEv8PokXNuNbDa+/qibo5xwJe6eewe4J4ohTdoqclJXDpvHI+8VU5jSxuZaUrlRERiTblF5PQ3t9BPPRGRKCitrGPW2BySk7rqCQhTAtkA7Dl8jMXB/FiGNuykpqYSDAb9DkMkYlbOL+K+tXv529ZDrCwp8jscEZFhR7mFf7SEREQkwpxzbK2o67aBJ8D4USNITTZ2q5GniPTT4mA+Y3LTWaVlJCIiMsyogCEiEmH7jx6nvrmt2waeAMlJxuSCLMoON8QwMhFJBMlJxmXzinhxexW1x1v9DkdERCRmVMAQEYmwd7wGnj3NwABtpSoiA3fFgiJa2jv46zsH/A5FREQkZlTAEBGJsNLKOpIMZo7pegeSsCmBLPYcaaS9Qx2sRaR/SiaMZGL+CC0jERGRYUUFDBGRCCutqCMYyGJEWnKPxxUHsmhp66Ci5niMIhORRGFmrJxfxKu7jnC4odnvcERERGJCBQwRkQjbWlnHnKLu+1+Edd5KVUSkv65YUER7h+OpzZV+hyIiIhITKmCIiERQbWMr5TXHmTOu5/4XEFpCAipgiMjAzByTw/TR2Ty+UQUMEREZHlTAEBGJoNLKvjXwBCjMSScrLVkFDBEZEDPjipIi3thTTWWtlqKJiEjiUwFDRCSCThQw+jADw8wIFmonEhEZuMtLigB4QrMwRERkGFABQ0Qkgkor6ijMSacwJ71PxwcD2SpgiMiABQNZzBs/ksc3aTcSERFJfCpgiIhEUGllXZ9mX4QFA1nsP9pIc1t7FKMSkUR2RUkRm/bXqhgqIiIx8+iGcpbe9jzBm//C0tue59EN5TF5XRUwREQipKWtg52H6pndjwLGlEAWHQ72VTdGMTIRSWSXzR8HwBMbNQtDRESi79EN5dzy8GbKa47jgPKa49zy8OaYFDFUwBARiZCdhxpobXd9auAZVuztRLK7Sp+cisjAFOWNYHFxPqs2VuCc8zscERFJcD98ZjvHW0+ePXy8tZ0fPrM96q+tAoaISIT0p4FnWLBAW6mKyOCtLBnHjkMNbD9Y73coIiKS4Cpqut75qrvxSFIBQ0QkQkor6shITSLozaroi5GZqRRkpamAISKDcum8cSQnGY9rGYmIiERZUd6Ifo1HkgoYIiIRUlpZy6yxuSQnWb+eFwxksVsFDBEZhEB2Ou+bWsDjGyu1jERERKLq65fM4NRsd0RqMjctnxn111YBQ0QkApxzlFbU9av/RVgwkMUeFTBEZJBWlhSxt7qRjftr/Q5FREQS2MT8TByQNyIVA8bnjeAHH5nHVWeOj/prp0T9FUREhoHymuPUNbX1aweSsGBhFg+u309DcxvZ6XpbFpGBWT53LN98ZAur3q5gwcQ8v8MREZEEdd/avWSnp/DKzReRFePcVTMwREQioLSi/w08w6Z4PTM0C0NEBmPkiFTOn1nIE5sqaO/QMhIREYm8msYWnthcyVVnFsW8eAEqYIiIRMTWynrMYNbYnH4/NxjIBlAfDBEZtJUlRRyqb+bNPdV+hyIiIgnoz2+V09LWwScWT/bl9VXAEBGJgNLKWoIFWQOqRE8uyMQMyqpUwBCRwfnA7NGMSE1mlXYjERGRCHPOcd/a9zhzUt6A+r5FggoYIiIRUFpZx+wBvpFnpCZTNHIEZYcbIhyViAw3mWkpfGDOGJ7aXElre4ff4YiISAJZW1bNrqpjfGLxJN9iUAFDRGSQao+3sq/6+ID6X4QFA1mUaQmJiETAyvnjONrYypqdh/0ORUREEsh9a/eSk5HC5fOLfItBBQwRkUHaVuk18BzEVLpgIIvdh4/hnBrvicjgnD+zkJyMFB7XMhIREYmQ6mMtPL3lAP+wcAIj0pJ9i0MFDBGRQSqtHPgOJGHBQBb1TW0cOdYSqbBEZJhKT0lmxdyx/PWdgzS1tvsdjoiIJICH1u+jpb2DT5zj3/IRUAFDRGTQSivqKMhKY3RO+oDPESzUVqoiEjlXLCiiobmN1dsP+R2KiIgMcc457n9jH4smj2LGmP7vuBdJKmCIiAzS1gN1zCnKxcwGfI4pgVABQ1upikgknDulgIKsNB7fWOl3KCIiMsS9tusIZYeP+T77AlTAEBEZlNb2Dt490DCo5SMA4/NGkJpsauQpIhGRkpzEh+aN47mtB2lobvM7HBERGcLufWMveZmpfGjeOL9DUQFDRGQwdlU10NLeMei9sFOSk5iUn0lZlQoYIhIZVywoormtg+dKD/odioiIDFFV9c084zXvzEj1r3lnmAoYIiKDUFox+AaeYcFAtmZgiEjEnDVpFONGZmg3EhERGbAH1++jrcNx7WL/l4+AChgiIoNSWlFHekoSQa+HxWAEA5mUHTlGR4e2UhWRwUtKMlaWFPHSjipqGrXDkYiI9E9Hh+OBN/ZxTjCfaaOz/Q4HUAFDRGRQSivrmDk2h5Tkwb+dBgPZtLR1UFF7PAKRiYjAyvlFtLY7nt5ywO9QRERkiFmz8zB7qxvjonlnmAoYIiID5JyjtLIuIstHgBOzOLSMREQi5YzxuRQXZLJKy0hERKSf7lu7l/ysNFacMdbvUE5QAUNEZIAqa5uoaWwddAPPsCmFKmCISGSZGVeUFPHa7iMcqmvyOxwRERkiDtU18ezWg1x91gTSU/xv3hnmSwHDzPaY2WYze9vM1nlj+Wb2rJnt8P4c5Y2bmd1hZjvNbJOZLex0nuu943eY2fV+XIuIDF9bKyPXwBNgdE46mWnJKmCIDJCZJZvZBjN7wrsfNLO1Xp7wRzNL88bTvfs7vceLO53jFm98u5kt9+dKImtlSRHOwZObK/0ORUREhog/rdtHexw17wzzcwbGhc65Bc65Rd79m4G/OeemA3/z7gNcCkz3bjcAv4BQwQO4FTgHWAzcGi56iIjEQngHklkRKmCYGcFAlgoYIgP3VWBrp/v/Cdzu5RZHgc96458FjjrnpgG3e8dhZnOAa4C5wArg52YWPx87DdD0MTnMGpujZSQiItIn7R2O+9/Yx9JpBRFpVB9J8bSE5Ergd97XvwOu6jT+exfyOpBnZuOA5cCzzrlq59xR4FlCyYaISEyUVtZRXJBJdnpKxM6pAobIwJjZBOAy4NfefQMuAh7yDjk1twjnHA8BF3vHXwk84Jxrds6VATsJfUgy5K0sKeKtvTXsq270OxQREYlzL71bRXnNcT6xeLLfoZzGrwKGA/5qZuvN7AZvbIxzrhLA+3O0Nz4e2Nfpufu9se7GT2NmN5jZOjNbV1VVFcHLEJHhrLSyLmL9L8KmBLLYV91IS1tHRM8rMgz8BPgXIPyfpwCocc61efc75wkncgjv8Vrv+D7lFkMxr1g5vwiAJzZpGYmIiPTs3rV7CWSn8cE5Y/wO5TR+FTCWOucWEloe8iUze38Px1oXY66H8dMHnbvLObfIObeosLCw/9GKiJyivqmV9440MntsZAsYxYEsOhzs1aekIn1mZpcDh5xz6zsPd3Go6+WxPuUWQzGvmFSQyYKJeTyuZSQiItKDytrjPL/tIB9dNJG0lHhasBHiS0TOuQrvz0PAI4SmZx70lobg/XnIO3w/MLHT0ycAFT2Mi4hE3bYD9QARn4GhrVRFBmQpcIWZ7QEeILR05CeElp2G13h1zhNO5BDe4yOBahI8t1hZUkRpZR07DzX4HYqIiMSpP765jw4H154dX807w2JewDCzLDPLCX8NXAJsAVYB4Z1Ergce875eBXza241kCVDrLTF5BrjEzEZ5zTsv8cZERKIu3MAzegUM/YIh0lfOuVuccxOcc8WEmnA+75y7DngBuNo77NTcIpxzXO0d77zxa7xdSoKEGoi/EaPLiLrL54/DDM3CEBGRLrW1d/DHN/dx3vQAkwoy/Q6nS37MwBgDrDGzjYSSgr84554GbgM+aGY7gA969wGeBHYTaqT1K+BGAOdcNfAd4E3v9m1vTEQk6rZW1jEqM5WxuRkRPW9eZhr5WWmagSESGd8AvmZmOwn1uLjbG78bKPDGv4a385lz7h3gT0Ap8DTwJedce8yjjpIxuRmcE8zn8U0VhOo1IiIif7d6exWVtU1cd058zr4AiFzr/D5yzu0GSroYPwJc3MW4A77UzbnuAe6JdIwiIr0JN/AMbVwQWcFAFrurVMAQGQjn3Gpgtff1brrYRcQ51wR8tJvnfw/4XvQi9NfKkiL+7ZEtvFNRxxnjR/odjoiIxJH73tjL6Jx0Lp4df807w+KvK4eISJxra+9g24F65oyL7PKRsGAgiz1HVMAQkci79IxxpCQZj2/SMhIREfm7/UcbeWH7IT5+9kRSk+O3TBC/kYmIxKndh4/R0tbB7CgWMA7WNXOsua33g0VE+iE/K41l0wM8sbFSy0hEROSEP74Z2kX842dP7OVIf6mAISLST9Fq4Bk2RTuRiEgUrZxfRHnNcd7ae9TvUEREJA60es07L5hRyIRR8dm8M0wFDBGRfiqtrCMtOYmphdlROX+wUAUMEYmeS+aOIS0licc3VvodioiIxIG/bT3EofpmPnHOZL9D6ZUKGCIi/VRaUceMsdlRWx84OV8FDBGJnpyMVC6aOZonNlXS1t7hdzgiIuKz+97Yy9jcDC6cWeh3KL1SAUNEpB+cc2ytrItaA0+AEWnJFI3MUAFDRKJm7Mh0Djc0M+3fnmLpbc/z6IZyv0MSEREf7Ktu5OUdVXz87ImkxHHzzrD4j1BEJI4cqm/myLGWqBYwILSMZLcKGCISBY9uKOcBr1kbQHnNcW55eLOKGCIiw9D9b+zFgGsWx3fzzjAVMERE+uHvDTxHRvV1goEsyqoatEuAiETcD5/ZTlPryUtHjre288NntvsUkYiI+KGlrYM/rdvHRbPGMG7kCL/D6RMVMERE+qG0MlTAmDUuJ6qvEwxkU9fURvWxlqi+jogMPxU1x/s1LiIiienZ0oMcbmjhunMm+R1Kn6mAISLSD6UVdUzMH0FuRmpUXye8leqeI1pGIiKRVZTX9ads3Y2LiEhiuu+N9xifN4L3z4j/5p1hKmCIiPRDaZQbeIYFvQLG7ioVMEQksm5aPpMRqcknjY1ITeam5TN9ikhERGJtz+FjvLLzCNecPZHkJPM7nD5TAUNEpI+ONbex58gx5oyLbv8LgAmjRpCSZNqJREQi7qozx/ODj8xjfKcZF/922SyuOnO8j1GJiEgs3f/GXpKTjI+dPTSad4apgCEi0kfbDtTjHMwpiv4MjJTkJCYVZKqAISJRcdWZ43nl5ot45Mb3AZA7Is3niEREJFaa29p5cP1+PjB7NGNyM/wOp19UwBAR6aNwA89YFDAAggVZKmCISFTNGz+SnIwUXtlx2O9QREQkRp555yDVx1r4xDmT/Q6l31TAEBHpo9KKOkaOSKVoZGwq1cFAqIDR0aGtVEUkOlKSk3jf1ALW7DysbZtFRIaJe19/j4n5IzhvWsDvUPpNBQwRkT4qraxj9rgczGLT6ChYmEVzWweVdU0xeT0RGZ6WTS+kvOa4ZnyJiAwDOw81sLasmmsXTyJpCDXvDFMBQ0SkD9raO9hWWReOc0MoAAAgAElEQVSTBp5h4Z1IyrQTiYhEUfgTuDU7tYxERCTR3f/GXlKSjI+eNbSad4apgCEi0gd7jhyjua0jZv0vAKYEsgEoO6IChohEz+SCTMbnjeBl9cEQEUloTa3t/Pmt/SyfO5bCnHS/wxkQFTBERPrgnQqvgee42BUwxuSmMyI1WTMwRCSqzIzzpgd4fdcR2to7/A5HRESi5KktldQ0tvKJcyb5HcqAqYAhItIHWyvrSU02po3OjtlrmpnXyLMhZq8pIsPTsukB6pvb2Li/1u9QREQkSu5bu5figkzOnVLgdygDpgKGiEgflFbWMX10DmkpsX3bDBZqK1URib6lUwOYwRotIxERSUjvHqznzT1Hh2zzzjAVMERE+qC0oi6m/S/CggVZ7Dt6nJY2TesWkegZlZXGGUUjWbOzyu9QREQkCu5bu5e05CSuPmuC36EMigoYIiK9OFTfxOGGZmbHsP9FWDCQRXuHY9/Rxpi/togML0unBdiwt4aG5ja/QxERkQg63hJq3rnijLEUZA/N5p1hKmCIiPSi1IcGnmHBQm2lKiKxcd70AG0djrW7j/gdioiIRNATmyqob2ob0s07w1TAEBHpRWmlfwWMKQGvgKE+GCISZWdNHkV6SpK2UxURSTD3vbGXqYVZnBPM9zuUQVMBQ0SkF6UVdYzPG8HIzNSYv3ZeZhqjMlPZrQKGiERZRmoyi4P5rNmpAoaISKLYWlnHhr01XLt4EmZDt3lnmAoYIiK92FrpTwPPsGAgiz1xWMB4dEM5S297nuDNf2Hpbc/z6IZyv0MSkUFaNi3AzkMNHKht8jsUERGJgPvW7iUtZeg37wxTAUNEpAeNLW3sPnzMl+UjYcFAdtwtIXl0Qzm3PLyZ8prjOKC85ji3PLxZRQyRIW7Z9ACAZmGIiCSAxpY2Ht1QzmXzxpGXmeZ3OBGhAoaISA+2H6jHOXydgTGlMIsDdU0ci6OdAX74zHaOt7afNHa8tZ0fPrPdp4hEJBJmj82lICuNNTu0naqIyFD3+MYK6pvbuC4BmneGqYAhItIDPxt4hgW9Rp57jsTPLIyKmuP9GheRoSEpyVg6LcCanUdwzvkdjoiIDMK9a/cyY0w2Z00e5XcoEaMChohID0or6shJT2HCqBG+xVBcEH87kRTldf330d24iAwdy6YFONzQzPaD9X6HIiIiA7SlvJZN+2v5RII07wxTAUNEpAellXXMLsr19Y2/OJAJQFlV/BQwblo+k7Tkk3+EjEhN5qblM32KSEQi5UQfDG2nKiIyZN27di8ZqUl8eGFiNO8MUwFDRKQb7R2ObZX1vi4fAchMS2HcyIy4moFx1ZnjOWtyHuGyTmqy8YOPzOOqM8f7GpeIDF5R3gimFGbxsgoYIiJDUkNzG6veLufy+UWMHJHqdzgRleJ3ACIS/x7dUM4Pn9lORc1xivJGcNPymcPiF9X3jhzjeGu7rw08w4KBLHbHUQGjvcPx7sEGLi8poiArjT+t28cVJUV+hyUiEXLetAB/XLeP5rZ20lOS/Q5HRET6IJyzl3s9yfxcAh0tmoEhIj0azttlxkMDz7BgICuuZmC8uaeaI8daWDF3LLPG5tDY0s7+o2rgKf4wswwze8PMNprZO2b2H974b82szMze9m4LvHEzszvMbKeZbTKzhZ3Odb2Z7fBu1/t1TX5bOi1AU2sHb71X43coIiLSB51z9rBfvrgr4XJ2FTBEpEfDebvM0oo6UpKM6WOy/Q6FYCCL2uOtHD3W4ncoADy95QDpKUlcMLOQGWNzANh2oM7nqGQYawYucs6VAAuAFWa2xHvsJufcAu/2tjd2KTDdu90A/ALAzPKBW4FzgMXArWaWOK3b+2HJ1AKSk4w1O7WdqojIUNB1zt6RcDm7Chgi0qPhvF1maWUd00Znx8X06SmFoZ1I4mEZSUeH4+ktBzh/RiFZ6SnMGBMqYLyrHQvEJy6kwbub6t162gP0SuD33vNeB/LMbBywHHjWOVftnDsKPAusiGbs8So3I5UFE/PUyFNEZIgYLjm7Chhymkc3lLP0tucJ3vwXlt72fMJNO5L+Gc7bZZZW1MXF8hGAYCA0CyQelpFs3F/DgbomVpwxFoBsb5vZ7QcbenmmSPSYWbKZvQ0cIlSEWOs99D1vmcjtZpbujY0H9nV6+n5vrLvxYWnZtACbymupaYyPmV8iItK94ZKzq4AhJxnO/Q6kazctn8mI1JNnIKQlW8Jvl1lV38yh+ua4aOAJoSZMKUlG2WH/iwRPbzlAarJx8ewxJ8Zmjslhu5aQiI+cc+3OuQXABGCxmZ0B3ALMAs4G8oFveId3tS+y62H8JGZ2g5mtM7N1VVWJu8Ri2fQAzsFru474HYqIiPTipuUzSUk6+cdYIm5xrwKGnGQ49zuQrl115ni+/+EzTmT1yUnG6Jx0rlyQ2DtObI2jBp4AqclJTMzP9H0GhnOOp7Yc4H1TAydtyzVjbA67q47R0tbhY3Qi4JyrAVYDK5xzld4ykWbgN4T6WkBoZsXETk+bAFT0MH7qa9zlnFvknFtUWFgYhauIDwsm5pGdnsLLO7WMRERiQzPBB27FGWNJSzYyUpIwYHzeiITc4l7bqMpJhsvaKemfMyeNwgE/+Mg8kgy+8efNvLTjMOfPSNzEPVzAmB0nBQzwtlKt8reAsbWynr3VjXzxgqknjc8am0Nbh6Ps8DFmek09RWLFzAqBVudcjZmNAD4A/KeZjXPOVZqZAVcBW7ynrAK+bGYPEGrYWesd9wzw/U6NOy8hNItjWEpNTmLJlHz1wRCRmAjPBA9/mBqeCQ4k3C/h0fDIhnIaWzt44IYlLJlS4Hc4UaMZGHKS4bJ2SvpnU3ktAPPGj+TDZ05gbG4GP39hp89RRVdpZR1FIzMYlZXmdygnBANZ7DlyjI6OnnoTRtfTWypJMrhkzpiTxsONPLUTifhkHPCCmW0C3iTUA+MJ4F4z2wxsBgLAd73jnwR2AzuBXwE3AjjnqoHveOd4E/i2NzZsLZsWYG91I3uPNPodiogkOM0EHzjnHPesKWNuUS7nBPP9DieqVMCQk9y0fCanLJ1KyLVT0j9bymtJS0lixpgc0lKS+Nx5QdaWVbP+vaN+hxY1pRV1cTX7AkIFjKbWDg7UNfkWw1NbDrA4mE9BdvpJ41MKs0hOMu1EIr5wzm1yzp3pnJvvnDvDOfdtb/wi59w8b+yT4Z1KvGUlX3LOTfUeX9fpXPc456Z5t9/4dU3xYtn00Ey7NVpGIiJRppngA/fyjsPsONTAPy0NEpp0mLhUwJCTfGjeuJM6mCXq2inpn037a5g9Lpe0lNBbxrWLJzEqM5VfrE7MWRhNre3sqmqImwaeYVMCoa1U/eqDsfNQAzsONbBi7tjTHktPSWZKIIvtB/xvMioikTO1MIuxuRms2Zm4zUpFJD5oJvjA3b2mjMKcdC4vGed3KFE34AKGmX3UzHK8r79pZg+b2cLIhSZ+eKeilnYHiyaHlv8+9uWlKl4Mcx0dji3ldcwfP/LEWFZ6Cp95X5Dnth5KyCUD2w/U0+Hip4FnWLDQ3wLGM+8cAGDFGV3/cJwxNoftBxPv+0FiR7lF/DEzlk0P8MrOI7T7uHxNRBKfZoIPzM5D9bz4bhWfWjKZ9JTk3p8wxA1mBsa3nHP1ZrYMWA78DvhFZMISv4SXBHzinEkAvHtA08GHu7Ijx2hobmPehJEnjV//vslkpSXzi9W7fIosekrDO5DE2QyMMTkZjEhN9q2A8dSWSs6clMfYkRldPj5rTA77qo9zrLktxpFJAlFuEYfOmx6g9ngrW7x+SCIi0XB2MJ8OB7kZoX0mUpJMM8H74J5X9pCWksR13u9viW4wBYxwh5XLgF845x4D4qfbnQzIuj1HmZg/gmXTAwBsUwFj2Nu8P5Swzj+lgJGXmcYnzpnE4xsrEq65W2lFHdnpKUwclel3KCdJSjKKA1m+FDD2VTeypbyuy+UjYTO83Ud2HNIyEhkw5RZx6H1TQzmB+mCISDT9ZVNo1+pVX17GTctn0tbhOM/7nUS6dvRYCw+/tZ8PLxh/Wn+yRDWYAka5mf0S+BjwpJmlD/J84jPnHOv3HmXR5HwKs9MZlZmqhnzCpv21ZKQmMa0w+7THPnfeFFKSkvjlS4k1C2NrZR2zx+WQdOo8xjgQDGT6UsAILx+5tJvlIwAzvZ1ItifgsiKJGeUWcagwJ51ZY3O0naqIRNWqjRXMnzCS4kDWiW1A15YN642genXfG3tpau3gn5YF/Q4lZgaTFHwMeAZY4ZyrAfKBmyISlfhiX/VxquqbWTh5FGbGzLE5moEhbC6vYW7RSFKST3+7GJObwT+cNYEH1+/nkI87Y0RSR4dja2Vd3PW/CAsGsthb3Uhre0dMX/epLQeYMy6XSQXdz0qZlJ9JRmqSGnnKYCi3iFPnTQ+w/r2jHG9p7/1gEZF+2l3VwJbyOq4oKQJCM38z05J5bdcRnyOLX63tHfz+tT0smxZgpjcLdjgYcAHDOdcIHAKWeUNtwI5IBCX+WPdeqMIZbuA5c0wOOw7W06GmXcNWu9fAc974kd0e84Xzp9DW3sHda8piGFn07K1u5FhLe9xtoRoWDGTT3uHYVx27ZTuH6ppY/95RVpzR/fIRCC1xmTEmRzO3ZMCUW8SvZdMLaWnvYG2ZfpkQkch7YlMlZnDZ/NBMz9TkJM4uzue13XrP6c6Tmys5WNfMZ4fR7AsY3C4ktwLfAG7xhlKBP/Tj+clmtsHMnvDuB81srZntMLM/mlmaN57u3d/pPV7c6Ry3eOPbzWz5QK9FQta/d5Sc9BRmeNPAZ4zN4VhLO+Xae3nY2lXVwPHW9tP6X3Q2uSCLy+YX8YfX36O2sTWG0UVHvDbwDAv6sJXq35eP9FzAAJgxRjO3ZOAGm1tI9CwuzictOYlX1AdDRCLMOceqjRWcXZzPuJF/3zL1fVML2HmoIWFm+UaSc46715QxpTCL82cU+h1OTA1mCcmHgSuAYwDOuQqgP3NXvgps7XT/P4HbnXPTgaPAZ73xzwJHnXPTgNu94zCzOcA1wFxgBfBzM0v8fWOiaP17R1kwKY9kb93/rLHh9ez6ZWS42tRNA89TffH8qRxraef3r+2JflBRVlpRR7I3kyAeTfGhgPHUlgNMKcxi2ujT+6CcatbYHA43NHOkoTkGkUkCGmxuIVEyIi2ZsyaP4mX1wRCRCNt2oJ6dhxpY6S0fCTt3aqgPhmZhnG79e0fZtL+Wf1wajMuebdE0mAJGi3POAQ7AzLL6+kQzm0Cow/ivvfsGXAQ85B3yO+Aq7+srvft4j1/sHX8l8IBzrtk5VwbsBBYP4nqGtbqmVrYfrGfR5PwTY9PDDfk0HXzY2ry/hqy0ZIKBnn9xnVOUy0WzRvObV/fQ2DK0t9AsraxjamEWGanxWQ8dlZVGXmYqu2NUwKg+1sLasmouPWMsobfenoULP+8eVB8MGZAB5xYSfcumB9h2oJ6qehUoRSRyVm2sIDnJ+NApMz3nFo0kJyOF11XAOM09r5QxckQq/7Bw+G0xO5gCxp+8TuF5ZvZ54DngV3187k+AfwHCXegKgBrnXPg3n/1A+F9jPLAPwHu81jv+xHgXzzmJmd1gZuvMbF1VVVUfQxxeNuytwTlYVDzqxFhuRirj80ZoPfswtqm8lrnjR56YldOTGy+YSvWxFh54Y1+vx8az0or4beAZFgxksSdGBYznSg/S3uF63H2ks5ljtROJDMpgcguJsvB2hlpGIiKR4pzj8Y0VLJ0WOG0b0OQk45xgvhp5nmJfdSNPbznAtYsnkZmW4nc4MTeYJp4/IjQj4s/ATOB/O+d+1tvzzOxy4JBzbn3n4a5eopfHenrOqbHe5Zxb5JxbVFg4vNYI9dX6PdUkGZRMzDtpfMaYbC0hGaZa2zsorahjfg8NPDtbVJzP4uJ8fvXyblraYrtDRqRUH2vhQF1T3Pa/CAsGsmK2hOSpLZVMGDWCuX38Oxmdk05eZirbNQNDBmCguYXExtyikeRlprJGBQwRiZC399Ww/+hxVs7v+oOSJVMK2HOkkcpa9eQL+/1rezAzrn/fZL9D8cWg9lZ3zj3rnLvJOfd159yzfXzaUuAKM9sDPEBo6chPCH3aEi4hTQAqvK/3AxMBvMdHAtWdx7t4jvTT+r1HmT0ul+z0k6t4M8bmsKuqIeZbNor/dhxsoLmtg3m99L/o7IsXTqWytolH3y6PYmTRszXcwHNc36/ZD1MCWVTWNkV9uU5dUyuv7DzCirl9Wz4CYBbqH6IZGDJQA8wtJAaSk4z3TS1gzY7DhFb6iIgMzqqNFaQlJ7G8m0bhJ/pgaBYGAA3NbTzwxj4+NG/cSQ1Ph5N+FzDMrN7M6rq41ZtZrxmrc+4W59wE51wxoSaczzvnrgNeAK72DrseeMz7epV3H+/x5731sauAa7xdSoLAdOCN/l6PQFt7Bxv21pzYPrWzWWNzaG13MZuuLvFjc3kNAPMn5PVy5N9dMKOQOeNyufPFXbQPwe13SytCb2Gzx8V3z8Bir5HnnsPR3Ur1hW2HaGnv4NJ5ve8+0tnMMTm8e7BBv+BInw02t5DYWTatkAN1Teyq0iwrERmc9g7HXzZVcsHMQnIzUrs8ZvbYXPIyU1XA8Dy0bh/1zW3DbuvUzvpdwHDO5Tjncru45TjnBjPv+hvA18xsJ6EeF3d743cDBd7414CbvTjeAf4ElAJPA19yzrUP4vWHrW0H6mlsaWdhFwWMcEM+bYs4/GzaX0tORgqT8zP7/Bwz48YLp7K76tiJrTeHktLKOsbkpp+2BjPexGor1ac2H2B0TjpnTjz9vaEnM8fm0NDcRkWttj2TvolibiERFu6Dod1IRGSw3iir5lB982m7j3SWFO6DoUaetHc4fvPqHhZOymPBxL5/wJhoBrWEBMDMRpvZpPCtP891zq12zl3ufb3bObfYOfd/2bvz+CjLe+/jn2smezLJkD1kIXvYAokgEsB9QeuG1rbWHk9tbfXY/dja6mn7PN2sttYuT11arUtre7S2tSoq4C67QExIwhIICdn3zEz2ZJbr+WNmIGACWWaf6/165QW5MzP3FYVw37/5Xd9fvpTyU1LKMcfxUcfn+Y6v1094/v1SyjwpZZGUctNcv5dgVd5oAOwZBqfLS4pBqxEqyDMIVbeaKE6Pm/FopquWppGTGM1j79f53Tvw/hDgCZCd4CxguO8d0JFxK+8f6WL9ktQZ/xlQQZ7KXM3l2kJxr8z4KBYkRKkgT0VR5uzV/W1EhWm5dFHyGR+3Ji+RFsMIzX3u7Tz1de8e7qKxd5jb1+V6eyleNesChhDiOiHEUaAB+AA4Dqgigh/a12ggLS6CdP3H91FFhGrJTohSQZ5BZsxi5VB7/4zyL5y0GsGdF+RS09rvV+/QjZqt1HUP+nyAJ0B0eAipsRFuHaX6wZEuRs02rppiT+qZFCY7CxiqxVyZGXVt4R/W5ieyu75P5WMpijJrZquNTTXtXLYo5ayTNFQOht1T2+tJ10eyfkmKt5fiVXPpwPgpsBo4IqXMAS4FdrhkVYpHlR/vm3T7iFNRqo5a1YERVI50DGK2SoqnOYHkdDeck05qbASPvV/n4pW5z9HOQaw26fMBnk7unkSyqaaDeVGhrMr5eGfW2cRFhZIWF6E6t5TZUNcWfuD8/EQGxyxUNhu9vRRFUfzU9qM9GIfNXHeG7SNOBckxJMaEBfU2kgNtJnbX9/H5NQsI0c55E4Vfm8t3b5ZS9gIaIYRGSvkeUOKidSke0mYcoc00OmmAp1Nhio6mvmG3TzxQfEeVM8AzfXb768JDtHzp/Bx21/ed2KLk605MIPGDDgyAnCT3FTDGLFbePdTF5YtTZv2PZGGKTmXnKLOhri38wJq8RDRC5WAoijJ7G/e3ERsRwvmFiWd9rBCC83IT2HWs1++2J7vK09uPExWm5TPnql2VcylgGIUQMcBW4G9CiN8B6g7XzzhvLlecoYCxMFWHlPZ3qJXgUN1iIi4ylMz42Y9n+uyqLPRRoTzuJ10YB9v7iQrTzii01JtyE6MxDpsxDI27/LV31vUyMGbhqqWTz2SfjoWpOo51DWJRLebKzKhrCz8QFxVKcYZe5WAoijIro2Yrbx7s5MqlqYSHaKf1nLLcBDr6RzneG3w5GF0Do2zc38anVmQQFzn5tJZgMpcCxvXAMPDf2KeAHAOudcWiFM8pbzQQGapl0RmCC52TSNQ2kuBR1WJiWUYcQswsvHGi6PAQbluTzduHuvwiQ+VgWz8LU3UzDqz0lhOTSHpd34WxqaYdXXgIa/ITZv0ahSk6xq22oLzQUOZEXVv4ifPzE6lsNtI/avb2UhRF8TPvHe5icMzCdcvTp/2cYM7B+OvuJsw2G7etDd7RqRPNeQONlNIC7MIetKUi5/1MeaOBkkw9oWdoE1+QEE14iIYjfnATqszdqNnKkc6BWedfTHTbmmyiwrQ+34Vhs0kOtvf7zfYRmFDA6HZtAcNitfHWwU4uWZQ87XdFJnNyEon6uaHMnLq28H1r8xOx2iS7g/BmQlGUudlY1UZiTBirc6efs5WbGE2yLjzocjBGzVb+truRSxcmn7j2C3ZzKWBsBSKEEOnAO8AXgGddsSjFM4bGLBxs7z/j9hGwT5UoSIlRHRhB4nDHABabZNksJpCcTh8VxufOy2JjVTtNPvxOfIthhMExi98EeIJ9lKFWI1yeg7GnoQ/DsHlW00cmyk+OQSNU55YyY+rawk+cs0BPZKiW7WobiaIoMzAwauadQ11cXZw2o5wtIQRr8oIvB+PVyjZ6h8b5ouq+OGEuBQwhpRwGbgR+L6W8AVjsmmUpnrC/2YjVJlmRfeYCBtjbwdU7qcGhusUe4FmcMbsAz9N96fxctELwx63HXPJ67nCw3QT4T4AnQKhWQ+a8SJcXMDbVdBARquGCwqQ5vY59BHO06txSZkpdW/iJ8BAt5+XGqwKGoigz8vahTsYsNq6dxvSR05XlJdAzOEZdV3Dk8kkpeXpHAwtTdSe20ChzLGAIIcqAzwGvO46deYiv4lOcAZ7nZJ29gLEwVUfXwJhbAgMV31LVYiIhOoz5cREueb2U2Ag+uSKdf5S30DUw6pLXdLWDbf1oBBQ58l78RU5iNPUuLGDYbJItBzq4qDD5rDPZp6MwRY1gVmZMXVv4kXX5idR3D9FmHPH2UhRF8RMb97czPy5iWvcfpyvLtU8sCZZtJDuP9XK4Y4AvrsuZUy5doJlLAeObwH3Av6WUB4QQucB7rlmW4gn7Gg0UpsRMK81WBXkGj+pWE8VzDPA83Z0X5GGx2nhqe4PLXtOVDrYPkJsUQ2TY7DMfvCEnMYbjPUPYbK5ppaxoNtA1MMZVxXPbPuJUlKrjeO8Qo2arS15PCQrq2sKPrCuw30xsV+NUFUWZBsPQOFuPdHPt8vmzCk3PjI8kXR8ZNEGeT29vIDEmjOtm0a0SyGZdwJBSbpVSXiel/IXj83op5TdctzTFnWw2yUdNBlYsmF54jjOQ74gqYAS0kXF7gOcyFwR4TpSdGM3Vy+bzt91NmIZ9L7H+UHv/GSfx+KqcpGhGzFY6XdTZsqm6g1Ct4OKFyS55vSLHCOZgafVU5k5dW/iXohQdSbpwtqltJIqiTMPmAx1YbHJW20fAnoOxOjeB3fW9LnvzxlfVdw/yzuEuPnfeAiJC/esNNneb8xQSxT8d7RpkYNTCyrMEeDqlxkYQGxGicjAC3MF2EzbpuvyLie66MI/BMQt/2XXc5a89F8bhcVqNIyz2wwJGrnMSiQu2kUgp2Xygg3X5icRGuGbGuLNz67D6uaEoAUkIwbr8RHbW9QT8zYSiKHP3amUbuYnRLJlD5lhZXgKGYXPAd4U/u/M4YVoN/7F6gbeX4nNUASNI7WvsAzjrBBInIQRFqSrIM9BVtdjDLF0xgeR0i+fHcnFREs/sPM7IuG9sKXi5opXLfv0BAH/aVs/LFa1eXtHM5LiwgHGgrZ8WwwhXLU2b82s5ZSdEERaiUZ1bihLA1uUn0js0zqEONe1WUZSpdfWPsruhl2uWz5/TNmVnmGUgbyMxDZv5x74WriuZT5Iu3NvL8TmzLmAIIdZO55jim8qPG0iMCWNBQtS0n+MM5Aum0UXBprrFRLIunJRY1wR4nu4rF+fTNzTOC3ub3PL6M/FyRSv3vVRNz6A9mLZ3aJz7Xqr2qyJGamwEEaEaGrrnXsDYVNOOViO4bHGKC1ZmF6LVkJ8UowqfyrSpawv/szZf5WAoinJ2r1e3IyVct3xub5Sk6yNZkBAV0EGeL+xtYsRsVaNTpzCXDozfT/OY4oPKmwysWDBvRhXQhak6BkYtdPT75iQJZe6qWk1u6b5wOjc7nlXZ8Ty5tZ5xi81t55mOh7bUMnJauOSI2cpDW2q9tKKZ02gE2QnRLunA2FzTwXk58cRHh7lgZSepzi1lhtS1hZ9JjYugIDlGjVNVFOWMXt3fxqK0WPKT5z7xrcyRg2ENwK1rFquNP+88TlluAovnsNUmkM24gCGEKBNCfBtIEkLcPeHjR4BKGPED3QNjNPYOT3v7iJPazx7YBscsHOsepDjd9fkXE911cR5tplFervRup8NUY//8bRxgTuLcCxhHOwc41j3EVUtdM31koqJUHR39oz4Z3qr4DnVt4d/WFSSyp6FPTRxSFGVSzX3DVDQZuXaO3RdOZXkJDIxaONgWeFvXNh/ooM00yu3rVPfFVGbTgREGxGCfy66b8NEP3OS6pSnuUn4i/2J6E0icTkwiUQWMgHSg1YSU7sm/mOiiwiQWp8Xyhw+OeaVyLqXkhT1Tb2GZr4/04GrmLicxmqa+YczW2Xe0bKrpAOCKJW4oYKgRzGf0ckUrax98l5x7X2ftg+/61RYmF1PXFn7s/IJExiw2yhsN3l6Koig+6LWqdgCuXeaacaBluY4cjPrA6/x6ansD2QlRXOKiiXCBKENbUHIAACAASURBVGSmT5BSfgB8IIR4VkrZ6IY1KW5W3mggLETD0vSZtSXpo8JIiQ1XNyIBqrrVHuC51MUjVE8nhOCui/L4+vMVvHmgg6uKXRcaeTa9g2Pc+1I1bx3spCA5mua+EUYnbGWJDNVyz/oij63HFXISo7HYJC2GkROhnjO1uaaDFQvmuSX7pDD1ZAFjVc7MiqaBzpnD4tzK1Goc4b6XqgHYUJruzaV5nLq28G+rchII0Qi2He05kYmhKIri9Or+Nkqz9GTGTz9770ySYyPITYpm17Fe7rggzyWv6Qs+ajJQ0WTkx9ctQaOZfdBpoJvNFpLfOn77iBDi1dM/XLw+xQ32NRpYlh5HeMjMu3ILU9R+9kBV1WJiflyER9KOP1GcRnZCFI+9f8xjobDv1Xax/rfb+KC2mx9cvYgt37qQBz+5jHR9JAJ7KNQDNxb73Y1jbpJzEsngrJ7f1DvMwfZ+t2wfAZgfF4EuPER1bk0iEHJYXGWu1xZCiAghxB4hxH4hxAEhxI8dx3OEEB8KIY4KIf4uhAhzHA93fF7n+Hr2hNe6z3G8Vgix3g3fbsCJCQ/hnKx5bK/r9vZSFEXxMXVdAxxq73dZ94VTWW4Ce48bsMyhA9XXPL29AV1ECDetyPD2UnzajDswgOccv/7KlQtRPGPUbKWm1cQXZ7mvamGqjj/vasRqk2hVZTCgVLeaKHbz9hEnrUbwXxfmce9L1Ww72sMFhUluO9fIuJUHNh3iL7saKUrR8dztq1iUZu8+2lCa7ncFi9PlJMYAUN89xCULZ/78zQfsbZ3r3bB9BOwdN4UqyHNSgZLD4iJzvbYYAy6RUg4KIUKB7UKITcDdwG+klC8IIf4A3A487vjVIKXMF0LcDPwC+IwQYjFwM7AEmA+8LYQolFKqcIezWFeQyG/ePkLf0LjLw4AVRfFfr+5vRwi4ZplrO27L8hL424dNVLeaKM2aWa6fL2ozjrCppoPb1+UQHT6bW/TgMeMODClluePXDyb7cP0SFVeqbjVhtkpWzjD/wqkwRce4xcbx3rlPPVB8h2nETEPPEMsy3BvgOdEN56STEhvOY+/Xue0cNa0mrn1kO3/Z1cjt63J45WtrTxQvAsW8qFDiIkNn/XdyU00HS9NjXdbWOZmiVDWCeTJT5a34Ww6LK8z12kLaOduQQh0fErgE+Kfj+J+BDY7fX+/4HMfXLxX2sVzXAy9IKceklA1AHbDKBd9iwFtXkIiUsPNY4O1JVxRldqSUvLa/jdU5CSS7eJvq6hM5GIExTvXPu44jpeQ/yxZ4eyk+b9ZjVIUQa4UQbwkhjggh6oUQDUKIelcuTnG9fcftAVvnZM3uRlUFeQamA478i2I3519MFB6i5cvn57K7vo+Pmlwb/Ga1SR5//xg3PLaDgVEzf739PH54zWIiQgNvmIEQYtaTSNpNI1Q0GblqqXtzSIpSdJhGzHQNjLn1PP7mnvVFaE8bZe2POSyuNJdrCyGEVghRCXQBbwHHAKOU0uJ4SAvgbLlKB5oBHF83AQkTj0/yHOUMlqXHoYsIYftRVcBQFMXuQFs/9T1DXFfi2u0jAIkx4RSl6Nh1zP8LGMPjFp7/sImrlqaRMc99bygFilkXMICngF8D64BzgZWOXxUfVt7YR25iNAkxs8s5KEjWIYSaKBBoqrxQwAD47Kos9FGhPPbeMZe9ZqtxhFue3M0vNh/m8sUpbPnWBawrCOxQudzEaBq6Z17AePNAJ+C+7SNOagTz5DaUppMQE4ZzN15CdJhf5rC42KyvLaSUVillCZCBvWti0WQPc/w62R5IeYbjpxBC3CGE2CeE2NfdrXIfAEK0GspyE9h2tEd1WymKAsDG/W2EaARXuuk6oywvgX3HDYxb/DsH41/lLfSPWvjiumxvL8UvzKWAYZJSbpJSdkkpe50fLluZ4nJSSsobDaxYMPt9YpFhWhbER6n97AGmusVEZnwk8zy8bzk6PITb1mTz9qFOl/yZeqWylSt/u5WaVhO/+tRyHr3lHPRRgb8XOzsxmjbTKCPjM9umv6mmnYLkGPKTY9y0MjvVuTW5/lEz3YNj3HFBHhoBnzsvK9iLF+CCawsppRF4H1gN6IUQzs3EGUCb4/ctQCaA4+txQN/E45M8Z+I5npBSrpRSrkxKcl+Gj785vyCRVuMIjb3D3l6KoiheZrNJXqtq5/yCRLddX67OTWDEbGV/i9Etr+8JNpvkmR3HWZ6p55wAyPLwhLkUMN4TQjwkhCgTQpzj/HDZyhSXq+8ZwjBsnlMBAxyTSFQHRkCpajWyLN1z+RcT3bYmm6gwLY/PIQvDNGLmG89X8M0XKilM0bHpmxdw04oMhAiOoFnn+NSZ5GD0Do6xp6HPbdNHJoqPDiNJp0Ywn66q2YSUsCYvgcIUHRXN/nsB5kKzurYQQiQJIfSO30cClwGHgPeAmxwP+zzwiuP3rzo+x/H1d6W9beBV4GbHlJIcoADY46pvLtCtK7AXc7bVqW0kihLsPmoy0Gocccv2EafVufEIgV9vI3n/SBf1PUN8cW120Fy3ztVcIk7Pc/y6csIxZ2CW4oPKHfkXK7PnVsBYmKrj7UOdjJqtAZkpEGwMQ+M0943wufO8ExqkjwrjllVZPLPzON++omjGYZK7jvXy7Rcr6RoY4ztXFPJfF+YRop1Lbdb/OAsYDT1D0w4pfetgJzYJ6z1QwAB7Dobq3DpVZbP9Z/LyTD2lWXper2rHZpPBPvt9ttcWacCfhRBa7G/OvCilfE0IcRB4QQjxM6AC+xYVHL8+J4Sow955cTOAlPKAEOJF4CBgAb6qJpBMX3ZCFOn6SLYf7ebW1SqITlGC2cb9bYSHaLh8sfuuM/RRYSxKjWXXsV6+cWmB287jTk9vP05qbASfKHZvHlkgmXUBQ0p5sSsXorhfeaMBfVQouYlzaxcvTNVhk1DXNchSD2cmKK5X7ci/WObF/5dfOj+XP+86zh+3HuNnG4qn9Zwxi5Vfv3WEJ7bWk50Qzb/uWsPyTO90kXjbxALGdG2q6SArPorFHprKUpSq428fqhHME1U2G8lNiiYuMpSSTD3P72mmoXeIvCT3bunxZbO9tpBSVgGlkxyvZ5IpIlLKUeBTU7zW/cD9s1lHsBNCsC4/kTdq2rFYbUFXTFYUxc5itfF6dTuXLkomxs0jQcvyEnhud6NfvrF6uKOf7XU9fPfKIkLVz8tpm/WfKCHE/5nsuJTyJ7NfjuJO+xr7OCdr3pzf3StyBPId6RxQBYwA4CxgLPHi/8vUuAhuWpHBi/ta+MalBSTrzjxq62jnAN98oZKD7f3ccl4WP7h6EVFhwTszOzo8hJTYcOqnGeRpGjGz81gPX1yb47F2xaIUHaNmG819w2Q7Ci7BTEpJRZORC4vsLfclmfbOuMomY1AXMNS1hf9bV5DI3/c1U91qolTt51aUoLS7vo+ewXGuXea+7SNOZbkJPLW9gYomI2V5CW4/nys9s/04EaEablmV5e2l+JW5lHqGJnxYgauAbBesSXEDw9A4x7qH5px/AfbAwDCtRu1nDxBVLUayE6KIiwz16jruvCAPi9XGU9sbpnyMlJJndzRwze+309k/yp/+cyU/v6E4qIsXTvZRqoPTeuy7hzsxW6XHto+AvXML1CQSpxbDCL1D4ydu8PKTY4gJD6Gi2bUjhf2Qurbwc2vzExECNU5VUYLYxv1txISHcPHCZLefa1VuPBoBu+r9KwejZ3CMf1e28slzMoIicN6VZl3AkFI+POHjfuAi1Kx0n/VRkyP/wgUFjFCthtykaLWfPUDUtPZTnOH9rRfZidF8ojiNv+1uwjRi/tjXu/pHue2Zvfxo40HW5iey+VsXcNniFC+s1DflJMZwfJrJ/5uqO0iNjaDEg//fC1PsXQVHVOET4ERgZ6lj25NWI1iWEUdlkAd5qmsL/xcfHcaS+bEqyFNRgtSYxcqmmnauWJzikS0dsRGhFKfHseuYf/3M+d8Pmxi32PjC2hxvL8XvuHKzTRSQ68LXU1xoX6OBEI1gmYtuWBam6tRIxADQMzhGq3HEq/kXE911UR6DYxae23X8lONbDnSw/rdb+bChl59uWMpTn19Jki7cK2v0VbmJ0fQNjWMcHj/j44bGLHxwpJsrl6Z6NCwyKiyErPgo1bnlUNlkJDxEc2LELEBJpp7D7QOMmlVm5ATq2sIPrc1PpKLJwNCYxdtLURTFw7Yd6aF/1MK1y92/fcRpdV4Clc3GGY+T95Yxi5XndjdyUVGS20fZB6JZFzCEENVCiCrHxwGgFvid65amuFJ5o4El6XFEhrmmElqYqqPNNEr/6MffKVf8hzP/ojjDNwoYS+bHsShVx6/fOkLOva9T9sA73PzHXdz5XDnp8yJ57evnc+vqBWrM1CSmG+T5wZFuxiw21i/x3PYRp0I1ieSEymYDxelxp4R2lWTqsdgkNY6/l8FIXVsEhvPzkzBbJXsa+ry9FEVRPGxjVRv6qFDWFSR67JxluQmYrZJ9jf7xM+e1/e10D4xx+zrVfTEbc+nAuAa41vFxBTBfSvmIS1aluNS4xcb+ZqNLto84nQjyVDcjfq26xYQQsGS+ZyZRnM3LFa0c6xnCJu1zE9tNo+xu6OPyRcm8dNdaVaU+g+xpFjA21XSQEB3Gqpx4TyzrFAtTdTT0DDFm8Y93SNxl3GKjpq2fktOm5pRk2T8P8m0k6toiAKzMnkd4iIZtKgdDUYLKyLiVtw52ctXSNI9O1Tg3O54QjWDXMd/PwZBS8vSOBgpTYliX77kiTyCZSwZG44SPViml6hP0UQfaTIxZbC4J8HRytj2rdnD/VtViIjcxGl2EdwM8nR7aUsu4xfax4wfbBwgLUeOlziQrPgqNOHMBY9Rs5d1DnVyxJMUro0wLU3VYbZJjXdMf9xqIDnf0M26xnShYOCXrIkjXR57IxwhG6toiMESEalmVE8/2um5vL0VRFA9653Anw+NWrl2e5tHzRoeHsCwjzi+CPD9s6ONAW79HJ8EFGnVHEATKG10X4OmUro8kOkyrOjD8XHWr0WW5KK7QZhyZ0XHlpLAQDZnxUdSfoYCxo66HoXGrV7aPwKkjmIOZs8Pi9A4M57HKpuAtYCiBY21+Ikc6B+nsH/X2UhRF8ZCN+9tI1oVzXo7nx5mW5SVQ1WJi0Mezd57e3kB8dBgbSlU+9WypAkYQKG80kBkfSXJshMteUwhBYapOjUT0Y539o3T2j1HsIwGeAPP1kTM6rpwqJzGahu6pCxibajrQRYSwJs87LYs5idGEakXQd25VNhlJjAknfZI/1yWZelqNI3QPjHlhZYriOs7W6B1qGomiBIX+UTPv1XZz9bI0r3R5rslLxGqT7D3umzkYL1e0ct7P3+bNg52YLTY213R4e0l+SxUwApyUkn2NBlZkua77wmlhqo4jnQNIKV3+2or7VbfYgwKX+UiAJ8A964uIPG3kVmSolnvWF3lpRf4lJzGahp6hSf9Omq023jrYyeWLUry2HScsRENuYkzQB3lWNhspydRP2jqqcjCUQLE4LZaE6DC2qxwMRQkKbx7oZNxi4zoPTh+ZaMWCeYRpNT6Zg/FyRSv3vVRNZ7/9zYmBMQv3vVTNyxWtXl6Zf1IFjADXYrC/k7ci2/WBfYUpOgzDZvVOoZ+qajWhEbDYRwI8ATaUpvPAjcWk6yMR2LcqPXBjsWqzm6bcxGhGzNYT/0BO9GF9H6YRM+uXemf7iFNRanBPIjEOj1PfM0Rp1uRbt5bOjyNEI6hoMnh4ZYriWhqNYE1+ItvretQbHYoSBF7d30bGvMhJt0d6QkSolpIsvU8WMB7aUsvIaSPSR8xWHtpS66UV+bcQby9AcS/nOCFX5l84Ofez13YOuHR7iuIZ1S1GCpJ1RIX51o+BDaXpqmAxSzmJ9iktDT1DpMad+ndyU007UWFaLixM8sbSTihK1fHq/jYGRs0+Ex7rSc7OitIpLvAiw7QsTNOpDgwlIKzLT2Dj/jaOdA6eCP9WFCXw9A6OsaOuhzsuyPVqMGVZbgK/f/cophEzcZG+c42hMt5cS3VgBLh9xw3owkMoTHH9hcOJSSRB/G6qv5JSUt1qotiHto8oc5eTNPkoVatNsuVAJxcXJRNx2hYdTys8EeQ56NV1eEtlsxEhOOPfvZJMPVUtJqw29a614t/WFdgLptuOqmkkihLINtV0YLVJr20fcSrLS8AmYU+Db+VgpEzxRq/KeJsdVcAIcOWNBkqy9G4J00mICScxJkwVMPxQu2mUnsFxn8q/UOYuLTaC8BANDT2nFgc+ajLQMzjm9e0jYM/OgeCdRFLZbKQgOeaM3SclmfMYHLNwrDs4izxK4EjXR5KbGK2CPBUlwL26v4385JgT/8Z7S2mWnvAQ38vByEmM+tgxlfE2e6qAEcD6R83Udg6wcoHr8y+cClN0QXsj4s+qHAGevjSBRJk7jUaQnRD9sQ6MTdUdhGk1XLIw2UsrOyldH0lUmDYoC59SSvY7AjzPxPl1NU5VCQTrChL5sKGPcYvN20tRFMUN2k0j7D3ex7XL5nt1+whAeIiWFQvmsavedwoYdV2DfNjQxwUFiSrjzUV8a/O74lIVTUaktKfyuktRqo4X9jRjs0k0XhiZpMxOdauREI1gUZrvBHgqrpGTGM2RrpPFASklWw50cH5BIjHh3v+Rr9EIClKCM8izsXcYw7CZkswz/0zOTYxGFxFCRbORT5+b6aHVKYp7rMtP5C+7GvmoycDq3ARvL0dRFBd7vaodKeHa5WneXgoAa/IS+NWbRzAMjTMvOszby+FXW2qJCgvhN58pISEm3NvLCQiqAyOAlTca0IiTY/ncoShFx4jZSrNh2G3nUFyvqsVEYYrO63kIiuvlJEXT1DuMxWp/t7O61USrcYQrfWD7iNPCIO3ccgZznq0DQ6MRlGTqVZCnEhBW5yWg1Qg1TlVRAtTG/W0sTY8lNynG20sB7DkYAB82eL8L46MmA5sPdHDHBbmqeOFCqoARwMob+1iUFuvWd11VkKf/cQZ4qvyLwJSTGI3FJmkx2JOtN9V0oNUILl+c4uWVnVSYqqN3aJyeweAawVzZbCQyVEthytkv8koy9dR29DM8bvHAyhTFfWIjQlmeEcd2lYOhKAGnsXeI/S0mr4d3TrQsQ09UmJadXs7BkFLy4KbDJMaEc/u6HK+uJdCoAkaAslhtVDQZ3bp9BKAgJbgD+fxRi2EE47BZTSAJULmJJyeRSCnZXNNBWW4C+ijvt1E6nRjBHGSFz4pmI8UZcYRoz/5Pb0mmHpuEakdejaL4s3UFSVS1GDENm729FEVRXGjj/jYArl7mOwWMUK2GldnxXg/yfL+2mz0NfXzz0nyifWALbyBRBYwAdbhjgOFxq9sLGDHhIWTMi+RwkN2I+DNngOeydPdtLVK8J8dRwKjvGeJI5yANPUM+tX0EgrNza9Rs5WCbidJpbuk7EeSptpEoAUFik7D8J2+y9sF3ebmi1dsLUhTFBTbub2flgnmk+9g40LLcBI52DdI94J1OT6tN8ovNh1mQEMXNq7K8soZApgoYAaq80QDAymz3TSBxWpganPvZ/VVVq5EwrYbCVN/Yq6i4Vnx0GLERIRzvGWJTTTtCwBVLfGf7CEBiTBjx0WFB9XPjYHs/Zquk9Cz5F04JMeFkxUdRoSaRKH7u5YpWnthaf+LzVuMI971UrYoYiuLnajsGqO0c4LoS3+m+cHLmYOz20jSSVypbOdwxwHeuKCJ0Gl2Xysyo/6IBal+jgdTYCObHRbj9XIUpOuq7h9SIND9R3WJiYZqO8BAV4BmIhBDkJMXQ0DPE5poOVi6YR7LO/T8HZkIIQWFKTFB1bjlHop5tAslEKshTCQQPball1Hzq9cGI2cpDW2q9tCJFUVxh4/42NAKuWuob00cmWjrfngHojXGqYxYrD795hKXpsVxd7Hv/bQKBxwsYQogIIcQeIcR+IcQBIcSPHcdzhBAfCiGOCiH+LoQIcxwPd3xe5/h69oTXus9xvFYIsd7T34sv+6jRwIrseR6Zx1yUqsNik9T3DLr9XMrc2Gz2AM/idJV/EchyEqL4qMnA4Y4BrvTBCwuAhamxHO0cwGaT3l6KR1Q2G0mNjSB1BkXlkkw9Hf2jdJhG3bgyRXGvNuPIjI4riuL7pJRsrGpjbX4iSTrfm64RotWwKiee3V7Iwfjr7iZajSPce+UiNBr334cFI290YIwBl0gplwMlwJVCiNXAL4DfSCkLAANwu+PxtwMGKWU+8BvH4xBCLAZuBpYAVwKPCSHUW8pAu2mEVuMIK92cf+EUjPvZ/VVj3zADoxY1gSTAjZptDI9bAXhi6zGfbNUuTNExNG6lNUhuYiqbjWcdn3o65wjsymaDO5akKB4xf4q98QkxvhMsrCjKzFS1mGjsHeZaHwrvPN2avATqe4Y8+iZA/6iZR949yrr8RNYVJHrsvMHG4wUMaed8qz7U8SGBS4B/Oo7/Gdjg+P31js9xfP1SYW8ruB54QUo5JqVsAOqAVR74FnzevuP2i113B3g65SbGEKIRqoDhB6pb7QGexSrAM2C9XNHKO4c7T3ze2T/mk/vNixwZLMHwc6N3cIymvuETBYnpWpwWS6hWUKG2kSh+7J71RUSGnvr+kgD6hsZ5YU+TdxalKMqcbNzfRqhWsH6Jb4WET7Q6156DsavecyOcn9xaj2HYzPeuXOixcwYjr2RgCCG0QohKoAt4CzgGGKWUzoH3LUC64/fpQDOA4+smIGHi8UmeE9TKGw1EhmpZlBbrkfOFhWjISYwOqkA+f1XdYiQ8RENBigrwDFQPbanFbD11W4Yv7jcvdI5SDYKfG/tbnPkXMytgRIRqWZwWeyI/Q1H80YbSdB64sZh0fSQCSNdHcv+NS1lXkMS9L1Xz440HsFhVhpai+AubTfJaVTsXFiYTFxXq7eVMaXFaLHGRoR4bp9o1MMqftjVwzbI0ilWns1t5ZSitlNIKlAgh9MC/gUWTPczx62Sbh+QZjn+MEOIO4A6ArKzAH2VT3migJFPv0dTbolTdiYt0xXdVtZhYlBarEpEDmL/sN9dFhJKujwyKDozKJiMawayyZ0oy9fyjvAWrTaJVe2kVP7WhNJ0Npae+x/TpFZn8/I3DPL2jgbquQR655RziIn33ZkhRFLu9x/vo6B/lvk/4dpeBRiM4LyfeY0Ge/++do5itNr5zRZFHzhfMvHoXI6U0Au8DqwG9EMJZUMkA2hy/bwEyARxfjwP6Jh6f5Dmnn+cJKeVKKeXKpKQkV38bPmVozMLB9n6PbR9xKkrR0dw3wuCY5ewPVrzCZpPUtJpU/kWAm2q/+VTHvakwJSYoOrcqmo0UpuiIDp/5ewYlWXqGx61B8d9JCS4hWg3/59rF/OKTxeyu7+WGR3dQ363CwBXF122saiMyVMvli31rRPtkyvISaO4bocUw7NbzNPQM8fyeZj67KovsxGi3nkvxzhSSJEfnBUKISOAy4BDwHnCT42GfB15x/P5Vx+c4vv6ulFI6jt/smFKSAxQAezzzXfiu/S1GrDbJimzPFjAKHUGeR9VFts+q7xliaNyqJpAEuMn2m0eGarlnve+9I1CUGsux7kHMAdw+brNJ9jcbKc2a3c/kUsfYVTVOVQlUnzk3i799aTXGETMbHt3BtqPd3l6SoihTMFttvFHdwaWLkokK80oj/4yU5TlyMNy8jeRXb9YSHqLh65fmu/U8ip03OjDSgPeEEFXAXuAtKeVrwPeAu4UQddgzLp5yPP4pIMFx/G7gXgAp5QHgReAgsBn4qmNrSlArdwR4njPLi+XZWugoYKh3CX1Xdav9BmhZhgrwDGST7Td/4Mbij7Vv+4Ki1BjMVklDz5C3l+I29T1D9I9aKJ1h/oXTgoQo5kWFqhwMJaCtyonnla+uJS0uktue2cuzOxqwv1elKIov2Xmsl76hca5d7rvTRyYqTNYRHx3m1m0kVS1GXq9q50vrckjWTX9UujJ7Hi+dSSmrgNJJjtczyRQRKeUo8KkpXut+4H5Xr3G6Xq5o5aEttbQZR5ivj+Se9UVev0nY12igMCXG4/tIM+dFERmq5XAQ7Gf3V1UtJiJDteQlqda2QDfZfnNfVJRiDxqu7Rg4EeoZaJydEzOdQOIkhGB5pp4KNUpVCXCZ8VH86ytr+NYLlfxo40FqOwf58XVLCAtRmU2K4m3Oe55W4wgCGBgxe3tJ06LRCMpyE9h9rBcpJfZBlq4jpeTBTYeJjw7jyxfkuvS1lampfxVm6eWKVu57qZpW4wgSaDWOeH1Uoc0m+ajJwIoF8R4/t0YjgmY/u7+qbjGxZH4sISrAU/ERuUnRaDUioH9uVDYbiAkPIS9p9pN/SjL1HO0aZGDUPy4YFWW2YsJDeOLWFXzlojye39PErU99SN/QuLeXpShBbeI9D9gnJvzwlQM+N559KqvzEmgzjdLY6/ocjG1He9h5rJevXZyPLkKFEHuKupOZpYe21DJiPnXHirdHFdovcC2s9HCAp1Nhio7aDhXA5YssVhsH2vrVWCfFp0SEaslOiArozq3KZiPLMuLmNEGkJFOPlPYipKIEOo1G8N0rF/Lbz5RQ0Wzk+ke3B8W0IkXxVb54zzMTZbmOHAwXbyOx2SS/2HyYjHmRfG514E+59CWqgDFLvjiqcF9jH4DHJ5A4FaXq6Bkco3dwzCvnV6Z2rHuIEbNVTSBRfM7C1NiA7cAYNVs53D5AySzzL5ycz69QQZ5KENlQms6Ld5YxarZx42M7eOdQp7eXpChByRfveWYiLymaJF24y4M8N1a1caCtn29fUUh4iPbsT1BcRhUwZskXRxWWNxpIjAljQUKUV85f5AjyrA3QmxF/VtViv/EpTlcBnopvKUzR0dQ3zPB44I1grmk1YbHJORcw9FFh5CRGq0kkStApydTz6tfWkpsUw5f+so8/fHBMhXsqiof54j3PTAhhT85QzAAAIABJREFUz8HYVd/rsp8f4xYbD795hEVpsVy/3PczxwKNKmDM0mSjCkM0wqujCssbDaxYMM/lATXTVeQI4TuiWj19TnWriegwLblqNrXiY4pSY5ASjnYG3vazuQZ4TlSSqaey2ahu3qYghMgUQrwnhDgkhDgghPim4/iPhBCtQohKx8cnJjznPiFEnRCiVgixfsLxKx3H6oQQ93rj+1FOSouL5MU7y7i6OI0HNx3m2y/uZ9Qc9EPnFMVj7llfRNhp+Wm+Op59KmV5CXQPjHGs2zVTz57f00RT3zDfvbIIzRy2iCqzowoYs3T6qMLIUC1WmyQz3jvdD90DYzT2Dntt+whAki4cfVSo6sDwQVUtJpamx6kfsorPKUp1TCIJwJ8bFc1G0vWRLhmrVpKpp3tgjDbTqAtWFpAswLellIuA1cBXhRCLHV/7jZSyxPHxBoDjazcDS4ArgceEEFohhBZ4FLgKWAx8dsLrKF4SGabl958t5e7LC3mpopXPPrmbrgH1d0FRPGFDaTqlWXEI8Pnx7FNxZQ7G4JiF3797lNW58VxUmDTn11NmThUw5mBDaTo77r2Ehgev5sPvX0r6vEj++++VDI55vhW6vNE+Ys8bE0ichBAUpehU2JaPMVttHGzvV/kXik/Kio8iPEQTkJ1blU1Gl3RfwMkcjMomtY1kMlLKdinlR47fDwCHgDNdXV8PvCClHJNSNgB12Ee5rwLqpJT1Uspx4AXHYxUvE0LwjUsLePxz53C4fYDrH9lBTasKtlUUd7NYbRzpHOS6kvk0PHg1O+69xK+KFwALEqKYHxfBbhfkYPxpWz09g+N878qFXut6D3aqgOEisRGh/PYzJbQYhvnRqwc8fv7yxj7CQjQsTY/1+LknKkrVcaRzULU5+5AjnQOMW2wUZ6j8C8X3aDWCgpSYgOvA6BoYpdU4Qul08i/a2+HCC6GjY8qHLEqLJSxEQ2WzwYWrDExCiGygFPjQcehrQogqIcTTQghnm2I60DzhaS2OY1MdP/0cdwgh9gkh9nV3d7v4O1DO5KriNP55VxkCuOkPO3m9qt3bS1KUgLanoQ/DsJmrlqZ6eymzJoRgdV4Cu+t7sdlmf4/SMzjGk1vruWppKqVZ3ut6D3aqgOFCK7Pj+drF+fyzvIXXqto8eu7yRgPL0uO8noJbmKJjcMxyYla04n3O0YvL0lUHhuKbilJiA65zy9kpMa0Az5/+FLZvt/86hbAQDUvnx6ogz7MQQsQA/wK+JaXsBx4H8oASoB142PnQSZ4uz3D81ANSPiGlXCmlXJmUpFqIPW3J/Dhe+do6FqfF8tX//Yjfvn1kTjcliqJMbfOBDiJCNVzg59slynIT6B0a50jX7K83Hnm3jlGLje/4Uf5HIFIFDBf7+qUFlGTq+Z+Xqj02XmjUbKWmtZ8V2d6vBC50TCIJ1LGI/qiq1YQuIsRr02kU5WyKUmPoGhjDMDTu7aW4TGWzkRCNYOnZCoeHD8MTT4DNZv/1gQfg3/+GY8c+9tCSzHlUtZgwW21uWrV/E0KEYi9e/E1K+RKAlLJTSmmVUtqAJ7FvEQF7Z0XmhKdnAG1nOK74mCRdOM/fsZpPnpPBb98+ytefr2BkXIV7Koor2WySzTUdXFSYTFRYiLeXMydleY4cjFluI2nqHeZvHzby6ZWZ5CXFuHJpygypAoaLhWo1/O7mEqw2yX//vRKrB94RqG41MW61sdKL+RdOBY5JJIcD7N1Uf1bdYmJZRpzap6f4rMKUwBvBXNlsZGGajoiJ06pGRuDNN+Hb34af/9x+7He/A6vjpstigf/5H7jxRnj6afuxvj5ISYHSUu785df50cbfYvjOfbB3r/3rZjO0tdmfOx3T2K7ij4T9B9xTwCEp5a8nHE+b8LAbgBrH718FbhZChAshcoACYA+wFygQQuQIIcKwB32+6onvQZm58BAtv/rUMr7/iUW8UdPOTX/YyTM7Glj74Lvk3Ps6ax98l5crWr29TEXxWxXNRroGxrjSj7ePOGXMiyIzPnLWBYyH36pFqxF867ICF69MmSlVwHCDBQnR/Oi6JXzY0McTW+vdfj5ngOc5LgqLm4u4yFDmx0UEZCCfPxqzWDnc0U9xuvf/bCjKVBY6JpEESueW1SapajGd3D7y7LNw5ZUQHw/r18Mjj8CRI/ZiwrPPnvrkiAh7kePLX3a8mBU2bICMDOYN9HHpsb0k/e4h2LfP/vVDhyA9HcLDYf58WLkSrrsOtm2zf72nB954AyorobMTfvKTs25X8VNrgVuBS04bmfpLIUS1EKIKuBj4bwAp5QHgReAgsBn4qqNTwwJ8DdiCPQj0RcdjFR8lhODLF+Ty9OfPpa5zgB9vPEircQQJtBpHuO+lalXEUJRZ2lzTTqhWcMmiZG8vxSXKchP4sKFvxlvOalpNvFLZxhfX5pASO/fJYsrc+HcvkA+7aUUG79d28/CbtazLT6TYjRMg9h03kJsYTUJMuNvOMROFqTpqOwe9vQwFqO0YwGyVagKJ4tNSYsOJjQg5ew5GezvcfDP8/e+Q6qPvBvX10fnPjdz9+kvEXvuI/di2bdDYCHfeaS9gXHghREXBV75i3zoykc0GL78Mjz5q/zwpCf74RwBCpeSqn73NJfnxPHTjUvvXU1Ph8cftXRjOj6YmGHWMmPzwQ7jmmpOvHxJiP8czz8APf+i7/x1nSEq5ncnzK944w3PuB+6f5PgbZ3qe4psuXphMXFQYXQNjpxwfMVt5aEut301NUBRvk1Ky+UAHa/MTiY0I9fZyXKIsL4EX97VwsL3/7Fs8J/jlllr0UaHceWGeG1enTJcqYLiJEIL7b1jKR00GvvlCBa99Y51b9o5JKfmoycClC32nMlqUomNnXS9mq41QrWry8aYqR4BnsQrwVHyYEIKi1GmMYJ4Ydum8wfcFdXXw17/Cli2wZw/zbTZuCo+mTztk//rjj0NY2Meft2sXjJ+W+zE+Djt3TnoaIQQlmXo+ahuwd1wAJCfDf/3X1Gs7/3z7edrb4de/thc0wN7Z4Wv/HRVljrpPK144tRpHuPvvlSzLiGNZpp7FabGnbu9SFOVjDrT109w3wtcuzvf2UlymLDcRgN31vdMuYOys62HrkW6+/4lFxEUGRiHH36m7SzfSR4Xx8KeX09A7xM9eP+SWczT0DNE3NM6KBd4P8HQqStUxbrXR2Dvk7aUEveoWE/OiQsmYF+ntpSjKGRWl6qjtHJh6BHN7Ozz1lL174Mkn7dkRmzadzI84vZNhrs6UFdHUBH/6k30bCMCBA/ZigJTwgx/w+P1/4YLvvkjWckdK+WTFC4CKCvtzTv+oqJhyWSWZeo51D2EaMU/v+4iNhdWr7R/79tkzM8BeKHnmmYDLwlCC23z95P/WRYRo2FbXw482HuTGx3ay9P9u4er/t437XqrmhT1NHGzrx6LCcRXlFFsOdKARcNmiFG8vxWVS4yLITYyedg6GlJIHNx9mflwEt5YtcPPqlOlSHRhutiYvkTsvyOMPHxzjosIkrlji2nbdfY78i5U+MIHE6UQgX8cg+ck6L68muFW1mijO0KsAT8XnFaXoGBi10G4anfwm5LbbTnYrmM3wrW/Zt0M4j915J7z4or0jwfmRlWUvdIC982B4+OTX4uNBe4Z3YCd2e/z2t/D22/YOiy1b7JNDAB5+GO6+274tpLvb/prAq7/bRnFKGBqN6//elTiyjqpajJxfMIORdj/96ceLPKoLQwkw96wv4r6Xqhkxn5xGEhmq5YEbi7m+ZD4d/aPsbzZR1WKkqsXE61VtPL+nCYCIUA1L5sexLCOO5Rl6lmXEkZ0Q7Za/x4riDzbVdHBeToLPbFF3ldV5Cbxa2YbFaiPkLJ3ib1R3UNVi4qGblqmuLR+iChgecPflhWyv6+Z7/6qiJFNPsgvDX8qPG9BHhZKb6DvjfPKTY9AIqO3o5+plaWd/guIWo2YrRzsHfGp7kaJMpcgR5FnbOfDxAkZ7O2zdeuqxiAj7uFFnce6KK+y5Et3d0NVlH0Pa0nLy8T/6EWzefPJzjQbOPRd277Z//rOf2Z+XnGzfnuHs9njmGfvUkBtusJ/rwgvhjjvs51u8+ORaIuw/14fHLdR29HO5m1pul2fqEQIqm2ZYwJjhdhVF8UfOnIuHttTSZhxhvj6Se9YXnTieFhdJWlzkiYkKUkqO9w5T1WI8Udh4fk8Tz+w4DoAuIsS+7SRDz3LHr2lxESfeFHi5onXKcymKP6vrGqCua5BbVwde10FZbgL/+2ETNW39J8O2J2G22vjVm7UUpsRw4zkZHlyhcjaqgOEBYSEafvuZUq75/Ta+/Y/9/PkLq1xW0S9vMnBO1jyfeocgIlRLdmJ0QI1E9EeH2vux2KRbA2QVxVUKU+xF2CMdA1xc5Ci69fTYcxv6+iYPu9y40T7dA+BTn7J/TOWRR6C52V6kcH7ETCj87t5t77gwmU59ntVq77TYvh2WLIHIM2/HqmoxYZMnOyVcLTYilLykGCqbjTN74hm2pShKINlQmj7tIoIQgpzEaHISo7m+xP4ci9XG0a5Be1GjxV7UeHJrPRbH1ILEmHCWZ8QRqtXw7uEuxh1bT5wTT5xrUBR/trnGvr1wvYs7x33B6twEAHYd6z1jAePve5tp6BniT/+5Eq0P3WcpqoDhMfnJMfzwmsV8/981PLPzOLevy5nzaxqHx6nrGuQGH/yHsihFx2E1StWrqlvtN2JqAoniD/RRYaTEhp8M8qypsY8DbWuDBQvm3j2Ql2f/mMprr9l/PX4cFi6EsbGT53FO7DhL8QI4UVhYnuG+0cUlmXrePdyFlFJtD1MUFwvRaliUFsuitFg+c6792KjZyqH2fqpaTOx3bD+p6/r4tDU18UQJFJsPdFCapSc1LvBGhibpwilIjmFXfS93XTT5dcHwuIXfvXOUc7PncWmAjJANJCrE04NuWZXFZYtS+MWmwxxq75/z65U78y98KMDTqTBFx/HeIUbGrWd/sOIWVS0mEmPCSVXzqhU/UZQaa+/cevVVKCuzjwLduhVqa2ccdjlrv/yl/bUncmZFTENlk5Gs+Ci37hkuydTTNzROc9+I286hKMpJEaFaSrPm8fk12fz60yW8ffeFk87sBWgzqr+Xin9r7humprWfq5YGXveFU1leAvuO92GeIrz36e0NdA+Mce9VC9UbBT5IFTA8SAjBLz5ZTFxUKN98oYJR89xu7ssbDYRoBMvc+E7fbC1M1SElk75DoXhGdYuJZRlx6gev4jeKUmJYuenvyA0b7F0Qe/fCqlWeXcQcsyIqm41nbEl1BefrVzQb3HoeRVGmNtXEk6mOK4q/2HLAvn3kyiWBm2NXlpvA8LiVqpaPb8fsGxrnjx/Uc/niFFYsiPfC6pSzUQUMD0uICedXn1rOkc5BHtx0eE6vta/RwJL0OCLDfC8VtzDVMYlE5WB4xfC4haNdA9Oeca0ovqAwRUd5SgEDn7nF3nmR7oU27FmMNnXqMI3S0T/q9gLGwlQdEaGamedgKIriMvesLyJykqkEN56jto/4opcrWln74Lvk3Ps6ax98l5crWr29JJ+1qaaDxWmxZCVEeXspbjMxB+N0j75Xx9C4he+uL/L0spRpUgUML7iwMIkvrs3h2Z3Hea+2a1avMW6xsb/Z6JPbRwAWxEcRFqKhtmPuW2WUmTvY1o9NwjJVwFD8QVsbPPIIC1NjqUnNZ8f/PDStvAlfU+noiHBXgKdTiFZDcXqcKmAoihdtKE3ngRuLSddHIoC0uAiSdGH8ZVcjdV3qzRtf8nJFK/e9VE2rcQTJycBVVcT4uK7+UcobDScm9QSqedFhLEqLZVf9qQWMFsMwz+1q5KYVGRSk6Ly0OuVsVAHDS757ZRELU3Xc848qegbHZvz8g+39jFlsrPDRAkaIVkNBcgy1nWoLiTdUtdgDPNUEEsXn7d1rH2d6330UjBkQwn87tyqajYRqBYvTYt1+rpJMPQfa+hm3TL5/V1EU99tQms6Oey+h4cGr2XXfpbx011pCtRo+//ReugZGvb08xeGhLbWMnLZt2xm4qpzKuX0kkPMvnMpyE9h33MCY5eSfjV+/dQQEfOuyQi+uTDkbVcDwkohQLb+7uZT+UTPf+2cV8vTQuLPYd7wP8M0AT6eiFB1H1CQSr6huNZESG06KCvBUfNnzz8MFF0BYGOzcSUReNgvio05OIvEzlU1GFs+PI2KStnJXK8mcx7jF5pJAaEVRXCMzPoqnb1tJ39A4X3x2L0NjFm8vSWHqYFUVuPpxmw90kJsUTX5yzNkf7OfK8hIYs9ioaLJ3Mx5q7+ffFa18YU22yrLxcaqA4UVFqTruu2oh7xzu4q8fNs3oueWNBjLjI0n24RvUolQdHf2jmIbN3l5K0KlqMVKc7nvhropywk9+ArfcYg/p3LMHiosB+88Nf+zAsFhtVLeaKHVz/oVTqWObitpGoii+ZVmGnkc/V8rBtn6++r8fYZliyoHiOVPdjKbpffca2hsMQ+Psru/jqqWpQREAvyonHo04mYPx0JZadOEhU45WVXyHKmB42W1rsrmwMImfvXZw2nsmpZTsazSwIst3uy9ABXl6y8ComfqeIZap7SOKLysshDvugLfegqSkE4eLUnQc7xma85QmTzvSOcjwuNXtAZ5OaXERJOvCVQFDUXzQJQtT+NmGYt6v7eYHL9fMuMtWca2vXTL5DWlsRCj9o+pNNqe3DnVitUmuWhq400cmiosMZcn8OHbV9/JhfS/vHu7irovy0UeFeXtpylmoAoaXCSF46FPLiA4P4RvPV56yD2sqLYYRugfGWJHt26N9ihzhNyrI07MOtPUjpcq/UHxQQwP8+9/23998M/zxj/btIxMUpuqw+eEIZmchwVMFDCEEJZl6VcBQFB91y3lZfPXiPF7Y28wj79Z5ezlBrW/IXqRI0oUjgHR9JJ9amUFd1yA3PLqD4z1D3l2gj9hc00HGvEiWzHd/jpOvSIwJY09DH595YjcaAQnRqnjhD1QBwwck6yL45SeXcbC9n4ffPHLWx+9r9P38C7C/Q6iLCFEdGB5W7QzwVBNIFF/ywQf27SJ33QVDU18sLnR0bh3xs58blc0G5kWFssCDY+dKsvQ09AxhGBr32DkVRZm+71xRxA2l6Tz81hH+Vd7i7eUEpeFxC09tb+CioiT2fv8yGh68mh33XsJDNy3nudvPo3donOsf3cHOuh5vL9WrBkbNbD/aw5VLgmP7CNin0+yoOzmFxCbh/756QE2n8QOqgOEjLlucwn+szuKJrfXsOMsP0fJGA7rwEAp9fLyPEMIR5Olf76T6u6pWE+n6SBJjwr29FEWxe+IJuOwySEiAbdsgOnrKhy5IiCZMq/G7wmdls5HlmXqPXvg5uz0qW1QXhqL4IiEEv/jkMtbkJfC9f1Wx/Whw3yR7w/N7mukbGudrF+d/7GtleQm8+tV1JOvCufXpPTy367jH1+cr3j3cxbjVFvDjUyd6aEst46dl1KjpNP5BFTB8yPc/sZi8pGi+/eL+M76jtu+4gZIsPVqN71dIC1N1HO7oV/s/Pai6xai6LxTfICV84xtw5532Asbu3VBQcManhGo15CZF+9UkkoFRM0e7Bj22fcRpWYYeIezTTxRF8U1hIRr+cOsK8pJi+K+/lqvJQR40arbyxNZjrM6NZ+UU266zEqJ46StruKgwiR++coAfvFyNOQiDV7cc6CBJF845Pp6v50pqOo3/UgUMHxIZZh+t2js0xv/8u3rSm/7+UTO1nQOsXODb+RdOC1N19I9a6Owf8/ZSgoJp2Mzx3mGVf6H4BiEgJgbuvhteew3007vBX5jqXyOYq1tMSOm5/AunmPAQCpN1KgdDUXxcbEQoz3zhXGLCQ/jCM3tpN6kbJE/4Z3kLnf1jfP2SMxfOdRGhPPGfK7nzwlz+uruJ/3xqT1BtzRsZt/Le4W7WL0lB4wdvjrrKVNNp1AhV36cKGD5maXoc37miiE01Hfxj38f3S1Y2GZESVvh4/oWTc5uLv7WD+6uaNnv+hZpAonjV4cP20agA998PDz8MWu20n16YqqPNNIppxD/S4Ss8HOA5UUmmnv0tRtXlpig+br4+kme+cC6DYxZue3qvmn7hZmarjT98cIySTD1r8hLO+nitRnDfVYt4+FPLKW80sOGxHdOeDujvth7tZsRsDZrpI073rC8iMvTUa5PIUC33rC/y0oqU6VIFDB/05fNzWZOXwI82HqDhtGTkfY0GNMIe3uYP1CQSz6pSAZ6Kt23aBOedB1/+Mths9i6MGXIGeR71k8JnZbOR3MRor4xeK8nSY3R0XimK4tsWpcXy+H+cw7HuQe76aznjluDbquApr1S20WIY4euX5M8om+iTKzJ4/o7VDI1ZueHRnbx3uMuNq/QNm2s60EeFsirHP7q7XWVDaToP3FhMuj7yxHSaB24sZkNpureXppyFKmD4II1G8PCnlxOq1fCtFypO2YtX3tjHorRYYsJDvLjC6ZsXHUayLpxaFeTpEdWtRrLio9QMawXa2+HCC6GjwzPnaW+HX/8arrkGcnNh40bQzO6fGH/q3JJSUtls9Er3BUCpo5hd2WzwyvkVRZmZ8wuSePCTy9hR18u9/6pS3VNuYLVJHnuvjkVpsVyyMHnGz1+xYB6vfG0tmfFRfPHPe3lya33A/n8at9h4+1Anly9KIVQbfLeFG0rT2XHvJSem06jihX8Ivj+pfiItLpIHbyxmf4uJ3719FACL1UZlk9Fvto84FaXq/G4kor+qajG5L//CUzfEnj5XoPrpT2H7dvuvnjjPxRfDt78NN9xg/zwra9Yvma6PJCY8xC+CPNtMo3QPjHmtK64gWUd0mFYFeSqKH7lpRQZ3X17ISxWtPPzmEW8vJ+C8Ud1Ofc/QjLsvJkrXR/LPu8q4ckkq979xiHv+WcWYxerilXrfzmM9DIxagmr6iOL//ONt/CB1VXEan16ZwaPv13F+QSLR4SEMjVv9r4CRouO53Y1YbdIvJqf4q76hcVoMI9y6eoF7TjDxhvjRR91zDm+cq70dbr4Z/v53SHXzP+CuOJeUMDICg4P2j/h4ezhmby+89x4MDEBrKzz5pH0Lx1NPwQ9/aH/O889//PU+8xnIy4PaWvjXvz7+9VtvhcxMqK62d1VM1N8PzzxjP8/Ro/YCxi9/OevOCychBIUpMX5RwKhosnc+eKsDQ6sRFGfEqSBPRfEzX78knzbjCI+8V8d8fSS3nDf7oq9yks0mefS9OvKSorlyydz+TY8KC+HRW87hd+8c5XfvHKWhZ4g//McKknSBM6Z+y4EOYsJDWJuf6O2lKMq0qQKGj/u/1y5hT0Mfdz5Xjs3RvvbzNw4hJX7T5lSYqmPMYqOpb5icxGhvLydgVbc68i/c0YHR3n7yRvWxx+Df/z41lPHYMQgLs98oP/vsqc8NC7N/HezTKP7xj1O/rtfbb44B7rjDPq2ivd3++eOPQ1UVbPv/7d13eJRV9sDx70kPSUiAQIAQOglKC9WCougqqKuCvfe14eq69nVdd1dd9WfvnSJWVETdRRDFggjSEnqvSUjogQDpub8/7gwJkIQkzMw7Mzmf55knyTvt3pTJfc+ce84M+/X990NmJkRGVl46dYLHH7fXv/66vW9EROX17drBhRfa67//3p7IR0ZW3uaVVyqDJXfcAaWHFFaLj4cOrqDQ0qVQfsg7MM2a2RN8sGM9VIsWkJxsv3d33WXncvPNcMMNNtjQsyf07Qv5+fDvf1cGJgoK7Mdbb7VBhhUrbG2JvXvtY7m9+y7ceCOsWQMXX3z485eX27mNHAkPP3z49X372gDGsmXVXz9kiJ1fRkb114eH249hYfZ7e5TBC7e01nFMWZKHMabB76D5QuamfCLCQujeuqljY0hPacZ7v66jqLScqPC6F0tVSjlHRHhsRE9ydxfxyFdLaBMfxdAGbHdQB/thxVZW5BXw/CV9PNJRIyREuPuMVFKT4rjns0xGvDaTd64ZwLFtnXvN95TyCsN3S7cwtHsr/d+hAooGMPxcTGQYI/sl88K01QeObdlTzEMT7QlfIAQxKgt5FmgAw4uWuAIYPb1RwPOxxypPmkNC7En9iSdWXu8+ae3eHc488+D7hlV5menRA3bvPvj6mCq/E3362BP8LVsqC0DuqrK3v7TUnthv3w7FxfayY0fl9R99BL/9ZrMU3AYPrgxg3HknLF9+8POHhNjnGjPGBmbcwRO3iy+GCRPs5yedZAMNVd1wg81yAOjX7/AAx513wksvwYYNlcGbb76pzGZ4+GEbRCgttVkTcXG29aj7o1uLFnD99QdfFxdX+XPo1csGgvbvt9tviors8bIyO7eHH7bfr0O5fz7nn1/79VdeabNH3HJz7c/b/TwlJfZ5HnnEI5ksqUlxfDwni20FxbRqGnXUj+ctmVn59GzblIgw53ZkpqckUFpuWLp5T8Bl6CnVmIWHhvDalf249K1ZjPpoAZ/efIK2QT8Kxhhenb6alObRnNenrUcf+5zebejQogk3jZvHhW/8xguXpte+7cKX2Z0NNHfDTnbsK+Es3T6iAowGMALAhLmHt1MtLC3nmakrAyKA0S0pFhEbwNA9dt6zKNt2QmgaFe7ZB3ZnX5S4eqJXVMDGjXa7wqH/lK+80l5qcuON9lKTCy6Ae++tDJZUVMC6dbYWRuvW8MILtY/1119t8KK8vDLAUdWkSTYA4r7uuefgu+/s85SX2wDEDTccfJ/kKn9j779/eIZGhypbdj777ODgCdjsBrBbK8LCbEAhPBxGjLAtRlu53nFr2dKOrSYtW8KLL9Z8fZMmNpvj9tsPztAAO7cnnqh9O05IiM1KqUlo6MFZN08/Xf3zeGjbT1rrykKe/hrAKC2vYHHObq48zkvbtuqospBn4NVIUqqxi40MY8x1Axn5+m9cP3YuX95+IinNmzg9rID065oKyHitAAAgAElEQVTtLMzezZMX9CLMCwUpeybH8/Udg7l5/Hxu/WA+95yRyh011dnw5VbYBpqyJI/IsBBOSW3p9FCUqhcNYASAzfmF9Trub5pEhNG+eRMt5Olli7N3M9AbLbCqZl+4efBE1ePPJWIDBWFhB2d3AKSmVn6emws//FAZkCgpgenT7ZaMmt4tOffc2p975Mjqj+fmwrhxNngB9jn/+194+WWbzeJJs2ZVBpvcSkpsZkoAPU/VzK2Tu/nn4mplXgHFZRWOt7VOahpFm/gorYOhVIBq1TSKcTcM5ILXf+PaMXOYeNuJ2k2sAV6ZvobWTaO4oJ/33txr1TSKT24+nocmLua5aatYtXUvz1zUmygx9s2d1athwYLKbbfvvQennQbp6XY7Zm1vFPhQRYVhypI8TkltSUyAdDZUyk27kASAtgnR9Truj1KT4liRt8fpYQStbQXFbN5dRC9vbB/x1Qmxr5+rtmBJID9XRobNAjn0kpERUM/TIjaSxNgIvy7kmeEKGPR1qIBnVekpCdpKVakA1rVVHO9cM4DsnYX86f15FJUGX8cLb/p93Q5bM+6UzkSGeameQ2kprF5N1Pff8fw5XXlgeHfCP/qAHckdMdHR0LUrnHWW3bLp/p9fWgoXXWSvi4qyWZ2DB1duN/31V/uGxqJFh2+xrYsGdm1bmJ1P3p4izYxWAUkDGAHgvmFpRB9SXCc6PJT7hqU5NKL66946jg079us/ZC9x17/o3c4LJ1K+OiH29XMFa2AmiPh7C+bMTfkkxkbQrpnzweT0lASydhayY281dUyUUgHhuM4tePaSPszdsIt7PltIRYU58p0UAK/+uIbE2AguG1hLN5e6nOyXlsKqVZXBhFmzbFCiWzeIjraZnGefjSxbxm2nduHqc/qxOLETYwdfzKZnX4WJE22gouq224gIuwX00Udh2DBo06ZyS+bzz9vszj59bFHz+Hg4/fTK8UyaBJ98YsexefPhb4Y0sGX6lCV5hIUIpx+TVK/7KeUPNGcoALjrXDwzdSWb8wtpmxDNfcPSAqL+hVtqUhzlFYZ12/YFReVmf7Moezci0EO/t3XnjaCIPzxXEElNiuOTOVlUVBiPVJP3tMysXaSnJPhFlxR3G9fMrHxdkCoVwM7r05bc/EKe/HYFbeOjePicY50ekt9bmJXPjNXbefCs7kRH1JJ94T7Z/+c/beCgSRO77eOZZ2wnr9Wr7dfl5bam1UUX2YDBli22RtYll9hARteutig50PfWK1k54jweHzeXJ3cVM+Xjj+lQXkHVUZQbCF21qvqtsG+9BQ88AJs22efetKmyuxfY+lXz5lV+HR4OZ59tAxu5uXbbq3uryoMPVnZFq4UxhilL8zixayLx0R6umxaIAqDgqjqYBjACxIi+yQEVsDhUZUG+PRrA8ILFOfl0bRmr+xhVUElLiqOwtJysXfvp0MK/OhjtLixl7bZ9jPST1+Ve7eIJDRENYCgVBG4e0pmc/ELembGe5IRorhvcyekhec/Rnjwawwdf/Ea/ghyuKY+Bb5bZbmFpaTBoEOzZY1uY5+bCtGn2ZP+tt6BtW/jHP2zGxfjxNjAxaBBccYUNUBx3nH38wYNtTYtapLWO46tRg7ntwwUUvjOT0NKDMy5DS0vI/+Fnqs2RbdnSXtzPd6jp0yuDG+4Ah7v492OPVdbxKi62beUHDYKrr4bbbjvw/eGQIPvy3AI27tjPrad0qXVejUYAFFxVB9OzHeUTnRJjCA8VVubtdXooQWlR9m5O6pbo9DCU8qgDgc+8Ar8LYCx01b9IT/GPrh9NIsJIS4rTQp5KBQER4dFze5C7u4h//XcZreOjg7dWgfvk8d574e67bfBh1y57kj5kiL3NXXfB1q2V1+Xn26LZTz7J8uxdPHP3OfZ2r1d53L/+1Z7Mh4bagt379lV2CQsNtTUnwHYKy88/7CS/vlrERvLBjceRnvMa+0sO3y6dnBDNzIY8cFyczfZwZXwc4O4QV5WIzR7Zvt1+vW8ftG9vt6cMHHjgMmVlESECZxzbiIPdxtitRL/8cnAWyz33QOfOTo9OHYEGMJRPhIeG0KVlrF/vZw9UW/YUsbWgmN7eKOCplIO6uTqRrNpSwJk9/GvxnpmVjwj0TvGfv7v09gl8s3Cz3265UUrVXWiI8PJlfbn8ndnc9UkGH/3p+OBrk+w+Ca+ogA8/tBe3ESMqAxjTptkuXgkJ0KyZbV/etSsAr83YQPNz/8IDFw8kpnVLe5uEBEhynZzHxMDvv9uTUncAo7wcJk+ubNHuIRFhIRRWE7wAL3QOrK44eEgIDBgAjzxiv96/3257mTvXtqF3ZWuEX/gXBp5xCYnFe+Hn721gw53VEWyMscGvpUvt5ayz7O/OZ5/BpZcefNviYrj/fvj8c5g61d6me/fKS8eOtsOdcpwW8VQ+k5oU59cdBQLVomxbaKpXO/85kVLKE2Ijw2jXLJoVfvi6kZmVT5eWsTSN8p/9w+kpCRQUlbFu+z6nh+JzIpIiIj+KyHIRWSoid7mONxeRaSKy2vWxmeu4iMjLIrJGRBaJSL8qj3Wt6/arReRap+akVHREKO9dO4A28VHcNG4u64Ptb7vqSXhoqD25/Plnmx3x5puVt1u2zBbWnDPHnlh++inceCNrt+3lf4tzibn9FmKuvgLOOMOejHfrBk2bVv88bl7qBOazzoF1KQ7esiW88YatoVFQAHPmsO2p5/myWXeb0fPTT/DHP9pgT4cOtubH00/XXOS0gR1PfGbbtsrsk7Vr7VhbtrRBqtNPhzvvhBkz7PUnnGB//oe2tXUHtjZsgK+/hvvus0VWu3WzNVPcj//TT/D++/Z3sj7dY/z9exggNIChfCatdRw5+YUUFJU6PZSgMSkjh79+mgnAnz/KYFJGjsMjUsqzuvthJxJjDJlZ+QcKZ/oLdzvXjE2Nsp1qGXCPMeYY4HhglIgcCzwI/GCM6Qb84Poa4Cygm+tyM/AG2IAH8ChwHDAIeNQd9FDKCS1iIxl7/SBEhOvGzGF7sHQacmdfuE/Cy8vtSWFqKvTqVZlBUYs3flpLZFgIN550hBohPuwEVl3nwKiwEM93Dqxv17bISBg4kAmDzmVdi3YM69Eahg+3AaNnn7Un9BkZthDoXtd27wkTbD2Nl1+238NHH21Qx5MGOdKJfkmJDc7ccQcMHWozSFq1qqxh0ayZzdq54ALbAWbaNNvF5brr7PUpKfbrQ7kDW7fcYjM3duywvyejR9tgRosW9nZjxsC119raJQkJtrPM8OGVWT7LltkgiIe6xqiDaR6M8pm0A+nge4MvDdIBkzJyeGjiYgpdrWk37y7ioYmLAQK64KtSVaUmxfHTym2UlFUQEeYfMfesnYXs3FfidwGMLi1jiYsMIzMrn4sHHLkSfTAxxuQCua7PC0RkOZAMnA+c6rrZOOAn4AHX8feNMQaYLSIJItLGddtpxpidACIyDRgOfOyzySh1iI6JMbx77QCueGc2N46bxyd/Or72bhuBoLasiDoUUszauZ8vM3K45oQOJMZG1n5jH3YCO7RzoAE6JMZwfnpbn42hNlOX5tEnJaEyI2TIkMqtOmBP2Js3t5/n5dn6IR98cPCDjBljt6mMG2czGqKi7CU6GhIT4ckn7e0+/9wWHnVfFxVlAwDDhtnrly+32zbc94+Kslt+4uIqT/Rvu80GBtxbQAYOhKeesls57r3XZu4ce6zNkujZ0wYzwM5h5hGqjtQlsNW8uQ3unHDCwbd75x146CFYuRJWrLCXkpLKWip33AE//mjnlJpqt6D06FG5Zcr9PdSuJw3i8wCGiKQA7wOtgQrgbWPMS653PT4FOgIbgEuMMbvE9qd7CTgb2A9cZ4xZ4Hqsa4G/ux76cWPMOF/ORdVP1YJ8GsA4es9MXXkgeOFWWFrOM1NXagBDBY201nGUVRjWbd9L99b+0cEoI8tmOPhbACMkROidEt/oC3mKSEegL/A7kOQKbmCMyRUR90bvZCCryt2yXcdqOq6Uo/q1b8ZLl/Xl1g/mc+cnGbx5VX9CA7nWzVFmRbz581pCRbh5iP8VXKzaOXD8rA088tVSPvx9E1cd38HRcWXv2s+i7N08eFb3mm/kzjAAu+XizjshJwduuslmMZSXVwaamje3QY6iospL8+aVAYzRo+Hbbw9+/LQ0e7IPNsvBvaXDrX9/+OabyhP9SZPsJTbWBiriXdulQ0LsNpGkpIYXYD2awFZERGVtjPPPP/z6p56ChQsrgxvz59vfeXfQrqTEFq39WGPjDeFEBoY7zXOBiMQB813vcFyHTfN8SkQexKZ5PsDBaZ7HYdM8j6uS5jkAMK7H+doY0yhzZwNBckI0MRGhfpcOHqhqKgjl8UJRSjmoauDTXwIYmVn5RIWH0N01Nn+SnpLAmz+vo7CkPPDfoW0AEYkFvgD+YozZIzUvbKu7wtRy/NDnuRm79YT27ds3bLBK1dOwHq3557k9ePTrpfzrm6X867we1PI77t+O4uRxy54iPpuXzUUD2tEm3sO1JTzsquM7MG35Vp7433IGd02kU6JzHbWmLt0CwPD6FsUOCbHbe8pdb5qVlNgAw7p1tW+FmDTp4OBG4SHr06efhi1b7PGqAZCq2TlhYbbN7rhxdhxV+XP2wqBB9uKWm2sLyVbdMvXJJzY4dN99cPbZNpskEB1tK+QG8Hk+rjEm151BYYwpAKqmebozKMYBI1yfH0jzNMbMBtxpnsNwpXm6ghbuNE/lp0JChG4OFfKclJHD4Kem0+nB/zH4qelBUSuiRWxEtcc9XihKKQd1TowlLET8qgBwZlY+vZMTCAv1jy0tVaWnNKO8wrBkcz2KigUJEQnHBi8+NMZMdB3e4loz4Pq41XU8G6i6z6YdsLmW4wcxxrxtjBlgjBnQsmVLz05EqVpce2JHbh7SmfdnbeTtX9Y5PRxHvP3LOsqN4bZTujg9lCMSEf7vwt5EhIXw1wmZlJVXHPlOXjJlSS7dW8fRsb5BlIYWQY2IsMVUW7Wy7VzT0uzF7YQTbLeZyy+H66+320WGDDm4NkpZGXzxha1HEciq+x6GhsKCBXDeebZIqA+3OnmUA3U9HF191ZbmCWiaZxDq3jqOlVsKMOawN7S8xl0rIse1FzEnv5CHJi4O6CDGlj1FFJWWH/ZWYXR4qOcLRSnloIiwEDq3jPGbzK2SsgqWbt5Denv/2j7i5t7WkrmpcW0jcW03fQ9Ybox5vspVXwPuTiLXAl9VOX6NqxvJ8cBu19pjKnCmiDRzFe8803VMKb/x4PDu/LF3G578dgVfL6ymEGEQ27G3mA9/38j56W1Jad7E6eHUSev4KB4b0ZOMTfm8+fNaR8awtaCIeRt32e4j9eXDIqi+7BjjU9V9D8vLbUvXCRNsnYwuroDcr7/CkiW+H2N9lJbC4sUHt0IeM8Zn3VUcC2AcmuZZ202rOVbnNE/Xc90sIvNEZN62bdvqP1jlMalJcezcV8L2vSVHvrGH1FYrIhCVlFVw2wfzqTBw//A0khOiEewWnScv6KX1L1TQSU2ygU9/sDx3DyVlFX5X/8KtZVwk7ZpFN8Y6GIOBq4HTRCTTdTkbeAo4Q0RWA2e4vgaYDKwD1gDvALcDuIp3PgbMdV3+7S7oqZS/CAkRnr24D4M6NefeCQuZvW6H00Pymfd+XU9xWQW3n9rV6aHUy3l92nJun7a8+P1qluT4PkPuu6VbMAbO6tmm/neub8eTo+HLYIkv1fQ9zMyEiy+GKVMqW//ec4/txHPaafDll5Vbd5yWn2+3vVxxhW1Pe8IJtjONO+Dkw0CTIwEMX6Z5gqZ6+hP3fnZfvpsabLUiHvvvMhZsyueZi/pw26ldmfngaax/6hxmPniaBi9UUEpLiiNrZyF7i8ucHsqBFqX+GsAAO7bGFsAwxvxqjBFjTG9jTLrrMtkYs8MYc7oxppvr407X7Y0xZpQxposxppcxZl6VxxptjOnquoxxblZK1SwqPJS3r+5PSvNobn5/Hqv9JMjrTbv3l/L+rI2c3asNXVvFOj2cenvs/B60iI3g7k8zKSr17Unp1KV5dEqMITXJz79vvgyW+KvJk20R0LVrbRvYLl3go4+cGYs7Y/6992zQ4vLL4fvv7bheegnGj68MOLlro/ggC8PnAQxN82zc3AGMFT7az76vuIzIGlovBmKtiM/mZTF+9kZuGdKZc3o3IIquVAByv274wwI9MyufVnGRtImPcnooNUpPSSAnv5Cte4qcHopSyosSmkQw9vpBRIaHct2YuUH/Nz9u1gb2FpcxKsCyL9wSmkTwzEV9WL11L8/6MAs4f38Js9buYHjP1oFb9LUxadECHnjABjAmToROnSqzMPLzbXcTb6mogNmz4eGHbRbIVNep9YABNjNk5ky7bWT0aNtZxaHtPk5kYGiaZyOWGBtJi5gIVvkggLE5v5CL3pxFUVkF4aGHv2AP75Hk9TF40uLs3Tw8aQkndmmhdS5Uo+JE5lZNMrPySU9J8OtFYF9XfY6MRpaFoVRjlNK8CWOuG8iu/SVcP3auX2SqecPe4jJGz1zPH45pxbFt/aMjVUMMSW3J1cd34L2Z65m11jdbf75fvpWyClP/7iPKWWFhMHIk/PgjXHWVPfbuu5Ceboudfv65LXLqCbt321a5bdvarSFPP20zLiJcDQP69LFZISeeWNktxcHtPk50IdE0z0YuNSmOFV4+EVmYlc/5r80ka+d+xl4/kGcu6nOgVkSb+ChSmkfz4ZxNLNgUGF13d+4r4dYP5pMYE8Erl/f1y+4HSnlLSrMmRIeH+ixzqya79pWwYcd+vy3g6dajbTxhIdLotpEo1Vj1TI7ntSv7sSKvgFEfLqDUwU4X3vLh7I3k7y9l1NDAzL6o6qGzu9OxRQz3fraQPUWlXn++KUtyaRsfRe928V5/LuUl7jdNbrwRnn0WsrNt7YxOneDJJyu3etRVXp4Nhrz9tv06Lg5mzIChQ+HDD2HbNpg+3dbhqImD2330LEj5XFrrOFZvKaCiwjudSCYvzuWSt2YRGRbCF7edyKlprRjRN/lArYhZD53Ol7cPpnXTKG4cO5d12/Z6ZRyeUlZewZ8/XsC2vcW8eXV/WsRGOj0kpXwqJERITYp1PAMjM9sGBPy5/gXYvfHHtGna6DqRKNWYDU1rxRMjevLzqm38/cslPu325m1FpeW8M2M9J3dLpG/7Zk4P56g1iQjj+Uv6kLeniH99vcyrz7W3uIxfVm9nmG4fCQ7NmtmtHKtXw1dfQffu8NNPlQGO9evtx9xcOOWUg+tRLFsGTzwBxx0HbdrAn/4EH3xgrwsJgRUr4OOPbZHOZv79d6YBDOVzaa3j2F9STo6Hi2gaY3jtxzXc/uECeibHM2nU4AOp54dKjI1k3A2DCBHh2jFz2Frgv/tGn/1uFTPX7ODx83vSu51/nzgp5S2pSXGszHM22Ji5KZ8QISD+DtNTEliUnU+5lwLFSin/c9mg9vz5tK58Oi+LV6avcXo4HvPp3Cy27y0OiuwLt77tmzHq1C58sSCbKUu8V/TwxxVbKSmraFj3EeW/QkPhvPNg2jT4+mt7LDsbunWDwYPhmmtsO9Zbb63Mznj2Wfj7322w44knYNEi+PnnyscMoACXBjCUz3mjkGdxWTn3TFjIM1NXMiK9LR/edByJR8hU6NAihtHXDWR7QQk3jp3HPj/cN/rt4lze/HktVxzXnksGphz5DkoFqbTWcWzfW8yOvcWOjSEzK5/UpDhiI8McG0NdpacksK+knDVb/TvDTCnlWX89I5UL+iXz/LRVfD4/2+nhHLWSsgre/HktAzs247hOzZ0ejkf9+fRu9EqO529fLmZbgXf+t01ZmkdibAT9O/j3O+rqKES6znfi422QIifHdgqpqLBZGj/+aK//xz9sZsbs2fC3v9kinQEUtKhKAxjK57q5Wl95Kh18574Srnr3dyZm5PDXM1J54dJ0osJD63TfPikJvHZlX5bl7uF2P9s3unpLAfd+tpD0lAQePfdYp4ejlKPcgc+VDm0jMcawMDvf77ePuLkLeWZmBUadH6WUZ4gIT13Qm5O6JvLgF4uYsXqb00M6KhMXZJO7u4g7TusWdFsgwkNDeOHSPuwtLuOhiYs8vu2nqLScH1ds5cwerQkNCa7vnapGXBz85S9w1lm2ACjYj59+aj/v2BFaB0chVw1gKJ+LiwonOSGalR7IwFiztYARr81kUfZuXrm8L3eeXv9/cKd1T+I/I+2+0YcmLvaLfaN7ikq5Zfx8oiNCeeOqfkSG1S0go1SwWu+qVXPFO78z+KnpTMrI8enzb9ixn/z9pQETwOiUGEN8dLgW8lSqEYoIC+H1q/rRtVUst32wgGWb9zg9pAYpK6/gjZ/X0rtdPEO6JTo9HK/o2iqOB4Z35/vlW5kwL8ujjz1j9Xb2l5RzVs/gOGlVdZCbC2PHVnYnKSuD8eMProURBDSAoRzRvXXcUQcwZqzexsjXf2N/STmf3Hw85/Zp2+DHunRge+46vRufz8/mhWmrjmpcR6uiwnDPhIVs3LmfV6/oR5v4aEfHo5TTJmXk8OS3Kw58nZNfyEMTF/s0iJHh6ljk7x1I3ESEPikJZGghT6UapaZR4Yy5fiCxkWFcP3YOmz1cd8wX/rsol4079jNqaNegy76o6voTO3Jilxb8+5tlbNqx32OP++2SXOKjwzm+cwuPPabyc489ZreOVFVebo8HEQ1gKEekto5j7ba9lJQ1bMvG+NkbuW7MXJITopk06kSPVKX+yx+6cemAFF6evoaPft901I/XUG/8vJZpy7bwt7OP0X86SgHPTF1JYenBrxWFpeU8M3Wlz8aQmZVPTEQo3VpVXxjYH6WnJLBqS4Ff1vdRSnlfm/hoxt4wkP3F5Vw/Zi67C73fstNTKioMr/64hrSkOM44Jsnp4XhVSIjwzMV9CBHh3s8WeqT4cml5Bd8v28IfjkkiPFRP9xqNWbOgpOTgYyUl8NtvzozHS/Q3WjkiLSmOsgrDhh376nW/8grDv75ZyiOTlnBKaks+v+1E2jVr4pExiQiPj+zJ0LSW/H3SYr5ftsUjj1sfP6/axrPfreT89LbcMLijz59fKX9U0zuHvnxHMTMrn17t4gNqH3HflAQqDCzO2e30UJRSDuneuilvXt2fddv3cuv4+Q1+48jXpi7NY83WvYw6rSshAfS621DJCdH86/wezNmwk3dnrDvqx5u1dgd7isoYrttHGpeMDNt15NBLRobTI/MoDWAoRzSkE0lBUSk3jZvLmJkbuPGkTrxzzQCPdwMIDw3h1Sv60TM5njs+XnAgbdwXsnbu586PM0hLiuPJC3oFdbqkUvXRNqH6bVRH6jTkKUWl5SzP3UN6SmBVce/jqteh20iUatwGd03k6Qt7M2vdDh74wvPFIj3NGJt90SkxhnN6NZ72nyP7JnNWz9Y8990qluceXd2SKUvzaBIRyslBWjtENW4awFCO6NwyhtAQYVUdAxjZu/Zz0Ruz+GX1dh4f0ZNH/nis194JjYkMY/R1A2kVF8WN4+axfnv9skQaorCknFvGz8cYw1tX96dJhP+3aVTKV+4blkb0IZ2FBNi+t5iXvl9NmZe7By3dvIfSchMwBTzdmsdE0KFFE+1EopTign7tuPfMVL7MyOHZ73y3/a4hflq5jaWb93DbqV0CKuvtaIkIT4zsRdPocO7+NJPisvIGPU55heG7pXkM7d6qzl35lAokGsBQjogMC6VTYkydWiIu2LSLEa/NZPPuQsZdP4irju/g9fElxkYy7oZBAFw7eo7X+nODfafhb18uZnneHl66vC8dWsR47bmUCkQj+ibz5AW9SE6IRrCptv+5oCcj+ibzwveruPitWWys53a0+nB38ugbIAU8q0pPSdBOJEopAEYN7crlg1J47ce1jtb6qo0xhpenryY5IZqRfZOdHo7PNY+J4P8u6sWKvAJemLa6QY8xf+Mutu8t0e4jKmhpAEM5Jq0OnUi+XriZy96eTZOIML68fTAn+TAVrlNiDO9dO4CtBUXcOG6u1wrhjfttA19m5HD3H1IZmtbKK8+hVKAb0TeZmQ+exvqnzmHmg6dx+aAOvHBpOi9f3pc1W/dy9kszmDAvyyup0ZlZ+bSNjyKpaZTHH9vb0lMS2LKnmNzdgdeBQCnlWSLCY+dX1vqavsL3tb6OZNbaHWRsyufWU7s02uKTp3VP4vJBKbz1y1rmbthZ7/t/uySXiLAQTtU1pQpSjfOVQfmFtKQ4Nu3cz/6SwwMDxhhe/H4Vd36cQXq7BCaNGkzXVrE+H2Pf9s147Yp+LMnZzR0fLfB4qvqc9Tt5/H/L+cMxrbhjaFePPrZSjcF5fdoy5S9D6NUunvs/X8RtHyxg176SI9+xHjKzdgVM+9RDuTs0ZWodDKUUEOaq9dWjbTyjPsxgUbZ/vTa8+uMaWsVFcnH/dk4PxVF/P+dYUpo14Z4JC9lbjzfQjDFMXZLHkG4tPV4nTil/oQEM5ZjUJFvIc/WWvQcdLyot565PMnnx+9Vc2K8d428aRPOYCCeGCMDpxyTx+Ihe/LhyGw9/ucRj7/Bu2VPE7R8uIKV5E56/NL1RVNlWyhuSE6L56Kbj+dvZ3flhxRaGvfgLv6za5pHH3rG3mKydhQFX/8LtmDZxRISG6DYSpdQBMZFhvHfdAFrERnDD2Llk7dzv9JAAmL9xJ7+t3cHNQzo3+toNMZFhPH9JH7J27eeJ/y2r8/0WZe9m8+4i7T6igpoGMJRjurs6kVTdRrKtoJjL35nN1ws3c//wNJ69uDeRYc7/E7viuPb8+bSufDovixe/b9iexKpKyiq47YP57C8p482r+tM0KtwDo1Sq8QoJEW4e0oVJowYTHx3ONaPn8M+vl1JU2rAiaG7uE/9A60DiFhkWyrFtm5KhAQylVBWt4qIYe/0gSssN146Z4/HMtYZ4dfoamjUJ50bhz7YAABNHSURBVIrj2js9FL8woGNzbhnShY/nZPHD8rpt95myNI+wEOEPx+j2ERW8NIChHJPSvAlR4SEHCnmuzCtgxGszWZ67hzeu7Mftp3b1q1aifz0jlYv6t+OlH1bzyZyjK3712H+XsWBTPv93Ue8DLWWVUkevR9t4vvnzSVx3YkfG/raB8179lWWbG96OLmNTPqEhQq/keA+O0rfSUxJYnL3b691alFKBpWurWN65ZgDZuwr50/vzjjrgezSW5Ozmx5XbuOnkztqJrYq7z+hG99ZxPPDFYnbsrb2gvDGGKUvyOKFLCxKaOJe5rJS3aQBDOeabhZsprzC89+t6+j82jfNemUFpeQUTbjmBs/yw77eI8OQFvTgltSUPT1rS4OJXn83LYvzsjdw8pDN/7N3Ww6NUSkWFh/LP83ow7oZB7NpfyojXZvL2L2upqKj/9q/MrHzSkuKIjnA+E6yh+rZPoLC0nFWHbNdTSqlBnZrz/CV9mLdxF3+dkNmg10lPeO3HNcRFhXH1Cd7vNBdIIsNCefGydPYUlh5xG/OqLXtZv32fbh9RQU8DGMoRkzJyeGjiYkrL7Qvxjn0llJQbbh/ahd7t/HeveXhoCK9f2Y9j2sQx6sMMFtYzLXtJzm4enrSEEzq34P5haV4apVIK4JTUlkz9yxCGdm/Jfyav4Mp3f2dzft27cVRUGBZm5QdsAU83d/0OrYOhlKrOH3u35eGzj2Hy4jyemLzc58+/aksB3y7J47oTO+qW2mp0b92Ue85MZcrSPL7MyKnxdt8uyUUEzjg2yYejU8r3NIChHPHM1JUUHpKqaIB3flnvzIDqISYyjNHXDSQxzha/2rB9X53ut3NfCbeMn09iTASvXtGXsEbaHkwpX2oeE8GbV/Xn/y7szcLsfIa/+AtfL9xcp/uu276XguKygC3g6da+eROax0SQsWmX00NRSvmpm07uxHUnduS9X9fT519T6fTg/xj81HQm1XLCfLQmZeQw+KnpnPnCLwgEZKtqX7np5M4M6ticR79aSk4NgfgpS/IY2KE5reL0+6iCm55BKUfU9C5ofd4ddVKruCjGXT+ICmOLX20/wr7E8grDnR9nsK2gmDeu6k+L2EgfjVQpJSJcMjCFb+86mS6tYrnz4wzu/jSTPUWltd4vw9V6tF+AZ2CICH3axWsGhlKqRiJC7+R4QgR2F5ZhgJz8Qh6cuIiP5mxkd2GpRy8fzdnIgxMXHTgZN8AT/1vu1YBJIAsNEZ67pA8VxnDvhIWHbfXZsH0fK/IKGKbbR1QjoFVylCPaJkRXG0FumxDtwGgapnPLWN69diBXvDObG8fO5eObj6+x8NSz363k1zXbefrCXvQJ8HdzlQpUHVrE8NktJ/Daj2t5efpq5qzfyfOX9OG4zi2qvX1mVj5xUWF0Toz18Ug9Lz2lGT+t2kZBUSlxmqKtlKrGc9NWcWgJjKLSCv42cQl/m7jE689fWFrOM1NXMqJvstefKxClNG/CP849lge+WMyY3zZw40mdDlz37ZI8AK1/oRoFDWAoR9w3LI2HJi4+aBtJdHgo9wVYXYj+HZrxyuV9ufWD+dzxUQZvX93/sK0h3y7O5Y2f1nL5oPZcOlBbgynlpLDQEO76QzdOTk3k7k8zueyd2dx6Shfu/kMqEWEH/+1mZuXTp10CISH+0w2pofq2T8AYWJS9m8FdE50ejlLKD9WWBfvIH4/16HM99t9l9R6DgksGpDBt2RaenrKCId0S6ZZkO9lNWZpH73bxJAfQG4FKNZQGMJQj3NH1Z6auZHN+IW0TorlvWFpARt3P7NGaf5/fk79PWsIjXy3hPyN7HWj/umZrAfd+tpD0lAT+eZ5n//krpRquX/tmTL7zZB777zLe+GktM1Zv48VL+9K1lc22KCwpZ0VeAbed0sXhkXpGnyqFPDWAoZSqTk3ZsckJ0Qe92+8Jo39dH/CZuE6wHfF6M/zFX7h7QiZf3j6YbQXFLMzK5/7hgfUmoFINpQEM5ZgRfZMDMmBRnauO70Du7kJe+3Etu/aVsjhnN5vzCwkNEaLCQ3jjqn5EhgVuG0alglFMZBhPXdibod1b8eAXi/jjKzN4+OxjiI0M44nJyymvMHw0ZxNdW8UG/GtVfHQ4LeMieOWH1Tw7dWVAB42VUt7hy+zYYMnEdULLuEj+c0Evbhk/n37/nkZBcRkAYUGQLahUXWgAQykPuffMNH5ft4MpS/MOHCurMJSUG35ft1NPFJTyU8N6tKZvSgL3fb6IR75aSohwYB/4zn0lPDRxMUBA/w1Pyshh595Syo2dWE5+YVDMSynlOb7Mjg2mTFwnFJaUEypyIHgB8MK01bSKi9LvoQp6Yow58q2CyIABA8y8efOcHoYKUic++QObdxcddjw5IZqZD57mwIiUUnVljCH939PYXXh4d5JA/xse/NT0GlPDPTEvEZlvjBlw1A8UgHRdoZTyNW+/pivlD2paW2gbVaU8KLea4AVoUSqlAoGIsKea4AUE/t9woLeuVkopVUlf01VjpgEMpTyopuJTWpRKqcAQrH/DwTovpZRqjPQ1XTVmGsBQyoPuG5ZGdPjBxTq1KJVSgSNY/4aDdV5KKdUY6Wu6asy0iKdSHqRFqZQKbMH6Nxys81JKqcZIX9NVY6ZFPJVSSil1VLSIp64rlFJKKU/SIp5KKaWUCkoiMlpEtorIkirH/ikiOSKS6bqcXeW6h0RkjYisFJFhVY4Pdx1bIyIP+noeSimllKqdBjCUUkopFejGAsOrOf6CMSbddZkMICLHApcBPVz3eV1EQkUkFHgNOAs4FrjcdVullFJK+QmtgaGUUkqpgGaM+UVEOtbx5ucDnxhjioH1IrIGGOS6bo0xZh2AiHziuu0yDw9XKaWUUg2kGRhKKaWUClZ3iMgi1xaTZq5jyUBWldtku47VdFwppZRSfkIDGEoppZQKRm8AXYB0IBd4znVcqrmtqeX4YUTkZhGZJyLztm3b5omxKqWUUqoONIChlFJKqaBjjNlijCk3xlQA71C5TSQbSKly03bA5lqOV/fYbxtjBhhjBrRs2dLzg1dKKaVUtTSAoZRSSqmgIyJtqnw5EnB3KPkauExEIkWkE9ANmAPMBbqJSCcRicAW+vzal2NWSimlVO20iKdSSimlApqIfAycCiSKSDbwKHCqiKRjt4FsAG4BMMYsFZEJ2OKcZcAoY0y563HuAKYCocBoY8xSH09FKaWUUrXQAIZSSimlApox5vJqDr9Xy+2fAJ6o5vhkYLIHh6aUUkopDxJjqq1PFbREZBuw0cMPmwhs9/Bj+oNgnFcwzgmCc17BOCcIznkF45xA51UfHYwxjbIYhJfWFRCcv3/BOCcIznkF45wgOOcVjHMCnVcg8dacql1bNLoAhjeIyDxjzACnx+FpwTivYJwTBOe8gnFOEJzzCsY5gc5LOSsYf07BOCcIznkF45wgOOcVjHMCnVcg8fWctIinUkoppZRSSiml/J4GMJRSSimllFJKKeX3NIDhGW87PQAvCcZ5BeOcIDjnFYxzguCcVzDOCXReylnB+HMKxjlBcM4rGOcEwTmvYJwT6LwCiU/npDUwlFJKKaWUUkop5fc0A0MppZRSSimllFJ+TwMYdSAio0Vkq4gsqea6e0XEiEii62sRkZdFZI2ILBKRfr4f8ZHVc05XuuaySER+E5E+vh9x3dRnXlWODxSRchG5yHcjrbv6zklEThWRTBFZKiI/+3a0dVfP38F4EflGRBa65nW970d8ZNXNSUT+KSI5rp9JpoicXeW6h1yvFStFZJgzoz6y+sxLRM4Qkfkistj18TTnRl67+v68XNe3F5G9InKv70d8ZA34HewtIrNcf1eLRSTKmZE3LsG4roDgXFsE47oCdG3h+lrXFg4KxrVFMK4rwP/WFhrAqJuxwPBDD4pICnAGsKnK4bOAbq7LzcAbPhhfQ4yl7nNaD5xijOkNPIZ/790aS93nhYiEAk8DU30xuAYaSx3nJCIJwOvAecaYHsDFPhpjQ4yl7j+rUcAyY0wf4FTgORGJ8MEY62ss1cwJeMEYk+66TAYQkWOBy4Aervu87vp99EdjqeO8sH3AzzXG9AKuBcb7aIwNMZa6z+vAdcC3Xh9Zw42l7r+DYcAHwK2u14tTgVJfDbSRG0vwrSsgONcWYwm+dQXo2gJ0beG0sQTf2mIswbeuAD9bW2gAow6MMb8AO6u56gXgfqBqIZHzgfeNNRtIEJE2PhhmvdRnTsaY34wxu1xfzgbaeX+EDVPPnxXAn4EvgK1eHlqD1XNOVwATjTGbXPcNlnkZIE5EBIh13a/M64Osp1rmVJ3zgU+MMcXGmPXAGmCQ1wZ3FOozL2NMhjFms+vLpUCUiER6bXBHoZ4/L0RkBLAOOy+/VM85nQksMsYsdN13hzGm3GuDUwcE47oCgnNtEYzrCtC1hfvm6NrCMcG4tgjGdQX439pCAxgNJCLnATnuH04VyUBWla+zXcf8Xi1zqupG/D9KeJCa5iUiycBI4E1HBnYUavlZpQLNROQnV4rdNQ4Mr8FqmderwDHAZmAxcJcxpsLX4zsKd7jSpEeLSDPXsYB9raiiunlVdSGQYYwp9vXAjtJh8xKRGOAB4F/ODq3BqvtZpQJGRKaKyAIRud/JATZ2wbiugOBcWwTjugJ0bYGuLfxFMK4tgnFdAQ6tLTSA0QAi0gR4GPhHdVdXc8zvW70cYU7u2wzFLjIe8NW4jtYR5vUi8ECgveN4hDmFAf2Bc4BhwCMikurD4TXYEeY1DMgE2gLpwKsi0tSHwzsabwBdsOPOBZ5zHQ/I14oqapoXACLSA5tGfYvvh3ZUaprXv7CpknudGthRqGlOYcBJwJWujyNF5HRHRtjIBeO6AoJzbRGM6wrQtQW6tvAXwbi2CMZ1BTi4tgjz5IM1Il2ATsBCm3VGO2CBiAzCRjpTqty2HTay6+9qnJMxJk9EegPvAmcZY3Y4OM76qu1nNQD4xHU8EThbRMqMMZOcGmwdHen3b7sxZh+wT0R+AfoAq5wabD3UNq/rgaeM7fu8RkTWA92BOU4Ntq6MMVvcn4vIO8B/XV8G6msFUOu8EJF2wJfANcaYtQ4Mr8FqmddxwEUi8n9AAlAhIkXGmFcdGGa9HOF38GdjzHbXdZOBfsAPPh+kCsZ1BQTn2iIY1xWgawtdW/iBYFxbBOO6ApxdW2gGRgMYYxYbY1oZYzoaYzpif1D9jDF5wNfANWIdD+w2xuQ6Od66qG1OItIemAhcbYwJhH9WB9Q2L2NMpyrHPwduD4RFxhF+/74CThaRMNe7DscByx0cbp0dYV6bgNMBRCQJSMPuGfR7cvBe9ZGAu4Lz18BlIhIpIp2wBfr8ftHkVtO8xBZ7+x/wkDFmphNjOxo1zcsYc3KV380Xgf8EyiKjlt/BqUBvEWkitujWKcAyX49PBee6AoJzbRGM6wrQtYWuLfxDMK4tgnFdAc6uLTQDow5E5GNsBdVEEckGHjXGvFfDzScDZ2OL5uzHRnf9Tj3n9A+gBbaSMUCZMWaATwZaT/WcV0Coz5yMMctFZAqwCKgA3jXGHNZKzB/U82f1GDBWRBZj0yMfcEd2/Ul1cwJOFZF0bArnBlxpj8aYpSIyAfuiXgaM8te04/rMC7gD6IpNMX7EdexM44dF3+o5r4BQz9/BXSLyPDDXdd1kY8z/nBh3YxOM6woIzrVFMK4rQNcWLrq2cFAwri2CcV0B/re2EJs1pZRSSimllFJKKeW/dAuJUkoppZRSSiml/J4GMJRSSimllFJKKeX3NIChlFJKKaWUUkopv6cBDKWUUkoppZRSSvk9DWAopZRSSimllFLK72kAQykVkETkJxHxu5Z7SimllAo8uq5QKjBoAEMppZRSSimllFJ+TwMYSimfEJH7ReRO1+cviMh01+eni8gHInKmiMwSkQUi8pmIxLqu7y8iP4vIfBGZKiJtDnncEBEZJyKP+35WSimllHKCriuUapw0gKGU8pVfgJNdnw8AYkUkHDgJWAz8HfiDMaYfMA/4q+v6V4CLjDH9gdHAE1UeMwz4EFhljPm7b6ahlFJKKT+g6wqlGqEwpweglGo05gP9RSQOKAYWYBccJwNfA8cCM0UEIAKYBaQBPYFpruOhQG6Vx3wLmGCMqbr4UEoppVTw03WFUo2QBjCUUj5hjCkVkQ3A9cBvwCJgKNAFWA9MM8ZcXvU+ItILWGqMOaGGh/0NGCoizxljirw2eKWUUkr5FV1XKNU46RYSpZQv/QLc6/o4A7gVyARmA4NFpCuAiDQRkVRgJdBSRE5wHQ8XkR5VHu89YDLwmYhoQFYppZRqXHRdoVQjowEMpZQvzQDaALOMMVuAImCGMWYbcB3wsYgswi48uhtjSoCLgKdFZCF2UXJi1Qc0xjyPTRsdLyL6mqaUUko1HrquUKqREWOM02NQSimllFJKKaWUqpVGFZVSSimllFJKKeX3NIChlFJKKaWUUkopv6cBDKWUUkoppZRSSvk9DWAopZRSSimllFLK72kAQymllFJKKaWUUn5PAxhKKaWUUkoppZTyexrAUEoppZRSSimllN/TAIZSSimllFJKKaX83v8Dh/7VOwrQtPgAAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "num_samples = 6\n", - "min_week = 140\n", - "sales = pd.read_csv(os.path.join(DATA_DIR, \"yx.csv\"))\n", - "sales[\"move\"] = sales.logmove.apply(lambda x: round(math.exp(x)) if x > 0 else 0)\n", - "\n", - "result_df[\"move\"] = result_df.predictions\n", - "plot_predictions_with_history(\n", - " result_df,\n", - " sales,\n", - " grain1_unique_vals=store_list,\n", - " grain2_unique_vals=brand_list,\n", - " time_col_name=\"week\",\n", - " target_col_name=\"move\",\n", - " grain1_name=\"store\",\n", - " grain2_name=\"brand\",\n", - " min_timestep=min_week,\n", - " num_samples=num_samples,\n", - " predict_at_timestep=145,\n", - " line_at_predict_time=False,\n", - " title=\"Prediction results for a few sample time series\",\n", - " x_label=\"week\",\n", - " y_label=\"unit sales\",\n", - " random_seed=2,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "forecasting_env", - "language": "python", - "name": "forecasting_env" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.10" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/grocery_sales/python/03_model_tune_deploy/aml_scripts/train_validate.py b/examples/grocery_sales/python/03_model_tune_deploy/aml_scripts/train_validate.py deleted file mode 100644 index d7d0e23d..00000000 --- a/examples/grocery_sales/python/03_model_tune_deploy/aml_scripts/train_validate.py +++ /dev/null @@ -1,275 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -""" -Perform cross validation of a LightGBM forecasting model on the training data of the 1st forecast round. -""" - -import os -import math -import argparse -import datetime -import numpy as np -import pandas as pd -import lightgbm as lgb -from azureml.core import Run -from sklearn.model_selection import train_test_split -from fclib.feature_engineering.feature_utils import week_of_month, df_from_cartesian_product, combine_features - - -FIRST_WEEK = 40 -GAP = 2 -HORIZON = 2 -FIRST_WEEK_START = pd.to_datetime("1989-09-14 00:00:00") - - -def create_features(pred_round, train_dir, lags, window_size, used_columns): - """Create input features for model training and testing. - - Args: - pred_round (int): Prediction round (1, 2, ...) - train_dir (str): Path of the training data directory - lags (np.array): Numpy array including all the lags - window_size (int): Maximum step for computing the moving average - used_columns (list[str]): A list of names of columns used in model training (including target variable) - - Returns: - pd.Dataframe: Dataframe including all the input features and target variable - int: Last week of the training data - """ - - # Load training data - default_train_file = os.path.join(train_dir, "train.csv") - if os.path.isfile(default_train_file): - train_df = pd.read_csv(default_train_file) - else: - train_df = pd.read_csv(os.path.join(train_dir, "train_" + str(pred_round) + ".csv")) - train_df["move"] = train_df["logmove"].apply(lambda x: round(math.exp(x))) - train_df = train_df[["store", "brand", "week", "move"]] - - # Create a dataframe to hold all necessary data - store_list = train_df["store"].unique() - brand_list = train_df["brand"].unique() - train_end_week = train_df["week"].max() - week_list = range(FIRST_WEEK, train_end_week + GAP + HORIZON) - d = {"store": store_list, "brand": brand_list, "week": week_list} - data_grid = df_from_cartesian_product(d) - data_filled = pd.merge(data_grid, train_df, how="left", on=["store", "brand", "week"]) - - # Get future price, deal, and advertisement info - default_aux_file = os.path.join(train_dir, "auxi.csv") - if os.path.isfile(default_aux_file): - aux_df = pd.read_csv(default_aux_file) - else: - aux_df = pd.read_csv(os.path.join(train_dir, "auxi_" + str(pred_round) + ".csv")) - data_filled = pd.merge(data_filled, aux_df, how="left", on=["store", "brand", "week"]) - - # Create relative price feature - price_cols = [ - "price1", - "price2", - "price3", - "price4", - "price5", - "price6", - "price7", - "price8", - "price9", - "price10", - "price11", - ] - data_filled["price"] = data_filled.apply(lambda x: x.loc["price" + str(int(x.loc["brand"]))], axis=1) - data_filled["avg_price"] = data_filled[price_cols].sum(axis=1).apply(lambda x: x / len(price_cols)) - data_filled["price_ratio"] = data_filled["price"] / data_filled["avg_price"] - data_filled.drop(price_cols, axis=1, inplace=True) - - # Fill missing values - data_filled = data_filled.groupby(["store", "brand"]).apply( - lambda x: x.fillna(method="ffill").fillna(method="bfill") - ) - - # Create datetime features - data_filled["week_start"] = data_filled["week"].apply( - lambda x: FIRST_WEEK_START + datetime.timedelta(days=(x - 1) * 7) - ) - data_filled["year"] = data_filled["week_start"].apply(lambda x: x.year) - data_filled["month"] = data_filled["week_start"].apply(lambda x: x.month) - data_filled["week_of_month"] = data_filled["week_start"].apply(lambda x: week_of_month(x)) - data_filled["day"] = data_filled["week_start"].apply(lambda x: x.day) - data_filled.drop("week_start", axis=1, inplace=True) - - # Create other features (lagged features, moving averages, etc.) - features = data_filled.groupby(["store", "brand"]).apply( - lambda x: combine_features(x, ["move"], lags, window_size, used_columns) - ) - - # Drop rows with NaN values - features.dropna(inplace=True) - - return features, train_end_week - - -if __name__ == "__main__": - # Parse input arguments - parser = argparse.ArgumentParser() - parser.add_argument("--data-folder", type=str, dest="data_folder", default=".", help="data folder mounting point") - parser.add_argument("--num-leaves", type=int, dest="num_leaves", default=64, help="# of leaves of the tree") - parser.add_argument( - "--min-data-in-leaf", type=int, dest="min_data_in_leaf", default=50, help="minimum # of samples in each leaf" - ) - parser.add_argument("--learning-rate", type=float, dest="learning_rate", default=0.001, help="learning rate") - parser.add_argument( - "--feature-fraction", - type=float, - dest="feature_fraction", - default=1.0, - help="ratio of features used in each iteration", - ) - parser.add_argument( - "--bagging-fraction", - type=float, - dest="bagging_fraction", - default=1.0, - help="ratio of samples used in each iteration", - ) - parser.add_argument("--bagging-freq", type=int, dest="bagging_freq", default=1, help="bagging frequency") - parser.add_argument("--max-rounds", type=int, dest="max_rounds", default=400, help="# of boosting iterations") - parser.add_argument("--max-lag", type=int, dest="max_lag", default=10, help="max lag of unit sales") - parser.add_argument( - "--window-size", type=int, dest="window_size", default=10, help="window size of moving average of unit sales" - ) - args = parser.parse_args() - args.feature_fraction = round(args.feature_fraction, 2) - args.bagging_fraction = round(args.bagging_fraction, 2) - print(args) - - # Start an Azure ML run - run = Run.get_context() - - # Data paths - DATA_DIR = args.data_folder - TRAIN_DIR = os.path.join(DATA_DIR, "train") - - # Data and forecast problem parameters - TRAIN_START_WEEK = 40 - TRAIN_END_WEEK_LIST = list(range(135, 159, 2)) - TEST_START_WEEK_LIST = list(range(137, 161, 2)) - TEST_END_WEEK_LIST = list(range(138, 162, 2)) - # The start datetime of the first week in the dataset - FIRST_WEEK_START = pd.to_datetime("1989-09-14 00:00:00") - - # Parameters of GBM model - params = { - "objective": "mape", - "num_leaves": args.num_leaves, - "min_data_in_leaf": args.min_data_in_leaf, - "learning_rate": args.learning_rate, - "feature_fraction": args.feature_fraction, - "bagging_fraction": args.bagging_fraction, - "bagging_freq": args.bagging_freq, - "num_rounds": args.max_rounds, - "early_stopping_rounds": 125, - "num_threads": 16, - } - - # Lags and used column names - lags = np.arange(2, args.max_lag + 1) - used_columns = ["store", "brand", "week", "week_of_month", "month", "deal", "feat", "move", "price", "price_ratio"] - categ_fea = ["store", "brand", "deal"] - - # Train and validate the model using only the first round data - r = 0 - print("---- Round " + str(r + 1) + " ----") - # Load training data - default_train_file = os.path.join(TRAIN_DIR, "train.csv") - if os.path.isfile(default_train_file): - train_df = pd.read_csv(default_train_file) - else: - train_df = pd.read_csv(os.path.join(TRAIN_DIR, "train_" + str(r + 1) + ".csv")) - train_df["move"] = train_df["logmove"].apply(lambda x: round(math.exp(x))) - train_df = train_df[["store", "brand", "week", "move"]] - - # Create a dataframe to hold all necessary data - store_list = train_df["store"].unique() - brand_list = train_df["brand"].unique() - week_list = range(TRAIN_START_WEEK, TEST_END_WEEK_LIST[r] + 1) - d = {"store": store_list, "brand": brand_list, "week": week_list} - data_grid = df_from_cartesian_product(d) - data_filled = pd.merge(data_grid, train_df, how="left", on=["store", "brand", "week"]) - - # Get future price, deal, and advertisement info - default_aux_file = os.path.join(TRAIN_DIR, "auxi.csv") - if os.path.isfile(default_aux_file): - aux_df = pd.read_csv(default_aux_file) - else: - aux_df = pd.read_csv(os.path.join(TRAIN_DIR, "auxi_" + str(r + 1) + ".csv")) - data_filled = pd.merge(data_filled, aux_df, how="left", on=["store", "brand", "week"]) - - # Create relative price feature - price_cols = [ - "price1", - "price2", - "price3", - "price4", - "price5", - "price6", - "price7", - "price8", - "price9", - "price10", - "price11", - ] - data_filled["price"] = data_filled.apply(lambda x: x.loc["price" + str(int(x.loc["brand"]))], axis=1) - data_filled["avg_price"] = data_filled[price_cols].sum(axis=1).apply(lambda x: x / len(price_cols)) - data_filled["price_ratio"] = data_filled["price"] / data_filled["avg_price"] - data_filled.drop(price_cols, axis=1, inplace=True) - - # Fill missing values - data_filled = data_filled.groupby(["store", "brand"]).apply( - lambda x: x.fillna(method="ffill").fillna(method="bfill") - ) - - # Create datetime features - data_filled["week_start"] = data_filled["week"].apply( - lambda x: FIRST_WEEK_START + datetime.timedelta(days=(x - 1) * 7) - ) - data_filled["year"] = data_filled["week_start"].apply(lambda x: x.year) - data_filled["month"] = data_filled["week_start"].apply(lambda x: x.month) - data_filled["week_of_month"] = data_filled["week_start"].apply(lambda x: week_of_month(x)) - data_filled["day"] = data_filled["week_start"].apply(lambda x: x.day) - data_filled.drop("week_start", axis=1, inplace=True) - - # Create other features (lagged features, moving averages, etc.) - features = data_filled.groupby(["store", "brand"]).apply( - lambda x: combine_features(x, ["move"], lags, args.window_size, used_columns) - ) - train_fea = features[features.week <= TRAIN_END_WEEK_LIST[r]].reset_index(drop=True) - - # Drop rows with NaN values - train_fea.dropna(inplace=True) - - # Model training and validation - # Create a training/validation split - train_fea, valid_fea, train_label, valid_label = train_test_split( - train_fea.drop("move", axis=1, inplace=False), train_fea["move"], test_size=0.05, random_state=1 - ) - dtrain = lgb.Dataset(train_fea, train_label) - dvalid = lgb.Dataset(valid_fea, valid_label) - # A dictionary to record training results - evals_result = {} - # Train LightGBM model - bst = lgb.train( - params, dtrain, valid_sets=[dtrain, dvalid], categorical_feature=categ_fea, evals_result=evals_result - ) - # Get final training loss & validation loss - train_loss = evals_result["training"]["mape"][-1] - valid_loss = evals_result["valid_1"]["mape"][-1] - print("Final training loss is {}".format(train_loss)) - print("Final validation loss is {}".format(valid_loss)) - - # Log the validation loss (MAPE) - run.log("MAPE", np.float(valid_loss) * 100) - - # Files saved in the "./outputs" folder are automatically uploaded into run history - os.makedirs("./outputs/model", exist_ok=True) - bst.save_model("./outputs/model/bst-model.txt") diff --git a/examples/grocery_sales/python/03_model_tune_deploy/azure_hyperdrive_lightgbm.ipynb b/examples/grocery_sales/python/03_model_tune_deploy/azure_hyperdrive_lightgbm.ipynb deleted file mode 100644 index cb591146..00000000 --- a/examples/grocery_sales/python/03_model_tune_deploy/azure_hyperdrive_lightgbm.ipynb +++ /dev/null @@ -1,1185 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Copyright (c) Microsoft Corporation.\n", - "\n", - "Licensed under the MIT License. " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Model Tuning and Deployment using Azure Machine Learning Service\n", - "\n", - "In this notebook, we perform hyperparameter tuning of a LightGBM retail sales forecast model using HyperDrive in Azure Machine Learning (AzureML). After the optimal hyperparameters are found, we further deploy the best model as a web service on Azure.\n", - "\n", - "To tune the hyperparameters, we carry out cross-validation with the Orange Juice data from week 40 to week 135. Specifically, we split the data into a training set and a validation set. Then, we train LightGBM models with different sets of hyperparameters on the training set and evaluate the accuracy of each model on the validation set. The set of hyperparameters which yield the best validation accuracy will be used to train forecast models when the data beyond week 135 is available, e.g., in the multi-round training examples provided in [examples/02_model](../02_model).\n", - "\n", - "## Prerequisites\n", - "\n", - "To run this notebook, you need to start from a conda environment where AzureML SDK is installed. In our case, we can first activate `forecasting_env` environment by\n", - "```\n", - "conda activate forecasting_env\n", - "```\n", - "as we have installed AzureML SDK in this environment. Then, we can start the notebook via\n", - "```\n", - "jupyter notebook --no-browswers\n", - "```\n", - "In addition, you need to install and enable AzureML widget extension in your environment by running the following commands.\n", - "```\n", - "jupyter nbextension install --py --user azureml.widgets\n", - "jupyter nbextension enable --py --user azureml.widgets\n", - "```\n", - "\n", - "Besides, you need to create an AzureML workspace and download its configuration file (`config.json`) by following the instructions in [configuration.ipynb](https://github.com/Azure/MachineLearningNotebooks/blob/master/configuration.ipynb) notebook." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Global Settings" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Installing /data/anaconda/envs/forecasting_env/lib/python3.6/site-packages/azureml/widgets/static -> azureml_widgets\n", - "Up to date: /data/home/chenhui/.local/share/jupyter/nbextensions/azureml_widgets/index.js\n", - "Up to date: /data/home/chenhui/.local/share/jupyter/nbextensions/azureml_widgets/extension.js\n", - "Up to date: /data/home/chenhui/.local/share/jupyter/nbextensions/azureml_widgets/packages/labextension/azureml_widgets-1.1.0.tgz\n", - "- Validating: \u001b[32mOK\u001b[0m\n", - "\n", - " To initialize this nbextension in the browser every time the notebook (or other app) loads:\n", - " \n", - " jupyter nbextension enable azureml.widgets --user --py\n", - " \n", - "Enabling notebook extension azureml_widgets/extension...\n", - " - Validating: \u001b[32mOK\u001b[0m\n" - ] - } - ], - "source": [ - "# Install and enable AzureML widgets\n", - "!jupyter nbextension install --py --user azureml.widgets\n", - "!jupyter nbextension enable --py --user azureml.widgets" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Turning diagnostics collection on. \n", - "Azure ML SDK Version: 1.0.85\n" - ] - } - ], - "source": [ - "import os\n", - "import sys\n", - "import json\n", - "import shutil\n", - "import azureml\n", - "import requests\n", - "import subprocess\n", - "import numpy as np\n", - "from azureml.core import (\n", - " Experiment,\n", - " ScriptRunConfig,\n", - ")\n", - "from azureml.telemetry import set_diagnostics_collection\n", - "from azureml.core.runconfig import (\n", - " RunConfiguration,\n", - " EnvironmentDefinition,\n", - " CondaDependencies,\n", - ")\n", - "from azureml.train.estimator import Estimator\n", - "from azureml.widgets import RunDetails\n", - "from azureml.train.hyperdrive import (\n", - " BayesianParameterSampling,\n", - " HyperDriveConfig,\n", - " quniform,\n", - " uniform,\n", - " choice,\n", - " PrimaryMetricGoal,\n", - ")\n", - "from azureml.core.webservice import AciWebservice\n", - "from azureml.core.model import Model, InferenceConfig\n", - "from fclib.common.utils import git_repo_path\n", - "from fclib.azureml.azureml_utils import (\n", - " get_or_create_workspace,\n", - " get_or_create_amlcompute,\n", - ")\n", - "from fclib.dataset.ojdata import download_ojdata, split_train_test\n", - "\n", - "cur_dir = os.getcwd()\n", - "if cur_dir not in sys.path:\n", - " sys.path.append(cur_dir)\n", - "from aml_scripts.train_validate import create_features\n", - "\n", - "# Opt-in diagnostics for better experience of future releases\n", - "set_diagnostics_collection(send_diagnostics=True)\n", - "\n", - "# Check core SDK version number\n", - "print(\"Azure ML SDK Version: \", azureml.core.VERSION)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "# Use False if you've already downloaded and split the data\n", - "DOWNLOAD_SPLIT_DATA = True\n", - "\n", - "# Get data directory\n", - "DATA_DIR = os.path.join(git_repo_path(), \"ojdata\")\n", - "\n", - "# Forecasting settings\n", - "N_SPLITS = 1\n", - "HORIZON = 2\n", - "GAP = 2\n", - "FIRST_WEEK = 40\n", - "LAST_WEEK = 138" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Initialize Workspace & Create an AzureML Experiment\n", - "\n", - "Initialize a [Machine Learning Workspace](https://docs.microsoft.com/azure/machine-learning/service/concept-azure-machine-learning-architecture#workspace) object from the workspace you created in the Prerequisites step. `get_or_create_workspace()` below creates a workspace object from the details stored in `config.json` that you have downloaded. We assume that you store this config file to a directory `./.azureml`. In case the existing workspace cannot be loaded, the following cell will try to create a new workspace with the subscription ID, resource group, and workspace name as specified in the beginning of the cell." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Workspace name: chhamlws\n", - "Azure region: westcentralus\n", - "Resource group: chhamlwsrg\n" - ] - } - ], - "source": [ - "# Please specify the AzureML workspace attributes below if you want to create a new one.\n", - "subscription_id = \"\"\n", - "resource_group = \"\"\n", - "workspace_name = \"\"\n", - "workspace_region = \"\"\n", - "\n", - "# Connect to a workspace\n", - "ws = get_or_create_workspace(\n", - " config_path=\"./.azureml\",\n", - " subscription_id=subscription_id,\n", - " resource_group=resource_group,\n", - " workspace_name=workspace_name,\n", - " workspace_region=workspace_region,\n", - ")\n", - "print(\n", - " \"Workspace name: \" + ws.name,\n", - " \"Azure region: \" + ws.location,\n", - " \"Resource group: \" + ws.resource_group,\n", - " sep=\"\\n\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "# Create an experiment\n", - "exp = Experiment(workspace=ws, name=\"tune-lgbm-forecast\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Data Preparation\n", - "\n", - "We need to download the Orange Juice data and split it into training and test sets. By default, the following cell will download and spit the data. If you've already done so, you may skip this part by switching `DOWNLOAD_SPLIT_DATA` to False. \n", - "\n", - "By passing `write_csv=True` to `split_train_test()` below, this function will write the training data and test data to three csv files: `train.csv`, `auxi.csv` and `test.csv`. The first two csv files contain the historical sales up to week 135 as well as auxiliary information such as future price and promotion. Here we assume that future price and promotion information up to a certain number of weeks ahead is predetermined and known. We will use these two files to implement cross-validation and search for the best model with HyperDrive." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Data already exists at the specified location.\n" - ] - } - ], - "source": [ - "if DOWNLOAD_SPLIT_DATA:\n", - " download_ojdata(DATA_DIR)\n", - " split_train_test(\n", - " DATA_DIR,\n", - " n_splits=N_SPLITS,\n", - " horizon=HORIZON,\n", - " gap=GAP,\n", - " first_week=FIRST_WEEK,\n", - " last_week=LAST_WEEK,\n", - " write_csv=True,\n", - " )" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Validate Script Locally\n", - "\n", - "A good practice is to test the model training and validation script on your local machine before you run the hyperparameter tuning job on a remote compute. To run the script locally, we need to correctly specify the path of the Python interpreter that has been installed in `forecasting_env` conda environment. In what follows, the script `train_validate.py` trains a model on the training set with the input arguments as specified in `ScriptRunConfig()` and computes the accuracy of the model on the validation set. Here we evaluate the model accuracy using mean-absolute-percentage-error (MAPE)." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "# Get Python interpreter path\n", - "python_path = subprocess.check_output(\"which python\", shell=True)\n", - "python_path = python_path.decode(\"utf-8\")[:-1]\n", - "\n", - "# Configure local, user managed environment\n", - "run_config_user_managed = RunConfiguration()\n", - "run_config_user_managed.environment.python.user_managed_dependencies = True\n", - "run_config_user_managed.environment.python.interpreter_path = python_path" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "# Directory of the local scripts\n", - "script_folder = \"./aml_scripts\"\n", - "\n", - "# Copy feature engineering utils\n", - "src_dir = os.path.join(git_repo_path(), \"fclib\", \"fclib\", \"feature_engineering\")\n", - "des_dir = os.path.join(script_folder, \"fclib\", \"feature_engineering\")\n", - "shutil.copytree(src_dir, des_dir)\n", - "\n", - "# Training script name and path\n", - "train_script_name = \"train_validate.py\"\n", - "train_script_path = os.path.join(script_folder, train_script_name)\n", - "\n", - "# Specify script run config\n", - "src = ScriptRunConfig(\n", - " source_directory=\"./\",\n", - " script=train_script_path,\n", - " arguments=[\"--data-folder\", DATA_DIR, \"--bagging-fraction\", \"0.8\"],\n", - " run_config=run_config_user_managed,\n", - ")\n", - "run_local = exp.submit(src)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'Running'" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Check job status\n", - "run_local.get_status()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We will wait until the local run finishes. Then, we print out the validation metric. Moreover, you can also use `run_local.get_details()` to get detailed information about this run." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'MAPE': 66.59144474679267}" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Check results\n", - "while run_local.get_status() != \"Completed\":\n", - " {}\n", - "run_local.get_metrics()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Run Script on Remote Compute\n", - "\n", - "After validating model training script locally, we can create a remote compute and further test the script on the remote compute." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Create a CPU cluster as compute target\n", - "\n", - "In the next cell, we create an AmlCompute target with a specific cluster name, VM size, and maximum number of nodes if the cluster does not exist. Otherwise, we will reuse an existing one. For more options of VM sizes, you can check information in this [link](https://docs.microsoft.com/en-us/azure/virtual-machines/sizes-general)." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found compute target: cpu-cluster\n" - ] - } - ], - "source": [ - "# Choose a name for your cluster\n", - "cluster_name = \"cpu-cluster\"\n", - "# VM Size\n", - "vm_size = \"STANDARD_D2_V2\"\n", - "# Maximum number of nodes of the cluster\n", - "max_nodes = 4\n", - "\n", - "# Create a new AmlCompute if it does not exist or reuse an existing one\n", - "compute_target = get_or_create_amlcompute(\n", - " workspace=ws,\n", - " compute_name=cluster_name,\n", - " vm_size=vm_size,\n", - " min_nodes=0,\n", - " max_nodes=max_nodes,\n", - " verbose=True,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Configure Docker environment\n", - "\n", - "The remote compute will need to create a [Docker image](https://docs.docker.com/get-started/) for running the script. The Docker image is an encapsulated environment with necessary dependencies installed. In the following cell, we specify the conda packages and Python version that are needed for running the script." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "env = EnvironmentDefinition()\n", - "env.python.user_managed_dependencies = False\n", - "env.python.conda_dependencies = CondaDependencies.create(\n", - " conda_packages=[\"pandas\", \"numpy\", \"scipy\", \"scikit-learn\", \"lightgbm\", \"joblib\"],\n", - " python_version=\"3.6.2\",\n", - ")\n", - "env.python.conda_dependencies.add_channel(\"conda-forge\")\n", - "env.docker.enabled = True" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Upload data to default datastore\n", - "\n", - "Each workspace comes with a default datastore. In the following, we upload the Orange Juice dataset to the workspace's default datastore, which will later be mounted on the cluster for model training and validation." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Datastore type: AzureBlob\n", - "Account name: chhamlws4931040064\n", - "Container name: azureml-blobstore-f799a640-1ca3-4877-ad24-08eef7bd307e\n" - ] - } - ], - "source": [ - "ds = ws.get_default_datastore()\n", - "print(\n", - " \"Datastore type: \" + ds.datastore_type,\n", - " \"Account name: \" + ds.account_name,\n", - " \"Container name: \" + ds.container_name,\n", - " sep=\"\\n\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "$AZUREML_DATAREFERENCE_dd1f71e8652d4d32b738e2c5aa03afdf" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Remote data path\n", - "path_on_datastore = \"data\"\n", - "ds.upload(\n", - " src_dir=DATA_DIR,\n", - " target_path=path_on_datastore,\n", - " overwrite=True,\n", - " show_progress=False,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "$AZUREML_DATAREFERENCE_5442e91c25f449ff9a9780f48c6d7792\n" - ] - } - ], - "source": [ - "# Get data reference object for the data path\n", - "ds_data = ds.path(path_on_datastore)\n", - "print(ds_data)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Create estimator\n", - "\n", - "Next, we will check if the remote compute target is successfully created by submitting a job to the target. This compute target will be used by HyperDrive for hyperparameter tuning later. Note that you may skip this part and directly go to [Tune Hyperparameters using HyperDrive](#tune-hyperparameters-using-hyperdrive) if you want.\n", - "\n", - "In the following cells, we first create an estimator to specify details of the job. Then we sumbit the job to the remote compute and check the status of the job." - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "script_params = {\"--data-folder\": ds_data.as_mount(), \"--bagging-fraction\": 0.8}\n", - "est = Estimator(\n", - " source_directory=script_folder,\n", - " script_params=script_params,\n", - " compute_target=compute_target,\n", - " use_docker=True,\n", - " entry_script=train_script_name,\n", - " environment_definition=env,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Submit job" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "# Submit job to remote compute\n", - "run_remote = exp.submit(config=est)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Check job status\n", - "\n", - "You can monitor the status of the remote run using the AzureML widgets. After the job is done, the following cell will display a dashboard similar as\n", - "\n", - "" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "e15887d5c8bb484d87506b7a5e3df4b5", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "_UserRunWidget(widget_settings={'childWidgetDisplay': 'popup', 'send_telemetry': True, 'log_level': 'INFO', 's…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/aml.mini.widget.v1": "{\"status\": \"Completed\", \"workbench_run_details_uri\": \"https://ml.azure.com/experiments/tune-lgbm-forecast/runs/tune-lgbm-forecast_1584652203_688e0e17?wsid=/subscriptions/9086b59a-02d7-4687-b3fd-e39fa5e0fd9b/resourcegroups/chhamlwsrg/workspaces/chhamlws\", \"run_id\": \"tune-lgbm-forecast_1584652203_688e0e17\", \"run_properties\": {\"run_id\": \"tune-lgbm-forecast_1584652203_688e0e17\", \"created_utc\": \"2020-03-19T21:10:05.552973Z\", \"properties\": {\"_azureml.ComputeTargetType\": \"amlcompute\", \"ContentSnapshotId\": \"c3e4a829-831a-45f4-973d-e5093d4639c7\", \"azureml.git.repository_uri\": \"git@github.com:microsoft/forecasting.git\", \"mlflow.source.git.repoURL\": \"git@github.com:microsoft/forecasting.git\", \"azureml.git.branch\": \"chenhui/hyperdrive_example_update\", \"mlflow.source.git.branch\": \"chenhui/hyperdrive_example_update\", \"azureml.git.commit\": \"a36fa88dade03b7811ab97e2f8d5120c643a2073\", \"mlflow.source.git.commit\": \"a36fa88dade03b7811ab97e2f8d5120c643a2073\", \"azureml.git.dirty\": \"True\", \"AzureML.DerivedImageName\": \"azureml/azureml_7842fd2c5e99a43f1cca1341b66a0ecb\", \"ProcessInfoFile\": \"azureml-logs/process_info.json\", \"ProcessStatusFile\": \"azureml-logs/process_status.json\"}, \"tags\": {\"_aml_system_ComputeTargetStatus\": \"{\\\"AllocationState\\\":\\\"steady\\\",\\\"PreparingNodeCount\\\":0,\\\"RunningNodeCount\\\":0,\\\"CurrentNodeCount\\\":4}\"}, \"script_name\": null, \"arguments\": null, \"end_time_utc\": \"2020-03-19T21:12:14.300468Z\", \"status\": \"Completed\", \"log_files\": {\"azureml-logs/55_azureml-execution-tvmps_64205f30874f615b57c2717938d4039e14020b51244803148cd50a8d15cabd4c_d.txt\": \"https://chhamlws4931040064.blob.core.windows.net/azureml/ExperimentRun/dcid.tune-lgbm-forecast_1584652203_688e0e17/azureml-logs/55_azureml-execution-tvmps_64205f30874f615b57c2717938d4039e14020b51244803148cd50a8d15cabd4c_d.txt?sv=2019-02-02&sr=b&sig=7%2FV%2F7Uf5suKYNSmAT2tod9rrpEQG02YGjGLWufjtR9M%3D&st=2020-03-19T21%3A02%3A23Z&se=2020-03-20T05%3A12%3A23Z&sp=r\", \"azureml-logs/65_job_prep-tvmps_64205f30874f615b57c2717938d4039e14020b51244803148cd50a8d15cabd4c_d.txt\": \"https://chhamlws4931040064.blob.core.windows.net/azureml/ExperimentRun/dcid.tune-lgbm-forecast_1584652203_688e0e17/azureml-logs/65_job_prep-tvmps_64205f30874f615b57c2717938d4039e14020b51244803148cd50a8d15cabd4c_d.txt?sv=2019-02-02&sr=b&sig=hAfvRB9EVc6%2BI%2FI1i91V3gw%2FPTQCjxEWFxztnvVK1Zk%3D&st=2020-03-19T21%3A02%3A23Z&se=2020-03-20T05%3A12%3A23Z&sp=r\", \"azureml-logs/70_driver_log.txt\": \"https://chhamlws4931040064.blob.core.windows.net/azureml/ExperimentRun/dcid.tune-lgbm-forecast_1584652203_688e0e17/azureml-logs/70_driver_log.txt?sv=2019-02-02&sr=b&sig=6gSOaifysap9NdTBY2fBOPPxns8WmwmTkSxK75giv0I%3D&st=2020-03-19T21%3A02%3A23Z&se=2020-03-20T05%3A12%3A23Z&sp=r\", \"azureml-logs/75_job_post-tvmps_64205f30874f615b57c2717938d4039e14020b51244803148cd50a8d15cabd4c_d.txt\": \"https://chhamlws4931040064.blob.core.windows.net/azureml/ExperimentRun/dcid.tune-lgbm-forecast_1584652203_688e0e17/azureml-logs/75_job_post-tvmps_64205f30874f615b57c2717938d4039e14020b51244803148cd50a8d15cabd4c_d.txt?sv=2019-02-02&sr=b&sig=Poow17KG%2BO1JDruHX2wImansTcQPtRtkC0VfCSLn1CI%3D&st=2020-03-19T21%3A02%3A23Z&se=2020-03-20T05%3A12%3A23Z&sp=r\", \"azureml-logs/process_info.json\": \"https://chhamlws4931040064.blob.core.windows.net/azureml/ExperimentRun/dcid.tune-lgbm-forecast_1584652203_688e0e17/azureml-logs/process_info.json?sv=2019-02-02&sr=b&sig=5SjpOATjDqF6j92anqPQEgkTM171%2FBRx8FTehTycICk%3D&st=2020-03-19T21%3A02%3A23Z&se=2020-03-20T05%3A12%3A23Z&sp=r\", \"azureml-logs/process_status.json\": \"https://chhamlws4931040064.blob.core.windows.net/azureml/ExperimentRun/dcid.tune-lgbm-forecast_1584652203_688e0e17/azureml-logs/process_status.json?sv=2019-02-02&sr=b&sig=KDk1VVeNgTRx37%2BBacOb%2FWVecpyoc%2FTJCbVLRmCOWCI%3D&st=2020-03-19T21%3A02%3A23Z&se=2020-03-20T05%3A12%3A23Z&sp=r\", \"logs/azureml/140_azureml.log\": \"https://chhamlws4931040064.blob.core.windows.net/azureml/ExperimentRun/dcid.tune-lgbm-forecast_1584652203_688e0e17/logs/azureml/140_azureml.log?sv=2019-02-02&sr=b&sig=cTNXDVJQhpbd5Y5%2BKlOOlYe7F4QZIHfTtriNZ1RZEbE%3D&st=2020-03-19T21%3A02%3A23Z&se=2020-03-20T05%3A12%3A23Z&sp=r\", \"logs/azureml/job_prep_azureml.log\": \"https://chhamlws4931040064.blob.core.windows.net/azureml/ExperimentRun/dcid.tune-lgbm-forecast_1584652203_688e0e17/logs/azureml/job_prep_azureml.log?sv=2019-02-02&sr=b&sig=TIVY4fks6XsO%2BQ50LbfoXWfWcD4JK0uekoE9%2FvQtA9I%3D&st=2020-03-19T21%3A02%3A23Z&se=2020-03-20T05%3A12%3A23Z&sp=r\", \"logs/azureml/job_release_azureml.log\": \"https://chhamlws4931040064.blob.core.windows.net/azureml/ExperimentRun/dcid.tune-lgbm-forecast_1584652203_688e0e17/logs/azureml/job_release_azureml.log?sv=2019-02-02&sr=b&sig=XvAIWqtaowr3HrYaVdz4HaQ6zARgk8mtZW8VP91Wp%2Fc%3D&st=2020-03-19T21%3A02%3A23Z&se=2020-03-20T05%3A12%3A23Z&sp=r\"}, \"log_groups\": [[\"azureml-logs/process_info.json\", \"azureml-logs/process_status.json\", \"logs/azureml/job_prep_azureml.log\", \"logs/azureml/job_release_azureml.log\"], [\"azureml-logs/55_azureml-execution-tvmps_64205f30874f615b57c2717938d4039e14020b51244803148cd50a8d15cabd4c_d.txt\"], [\"azureml-logs/65_job_prep-tvmps_64205f30874f615b57c2717938d4039e14020b51244803148cd50a8d15cabd4c_d.txt\"], [\"azureml-logs/70_driver_log.txt\"], [\"azureml-logs/75_job_post-tvmps_64205f30874f615b57c2717938d4039e14020b51244803148cd50a8d15cabd4c_d.txt\"], [\"logs/azureml/140_azureml.log\"]], \"run_duration\": \"0:02:08\"}, \"child_runs\": [], \"children_metrics\": {}, \"run_metrics\": [{\"name\": \"MAPE\", \"run_id\": \"tune-lgbm-forecast_1584652203_688e0e17\", \"categories\": [0], \"series\": [{\"data\": [66.59144474679267]}]}], \"run_logs\": \"2020-03-19 21:11:02,046|azureml|DEBUG|Inputs:: kwargs: {'OutputCollection': True, 'snapshotProject': True, 'only_in_process_features': True, 'skip_track_logs_dir': True}, track_folders: None, deny_list: None, directories_to_watch: []\\n2020-03-19 21:11:02,046|azureml.history._tracking.PythonWorkingDirectory|DEBUG|Execution target type: batchai\\n2020-03-19 21:11:02,053|azureml.history._tracking.PythonWorkingDirectory|DEBUG|Failed to import pyspark with error: No module named 'pyspark'\\n2020-03-19 21:11:02,054|azureml.history._tracking.PythonWorkingDirectory.workingdir|DEBUG|Pinning working directory for filesystems: ['pyfs']\\n2020-03-19 21:11:02,256|azureml._base_sdk_common.user_agent|DEBUG|Fetching client info from /root/.azureml/clientinfo.json\\n2020-03-19 21:11:02,257|azureml._base_sdk_common.user_agent|DEBUG|Error loading client info: [Errno 2] No such file or directory: '/root/.azureml/clientinfo.json'\\n2020-03-19 21:11:02,559|azureml.core.run|DEBUG|Adding new factory for run source azureml.scriptrun\\n2020-03-19 21:11:02,560|azureml.core.authentication.TokenRefresherDaemon|DEBUG|Starting daemon and triggering first instance\\n2020-03-19 21:11:02,567|msrest.universal_http.requests|DEBUG|Configuring retry: max_retries=3, backoff_factor=0.8, max_backoff=90\\n2020-03-19 21:11:02,567|azureml._restclient.clientbase|INFO|Created a worker pool for first use\\n2020-03-19 21:11:02,567|azureml.core.authentication|DEBUG|Time to expire 1814342.432059 seconds\\n2020-03-19 21:11:02,568|azureml._base_sdk_common.service_discovery|DEBUG|Found history service url in environment variable AZUREML_SERVICE_ENDPOINT, history service url: https://westcentralus.experiments.azureml.net.\\n2020-03-19 21:11:02,568|azureml._base_sdk_common.service_discovery|DEBUG|Found history service url in environment variable AZUREML_SERVICE_ENDPOINT, history service url: https://westcentralus.experiments.azureml.net.\\n2020-03-19 21:11:02,568|azureml._base_sdk_common.service_discovery|DEBUG|Found history service url in environment variable AZUREML_SERVICE_ENDPOINT, history service url: https://westcentralus.experiments.azureml.net.\\n2020-03-19 21:11:02,568|azureml._base_sdk_common.service_discovery|DEBUG|Found history service url in environment variable AZUREML_SERVICE_ENDPOINT, history service url: https://westcentralus.experiments.azureml.net.\\n2020-03-19 21:11:02,569|azureml._base_sdk_common.service_discovery|DEBUG|Found history service url in environment variable AZUREML_SERVICE_ENDPOINT, history service url: https://westcentralus.experiments.azureml.net.\\n2020-03-19 21:11:02,600|azureml._base_sdk_common.service_discovery|DEBUG|Found history service url in environment variable AZUREML_SERVICE_ENDPOINT, history service url: https://westcentralus.experiments.azureml.net.\\n2020-03-19 21:11:02,600|azureml._base_sdk_common.service_discovery|DEBUG|Found history service url in environment variable AZUREML_SERVICE_ENDPOINT, history service url: https://westcentralus.experiments.azureml.net.\\n2020-03-19 21:11:02,601|azureml._base_sdk_common.service_discovery|DEBUG|Found history service url in environment variable AZUREML_SERVICE_ENDPOINT, history service url: https://westcentralus.experiments.azureml.net.\\n2020-03-19 21:11:02,605|msrest.universal_http.requests|DEBUG|Configuring retry: max_retries=3, backoff_factor=0.8, max_backoff=90\\n2020-03-19 21:11:02,613|msrest.universal_http.requests|DEBUG|Configuring retry: max_retries=3, backoff_factor=0.8, max_backoff=90\\n2020-03-19 21:11:02,618|msrest.universal_http.requests|DEBUG|Configuring retry: max_retries=3, backoff_factor=0.8, max_backoff=90\\n2020-03-19 21:11:02,623|msrest.universal_http.requests|DEBUG|Configuring retry: max_retries=3, backoff_factor=0.8, max_backoff=90\\n2020-03-19 21:11:02,628|msrest.universal_http.requests|DEBUG|Configuring retry: max_retries=3, backoff_factor=0.8, max_backoff=90\\n2020-03-19 21:11:02,628|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.RunClient.get-async:False|DEBUG|[START]\\n2020-03-19 21:11:02,628|msrest.service_client|DEBUG|Accept header absent and forced to application/json\\n2020-03-19 21:11:02,628|msrest.http_logger|DEBUG|Request URL: 'https://westcentralus.experiments.azureml.net/history/v1.0/subscriptions/9086b59a-02d7-4687-b3fd-e39fa5e0fd9b/resourceGroups/chhamlwsrg/providers/Microsoft.MachineLearningServices/workspaces/chhamlws/experiments/tune-lgbm-forecast/runs/tune-lgbm-forecast_1584652203_688e0e17'\\n2020-03-19 21:11:02,629|msrest.http_logger|DEBUG|Request method: 'GET'\\n2020-03-19 21:11:02,629|msrest.http_logger|DEBUG|Request headers:\\n2020-03-19 21:11:02,629|msrest.http_logger|DEBUG| 'Accept': 'application/json'\\n2020-03-19 21:11:02,629|msrest.http_logger|DEBUG| 'Content-Type': 'application/json; charset=utf-8'\\n2020-03-19 21:11:02,629|msrest.http_logger|DEBUG| 'x-ms-client-request-id': '3c9a8626-ae8e-4e19-a9ed-2bb98b7ad7f6'\\n2020-03-19 21:11:02,629|msrest.http_logger|DEBUG| 'request-id': '3c9a8626-ae8e-4e19-a9ed-2bb98b7ad7f6'\\n2020-03-19 21:11:02,629|msrest.http_logger|DEBUG| 'User-Agent': 'python/3.6.2 (Linux-4.15.0-1067-azure-x86_64-with-debian-stretch-sid) msrest/0.6.11 azureml._restclient/core.1.0.85'\\n2020-03-19 21:11:02,629|msrest.http_logger|DEBUG|Request body:\\n2020-03-19 21:11:02,629|msrest.http_logger|DEBUG|None\\n2020-03-19 21:11:02,629|msrest.universal_http|DEBUG|Configuring redirects: allow=True, max=30\\n2020-03-19 21:11:02,629|msrest.universal_http|DEBUG|Configuring request: timeout=100, verify=True, cert=None\\n2020-03-19 21:11:02,629|msrest.universal_http|DEBUG|Configuring proxies: ''\\n2020-03-19 21:11:02,629|msrest.universal_http|DEBUG|Evaluate proxies against ENV settings: True\\n2020-03-19 21:11:02,683|msrest.http_logger|DEBUG|Response status: 200\\n2020-03-19 21:11:02,683|msrest.http_logger|DEBUG|Response headers:\\n2020-03-19 21:11:02,684|msrest.http_logger|DEBUG| 'Date': 'Thu, 19 Mar 2020 21:11:02 GMT'\\n2020-03-19 21:11:02,684|msrest.http_logger|DEBUG| 'Content-Type': 'application/json; charset=utf-8'\\n2020-03-19 21:11:02,684|msrest.http_logger|DEBUG| 'Transfer-Encoding': 'chunked'\\n2020-03-19 21:11:02,684|msrest.http_logger|DEBUG| 'Connection': 'keep-alive'\\n2020-03-19 21:11:02,684|msrest.http_logger|DEBUG| 'Vary': 'Accept-Encoding'\\n2020-03-19 21:11:02,684|msrest.http_logger|DEBUG| 'Request-Context': 'appId=cid-v1:2d2e8e63-272e-4b3c-8598-4ee570a0e70d'\\n2020-03-19 21:11:02,684|msrest.http_logger|DEBUG| 'x-ms-client-request-id': '3c9a8626-ae8e-4e19-a9ed-2bb98b7ad7f6'\\n2020-03-19 21:11:02,684|msrest.http_logger|DEBUG| 'x-ms-client-session-id': ''\\n2020-03-19 21:11:02,685|msrest.http_logger|DEBUG| 'Strict-Transport-Security': 'max-age=15724800; includeSubDomains; preload'\\n2020-03-19 21:11:02,685|msrest.http_logger|DEBUG| 'x-request-time': '0.034'\\n2020-03-19 21:11:02,685|msrest.http_logger|DEBUG| 'X-Content-Type-Options': 'nosniff'\\n2020-03-19 21:11:02,685|msrest.http_logger|DEBUG| 'Content-Encoding': 'gzip'\\n2020-03-19 21:11:02,685|msrest.http_logger|DEBUG|Response content:\\n2020-03-19 21:11:02,685|msrest.http_logger|DEBUG|{\\n \\\"runNumber\\\": 123,\\n \\\"rootRunId\\\": \\\"tune-lgbm-forecast_1584652203_688e0e17\\\",\\n \\\"experimentId\\\": \\\"3199ea18-1505-42f9-9092-777f27df73e5\\\",\\n \\\"createdUtc\\\": \\\"2020-03-19T21:10:05.5529735+00:00\\\",\\n \\\"createdBy\\\": {\\n \\\"userObjectId\\\": \\\"8157bc92-2d12-4bc9-9270-bab51c673493\\\",\\n \\\"userPuId\\\": \\\"10033FFF97A21586\\\",\\n \\\"userIdp\\\": null,\\n \\\"userAltSecId\\\": null,\\n \\\"userIss\\\": \\\"https://sts.windows.net/72f988bf-86f1-41af-91ab-2d7cd011db47/\\\",\\n \\\"userTenantId\\\": \\\"72f988bf-86f1-41af-91ab-2d7cd011db47\\\",\\n \\\"userName\\\": \\\"Chenhui Hu\\\"\\n },\\n \\\"userId\\\": \\\"8157bc92-2d12-4bc9-9270-bab51c673493\\\",\\n \\\"token\\\": null,\\n \\\"tokenExpiryTimeUtc\\\": null,\\n \\\"error\\\": null,\\n \\\"warnings\\\": null,\\n \\\"revision\\\": 9,\\n \\\"runUuid\\\": \\\"88b2e204-56cc-477e-bff4-cbb0c94c18d6\\\",\\n \\\"parentRunUuid\\\": null,\\n \\\"rootRunUuid\\\": \\\"88b2e204-56cc-477e-bff4-cbb0c94c18d6\\\",\\n \\\"runId\\\": \\\"tune-lgbm-forecast_1584652203_688e0e17\\\",\\n \\\"parentRunId\\\": null,\\n \\\"status\\\": \\\"Running\\\",\\n \\\"startTimeUtc\\\": \\\"2020-03-19T21:10:42.8166176+00:00\\\",\\n \\\"endTimeUtc\\\": null,\\n \\\"heartbeatEnabled\\\": false,\\n \\\"options\\\": {\\n \\\"generateDataContainerIdIfNotSpecified\\\": true\\n },\\n \\\"name\\\": null,\\n \\\"dataContainerId\\\": \\\"dcid.tune-lgbm-forecast_1584652203_688e0e17\\\",\\n \\\"description\\\": null,\\n \\\"hidden\\\": false,\\n \\\"runType\\\": \\\"azureml.scriptrun\\\",\\n \\\"properties\\\": {\\n \\\"_azureml.ComputeTargetType\\\": \\\"amlcompute\\\",\\n \\\"ContentSnapshotId\\\": \\\"c3e4a829-831a-45f4-973d-e5093d4639c7\\\",\\n \\\"azureml.git.repository_uri\\\": \\\"git@github.com:microsoft/forecasting.git\\\",\\n \\\"mlflow.source.git.repoURL\\\": \\\"git@github.com:microsoft/forecasting.git\\\",\\n \\\"azureml.git.branch\\\": \\\"chenhui/hyperdrive_example_update\\\",\\n \\\"mlflow.source.git.branch\\\": \\\"chenhui/hyperdrive_example_update\\\",\\n \\\"azureml.git.commit\\\": \\\"a36fa88dade03b7811ab97e2f8d5120c643a2073\\\",\\n \\\"mlflow.source.git.commit\\\": \\\"a36fa88dade03b7811ab97e2f8d5120c643a2073\\\",\\n \\\"azureml.git.dirty\\\": \\\"True\\\",\\n \\\"AzureML.DerivedImageName\\\": \\\"azureml/azureml_7842fd2c5e99a43f1cca1341b66a0ecb\\\",\\n \\\"ProcessInfoFile\\\": \\\"azureml-logs/process_info.json\\\",\\n \\\"ProcessStatusFile\\\": \\\"azureml-logs/process_status.json\\\"\\n },\\n \\\"scriptName\\\": \\\"train_validate.py\\\",\\n \\\"target\\\": \\\"cpu-cluster\\\",\\n \\\"uniqueChildRunComputeTargets\\\": [],\\n \\\"tags\\\": {\\n \\\"_aml_system_ComputeTargetStatus\\\": \\\"{\\\\\\\"AllocationState\\\\\\\":\\\\\\\"steady\\\\\\\",\\\\\\\"PreparingNodeCount\\\\\\\":0,\\\\\\\"RunningNodeCount\\\\\\\":0,\\\\\\\"CurrentNodeCount\\\\\\\":4}\\\"\\n },\\n \\\"inputDatasets\\\": [],\\n \\\"runDefinition\\\": null,\\n \\\"createdFrom\\\": null,\\n \\\"cancelUri\\\": \\\"https://westcentralus.experiments.azureml.net/execution/v1.0/subscriptions/9086b59a-02d7-4687-b3fd-e39fa5e0fd9b/resourceGroups/chhamlwsrg/providers/Microsoft.MachineLearningServices/workspaces/chhamlws/experiments/tune-lgbm-forecast/runId/tune-lgbm-forecast_1584652203_688e0e17/cancel\\\",\\n \\\"completeUri\\\": null,\\n \\\"diagnosticsUri\\\": \\\"https://westcentralus.experiments.azureml.net/execution/v1.0/subscriptions/9086b59a-02d7-4687-b3fd-e39fa5e0fd9b/resourceGroups/chhamlwsrg/providers/Microsoft.MachineLearningServices/workspaces/chhamlws/experiments/tune-lgbm-forecast/runId/tune-lgbm-forecast_1584652203_688e0e17/diagnostics\\\",\\n \\\"computeRequest\\\": {\\n \\\"nodeCount\\\": 1\\n },\\n \\\"retainForLifetimeOfWorkspace\\\": false,\\n \\\"queueingInfo\\\": null\\n}\\n2020-03-19 21:11:02,691|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.RunClient.get-async:False|DEBUG|[STOP]\\n2020-03-19 21:11:02,691|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17|DEBUG|Constructing run from dto. type: azureml.scriptrun, source: None, props: {'_azureml.ComputeTargetType': 'amlcompute', 'ContentSnapshotId': 'c3e4a829-831a-45f4-973d-e5093d4639c7', 'azureml.git.repository_uri': 'git@github.com:microsoft/forecasting.git', 'mlflow.source.git.repoURL': 'git@github.com:microsoft/forecasting.git', 'azureml.git.branch': 'chenhui/hyperdrive_example_update', 'mlflow.source.git.branch': 'chenhui/hyperdrive_example_update', 'azureml.git.commit': 'a36fa88dade03b7811ab97e2f8d5120c643a2073', 'mlflow.source.git.commit': 'a36fa88dade03b7811ab97e2f8d5120c643a2073', 'azureml.git.dirty': 'True', 'AzureML.DerivedImageName': 'azureml/azureml_7842fd2c5e99a43f1cca1341b66a0ecb', 'ProcessInfoFile': 'azureml-logs/process_info.json', 'ProcessStatusFile': 'azureml-logs/process_status.json'}\\n2020-03-19 21:11:02,691|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunContextManager|DEBUG|Valid logs dir, setting up content loader\\n2020-03-19 21:11:02,692|azureml|WARNING|Could not import azureml.mlflow or azureml.contrib.mlflow mlflow APIs will not run against AzureML services. Add azureml-mlflow as a conda dependency for the run if this behavior is desired\\n2020-03-19 21:11:02,692|azureml.WorkerPool|DEBUG|[START]\\n2020-03-19 21:11:02,692|azureml.SendRunKillSignal|DEBUG|[START]\\n2020-03-19 21:11:02,692|azureml.RunStatusContext|DEBUG|[START]\\n2020-03-19 21:11:02,692|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunContextManager.RunStatusContext|DEBUG|[START]\\n2020-03-19 21:11:02,692|azureml.WorkingDirectoryCM|DEBUG|[START]\\n2020-03-19 21:11:02,692|azureml.history._tracking.PythonWorkingDirectory.workingdir|DEBUG|[START]\\n2020-03-19 21:11:02,692|azureml.history._tracking.PythonWorkingDirectory|INFO|Current working dir: /mnt/batch/tasks/shared/LS_root/jobs/chhamlws/2c8fb242ef2843d9bed884402f5e3132/tune-lgbm-forecast_1584652203_688e0e17/mounts/workspaceblobstore/azureml/tune-lgbm-forecast_1584652203_688e0e17\\n2020-03-19 21:11:02,692|azureml.history._tracking.PythonWorkingDirectory.workingdir|DEBUG|Calling pyfs\\n2020-03-19 21:11:02,692|azureml.history._tracking.PythonWorkingDirectory.workingdir|DEBUG|Storing working dir for pyfs as /mnt/batch/tasks/shared/LS_root/jobs/chhamlws/2c8fb242ef2843d9bed884402f5e3132/tune-lgbm-forecast_1584652203_688e0e17/mounts/workspaceblobstore/azureml/tune-lgbm-forecast_1584652203_688e0e17\\n2020-03-19 21:11:03,544|azureml._base_sdk_common.service_discovery|DEBUG|Found history service url in environment variable AZUREML_SERVICE_ENDPOINT, history service url: https://westcentralus.experiments.azureml.net.\\n2020-03-19 21:11:03,544|azureml._base_sdk_common.service_discovery|DEBUG|Found history service url in environment variable AZUREML_SERVICE_ENDPOINT, history service url: https://westcentralus.experiments.azureml.net.\\n2020-03-19 21:11:03,544|azureml._base_sdk_common.service_discovery|DEBUG|Found history service url in environment variable AZUREML_SERVICE_ENDPOINT, history service url: https://westcentralus.experiments.azureml.net.\\n2020-03-19 21:11:03,544|azureml._base_sdk_common.service_discovery|DEBUG|Found history service url in environment variable AZUREML_SERVICE_ENDPOINT, history service url: https://westcentralus.experiments.azureml.net.\\n2020-03-19 21:11:03,544|azureml._base_sdk_common.service_discovery|DEBUG|Found history service url in environment variable AZUREML_SERVICE_ENDPOINT, history service url: https://westcentralus.experiments.azureml.net.\\n2020-03-19 21:11:03,544|azureml._base_sdk_common.service_discovery|DEBUG|Found history service url in environment variable AZUREML_SERVICE_ENDPOINT, history service url: https://westcentralus.experiments.azureml.net.\\n2020-03-19 21:11:03,545|azureml._base_sdk_common.service_discovery|DEBUG|Found history service url in environment variable AZUREML_SERVICE_ENDPOINT, history service url: https://westcentralus.experiments.azureml.net.\\n2020-03-19 21:11:03,550|msrest.universal_http.requests|DEBUG|Configuring retry: max_retries=3, backoff_factor=0.8, max_backoff=90\\n2020-03-19 21:11:03,551|azureml._run_impl.run_history_facade|DEBUG|Created a static thread pool for RunHistoryFacade class\\n2020-03-19 21:11:03,555|msrest.universal_http.requests|DEBUG|Configuring retry: max_retries=3, backoff_factor=0.8, max_backoff=90\\n2020-03-19 21:11:03,560|msrest.universal_http.requests|DEBUG|Configuring retry: max_retries=3, backoff_factor=0.8, max_backoff=90\\n2020-03-19 21:11:03,566|msrest.universal_http.requests|DEBUG|Configuring retry: max_retries=3, backoff_factor=0.8, max_backoff=90\\n2020-03-19 21:11:03,571|msrest.universal_http.requests|DEBUG|Configuring retry: max_retries=3, backoff_factor=0.8, max_backoff=90\\n2020-03-19 21:11:03,572|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.RunClient.get-async:False|DEBUG|[START]\\n2020-03-19 21:11:03,572|msrest.service_client|DEBUG|Accept header absent and forced to application/json\\n2020-03-19 21:11:03,572|msrest.http_logger|DEBUG|Request URL: 'https://westcentralus.experiments.azureml.net/history/v1.0/subscriptions/9086b59a-02d7-4687-b3fd-e39fa5e0fd9b/resourceGroups/chhamlwsrg/providers/Microsoft.MachineLearningServices/workspaces/chhamlws/experiments/tune-lgbm-forecast/runs/tune-lgbm-forecast_1584652203_688e0e17'\\n2020-03-19 21:11:03,572|msrest.http_logger|DEBUG|Request method: 'GET'\\n2020-03-19 21:11:03,572|msrest.http_logger|DEBUG|Request headers:\\n2020-03-19 21:11:03,572|msrest.http_logger|DEBUG| 'Accept': 'application/json'\\n2020-03-19 21:11:03,572|msrest.http_logger|DEBUG| 'Content-Type': 'application/json; charset=utf-8'\\n2020-03-19 21:11:03,572|msrest.http_logger|DEBUG| 'x-ms-client-request-id': 'a7d0b96f-400a-4641-b6ef-91c00e44e5d3'\\n2020-03-19 21:11:03,572|msrest.http_logger|DEBUG| 'request-id': 'a7d0b96f-400a-4641-b6ef-91c00e44e5d3'\\n2020-03-19 21:11:03,572|msrest.http_logger|DEBUG| 'User-Agent': 'python/3.6.2 (Linux-4.15.0-1067-azure-x86_64-with-debian-stretch-sid) msrest/0.6.11 azureml._restclient/core.1.0.85'\\n2020-03-19 21:11:03,573|msrest.http_logger|DEBUG|Request body:\\n2020-03-19 21:11:03,573|msrest.http_logger|DEBUG|None\\n2020-03-19 21:11:03,573|msrest.universal_http|DEBUG|Configuring redirects: allow=True, max=30\\n2020-03-19 21:11:03,573|msrest.universal_http|DEBUG|Configuring request: timeout=100, verify=True, cert=None\\n2020-03-19 21:11:03,573|msrest.universal_http|DEBUG|Configuring proxies: ''\\n2020-03-19 21:11:03,573|msrest.universal_http|DEBUG|Evaluate proxies against ENV settings: True\\n2020-03-19 21:11:03,625|msrest.http_logger|DEBUG|Response status: 200\\n2020-03-19 21:11:03,625|msrest.http_logger|DEBUG|Response headers:\\n2020-03-19 21:11:03,625|msrest.http_logger|DEBUG| 'Date': 'Thu, 19 Mar 2020 21:11:03 GMT'\\n2020-03-19 21:11:03,625|msrest.http_logger|DEBUG| 'Content-Type': 'application/json; charset=utf-8'\\n2020-03-19 21:11:03,626|msrest.http_logger|DEBUG| 'Transfer-Encoding': 'chunked'\\n2020-03-19 21:11:03,626|msrest.http_logger|DEBUG| 'Connection': 'keep-alive'\\n2020-03-19 21:11:03,626|msrest.http_logger|DEBUG| 'Vary': 'Accept-Encoding'\\n2020-03-19 21:11:03,626|msrest.http_logger|DEBUG| 'Request-Context': 'appId=cid-v1:2d2e8e63-272e-4b3c-8598-4ee570a0e70d'\\n2020-03-19 21:11:03,626|msrest.http_logger|DEBUG| 'x-ms-client-request-id': 'a7d0b96f-400a-4641-b6ef-91c00e44e5d3'\\n2020-03-19 21:11:03,626|msrest.http_logger|DEBUG| 'x-ms-client-session-id': ''\\n2020-03-19 21:11:03,626|msrest.http_logger|DEBUG| 'Strict-Transport-Security': 'max-age=15724800; includeSubDomains; preload'\\n2020-03-19 21:11:03,626|msrest.http_logger|DEBUG| 'x-request-time': '0.031'\\n2020-03-19 21:11:03,627|msrest.http_logger|DEBUG| 'X-Content-Type-Options': 'nosniff'\\n2020-03-19 21:11:03,627|msrest.http_logger|DEBUG| 'Content-Encoding': 'gzip'\\n2020-03-19 21:11:03,627|msrest.http_logger|DEBUG|Response content:\\n2020-03-19 21:11:03,627|msrest.http_logger|DEBUG|{\\n \\\"runNumber\\\": 123,\\n \\\"rootRunId\\\": \\\"tune-lgbm-forecast_1584652203_688e0e17\\\",\\n \\\"experimentId\\\": \\\"3199ea18-1505-42f9-9092-777f27df73e5\\\",\\n \\\"createdUtc\\\": \\\"2020-03-19T21:10:05.5529735+00:00\\\",\\n \\\"createdBy\\\": {\\n \\\"userObjectId\\\": \\\"8157bc92-2d12-4bc9-9270-bab51c673493\\\",\\n \\\"userPuId\\\": \\\"10033FFF97A21586\\\",\\n \\\"userIdp\\\": null,\\n \\\"userAltSecId\\\": null,\\n \\\"userIss\\\": \\\"https://sts.windows.net/72f988bf-86f1-41af-91ab-2d7cd011db47/\\\",\\n \\\"userTenantId\\\": \\\"72f988bf-86f1-41af-91ab-2d7cd011db47\\\",\\n \\\"userName\\\": \\\"Chenhui Hu\\\"\\n },\\n \\\"userId\\\": \\\"8157bc92-2d12-4bc9-9270-bab51c673493\\\",\\n \\\"token\\\": null,\\n \\\"tokenExpiryTimeUtc\\\": null,\\n \\\"error\\\": null,\\n \\\"warnings\\\": null,\\n \\\"revision\\\": 9,\\n \\\"runUuid\\\": \\\"88b2e204-56cc-477e-bff4-cbb0c94c18d6\\\",\\n \\\"parentRunUuid\\\": null,\\n \\\"rootRunUuid\\\": \\\"88b2e204-56cc-477e-bff4-cbb0c94c18d6\\\",\\n \\\"runId\\\": \\\"tune-lgbm-forecast_1584652203_688e0e17\\\",\\n \\\"parentRunId\\\": null,\\n \\\"status\\\": \\\"Running\\\",\\n \\\"startTimeUtc\\\": \\\"2020-03-19T21:10:42.8166176+00:00\\\",\\n \\\"endTimeUtc\\\": null,\\n \\\"heartbeatEnabled\\\": false,\\n \\\"options\\\": {\\n \\\"generateDataContainerIdIfNotSpecified\\\": true\\n },\\n \\\"name\\\": null,\\n \\\"dataContainerId\\\": \\\"dcid.tune-lgbm-forecast_1584652203_688e0e17\\\",\\n \\\"description\\\": null,\\n \\\"hidden\\\": false,\\n \\\"runType\\\": \\\"azureml.scriptrun\\\",\\n \\\"properties\\\": {\\n \\\"_azureml.ComputeTargetType\\\": \\\"amlcompute\\\",\\n \\\"ContentSnapshotId\\\": \\\"c3e4a829-831a-45f4-973d-e5093d4639c7\\\",\\n \\\"azureml.git.repository_uri\\\": \\\"git@github.com:microsoft/forecasting.git\\\",\\n \\\"mlflow.source.git.repoURL\\\": \\\"git@github.com:microsoft/forecasting.git\\\",\\n \\\"azureml.git.branch\\\": \\\"chenhui/hyperdrive_example_update\\\",\\n \\\"mlflow.source.git.branch\\\": \\\"chenhui/hyperdrive_example_update\\\",\\n \\\"azureml.git.commit\\\": \\\"a36fa88dade03b7811ab97e2f8d5120c643a2073\\\",\\n \\\"mlflow.source.git.commit\\\": \\\"a36fa88dade03b7811ab97e2f8d5120c643a2073\\\",\\n \\\"azureml.git.dirty\\\": \\\"True\\\",\\n \\\"AzureML.DerivedImageName\\\": \\\"azureml/azureml_7842fd2c5e99a43f1cca1341b66a0ecb\\\",\\n \\\"ProcessInfoFile\\\": \\\"azureml-logs/process_info.json\\\",\\n \\\"ProcessStatusFile\\\": \\\"azureml-logs/process_status.json\\\"\\n },\\n \\\"scriptName\\\": \\\"train_validate.py\\\",\\n \\\"target\\\": \\\"cpu-cluster\\\",\\n \\\"uniqueChildRunComputeTargets\\\": [],\\n \\\"tags\\\": {\\n \\\"_aml_system_ComputeTargetStatus\\\": \\\"{\\\\\\\"AllocationState\\\\\\\":\\\\\\\"steady\\\\\\\",\\\\\\\"PreparingNodeCount\\\\\\\":0,\\\\\\\"RunningNodeCount\\\\\\\":0,\\\\\\\"CurrentNodeCount\\\\\\\":4}\\\"\\n },\\n \\\"inputDatasets\\\": [],\\n \\\"runDefinition\\\": null,\\n \\\"createdFrom\\\": null,\\n \\\"cancelUri\\\": \\\"https://westcentralus.experiments.azureml.net/execution/v1.0/subscriptions/9086b59a-02d7-4687-b3fd-e39fa5e0fd9b/resourceGroups/chhamlwsrg/providers/Microsoft.MachineLearningServices/workspaces/chhamlws/experiments/tune-lgbm-forecast/runId/tune-lgbm-forecast_1584652203_688e0e17/cancel\\\",\\n \\\"completeUri\\\": null,\\n \\\"diagnosticsUri\\\": \\\"https://westcentralus.experiments.azureml.net/execution/v1.0/subscriptions/9086b59a-02d7-4687-b3fd-e39fa5e0fd9b/resourceGroups/chhamlwsrg/providers/Microsoft.MachineLearningServices/workspaces/chhamlws/experiments/tune-lgbm-forecast/runId/tune-lgbm-forecast_1584652203_688e0e17/diagnostics\\\",\\n \\\"computeRequest\\\": {\\n \\\"nodeCount\\\": 1\\n },\\n \\\"retainForLifetimeOfWorkspace\\\": false,\\n \\\"queueingInfo\\\": null\\n}\\n2020-03-19 21:11:03,630|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.RunClient.get-async:False|DEBUG|[STOP]\\n2020-03-19 21:11:03,631|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17|DEBUG|Constructing run from dto. type: azureml.scriptrun, source: None, props: {'_azureml.ComputeTargetType': 'amlcompute', 'ContentSnapshotId': 'c3e4a829-831a-45f4-973d-e5093d4639c7', 'azureml.git.repository_uri': 'git@github.com:microsoft/forecasting.git', 'mlflow.source.git.repoURL': 'git@github.com:microsoft/forecasting.git', 'azureml.git.branch': 'chenhui/hyperdrive_example_update', 'mlflow.source.git.branch': 'chenhui/hyperdrive_example_update', 'azureml.git.commit': 'a36fa88dade03b7811ab97e2f8d5120c643a2073', 'mlflow.source.git.commit': 'a36fa88dade03b7811ab97e2f8d5120c643a2073', 'azureml.git.dirty': 'True', 'AzureML.DerivedImageName': 'azureml/azureml_7842fd2c5e99a43f1cca1341b66a0ecb', 'ProcessInfoFile': 'azureml-logs/process_info.json', 'ProcessStatusFile': 'azureml-logs/process_status.json'}\\n2020-03-19 21:11:03,631|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunContextManager|DEBUG|Valid logs dir, setting up content loader\\n2020-03-19 21:11:32,573|azureml.core.authentication|DEBUG|Time to expire 1814312.426703 seconds\\n2020-03-19 21:11:49,120|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient|DEBUG|Overrides: Max batch size: 50, batch cushion: 5, Interval: 1.\\n2020-03-19 21:11:49,121|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.PostMetricsBatch.PostMetricsBatchDaemon|DEBUG|Starting daemon and triggering first instance\\n2020-03-19 21:11:49,133|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient|DEBUG|Used for use_batch=True.\\n2020-03-19 21:11:49,751|azureml.history._tracking.PythonWorkingDirectory.workingdir|DEBUG|Calling pyfs\\n2020-03-19 21:11:49,752|azureml.history._tracking.PythonWorkingDirectory|INFO|Current working dir: /mnt/batch/tasks/shared/LS_root/jobs/chhamlws/2c8fb242ef2843d9bed884402f5e3132/tune-lgbm-forecast_1584652203_688e0e17/mounts/workspaceblobstore/azureml/tune-lgbm-forecast_1584652203_688e0e17\\n2020-03-19 21:11:49,752|azureml.history._tracking.PythonWorkingDirectory.workingdir|DEBUG|Reverting working dir from /mnt/batch/tasks/shared/LS_root/jobs/chhamlws/2c8fb242ef2843d9bed884402f5e3132/tune-lgbm-forecast_1584652203_688e0e17/mounts/workspaceblobstore/azureml/tune-lgbm-forecast_1584652203_688e0e17 to /mnt/batch/tasks/shared/LS_root/jobs/chhamlws/2c8fb242ef2843d9bed884402f5e3132/tune-lgbm-forecast_1584652203_688e0e17/mounts/workspaceblobstore/azureml/tune-lgbm-forecast_1584652203_688e0e17\\n2020-03-19 21:11:49,752|azureml.history._tracking.PythonWorkingDirectory|INFO|Working dir is already updated /mnt/batch/tasks/shared/LS_root/jobs/chhamlws/2c8fb242ef2843d9bed884402f5e3132/tune-lgbm-forecast_1584652203_688e0e17/mounts/workspaceblobstore/azureml/tune-lgbm-forecast_1584652203_688e0e17\\n2020-03-19 21:11:49,752|azureml.history._tracking.PythonWorkingDirectory.workingdir|DEBUG|[STOP]\\n2020-03-19 21:11:49,752|azureml.WorkingDirectoryCM|DEBUG|[STOP]\\n2020-03-19 21:11:49,752|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17|INFO|complete is not setting status for submitted runs.\\n2020-03-19 21:11:49,752|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.FlushingMetricsClient|DEBUG|[START]\\n2020-03-19 21:11:49,752|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient|DEBUG|Overrides: Max batch size: 50, batch cushion: 5, Interval: 1.\\n2020-03-19 21:11:49,752|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.PostMetricsBatch.PostMetricsBatchDaemon|DEBUG|Starting daemon and triggering first instance\\n2020-03-19 21:11:49,752|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient|DEBUG|Used for use_batch=True.\\n2020-03-19 21:11:49,753|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.PostMetricsBatch.WaitFlushSource:MetricsClient|DEBUG|[START]\\n2020-03-19 21:11:49,753|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.PostMetricsBatch.WaitFlushSource:MetricsClient|DEBUG|flush timeout 300 is different from task queue timeout 120, using flush timeout\\n2020-03-19 21:11:49,753|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.PostMetricsBatch.WaitFlushSource:MetricsClient|DEBUG|Waiting 300 seconds on tasks: [].\\n2020-03-19 21:11:49,756|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.PostMetricsBatch|DEBUG|\\n2020-03-19 21:11:49,756|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.PostMetricsBatch.WaitFlushSource:MetricsClient|DEBUG|[STOP]\\n2020-03-19 21:11:49,756|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.FlushingMetricsClient|DEBUG|[STOP]\\n2020-03-19 21:11:49,756|azureml.RunStatusContext|DEBUG|[STOP]\\n2020-03-19 21:11:49,756|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.FlushingMetricsClient|DEBUG|[START]\\n2020-03-19 21:11:49,756|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.PostMetricsBatch.WaitFlushSource:MetricsClient|DEBUG|[START]\\n2020-03-19 21:11:49,756|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.PostMetricsBatch.WaitFlushSource:MetricsClient|DEBUG|flush timeout 300.0 is different from task queue timeout 120, using flush timeout\\n2020-03-19 21:11:49,757|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.PostMetricsBatch.WaitFlushSource:MetricsClient|DEBUG|Waiting 300.0 seconds on tasks: [].\\n2020-03-19 21:11:49,757|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.PostMetricsBatch|DEBUG|\\n2020-03-19 21:11:49,757|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.PostMetricsBatch.WaitFlushSource:MetricsClient|DEBUG|[STOP]\\n2020-03-19 21:11:49,757|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.FlushingMetricsClient|DEBUG|[STOP]\\n2020-03-19 21:11:49,757|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.FlushingMetricsClient|DEBUG|[START]\\n2020-03-19 21:11:49,757|azureml.BatchTaskQueueAdd_1_Batches|DEBUG|[Start]\\n2020-03-19 21:11:49,757|azureml.BatchTaskQueueAdd_1_Batches.WorkerPool|DEBUG|submitting future: _handle_batch\\n2020-03-19 21:11:49,757|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.PostMetricsBatch|DEBUG|Batch size 1.\\n2020-03-19 21:11:49,758|azureml.BatchTaskQueueAdd_1_Batches.0__handle_batch|DEBUG|Using basic handler - no exception handling\\n2020-03-19 21:11:49,758|azureml._restclient.clientbase.WorkerPool|DEBUG|submitting future: _log_batch\\n2020-03-19 21:11:49,758|azureml.BatchTaskQueueAdd_1_Batches|DEBUG|Adding task 0__handle_batch to queue of approximate size: 0\\n2020-03-19 21:11:49,758|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.post_batch-async:False|DEBUG|[START]\\n2020-03-19 21:11:49,758|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.PostMetricsBatch.0__log_batch|DEBUG|Using basic handler - no exception handling\\n2020-03-19 21:11:49,758|azureml.BatchTaskQueueAdd_1_Batches|DEBUG|[Stop] - waiting default timeout\\n2020-03-19 21:11:49,759|msrest.service_client|DEBUG|Accept header absent and forced to application/json\\n2020-03-19 21:11:49,759|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.PostMetricsBatch|DEBUG|Adding task 0__log_batch to queue of approximate size: 0\\n2020-03-19 21:11:49,760|azureml.BatchTaskQueueAdd_1_Batches.WaitFlushSource:BatchTaskQueueAdd_1_Batches|DEBUG|[START]\\n2020-03-19 21:11:49,760|msrest.universal_http.requests|DEBUG|Configuring retry: max_retries=3, backoff_factor=0.8, max_backoff=90\\n2020-03-19 21:11:49,760|azureml.BatchTaskQueueAdd_1_Batches.WaitFlushSource:BatchTaskQueueAdd_1_Batches|DEBUG|Overriding default flush timeout from None to 120\\n2020-03-19 21:11:49,760|msrest.http_logger|DEBUG|Request URL: 'https://westcentralus.experiments.azureml.net/history/v1.0/subscriptions/9086b59a-02d7-4687-b3fd-e39fa5e0fd9b/resourceGroups/chhamlwsrg/providers/Microsoft.MachineLearningServices/workspaces/chhamlws/experiments/tune-lgbm-forecast/runs/tune-lgbm-forecast_1584652203_688e0e17/batch/metrics'\\n2020-03-19 21:11:49,761|azureml.BatchTaskQueueAdd_1_Batches.WaitFlushSource:BatchTaskQueueAdd_1_Batches|DEBUG|Waiting 120 seconds on tasks: [AsyncTask(0__handle_batch)].\\n2020-03-19 21:11:49,761|msrest.http_logger|DEBUG|Request method: 'POST'\\n2020-03-19 21:11:49,761|azureml.BatchTaskQueueAdd_1_Batches.0__handle_batch.WaitingTask|DEBUG|[START]\\n2020-03-19 21:11:49,761|msrest.http_logger|DEBUG|Request headers:\\n2020-03-19 21:11:49,761|azureml.BatchTaskQueueAdd_1_Batches.0__handle_batch.WaitingTask|DEBUG|Awaiter is BatchTaskQueueAdd_1_Batches\\n2020-03-19 21:11:49,761|msrest.http_logger|DEBUG| 'Accept': 'application/json'\\n2020-03-19 21:11:49,761|azureml.BatchTaskQueueAdd_1_Batches.0__handle_batch.WaitingTask|DEBUG|[STOP]\\n2020-03-19 21:11:49,761|msrest.http_logger|DEBUG| 'Content-Type': 'application/json-patch+json; charset=utf-8'\\n2020-03-19 21:11:49,762|azureml.BatchTaskQueueAdd_1_Batches|DEBUG|\\n2020-03-19 21:11:49,762|msrest.http_logger|DEBUG| 'x-ms-client-request-id': '2b7451d4-0c71-4763-8101-a0e3fa53f3ac'\\n2020-03-19 21:11:49,762|azureml.BatchTaskQueueAdd_1_Batches.WaitFlushSource:BatchTaskQueueAdd_1_Batches|DEBUG|[STOP]\\n2020-03-19 21:11:49,762|msrest.http_logger|DEBUG| 'request-id': '2b7451d4-0c71-4763-8101-a0e3fa53f3ac'\\n2020-03-19 21:11:49,762|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.PostMetricsBatch.WaitFlushSource:MetricsClient|DEBUG|[START]\\n2020-03-19 21:11:49,762|msrest.http_logger|DEBUG| 'Content-Length': '341'\\n2020-03-19 21:11:49,762|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.PostMetricsBatch.WaitFlushSource:MetricsClient|DEBUG|flush timeout 300.0 is different from task queue timeout 120, using flush timeout\\n2020-03-19 21:11:49,762|msrest.http_logger|DEBUG| 'User-Agent': 'python/3.6.2 (Linux-4.15.0-1067-azure-x86_64-with-debian-stretch-sid) msrest/0.6.11 azureml._restclient/core.1.0.85 sdk_run'\\n2020-03-19 21:11:49,763|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.PostMetricsBatch.WaitFlushSource:MetricsClient|DEBUG|Waiting 300.0 seconds on tasks: [AsyncTask(0__log_batch)].\\n2020-03-19 21:11:49,763|msrest.http_logger|DEBUG|Request body:\\n2020-03-19 21:11:49,763|msrest.http_logger|DEBUG|{\\\"values\\\": [{\\\"metricId\\\": \\\"408d82dc-c710-4472-8aa0-38b854a71a92\\\", \\\"metricType\\\": \\\"azureml.v1.scalar\\\", \\\"createdUtc\\\": \\\"2020-03-19T21:11:49.120039Z\\\", \\\"name\\\": \\\"MAPE\\\", \\\"description\\\": \\\"\\\", \\\"numCells\\\": 1, \\\"cells\\\": [{\\\"MAPE\\\": 66.59144474679267}], \\\"schema\\\": {\\\"numProperties\\\": 1, \\\"properties\\\": [{\\\"propertyId\\\": \\\"MAPE\\\", \\\"name\\\": \\\"MAPE\\\", \\\"type\\\": \\\"float\\\"}]}}]}\\n2020-03-19 21:11:49,763|msrest.universal_http|DEBUG|Configuring redirects: allow=True, max=30\\n2020-03-19 21:11:49,763|msrest.universal_http|DEBUG|Configuring request: timeout=100, verify=True, cert=None\\n2020-03-19 21:11:49,763|msrest.universal_http|DEBUG|Configuring proxies: ''\\n2020-03-19 21:11:49,763|msrest.universal_http|DEBUG|Evaluate proxies against ENV settings: True\\n2020-03-19 21:11:50,292|msrest.http_logger|DEBUG|Response status: 200\\n2020-03-19 21:11:50,293|msrest.http_logger|DEBUG|Response headers:\\n2020-03-19 21:11:50,293|msrest.http_logger|DEBUG| 'Date': 'Thu, 19 Mar 2020 21:11:50 GMT'\\n2020-03-19 21:11:50,293|msrest.http_logger|DEBUG| 'Content-Length': '0'\\n2020-03-19 21:11:50,293|msrest.http_logger|DEBUG| 'Connection': 'keep-alive'\\n2020-03-19 21:11:50,293|msrest.http_logger|DEBUG| 'Request-Context': 'appId=cid-v1:2d2e8e63-272e-4b3c-8598-4ee570a0e70d'\\n2020-03-19 21:11:50,293|msrest.http_logger|DEBUG| 'x-ms-client-request-id': '2b7451d4-0c71-4763-8101-a0e3fa53f3ac'\\n2020-03-19 21:11:50,293|msrest.http_logger|DEBUG| 'x-ms-client-session-id': ''\\n2020-03-19 21:11:50,294|msrest.http_logger|DEBUG| 'Strict-Transport-Security': 'max-age=15724800; includeSubDomains; preload'\\n2020-03-19 21:11:50,294|msrest.http_logger|DEBUG| 'x-request-time': '0.419'\\n2020-03-19 21:11:50,294|msrest.http_logger|DEBUG| 'X-Content-Type-Options': 'nosniff'\\n2020-03-19 21:11:50,294|msrest.http_logger|DEBUG|Response content:\\n2020-03-19 21:11:50,294|msrest.http_logger|DEBUG|\\n2020-03-19 21:11:50,296|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.post_batch-async:False|DEBUG|[STOP]\\n2020-03-19 21:11:50,514|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.PostMetricsBatch.0__log_batch.WaitingTask|DEBUG|[START]\\n2020-03-19 21:11:50,515|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.PostMetricsBatch.0__log_batch.WaitingTask|DEBUG|Awaiter is PostMetricsBatch\\n2020-03-19 21:11:50,515|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.PostMetricsBatch.0__log_batch.WaitingTask|DEBUG|[STOP]\\n2020-03-19 21:11:50,515|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.PostMetricsBatch|DEBUG|Waiting on task: 0__log_batch.\\n1 tasks left. Current duration of flush 0.00025343894958496094 seconds.\\nWaiting on task: 0__log_batch.\\n1 tasks left. Current duration of flush 0.2507054805755615 seconds.\\nWaiting on task: 0__log_batch.\\n1 tasks left. Current duration of flush 0.5011677742004395 seconds.\\n\\n2020-03-19 21:11:50,515|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.PostMetricsBatch.WaitFlushSource:MetricsClient|DEBUG|[STOP]\\n2020-03-19 21:11:50,515|azureml._SubmittedRun#tune-lgbm-forecast_1584652203_688e0e17.RunHistoryFacade.MetricsClient.FlushingMetricsClient|DEBUG|[STOP]\\n2020-03-19 21:11:50,516|azureml.SendRunKillSignal|DEBUG|[STOP]\\n2020-03-19 21:11:50,516|azureml.HistoryTrackingWorkerPool.WorkerPoolShutdown|DEBUG|[START]\\n2020-03-19 21:11:50,516|azureml.HistoryTrackingWorkerPool.WorkerPoolShutdown|DEBUG|[STOP]\\n2020-03-19 21:11:50,516|azureml.WorkerPool|DEBUG|[STOP]\\n\\nRun is completed.\", \"graph\": {}, \"widget_settings\": {\"childWidgetDisplay\": \"popup\", \"send_telemetry\": true, \"log_level\": \"INFO\", \"sdk_version\": \"1.0.85\"}, \"loading\": false}" - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "RunDetails(run_remote).show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can check the validation metric after the job finishes. The validation metric should be the same as the one we obtained when the script was ran locally. For more details of the job, you can execute `run_remote.get_details()`." - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'MAPE': 66.59144474679267}" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Get metric value after the job finishes\n", - "while run_remote.get_status() != \"Completed\":\n", - " {}\n", - "run_remote.get_metrics()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "## Tune Hyperparameters using HyperDrive\n", - "\n", - "Now we are ready to tune the hyperparameters of the LightGBM forecast model by launching multiple runs on the cluster. In the following cell, we define the configurations of a HyperDrive job that does a parallel searching of the hyperparameter space using a Bayesian sampling method. HyperDrive also supports random sampling of the parameter space.\n", - "\n", - "It is recommended that the maximum number of runs should be greater than or equal to 20 times the number of hyperparameters being tuned, for best results with Bayesian sampling. Specifically, it should be no less than 180 in the following case. Nevertheless, we find that even with very small amount of runs Bayesian search can achieve decent performance. Thus, the maximum number of child runs of HyperDrive `max_total_runs` is set as `20` to reduce the running time." - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "For best results with Bayesian Sampling we recommend using a maximum number of runs greater than or equal to 20 times the number of hyperparameters being tuned. Current value for max_total_runs:20. Recommendend value:180.\n" - ] - } - ], - "source": [ - "# Increase this value if you want to achieve better performance\n", - "max_total_runs = 20\n", - "script_params = {\"--data-folder\": ds_data.as_mount()}\n", - "est = Estimator(\n", - " source_directory=script_folder,\n", - " script_params=script_params,\n", - " compute_target=compute_target,\n", - " use_docker=True,\n", - " entry_script=train_script_name,\n", - " environment_definition=env,\n", - ")\n", - "\n", - "# Specify hyperparameter space\n", - "ps = BayesianParameterSampling(\n", - " {\n", - " \"--num-leaves\": quniform(8, 128, 1),\n", - " \"--min-data-in-leaf\": quniform(20, 500, 10),\n", - " \"--learning-rate\": choice(\n", - " 1e-4, 1e-3, 5e-3, 1e-2, 1.5e-2, 2e-2, 3e-2, 5e-2, 1e-1\n", - " ),\n", - " \"--feature-fraction\": uniform(0.2, 1),\n", - " \"--bagging-fraction\": uniform(0.1, 1),\n", - " \"--bagging-freq\": quniform(1, 20, 1),\n", - " \"--max-rounds\": quniform(50, 2000, 10),\n", - " \"--max-lag\": quniform(3, 40, 1),\n", - " \"--window-size\": quniform(3, 40, 1),\n", - " }\n", - ")\n", - "\n", - "# HyperDrive job configuration\n", - "htc = HyperDriveConfig(\n", - " estimator=est,\n", - " hyperparameter_sampling=ps,\n", - " primary_metric_name=\"MAPE\",\n", - " primary_metric_goal=PrimaryMetricGoal.MINIMIZE,\n", - " max_total_runs=max_total_runs,\n", - " max_concurrent_runs=4,\n", - ")\n", - "\n", - "htr = exp.submit(config=htc)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "After the job finishes, you should see outputs from the AzureML widgets similar to the following. Note that you can rerun `RunDetails(htr).show()` after the job finishes to get the updated results on the dashboard in case it is not automatically refreshed.\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "5cc42add84c1482eb12f0659c0f0eb9d", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "_HyperDriveWidget(widget_settings={'childWidgetDisplay': 'popup', 'send_telemetry': True, 'log_level': 'INFO',…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/aml.mini.widget.v1": "{\"status\": \"Running\", \"workbench_run_details_uri\": \"https://ml.azure.com/experiments/tune-lgbm-forecast/runs/HD_2d2796ac-1717-4dce-896d-905dcbce2c1a?wsid=/subscriptions/9086b59a-02d7-4687-b3fd-e39fa5e0fd9b/resourcegroups/chhamlwsrg/workspaces/chhamlws\", \"run_id\": \"HD_2d2796ac-1717-4dce-896d-905dcbce2c1a\", \"run_properties\": {\"run_id\": \"HD_2d2796ac-1717-4dce-896d-905dcbce2c1a\", \"created_utc\": \"2020-03-19T21:12:15.556256Z\", \"properties\": {\"primary_metric_config\": \"{\\\"name\\\": \\\"MAPE\\\", \\\"goal\\\": \\\"minimize\\\"}\", \"resume_from\": \"null\", \"runTemplate\": \"HyperDrive\", \"azureml.runsource\": \"hyperdrive\", \"platform\": \"AML\", \"ContentSnapshotId\": \"c3e4a829-831a-45f4-973d-e5093d4639c7\"}, \"tags\": {\"max_concurrent_jobs\": \"4\", \"max_total_jobs\": \"20\", \"max_duration_minutes\": \"10080\", \"policy_config\": \"{\\\"name\\\": \\\"DEFAULT\\\"}\", \"generator_config\": \"{\\\"name\\\": \\\"BAYESIANOPTIMIZATION\\\", \\\"parameter_space\\\": {\\\"--num-leaves\\\": [\\\"quniform\\\", [8, 128, 1]], \\\"--min-data-in-leaf\\\": [\\\"quniform\\\", [20, 500, 10]], \\\"--learning-rate\\\": [\\\"choice\\\", [[0.0001, 0.001, 0.005, 0.01, 0.015, 0.02, 0.03, 0.05, 0.1]]], \\\"--feature-fraction\\\": [\\\"uniform\\\", [0.2, 1]], \\\"--bagging-fraction\\\": [\\\"uniform\\\", [0.1, 1]], \\\"--bagging-freq\\\": [\\\"quniform\\\", [1, 20, 1]], \\\"--max-rounds\\\": [\\\"quniform\\\", [50, 2000, 10]], \\\"--max-lag\\\": [\\\"quniform\\\", [3, 40, 1]], \\\"--window-size\\\": [\\\"quniform\\\", [3, 40, 1]]}}\", \"primary_metric_config\": \"{\\\"name\\\": \\\"MAPE\\\", \\\"goal\\\": \\\"minimize\\\"}\", \"platform_config\": \"{\\\"ServiceAddress\\\": \\\"https://westcentralus.experiments.azureml.net\\\", \\\"ServiceArmScope\\\": \\\"subscriptions/9086b59a-02d7-4687-b3fd-e39fa5e0fd9b/resourceGroups/chhamlwsrg/providers/Microsoft.MachineLearningServices/workspaces/chhamlws/experiments/tune-lgbm-forecast\\\", \\\"SubscriptionId\\\": \\\"9086b59a-02d7-4687-b3fd-e39fa5e0fd9b\\\", \\\"ResourceGroupName\\\": \\\"chhamlwsrg\\\", \\\"WorkspaceName\\\": \\\"chhamlws\\\", \\\"ExperimentName\\\": \\\"tune-lgbm-forecast\\\", \\\"Definition\\\": {\\\"Overrides\\\": {\\\"script\\\": \\\"train_validate.py\\\", \\\"arguments\\\": [\\\"--data-folder\\\", \\\"$AZUREML_DATAREFERENCE_5442e91c25f449ff9a9780f48c6d7792\\\"], \\\"target\\\": \\\"cpu-cluster\\\", \\\"framework\\\": \\\"Python\\\", \\\"communicator\\\": \\\"None\\\", \\\"maxRunDurationSeconds\\\": null, \\\"nodeCount\\\": 1, \\\"environment\\\": {\\\"name\\\": null, \\\"version\\\": null, \\\"environmentVariables\\\": {\\\"EXAMPLE_ENV_VAR\\\": \\\"EXAMPLE_VALUE\\\"}, \\\"python\\\": {\\\"userManagedDependencies\\\": false, \\\"interpreterPath\\\": \\\"python\\\", \\\"condaDependenciesFile\\\": null, \\\"baseCondaEnvironment\\\": null, \\\"condaDependencies\\\": {\\\"name\\\": \\\"project_environment\\\", \\\"dependencies\\\": [\\\"python=3.6.2\\\", {\\\"pip\\\": [\\\"azureml-defaults\\\"]}, \\\"pandas\\\", \\\"numpy\\\", \\\"scipy\\\", \\\"scikit-learn\\\", \\\"lightgbm\\\", \\\"joblib\\\"], \\\"channels\\\": [\\\"conda-forge\\\"]}}, \\\"docker\\\": {\\\"enabled\\\": true, \\\"baseImage\\\": \\\"mcr.microsoft.com/azureml/base:intelmpi2018.3-ubuntu16.04\\\", \\\"baseDockerfile\\\": null, \\\"sharedVolumes\\\": true, \\\"shmSize\\\": \\\"2g\\\", \\\"arguments\\\": [], \\\"baseImageRegistry\\\": {\\\"address\\\": null, \\\"username\\\": null, \\\"password\\\": null}}, \\\"spark\\\": {\\\"repositories\\\": [], \\\"packages\\\": [], \\\"precachePackages\\\": true}, \\\"databricks\\\": {\\\"mavenLibraries\\\": [], \\\"pypiLibraries\\\": [], \\\"rcranLibraries\\\": [], \\\"jarLibraries\\\": [], \\\"eggLibraries\\\": []}, \\\"inferencingStackVersion\\\": null}, \\\"history\\\": {\\\"outputCollection\\\": true, \\\"snapshotProject\\\": true, \\\"directoriesToWatch\\\": [\\\"logs\\\"]}, \\\"spark\\\": {\\\"configuration\\\": {\\\"spark.app.name\\\": \\\"Azure ML Experiment\\\", \\\"spark.yarn.maxAppAttempts\\\": 1}}, \\\"hdi\\\": {\\\"yarnDeployMode\\\": \\\"cluster\\\"}, \\\"tensorflow\\\": {\\\"workerCount\\\": 1, \\\"parameterServerCount\\\": 1}, \\\"mpi\\\": {\\\"processCountPerNode\\\": 1}, \\\"dataReferences\\\": {\\\"5442e91c25f449ff9a9780f48c6d7792\\\": {\\\"dataStoreName\\\": \\\"workspaceblobstore\\\", \\\"pathOnDataStore\\\": \\\"data\\\", \\\"mode\\\": \\\"mount\\\", \\\"overwrite\\\": false, \\\"pathOnCompute\\\": null}}, \\\"data\\\": {}, \\\"sourceDirectoryDataStore\\\": null, \\\"amlcompute\\\": {\\\"vmSize\\\": null, \\\"vmPriority\\\": null, \\\"retainCluster\\\": false, \\\"name\\\": null, \\\"clusterMaxNodeCount\\\": 1}}, \\\"TargetDetails\\\": null, \\\"SnapshotId\\\": \\\"c3e4a829-831a-45f4-973d-e5093d4639c7\\\", \\\"TelemetryValues\\\": {\\\"amlClientType\\\": \\\"azureml-sdk-train\\\", \\\"amlClientModule\\\": \\\"azureml.train.hyperdrive._search\\\", \\\"amlClientFunction\\\": \\\"search\\\", \\\"tenantId\\\": \\\"72f988bf-86f1-41af-91ab-2d7cd011db47\\\", \\\"amlClientRequestId\\\": \\\"460b5db2-3fac-4065-bb34-fa43a1fbcb53\\\", \\\"amlClientSessionId\\\": \\\"729a95f4-a59a-471a-94a6-cb8bffc6ea3a\\\", \\\"subscriptionId\\\": \\\"9086b59a-02d7-4687-b3fd-e39fa5e0fd9b\\\", \\\"estimator\\\": \\\"Estimator\\\", \\\"samplingMethod\\\": \\\"BayesianOptimization\\\", \\\"terminationPolicy\\\": \\\"Default\\\", \\\"primaryMetricGoal\\\": \\\"minimize\\\", \\\"maxTotalRuns\\\": 20, \\\"maxConcurrentRuns\\\": 4, \\\"maxDurationMinutes\\\": 10080, \\\"computeTarget\\\": \\\"AmlCompute\\\", \\\"vmSize\\\": null}}}\", \"resume_child_runs\": \"null\", \"all_jobs_generated\": \"false\", \"cancellation_requested\": \"false\", \"progress_metadata_evaluation_timestamp\": \"\\\"2020-03-19T21:12:16.513664\\\"\", \"progress_metadata_digest\": \"\\\"5fea6da115a28e7e578bf15f72768a97c8336919197281c1b5a301933aad5e22\\\"\", \"progress_metadata_active_timestamp\": \"\\\"2020-03-19T21:12:16.513664\\\"\", \"HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_0\": \"{\\\"--num-leaves\\\": 99, \\\"--min-data-in-leaf\\\": 50, \\\"--learning-rate\\\": 0.1, \\\"--feature-fraction\\\": 0.8132296030587374, \\\"--bagging-fraction\\\": 0.7418407116160189, \\\"--bagging-freq\\\": 16, \\\"--max-rounds\\\": 1930, \\\"--max-lag\\\": 17, \\\"--window-size\\\": 21}\", \"HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_1\": \"{\\\"--num-leaves\\\": 27, \\\"--min-data-in-leaf\\\": 120, \\\"--learning-rate\\\": 0.005, \\\"--feature-fraction\\\": 0.3706438325846761, \\\"--bagging-fraction\\\": 0.5542795601096442, \\\"--bagging-freq\\\": 18, \\\"--max-rounds\\\": 1760, \\\"--max-lag\\\": 33, \\\"--window-size\\\": 15}\", \"HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_2\": \"{\\\"--num-leaves\\\": 28, \\\"--min-data-in-leaf\\\": 310, \\\"--learning-rate\\\": 0.0001, \\\"--feature-fraction\\\": 0.4168752967156847, \\\"--bagging-fraction\\\": 0.6030886362720486, \\\"--bagging-freq\\\": 3, \\\"--max-rounds\\\": 720, \\\"--max-lag\\\": 21, \\\"--window-size\\\": 35}\", \"HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_3\": \"{\\\"--num-leaves\\\": 15, \\\"--min-data-in-leaf\\\": 390, \\\"--learning-rate\\\": 0.03, \\\"--feature-fraction\\\": 0.3668251847896762, \\\"--bagging-fraction\\\": 0.3815013306553845, \\\"--bagging-freq\\\": 12, \\\"--max-rounds\\\": 770, \\\"--max-lag\\\": 38, \\\"--window-size\\\": 28}\", \"environment_preparation_status\": \"PREPARED\", \"prepare_run_id\": \"HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_preparation\", \"HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_4\": \"{\\\"--num-leaves\\\": 13, \\\"--min-data-in-leaf\\\": 450, \\\"--learning-rate\\\": 0.02, \\\"--feature-fraction\\\": 0.2901471521383765, \\\"--bagging-fraction\\\": 0.3065771878629929, \\\"--bagging-freq\\\": 17, \\\"--max-rounds\\\": 1800, \\\"--max-lag\\\": 4, \\\"--window-size\\\": 37}\", \"HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_5\": \"{\\\"--num-leaves\\\": 80, \\\"--min-data-in-leaf\\\": 30, \\\"--learning-rate\\\": 0.03, \\\"--feature-fraction\\\": 0.9078890478449191, \\\"--bagging-fraction\\\": 0.45835888307459804, \\\"--bagging-freq\\\": 10, \\\"--max-rounds\\\": 650, \\\"--max-lag\\\": 27, \\\"--window-size\\\": 25}\", \"HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_6\": \"{\\\"--num-leaves\\\": 53, \\\"--min-data-in-leaf\\\": 250, \\\"--learning-rate\\\": 0.001, \\\"--feature-fraction\\\": 0.9904628066051346, \\\"--bagging-fraction\\\": 0.9205596553917755, \\\"--bagging-freq\\\": 15, \\\"--max-rounds\\\": 1340, \\\"--max-lag\\\": 22, \\\"--window-size\\\": 34}\"}, \"end_time_utc\": null, \"status\": \"Running\", \"log_files\": {\"azureml-logs/hyperdrive.txt\": \"https://chhamlws4931040064.blob.core.windows.net/azureml/ExperimentRun/dcid.HD_2d2796ac-1717-4dce-896d-905dcbce2c1a/azureml-logs/hyperdrive.txt?sv=2019-02-02&sr=b&sig=VLRUwMTzJCdY%2FZ4j9Zw%2Bh7AEBdIiFhexEtfRTmbjhzQ%3D&st=2020-03-19T21%3A07%3A21Z&se=2020-03-20T05%3A17%3A21Z&sp=r\"}, \"log_groups\": [[\"azureml-logs/hyperdrive.txt\"]], \"run_duration\": \"0:05:06\", \"hyper_parameters\": {\"--num-leaves\": [\"quniform\", [8, 128, 1]], \"--min-data-in-leaf\": [\"quniform\", [20, 500, 10]], \"--learning-rate\": [\"choice\", [[0.0001, 0.001, 0.005, 0.01, 0.015, 0.02, 0.03, 0.05, 0.1]]], \"--feature-fraction\": [\"uniform\", [0.2, 1]], \"--bagging-fraction\": [\"uniform\", [0.1, 1]], \"--bagging-freq\": [\"quniform\", [1, 20, 1]], \"--max-rounds\": [\"quniform\", [50, 2000, 10]], \"--max-lag\": [\"quniform\", [3, 40, 1]], \"--window-size\": [\"quniform\", [3, 40, 1]]}}, \"child_runs\": [{\"run_id\": \"HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_2\", \"run_number\": 126, \"metric\": 75.24892867, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2020-03-19T21:13:22.910343Z\", \"end_time\": \"2020-03-19T21:14:57.403012Z\", \"created_time\": \"2020-03-19T21:12:49.005902Z\", \"created_time_dt\": \"2020-03-19T21:12:49.005902Z\", \"duration\": \"0:02:08\", \"hyperdrive_id\": \"2d2796ac-1717-4dce-896d-905dcbce2c1a\", \"arguments\": null, \"param_--num-leaves\": 28, \"param_--min-data-in-leaf\": 310, \"param_--learning-rate\": 0.0001, \"param_--feature-fraction\": 0.4168752967156847, \"param_--bagging-fraction\": 0.6030886362720486, \"param_--bagging-freq\": 3, \"param_--max-rounds\": 720, \"param_--max-lag\": 21, \"param_--window-size\": 35, \"best_metric\": 75.24892867}, {\"run_id\": \"HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_1\", \"run_number\": 127, \"metric\": 39.54132385, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2020-03-19T21:13:22.100062Z\", \"end_time\": \"2020-03-19T21:15:46.098653Z\", \"created_time\": \"2020-03-19T21:12:49.08304Z\", \"created_time_dt\": \"2020-03-19T21:12:49.08304Z\", \"duration\": \"0:02:57\", \"hyperdrive_id\": \"2d2796ac-1717-4dce-896d-905dcbce2c1a\", \"arguments\": null, \"param_--num-leaves\": 27, \"param_--min-data-in-leaf\": 120, \"param_--learning-rate\": 0.005, \"param_--feature-fraction\": 0.3706438325846761, \"param_--bagging-fraction\": 0.5542795601096442, \"param_--bagging-freq\": 18, \"param_--max-rounds\": 1760, \"param_--max-lag\": 33, \"param_--window-size\": 15, \"best_metric\": 39.54132385}, {\"run_id\": \"HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_0\", \"run_number\": 128, \"metric\": 36.68529907, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2020-03-19T21:13:23.334757Z\", \"end_time\": \"2020-03-19T21:17:20.148798Z\", \"created_time\": \"2020-03-19T21:12:49.2375Z\", \"created_time_dt\": \"2020-03-19T21:12:49.2375Z\", \"duration\": \"0:04:30\", \"hyperdrive_id\": \"2d2796ac-1717-4dce-896d-905dcbce2c1a\", \"arguments\": null, \"param_--num-leaves\": 99, \"param_--min-data-in-leaf\": 50, \"param_--learning-rate\": 0.1, \"param_--feature-fraction\": 0.8132296030587374, \"param_--bagging-fraction\": 0.7418407116160189, \"param_--bagging-freq\": 16, \"param_--max-rounds\": 1930, \"param_--max-lag\": 17, \"param_--window-size\": 21, \"best_metric\": 36.68529907}, {\"run_id\": \"HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_3\", \"run_number\": 129, \"metric\": 37.11300476, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2020-03-19T21:13:21.844298Z\", \"end_time\": \"2020-03-19T21:15:02.636388Z\", \"created_time\": \"2020-03-19T21:12:49.556578Z\", \"created_time_dt\": \"2020-03-19T21:12:49.556578Z\", \"duration\": \"0:02:13\", \"hyperdrive_id\": \"2d2796ac-1717-4dce-896d-905dcbce2c1a\", \"arguments\": null, \"param_--num-leaves\": 15, \"param_--min-data-in-leaf\": 390, \"param_--learning-rate\": 0.03, \"param_--feature-fraction\": 0.3668251847896762, \"param_--bagging-fraction\": 0.3815013306553845, \"param_--bagging-freq\": 12, \"param_--max-rounds\": 770, \"param_--max-lag\": 38, \"param_--window-size\": 28, \"best_metric\": 36.68529907}, {\"run_id\": \"HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_5\", \"run_number\": 130, \"metric\": 41.06715386, \"status\": \"Running\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2020-03-19T21:15:55.338891Z\", \"end_time\": \"\", \"created_time\": \"2020-03-19T21:15:22.195216Z\", \"created_time_dt\": \"2020-03-19T21:15:22.195216Z\", \"duration\": \"0:02:00\", \"hyperdrive_id\": \"2d2796ac-1717-4dce-896d-905dcbce2c1a\", \"arguments\": null, \"param_--num-leaves\": 80, \"param_--min-data-in-leaf\": 30, \"param_--learning-rate\": 0.03, \"param_--feature-fraction\": 0.9078890478449191, \"param_--bagging-fraction\": 0.45835888307459804, \"param_--bagging-freq\": 10, \"param_--max-rounds\": 650, \"param_--max-lag\": 27, \"param_--window-size\": 25, \"best_metric\": 36.68529907}, {\"run_id\": \"HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_4\", \"run_number\": 131, \"metric\": null, \"status\": \"Running\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2020-03-19T21:15:56.362681Z\", \"end_time\": \"\", \"created_time\": \"2020-03-19T21:15:22.801193Z\", \"created_time_dt\": \"2020-03-19T21:15:22.801193Z\", \"duration\": \"0:01:59\", \"hyperdrive_id\": \"2d2796ac-1717-4dce-896d-905dcbce2c1a\", \"arguments\": null, \"param_--num-leaves\": 13, \"param_--min-data-in-leaf\": 450, \"param_--learning-rate\": 0.02, \"param_--feature-fraction\": 0.2901471521383765, \"param_--bagging-fraction\": 0.3065771878629929, \"param_--bagging-freq\": 17, \"param_--max-rounds\": 1800, \"param_--max-lag\": 4, \"param_--window-size\": 37, \"best_metric\": null}, {\"run_id\": \"HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_6\", \"run_number\": 132, \"metric\": null, \"status\": \"Running\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2020-03-19T21:16:25.813014Z\", \"end_time\": \"\", \"created_time\": \"2020-03-19T21:15:53.807197Z\", \"created_time_dt\": \"2020-03-19T21:15:53.807197Z\", \"duration\": \"0:01:28\", \"hyperdrive_id\": \"2d2796ac-1717-4dce-896d-905dcbce2c1a\", \"arguments\": null, \"param_--num-leaves\": 53, \"param_--min-data-in-leaf\": 250, \"param_--learning-rate\": 0.001, \"param_--feature-fraction\": 0.9904628066051346, \"param_--bagging-fraction\": 0.9205596553917755, \"param_--bagging-freq\": 15, \"param_--max-rounds\": 1340, \"param_--max-lag\": 22, \"param_--window-size\": 34, \"best_metric\": null}], \"children_metrics\": {\"categories\": [0], \"series\": {\"MAPE\": [{\"categories\": [126, 127, 128, 129, 130], \"mode\": \"markers\", \"name\": \"MAPE\", \"stepped\": false, \"type\": \"scatter\", \"data\": [75.2489286722675, 39.5413238482365, 36.68529906716439, 37.1130047640178, 41.06715386014593]}, {\"categories\": [126, 127, 128, 129, 130], \"mode\": \"lines\", \"name\": \"MAPE_min\", \"stepped\": true, \"type\": \"scatter\", \"data\": [75.2489286722675, 39.5413238482365, 36.68529906716439, 36.68529906716439, 36.68529906716439]}]}, \"metricName\": null, \"primaryMetricName\": \"MAPE\", \"showLegend\": false}, \"run_metrics\": [{\"name\": \"best_child_by_primary_metric\", \"run_id\": \"HD_2d2796ac-1717-4dce-896d-905dcbce2c1a\", \"categories\": [0], \"series\": [{\"data\": [{\"metric_name\": \"MAPE\", \"timestamp\": \"2020-03-19 21:15:15.945032+00:00\", \"run_id\": \"HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_3\", \"metric_value\": 37.1130047640178, \"final\": false}]}]}], \"run_logs\": \"[2020-03-19T21:12:15.797368][API][INFO]Experiment created\\r\\n[2020-03-19T21:12:16.647816][GENERATOR][INFO]Trying to sample '4' jobs from the hyperparameter space\\r\\n[2020-03-19T21:12:16.8347749Z][SCHEDULER][INFO]The execution environment is being prepared. Please be patient as it can take a few minutes.\\r\\n[2020-03-19T21:12:16.763629][GENERATOR][INFO]Successfully sampled '4' jobs, they will soon be submitted to the execution target.\\r\\n[2020-03-19T21:12:48.2820889Z][SCHEDULER][INFO]Scheduling job, id='HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_3'\\r\\n[2020-03-19T21:12:48.2795514Z][SCHEDULER][INFO]Scheduling job, id='HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_1'\\r\\n[2020-03-19T21:12:48.2806839Z][SCHEDULER][INFO]Scheduling job, id='HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_2'\\r\\n[2020-03-19T21:12:48.2772616Z][SCHEDULER][INFO]The execution environment was successfully prepared.\\r\\n[2020-03-19T21:12:48.2782467Z][SCHEDULER][INFO]Scheduling job, id='HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_0'\\r\\n[2020-03-19T21:12:49.1346437Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_2'\\r\\n[2020-03-19T21:12:49.2097647Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_1'\\r\\n[2020-03-19T21:12:49.5936141Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_0'\\r\\n[2020-03-19T21:12:49.6453613Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_3'\\r\\n[2020-03-19T21:15:19.237941][GENERATOR][INFO]Trying to sample '2' jobs from the hyperparameter space\\r\\n[2020-03-19T21:15:19.548213][GENERATOR][INFO]Successfully sampled '2' jobs, they will soon be submitted to the execution target.\\r\\n[2020-03-19T21:15:21.2292149Z][SCHEDULER][INFO]Scheduling job, id='HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_4'\\r\\n[2020-03-19T21:15:21.2308195Z][SCHEDULER][INFO]Scheduling job, id='HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_5'\\r\\n[2020-03-19T21:15:22.3288867Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_5'\\r\\n[2020-03-19T21:15:23.1078686Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_4'\\r\\n[2020-03-19T21:15:49.563444][GENERATOR][INFO]Trying to sample '1' jobs from the hyperparameter space\\r\\n[2020-03-19T21:15:49.950635][GENERATOR][INFO]Successfully sampled '1' jobs, they will soon be submitted to the execution target.\\r\\n[2020-03-19T21:15:53.3050388Z][SCHEDULER][INFO]Scheduling job, id='HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_6'\\r\\n[2020-03-19T21:15:53.8900932Z][SCHEDULER][INFO]Successfully scheduled a job. Id='HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_6'\\n\", \"graph\": {}, \"widget_settings\": {\"childWidgetDisplay\": \"popup\", \"send_telemetry\": true, \"log_level\": \"INFO\", \"sdk_version\": \"1.0.85\"}, \"loading\": false}" - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "RunDetails(htr).show()" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_0': {'MAPE': 36.68529906716439},\n", - " 'HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_1': {'MAPE': 39.5413238482365},\n", - " 'HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_10': {'MAPE': 39.48909471171353},\n", - " 'HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_11': {'MAPE': 32.65702658023425},\n", - " 'HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_12': {'MAPE': 32.93810233523822},\n", - " 'HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_13': {'MAPE': 37.62926074943101},\n", - " 'HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_14': {'MAPE': 42.136255340447754},\n", - " 'HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_15': {'MAPE': 72.76283423841294},\n", - " 'HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_16': {'MAPE': 38.453668227285334},\n", - " 'HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_17': {'MAPE': 41.00447880109714},\n", - " 'HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_18': {'MAPE': 34.26881841497758},\n", - " 'HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_19': {'MAPE': 50.750403770935534},\n", - " 'HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_2': {'MAPE': 75.2489286722675},\n", - " 'HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_3': {'MAPE': 37.1130047640178},\n", - " 'HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_4': {'MAPE': 36.51222909108317},\n", - " 'HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_5': {'MAPE': 41.06715386014593},\n", - " 'HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_6': {'MAPE': 46.580738749741506},\n", - " 'HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_7': {'MAPE': 29.75961762449498},\n", - " 'HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_8': {'MAPE': 51.76561288108014},\n", - " 'HD_2d2796ac-1717-4dce-896d-905dcbce2c1a_9': {'MAPE': 36.66639457885292}}" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "while htr.get_status() != \"Completed\":\n", - " {}\n", - "htr.get_metrics()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The best model and its hyperparameter values can be retrieved as follows" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['--data-folder', '$AZUREML_DATAREFERENCE_5442e91c25f449ff9a9780f48c6d7792', '--num-leaves', '108', '--min-data-in-leaf', '410', '--learning-rate', '0.05', '--feature-fraction', '0.418894687852234', '--bagging-fraction', '0.54813576760277', '--bagging-freq', '7', '--max-rounds', '1710', '--max-lag', '8', '--window-size', '25']\n" - ] - } - ], - "source": [ - "best_run = htr.get_best_run_by_primary_metric()\n", - "parameter_values = best_run.get_details()[\"runDefinition\"][\"arguments\"]\n", - "print(parameter_values)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can then register the folder (and all files in it) as a model named `lgbm-oj-forecast` under the workspace for deployment." - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [], - "source": [ - "model = best_run.register_model(\n", - " model_name=\"lgbm-oj-forecast\", model_path=\"outputs/model\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Deploy the Model in ACI\n", - "\n", - "Now we are ready to deploy the model as a web service running in Azure Container Instance [ACI](https://azure.microsoft.com/en-us/services/container-instances/). Azure Machine Learning accomplishes this by constructing a Docker image with the scoring logic and model baked in.\n", - "\n", - "### Create score.py\n", - "\n", - "First, we will create a scoring script that will be invoked by the web service call.\n", - "\n", - "* Note that the scoring script must have two required functions, `init()` and `run(input_data)`.\n", - " - In `init()` function, you typically load the model into a global object. This function is executed only once when the Docker container is started.\n", - " - In `run(input_data)` function, the model is used to predict a value based on the input data. The input and output to run typically use JSON as serialization and de-serialization format but you are not limited to that." - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Overwriting score.py\n" - ] - } - ], - "source": [ - "%%writefile score.py\n", - "import os\n", - "import json\n", - "import numpy as np\n", - "import pandas as pd\n", - "import lightgbm as lgb\n", - "\n", - "\n", - "def init():\n", - " global bst\n", - " model_root = os.getenv(\"AZUREML_MODEL_DIR\")\n", - " # The name of the folder in which to look for LightGBM model files\n", - " lgbm_model_folder = \"model\"\n", - " bst = lgb.Booster(\n", - " model_file=os.path.join(model_root, lgbm_model_folder, \"bst-model.txt\")\n", - " )\n", - "\n", - "\n", - "def run(raw_data):\n", - " columns = bst.feature_name()\n", - " data = np.array(json.loads(raw_data)[\"data\"])\n", - " test_df = pd.DataFrame(data=data, columns=columns)\n", - " # Make prediction\n", - " out = bst.predict(test_df)\n", - " return out.tolist()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Create myenv.yml\n", - "\n", - "We also need to create an environment file so that Azure Machine Learning can install the necessary packages in the Docker image which are required by your scoring script. In this case, we need to specify packages `numpy`, `pandas`, and `lightgbm`." - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "# Conda environment specification. The dependencies defined in this file will\r\n", - "# be automatically provisioned for runs with userManagedDependencies=False.\r\n", - "\n", - "# Details about the Conda environment file format:\r\n", - "# https://conda.io/docs/user-guide/tasks/manage-environments.html#create-env-file-manually\r\n", - "\n", - "name: project_environment\n", - "dependencies:\n", - " # The python interpreter version.\r\n", - " # Currently Azure ML only supports 3.5.2 and later.\r\n", - "- python=3.6.2\n", - "\n", - "- pip:\n", - " - azureml-defaults\n", - "- numpy=1.16.2\n", - "- pandas=0.23.4\n", - "- lightgbm=2.3.0\n", - "channels:\n", - "- conda-forge\n", - "\n" - ] - } - ], - "source": [ - "cd = CondaDependencies.create()\n", - "cd.add_conda_package(\"numpy=1.16.2\")\n", - "cd.add_conda_package(\"pandas=0.23.4\")\n", - "cd.add_conda_package(\"lightgbm=2.3.0\")\n", - "cd.save_to_file(base_directory=\"./\", conda_file_path=\"myenv.yml\")\n", - "\n", - "print(cd.serialize_to_string())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Deploy to ACI\n", - "\n", - "We are almost ready to deploy. In the next cell, we first create the inference configuration and deployment configuration. Then, we deploy the model to ACI. This cell will run for several minutes." - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Running............................\n", - "Succeeded\n", - "ACI service creation operation finished, operation \"Succeeded\"\n", - "Healthy\n", - "CPU times: user 385 ms, sys: 77.5 ms, total: 463 ms\n", - "Wall time: 2min 46s\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "inference_config = InferenceConfig(runtime=\"python\", entry_script=\"score.py\", conda_file=\"myenv.yml\")\n", - "\n", - "aciconfig = AciWebservice.deploy_configuration(\n", - " cpu_cores=1,\n", - " memory_gb=1,\n", - " tags={\"name\": \"ojdata\", \"framework\": \"LightGBM\"},\n", - " description=\"LightGBM model on Orange Juice data\",\n", - ")\n", - "\n", - "service = Model.deploy(\n", - " workspace=ws, name=\"lgbm-oj-svc\", models=[model], inference_config=inference_config, deployment_config=aciconfig\n", - ")\n", - "\n", - "service.wait_for_deployment(True)\n", - "print(service.state)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "> Tip: If something goes wrong with the deployment, you could look at the logs from the service by running this command `print(service.get_logs())`." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This is the scoring web service endpoint:" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "http://61ad47b3-7ad6-4093-8535-a5324b2238c7.westus.azurecontainer.io/score\n" - ] - } - ], - "source": [ - "print(service.scoring_uri)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "After the web service is successfully deployed, you will see a deployment in the Azure Machine Learning workspace on Azure portal\n", - "\n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Test the deployed model\n", - "\n", - "Let's test the deployed model. We create a few test data points and send them to the web service hosted in ACI. Note here we are using the run API in the SDK to invoke the service. You can also make raw HTTP calls using any HTTP tool such as curl.\n", - "\n", - "After the invocation, we print the returned predictions each of which represents the forecasted sales of a target store, brand in a given week as specified by `store, brand, week` in `used_columns`." - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "prediction: [12360.719047885588, 18866.86295711333, 5805.556346800228]\n" - ] - } - ], - "source": [ - "# Prepare features according to the input schema of the best model\n", - "train_dir = os.path.join(DATA_DIR, \"train\")\n", - "max_lag = int(parameter_values[parameter_values.index(\"--max-lag\") + 1])\n", - "lags = np.arange(2, max_lag + 1)\n", - "window_size = int(parameter_values[parameter_values.index(\"--window-size\") + 1])\n", - "used_columns = [\n", - " \"store\",\n", - " \"brand\",\n", - " \"week\",\n", - " \"week_of_month\",\n", - " \"month\",\n", - " \"deal\",\n", - " \"feat\",\n", - " \"move\",\n", - " \"price\",\n", - " \"price_ratio\",\n", - "]\n", - "GAP = 2\n", - "features, train_end_week = create_features(\n", - " 1, train_dir, lags, window_size, used_columns\n", - ")\n", - "test_fea = features[features.week >= train_end_week + GAP].reset_index(drop=True)\n", - "test_fea.drop(\"move\", axis=1, inplace=True)\n", - "\n", - "# Pick a few test data points\n", - "test_samples = json.dumps({\"data\": np.array(test_fea.iloc[:3]).tolist()})\n", - "test_samples = bytes(test_samples, encoding=\"utf8\")\n", - "\n", - "# Predict using the deployed model\n", - "result = service.run(input_data=test_samples)\n", - "print(\"prediction:\", result)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can also send raw HTTP request to the service." - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "POST to url http://61ad47b3-7ad6-4093-8535-a5324b2238c7.westus.azurecontainer.io/score\n", - "\n", - "input data: b'{\"data\": [[2.0, 1.0, 137.0, 4.0, 4.0, 0.0, 0.0, 0.0416446872, 1.1124927835293534, 12416.0, 28096.0, 15168.0, 20736.0, 31808.0, 25728.0, 43584.0, 14453.76], [2.0, 1.0, 138.0, 5.0, 4.0, 1.0, 1.0, 0.03734375, 0.9420125411290402, 12416.0, 12416.0, 28096.0, 15168.0, 20736.0, 31808.0, 25728.0, 14699.52], [2.0, 2.0, 137.0, 4.0, 4.0, 0.0, 0.0, 0.0519791667, 1.388567227553081, 11424.0, 4992.0, 7008.0, 6816.0, 5280.0, 7296.0, 5664.0, 9219.84]]}'\n", - "\n", - "prediction: [12360.719047885588, 18866.86295711333, 5805.556346800228]\n" - ] - } - ], - "source": [ - "headers = {\"Content-Type\": \"application/json\"}\n", - "\n", - "resp = requests.post(service.scoring_uri, test_samples, headers=headers)\n", - "\n", - "print(\"POST to url\", service.scoring_uri)\n", - "print(\"\")\n", - "print(\"input data:\", test_samples)\n", - "print(\"\")\n", - "print(\"prediction:\", resp.text)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Clean up\n", - "\n", - "After finishing the tests, you can delete the ACI deployment with a simple delete API call as follows." - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [], - "source": [ - "service.delete()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Additional Reading:\n", - "\n", - "\\[1\\] Training, hyperparameter tune, and deploy with TensorFlow: https://github.com/Azure/MachineLearningNotebooks/blob/master/how-to-use-azureml/ml-frameworks/tensorflow/deployment/train-hyperparameter-tune-deploy-with-tensorflow/train-hyperparameter-tune-deploy-with-tensorflow.ipynb
\n", - "\n", - "\\[2\\] AzureML HyperDrive package: https://docs.microsoft.com/en-us/python/api/azureml-train-core/azureml.train.hyperdrive?view=azure-ml-py" - ] - } - ], - "metadata": { - "author_info": { - "affiliation": "Microsoft", - "created_by": "Chenhui Hu" - }, - "kernelspec": { - "display_name": "forecasting_env", - "language": "python", - "name": "forecasting_env" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.10" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/grocery_sales/python/README.md b/examples/grocery_sales/python/README.md deleted file mode 100644 index 8e87201e..00000000 --- a/examples/grocery_sales/python/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Forecasting examples in Python - -This folder contains Jupyter notebooks with Python examples for building forecasting solutions. To run the notebooks, please ensure your environment is set up with required dependencies by following instructions in the [Setup guide](../../../docs/SETUP.md). - - -## Summary - -The following summarizes each directory of the Python best practice notebooks. - -| Directory | Content | Description | -| --- | --- | --- | -| [00_quick_start](./00_quick_start)| [autoarima_single_round.ipynb](./00_quick_start/autoarima_single_round.ipynb)
[azure_automl_single_round.ipynb](./00_quick_start/azure_automl_single_round.ipynb)
[lightgbm_single_round.ipynb](./00_quick_start/lightgbm_single_round.ipynb) | Quick start notebooks that demonstrate workflow of developing a forecasting model using one-round training and testing data| -| [01_prepare_data](./01_prepare_data) | [ojdata_exploration.ipynb](./01_prepare_data/ojdata_exploration.ipynb)
[ojdata_preparation.ipynb](./01_prepare_data/ojdata_preparation.ipynb) | Data exploration and preparation notebooks| -| [02_model](./02_model) | [dilatedcnn_multi_round.ipynb](./02_model/dilatedcnn_multi_round.ipynb)
[lightgbm_multi_round.ipynb](./02_model/lightgbm_multi_round.ipynb)
[autoarima_multi_round.ipynb](./02_model/autoarima_multi_round.ipynb) | Deep dive notebooks that perform multi-round training and testing of various classical and deep learning forecast algorithms| -| [03_model_tune_deploy](./03_model_tune_deploy/) | [azure_hyperdrive_lightgbm.ipynb](./03_model_tune_deploy/azure_hyperdrive_lightgbm.ipynb)
[aml_scripts/](./03_model_tune_deploy/aml_scripts) |
  • Example notebook for model tuning using Azure Machine Learning Service and deploying the best model on Azure
  • Scripts for model training and validation
| - diff --git a/fclib/README.md b/fclib/README.md index 75a72611..60ab6424 100644 --- a/fclib/README.md +++ b/fclib/README.md @@ -1,40 +1,11 @@ # Forecasting library -Building forecasting models can involve tedious tasks ranging from data loading, dataset understanding, model development, model evaluation to deployment of trained models. To assist with these tasks, we developed a forecasting library - **fclib**. You'll see this library used widely in sample notebooks in [examples](../examples). The following provides a short description of the sub-modules. For more details about what functions/classes/utitilies are available and how to use them, please review the doc-strings provided with the code and see the sample notebooks in [examples](../examples) directory. +A set of utility functions for forecasting. -## Submodules +## Install -### [AzureML](fclib/azureml) - -The AzureML submodule contains utilities to connect to an Azure Machine Learning workspace, train, tune and operationalize forecasting models at scale using AzureML. - - -### [Common](fclib/common) - -This submodule contains high-level utilities that are commonly used in multiple algorithms as well as helper functions for visualizing forecasting predictions. - -### [Dataset](fclib/dataset) -This submodule includes helper functions for interacting with datasets used in the example notebooks, utility functions to process datasets for different models tasks, as well as utilities for splitting data for training/testing. For example, the [ojdata](fclib/dataset/ojdata.py) submodule will allow you to download and process Orange Juice data set, as well as split it into training and testing rounds. - -```python -from fclib.dataset.ojdata import download_ojdata, split_train_test - -download_ojdata(DATA_DIR) -train_df_list, test_df_list, _ = split_train_test( - DATA_DIR, - n_splits=N_SPLITS, - horizon=HORIZON, - gap=GAP, - first_week=FIRST_WEEK, - last_week=LAST_WEEK -) +```bash +pip install -e . ``` -### [Evaluation](fclib/evaluation) -Evaluation module includes functionalities for computing common forecasting evaluation metrics, more specifically `MAPE`, `sMAPE`, and `pinball loss`. - -### [Feature Engineering](fclib/feature_engineering) -Feature engineering module contains utilities to create various time series features, for example, week or day of month, lagged features, and moving average features. This module is used widely in machine-learning based approaches to forecasting, in which time series data is transformed into a tabular featurized dataset, that becomes input to a machine learning method. - -### [Models](fclib/models) -The models module contains implementations of various algorithms that can be used in addition to external packages to evaluate and develop new forecasting solutions. Some submodules found here are: `lightgbm`, `dilated cnn`, etc. A more detailed description of which algorithms are used in our examples can be found in [this README](../examples/oj_retail/python/README.md). \ No newline at end of file +This will install the package fclib. \ No newline at end of file diff --git a/fclib/fclib/azureml/azureml_utils.py b/fclib/fclib/azureml/azureml_utils.py index e0c4aef8..c0f9707e 100644 --- a/fclib/fclib/azureml/azureml_utils.py +++ b/fclib/fclib/azureml/azureml_utils.py @@ -2,149 +2,6 @@ # Licensed under the MIT License. """ -This file contains utility functions for interacting with Azure ML Resources. -Reused code from -https://github.com/microsoft/nlp-recipes/blob/master/utils_nlp/azureml/azureml_utils.py +This file contains utility functions for using AzureML SDK in the +development of forecasting solutions. """ - -import os -from azureml.core.authentication import AzureCliAuthentication -from azureml.core.authentication import InteractiveLoginAuthentication -from azureml.core.authentication import AuthenticationException -from azureml.core import Workspace -from azureml.exceptions import ProjectSystemException -from azureml.core.compute import ComputeTarget, AmlCompute -from azureml.core.compute_target import ComputeTargetException - - -def get_auth(): - """ - Method to get the correct Azure ML Authentication type - - Always start with CLI Authentication and if it fails, fall back - to interactive login - """ - try: - auth_type = AzureCliAuthentication() - auth_type.get_authentication_header() - except AuthenticationException: - auth_type = InteractiveLoginAuthentication() - return auth_type - - -def get_or_create_workspace( - config_path="./.azureml", subscription_id=None, resource_group=None, workspace_name=None, workspace_region=None, -): - """ - Method to get or create workspace. - - Args: - config_path: optional directory to look for / store config.json file (defaults to current - directory) - subscription_id: Azure subscription id - resource_group: Azure resource group to create workspace and related resources - workspace_name: name of azure ml workspace - workspace_region: region for workspace - - Returns: - obj: AzureML workspace if one exists already with the name otherwise creates a new one. - """ - config_file_path = "." - - if config_path is not None: - config_dir, config_file_name = os.path.split(config_path) - if config_file_name != "config.json": - config_file_path = os.path.join(config_path, "config.json") - - try: - # Get existing azure ml workspace - if os.path.isfile(config_file_path): - ws = Workspace.from_config(config_file_path, auth=get_auth()) - else: - ws = Workspace.get( - name=workspace_name, subscription_id=subscription_id, resource_group=resource_group, auth=get_auth(), - ) - - except ProjectSystemException: - # This call might take a minute or two. - print("Creating new workspace") - ws = Workspace.create( - name=workspace_name, - subscription_id=subscription_id, - resource_group=resource_group, - create_resource_group=True, - location=workspace_region, - auth=get_auth(), - ) - - ws.write_config(path=config_path) - return ws - - -def get_or_create_amlcompute( - workspace, compute_name, vm_size="", min_nodes=0, max_nodes=None, idle_seconds_before_scaledown=None, verbose=False, -): - """ - Get or create AmlCompute as the compute target. If a cluster of the same name is found, - attach it and rescale accordingly. Otherwise, create a new cluster. - - Args: - workspace (Workspace): workspace - compute_name (str): name - vm_size (str, optional): vm size - min_nodes (int, optional): minimum number of nodes in cluster - max_nodes (None, optional): maximum number of nodes in cluster - idle_seconds_before_scaledown (None, optional): how long to wait before the cluster - autoscales down - verbose (bool, optional): if true, print logs - Returns: - Compute target - """ - try: - if verbose: - print("Found compute target: {}".format(compute_name)) - - compute_target = ComputeTarget(workspace=workspace, name=compute_name) - if len(compute_target.list_nodes()) < max_nodes: - if verbose: - print("Rescaling to {} nodes".format(max_nodes)) - compute_target.update(max_nodes=max_nodes) - compute_target.wait_for_completion(show_output=verbose) - - except ComputeTargetException: - if verbose: - print("Creating new compute target: {}".format(compute_name)) - - compute_config = AmlCompute.provisioning_configuration( - vm_size=vm_size, - min_nodes=min_nodes, - max_nodes=max_nodes, - idle_seconds_before_scaledown=idle_seconds_before_scaledown, - ) - compute_target = ComputeTarget.create(workspace, compute_name, compute_config) - compute_target.wait_for_completion(show_output=verbose) - - return compute_target - - -def get_output_files(run, output_path, file_names=None): - """ - Method to get the output files from an AzureML output directory. - - Args: - file_names(list): Names of the files to download. - run(azureml.core.run.Run): Run object of the run. - output_path(str): Path to download the output files. - - Returns: None - - """ - os.makedirs(output_path, exist_ok=True) - - if file_names is None: - file_names = run.get_file_names() - - for f in file_names: - dest = os.path.join(output_path, f.split("/")[-1]) - print("Downloading file {} to {}...".format(f, dest)) - run.download_file(f, dest) diff --git a/fclib/fclib/dataset/load_oj_data.R b/fclib/fclib/dataset/download_oj_data.R similarity index 57% rename from fclib/fclib/dataset/load_oj_data.R rename to fclib/fclib/dataset/download_oj_data.R index 9d780d97..dcbb5604 100755 --- a/fclib/fclib/dataset/load_oj_data.R +++ b/fclib/fclib/dataset/download_oj_data.R @@ -1,25 +1,20 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -# This script retrieves the orangeJuice dataset from the bayesm R package and saves the data as csv. -# -# Two arguments must be supplied to this script: -# -# RDA_PATH - path to the local .rda file containing the data -# DATA_DIR - destination directory for saving processed .csv files +# This script retrieves the orangeJuice dataset from the bayesm R package and saves the data as csv args = commandArgs(trailingOnly=TRUE) -# Test if there are at least two arguments: if not, return an error -if (length(args)==2) { - RDA_PATH <- args[1] - DATA_DIR <- args[2] -} else { - stop("Two arguments must be supplied - path to .rda file and destination data directory).", call.=FALSE) -} +# test if there is at least one argument: if not, return an error +if (length(args)==0) { + stop("At least one argument must be supplied (data directory).", call.=FALSE) +} else if (length(args)==1) { + DATA_DIR <- args[1] +} # Load the data from bayesm library -load(RDA_PATH) +library(bayesm) +data("orangeJuice") yx <- orangeJuice[[1]] storedemo <- orangeJuice[[2]] diff --git a/fclib/fclib/dataset/ojdata.py b/fclib/fclib/dataset/ojdata.py index 71342ae2..83aaf39a 100644 --- a/fclib/fclib/dataset/ojdata.py +++ b/fclib/fclib/dataset/ojdata.py @@ -8,16 +8,11 @@ import pandas as pd import math import datetime import itertools -import argparse -import logging -import requests -from tqdm import tqdm -from fclib.common.utils import git_repo_path from fclib.feature_engineering.feature_utils import df_from_cartesian_product DATA_FILE_LIST = ["yx.csv", "storedemo.csv"] -SCRIPT_NAME = "load_oj_data.R" +SCRIPT_NAME = "download_oj_data.R" DEFAULT_TARGET_COL = "move" DEFAULT_STATIC_FEA = None @@ -26,54 +21,25 @@ DEFAULT_DYNAMIC_FEA = ["deal", "feat"] # The start datetime of the first week in the record FIRST_WEEK_START = pd.to_datetime("1989-09-14 00:00:00") -# Original data source -OJ_URL = "https://github.com/cran/bayesm/raw/master/data/orangeJuice.rda" +def download_ojdata(dest_dir): + """Downloads Orange Juice dataset. -log = logging.getLogger(__name__) - - -def maybe_download(url, dest_directory, filename=None): - """Download a file if it is not already downloaded. Args: - dest_directory (str): Destination directory. - url (str): URL of the file to download. - filename (str): File name. + dest_dir (str): Directory path for the downloaded file + """ + maybe_download(dest_dir=dest_dir) + + +def maybe_download(dest_dir): + """Download a file if it is not already downloaded. + + Args: + dest_dir (str): Destination directory Returns: str: File path of the file downloaded. """ - if filename is None: - filename = url.split("/")[-1] - os.makedirs(dest_directory, exist_ok=True) - filepath = os.path.join(dest_directory, filename) - if not os.path.exists(filepath): - r = requests.get(url, stream=True) - total_size = int(r.headers.get("content-length", 0)) - block_size = 1024 - num_iterables = math.ceil(total_size / block_size) - - with open(filepath, "wb") as file: - for data in tqdm(r.iter_content(block_size), total=num_iterables, unit="KB", unit_scale=True,): - file.write(data) - else: - log.debug("File {} already downloaded".format(filepath)) - - return filepath - - -def download_ojdata(dest_dir="."): - """Download orange juice dataset from the original source. - - Args: - dest_dir (str): Directory path for the downloaded file - - Returns: - str: Path of the downloaded file. - """ - url = OJ_URL - rda_path = maybe_download(url, dest_directory=dest_dir) - # Check if data files exist data_exists = True for f in DATA_FILE_LIST: @@ -81,21 +47,13 @@ def download_ojdata(dest_dir="."): data_exists = data_exists and os.path.exists(file_path) if not data_exists: - # Call data loading script - repo_path = git_repo_path() - script_path = os.path.join(repo_path, "fclib", "fclib", "dataset", SCRIPT_NAME) - + # Call data download script + print("Starting data download ...") + script_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), SCRIPT_NAME) try: - print(f"Destination directory: {dest_dir}") - output = subprocess.run( - ["Rscript", script_path, rda_path, dest_dir], stderr=subprocess.PIPE, stdout=subprocess.PIPE - ) - print(output.stdout) - if output.returncode != 0: - raise Exception(f"Subprocess failed - {output.stderr}") - + subprocess.call(["Rscript", script_path, dest_dir]) except subprocess.CalledProcessError as e: - raise e + print(e.output) else: print("Data already exists at the specified location.") @@ -155,12 +113,12 @@ def split_train_test(data_dir, n_splits=1, horizon=2, gap=2, first_week=40, last Note that train_*.csv files in /train folder contain all the features in the training period and aux_*.csv files in /train folder contain all the features except 'logmove', 'constant', - 'profit' up until the forecast period end week. Both train_*.csv and auxi_*csv can be used for + 'profit' up until the forecast period end week. Both train_*.csv and aux_*csv can be used for generating forecasts in each split. However, test_*.csv files in /test folder can only be used for model performance evaluation. Example: - data_dir = "/home/ojdata" + data_dir = "/home/vapaunic/forecasting/ojdata" train, test, aux = split_train_test(data_dir=data_dir, n_splits=5, horizon=3, write_csv=True) @@ -216,7 +174,7 @@ def split_train_test(data_dir, n_splits=1, horizon=2, gap=2, first_week=40, last roundstr = "_" + str(i + 1) if n_splits > 1 else "" train_df.to_csv(os.path.join(TRAIN_DATA_DIR, "train" + roundstr + ".csv")) test_df.to_csv(os.path.join(TEST_DATA_DIR, "test" + roundstr + ".csv")) - aux_df.to_csv(os.path.join(TRAIN_DATA_DIR, "auxi" + roundstr + ".csv")) + aux_df.to_csv(os.path.join(TRAIN_DATA_DIR, "aux" + roundstr + ".csv")) train_df_list.append(train_df) test_df_list.append(test_df) @@ -478,12 +436,9 @@ def specify_retail_data_schema( if __name__ == "__main__": + data_dir = "/home/vapaunic/forecasting/ojdata" - parser = argparse.ArgumentParser() - parser.add_argument("--data-dir", help="Data download directory") - args = parser.parse_args() - - download_ojdata(args.data_dir) + download_ojdata(data_dir) # train, test, aux = split_train_test(data_dir=data_dir, n_splits=1, horizon=2, write_csv=True) # print((test[0].week)) diff --git a/contrib/tsperf/scripts/evaluate.py b/fclib/fclib/evaluation/evaluate.py similarity index 100% rename from contrib/tsperf/scripts/evaluate.py rename to fclib/fclib/evaluation/evaluate.py diff --git a/contrib/tsperf/energy_utils/train_utils.py b/fclib/fclib/evaluation/train_utils.py similarity index 100% rename from contrib/tsperf/energy_utils/train_utils.py rename to fclib/fclib/evaluation/train_utils.py diff --git a/contrib/tsperf/energy_utils/feature_engineering/base_ts_estimators.py b/fclib/fclib/feature_engineering/base_ts_estimators.py similarity index 100% rename from contrib/tsperf/energy_utils/feature_engineering/base_ts_estimators.py rename to fclib/fclib/feature_engineering/base_ts_estimators.py diff --git a/contrib/tsperf/energy_utils/feature_engineering/feature_engineering.py b/fclib/fclib/feature_engineering/feature_engineering.py similarity index 100% rename from contrib/tsperf/energy_utils/feature_engineering/feature_engineering.py rename to fclib/fclib/feature_engineering/feature_engineering.py diff --git a/fclib/fclib/feature_engineering/feature_utils.py b/fclib/fclib/feature_engineering/feature_utils.py index 006fb62f..55ffcc7e 100644 --- a/fclib/fclib/feature_engineering/feature_utils.py +++ b/fclib/fclib/feature_engineering/feature_utils.py @@ -11,17 +11,10 @@ import calendar import itertools import pandas as pd import numpy as np -import datetime from datetime import timedelta from sklearn.preprocessing import MinMaxScaler -from dateutil.relativedelta import relativedelta -ALLOWED_TIME_COLUMN_TYPES = [ - pd.Timestamp, - pd.DatetimeIndex, - datetime.datetime, - datetime.date, -] +from fclib.feature_engineering.utils import is_datetime_like # 0: Monday, 2: T/W/TR, 4: F, 5:SA, 6: S WEEK_DAY_TYPE_MAP = {1: 2, 3: 2} # Map for converting Wednesday and @@ -32,11 +25,6 @@ SEMI_HOLIDAY_CODE = 8 # days before and after a holiday DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S" -def is_datetime_like(x): - """Function that checks if a data frame column x is of a datetime type.""" - return any(isinstance(x, col_type) for col_type in ALLOWED_TIME_COLUMN_TYPES) - - def day_type(datetime_col, holiday_col=None, semi_holiday_offset=timedelta(days=1)): """ Convert datetime_col to 7 day types @@ -1014,81 +1002,3 @@ def normalize_columns(df, seq_cols, scaler=MinMaxScaler()): df_scaled = pd.DataFrame(scaler.fit_transform(df[seq_cols]), columns=seq_cols, index=df.index) df_scaled = pd.concat([df[cols_fixed], df_scaled], axis=1) return df_scaled, scaler - - -def get_datetime_col(df, datetime_colname): - """ - Helper function for extracting the datetime column as datetime type from - a data frame. - - Args: - df: pandas DataFrame containing the column to convert - datetime_colname: name of the column to be converted - - Returns: - pandas.Series: converted column - - Raises: - Exception: if datetime_colname does not exist in the dateframe df. - Exception: if datetime_colname cannot be converted to datetime type. - """ - if datetime_colname in df.index.names: - datetime_col = df.index.get_level_values(datetime_colname) - elif datetime_colname in df.columns: - datetime_col = df[datetime_colname] - else: - raise Exception("Column or index {0} does not exist in the data " "frame".format(datetime_colname)) - - if not is_datetime_like(datetime_col): - datetime_col = pd.to_datetime(df[datetime_colname]) - return datetime_col - - -def get_month_day_range(date): - """ - Returns the first date and last date of the month of the given date. - """ - # Replace the date in the original timestamp with day 1 - first_day = date + relativedelta(day=1) - # Replace the date in the original timestamp with day 1 - # Add a month to get to the first day of the next month - # Subtract one day to get the last day of the current month - last_day = date + relativedelta(day=1, months=1, days=-1, hours=23) - return first_day, last_day - - -def add_datetime(input_datetime, unit, add_count): - """ - Function to add a specified units of time (years, months, weeks, days, - hours, or minutes) to the input datetime. - - Args: - input_datetime: datatime to be added to - unit: unit of time, valid values: 'year', 'month', 'week', - 'day', 'hour', 'minute'. - add_count: number of units to add - - Returns: - New datetime after adding the time difference to input datetime. - - Raises: - Exception: if invalid unit is provided. Valid units are: - 'year', 'month', 'week', 'day', 'hour', 'minute'. - """ - if unit == "Y": - new_datetime = input_datetime + relativedelta(years=add_count) - elif unit == "M": - new_datetime = input_datetime + relativedelta(months=add_count) - elif unit == "W": - new_datetime = input_datetime + relativedelta(weeks=add_count) - elif unit == "D": - new_datetime = input_datetime + relativedelta(days=add_count) - elif unit == "h": - new_datetime = input_datetime + relativedelta(hours=add_count) - elif unit == "m": - new_datetime = input_datetime + relativedelta(minutes=add_count) - else: - raise Exception( - "Invalid backtest step unit, {}, provided. Valid " "step units are Y, M, W, D, h, " "and m".format(unit) - ) - return new_datetime diff --git a/contrib/tsperf/energy_utils/feature_engineering/lag.py b/fclib/fclib/feature_engineering/lag.py similarity index 100% rename from contrib/tsperf/energy_utils/feature_engineering/lag.py rename to fclib/fclib/feature_engineering/lag.py diff --git a/contrib/tsperf/energy_utils/feature_engineering/normalization.py b/fclib/fclib/feature_engineering/normalization.py similarity index 100% rename from contrib/tsperf/energy_utils/feature_engineering/normalization.py rename to fclib/fclib/feature_engineering/normalization.py diff --git a/contrib/tsperf/energy_utils/feature_engineering/rolling_window.py b/fclib/fclib/feature_engineering/rolling_window.py similarity index 100% rename from contrib/tsperf/energy_utils/feature_engineering/rolling_window.py rename to fclib/fclib/feature_engineering/rolling_window.py diff --git a/contrib/tsperf/energy_utils/feature_engineering/stats.py b/fclib/fclib/feature_engineering/stats.py similarity index 100% rename from contrib/tsperf/energy_utils/feature_engineering/stats.py rename to fclib/fclib/feature_engineering/stats.py diff --git a/contrib/tsperf/energy_utils/feature_engineering/temporal.py b/fclib/fclib/feature_engineering/temporal.py similarity index 100% rename from contrib/tsperf/energy_utils/feature_engineering/temporal.py rename to fclib/fclib/feature_engineering/temporal.py diff --git a/contrib/tsperf/energy_utils/feature_engineering/us_holidays.csv b/fclib/fclib/feature_engineering/us_holidays.csv similarity index 100% rename from contrib/tsperf/energy_utils/feature_engineering/us_holidays.csv rename to fclib/fclib/feature_engineering/us_holidays.csv diff --git a/contrib/tsperf/energy_utils/feature_engineering/utils.py b/fclib/fclib/feature_engineering/utils.py similarity index 100% rename from contrib/tsperf/energy_utils/feature_engineering/utils.py rename to fclib/fclib/feature_engineering/utils.py diff --git a/contrib/tsperf/energy_utils/feature_engineering/__init__.py b/fclib/fclib/tuning/__init__.py similarity index 100% rename from contrib/tsperf/energy_utils/feature_engineering/__init__.py rename to fclib/fclib/tuning/__init__.py diff --git a/contrib/tsperf/energy_utils/back_test_utils.py b/fclib/fclib/tuning/back_test_utils.py similarity index 100% rename from contrib/tsperf/energy_utils/back_test_utils.py rename to fclib/fclib/tuning/back_test_utils.py diff --git a/fclib/requirements.txt b/fclib/requirements.txt index 88eb009e..2ee724c7 100644 --- a/fclib/requirements.txt +++ b/fclib/requirements.txt @@ -1,5 +1,4 @@ pandas datetime scikit_learn -numpy -requests \ No newline at end of file +numpy \ No newline at end of file diff --git a/tests/ci/component_governance.yml b/tests/ci/component_governance.yml deleted file mode 100644 index 2a809587..00000000 --- a/tests/ci/component_governance.yml +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Pull request against these branches will trigger this build -pr: - - master - - staging - -# no CI trigger -trigger: none - -jobs: -- job: Component_governance - timeoutInMinutes: 20 # how long to run the job before automatically cancelling - pool: - vmImage: 'ubuntu-16.04' - - steps: - - bash: | - python tools/generate_requirements_txt.py - displayName: 'Generate requirements.txt file from generate_conda_file.py' - - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - task: notice@0 - inputs: - outputformat: 'text' - - - bash: | - ls -la - cat NOTICE.txt - git status - result=$(git status | grep NOTICE.txt) - if [[ $result ]]; then - echo "Notice file modified: $result" - echo `git diff NOTICE.txt` - BRANCH=NOTICE/`date +%s` - git checkout -b $BRANCH - git add NOTICE.txt - git commit -m "Notice file modified." - git push origin $BRANCH - else - echo "Notice file not modified." - fi - displayName: 'Check in notice file if modified.' \ No newline at end of file diff --git a/tests/ci/cpu_integration_tests_linux.yml b/tests/ci/cpu_integration_tests_linux.yml index 8fc5bb1f..2a720ccf 100644 --- a/tests/ci/cpu_integration_tests_linux.yml +++ b/tests/ci/cpu_integration_tests_linux.yml @@ -14,10 +14,10 @@ trigger: jobs: - job: cpu_integration_tests_linux - timeoutInMinutes: 60 # how long to run the job before automatically cancelling + timeoutInMinutes: 10 # how long to run the job before automatically cancelling pool: # vmImage: 'ubuntu-16.04' # hosted machine - name: $(Agent_Name) + name: ForecastingAgents steps: - bash: | diff --git a/tests/ci/cpu_unit_tests_linux.yml b/tests/ci/cpu_unit_tests_linux.yml index 3f66cba3..86206e28 100644 --- a/tests/ci/cpu_unit_tests_linux.yml +++ b/tests/ci/cpu_unit_tests_linux.yml @@ -17,7 +17,7 @@ jobs: timeoutInMinutes: 10 # how long to run the job before automatically cancelling pool: # vmImage: 'ubuntu-16.04' # hosted machine - name: $(Agent_Name) + name: ForecastingAgents steps: - bash: | diff --git a/tests/conftest.py b/tests/conftest.py index 494a9eb4..d2ad6136 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,21 +5,10 @@ from fclib.common.utils import git_repo_path @pytest.fixture(scope="module") def notebooks(): - """Get paths of example notebooks. - - Returns: - dict: Dictionary including paths of the example notebooks. - """ repo_path = git_repo_path() examples_path = os.path.join(repo_path, "examples") - usecase_path = os.path.join(examples_path, "grocery_sales", "python") - quick_start_path = os.path.join(usecase_path, "00_quick_start") - model_path = os.path.join(usecase_path, "02_model") + quick_start_path = os.path.join(examples_path, "00_quick_start") # Path for the notebooks - paths = { - "lightgbm_quick_start": os.path.join(quick_start_path, "lightgbm_single_round.ipynb"), - "lightgbm_multi_round": os.path.join(model_path, "lightgbm_multi_round.ipynb"), - "dilatedcnn_multi_round": os.path.join(model_path, "dilatedcnn_multi_round.ipynb"), - } + paths = {"lightgbm_quick_start": os.path.join(quick_start_path, "lightgbm_point_forecast.ipynb")} return paths diff --git a/tests/integration/test_notebooks_python.py b/tests/integration/test_notebooks_python.py index 98441373..db6afc43 100644 --- a/tests/integration/test_notebooks_python.py +++ b/tests/integration/test_notebooks_python.py @@ -2,6 +2,10 @@ # Licensed under the MIT License. import os + +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + import pytest import papermill as pm import scrapbook as sb @@ -19,31 +23,3 @@ def test_lightgbm_quick_start(notebooks): assert df.shape[0] == 1 mape = df.loc[df.name == "MAPE"]["data"][0] assert mape == pytest.approx(35.60, abs=ABS_TOL) - - -@pytest.mark.integration -def test_lightgbm_multi_round(notebooks): - notebook_path = notebooks["lightgbm_multi_round"] - output_notebook_path = os.path.join(os.path.dirname(notebook_path), "output.ipynb") - pm.execute_notebook( - notebook_path, output_notebook_path, kernel_name="forecast_cpu", parameters=dict(N_SPLITS=1), - ) - nb = sb.read_notebook(output_notebook_path) - df = nb.scraps.dataframe - assert df.shape[0] == 1 - mape = df.loc[df.name == "MAPE"]["data"][0] - assert mape == pytest.approx(36.0, abs=ABS_TOL) - - -@pytest.mark.integration -def test_dilatedcnn_multi_round(notebooks): - notebook_path = notebooks["dilatedcnn_multi_round"] - output_notebook_path = os.path.join(os.path.dirname(notebook_path), "output.ipynb") - pm.execute_notebook( - notebook_path, output_notebook_path, kernel_name="forecast_cpu", parameters=dict(N_SPLITS=2), - ) - nb = sb.read_notebook(output_notebook_path) - df = nb.scraps.dataframe - assert df.shape[0] == 1 - mape = df.loc[df.name == "MAPE"]["data"][0] - assert mape == pytest.approx(37.7, abs=ABS_TOL) diff --git a/tools/environment.yml b/tools/environment.yml index efa92ec4..27cd6222 100644 --- a/tools/environment.yml +++ b/tools/environment.yml @@ -3,7 +3,7 @@ # To create the conda environment: # $ conda env create -f environment.yaml -# +# # To update the conda environment: # $ conda env update -f environment.yaml # @@ -16,32 +16,30 @@ channels: - defaults - conda-forge dependencies: - - python=3.6.10 - - pip>=19.0.3 - - jupyter>=1.0.0 - - ipykernel>=4.6.1 - - jupyter_nbextensions_configurator=0.4.1 - - scipy=1.1.0 - - numpy=1.16.2 + - python=3.6 + - pip + - jupyter + - ipykernel + - scipy==1.1.0 + - numpy==1.16.2 - pandas=0.23.4 - xlrd=1.1.0 - urllib3=1.21.1 - scikit-learn=0.20.3 - - pytest>=3.6.4 - - tqdm>=4.43.0 - - pylint + - pytest - papermill>=1.0.1 - matplotlib=3.1.2 - - r-base>=3.3.0 + - r-base + - r-bayesm - pip: - - black>=18.6b4 - - flake8>=3.3.0 - - jupytext>=1.3.0 + - black + - flake8 + - jupytext==1.3.0 - lightgbm==2.3.0 - tensorflow==2.0 - tensorboard==2.1.0 - nteract-scrapbook==0.3.1 + - gitpython==3.0.8 - azureml-sdk[explain,automl]==1.0.85 - statsmodels==0.11.1 - pmdarima==1.1.1 - - gitpython==3.0.8 diff --git a/tools/environment_setup.bat b/tools/environment_setup.bat deleted file mode 100644 index 3eb7a824..00000000 --- a/tools/environment_setup.bat +++ /dev/null @@ -1,24 +0,0 @@ -REM Copyright (c) Microsoft Corporation. -REM Licensed under the MIT License. - -REM Please follow instructions in this link -REM https://docs.conda.io/projects/conda/en/latest/user-guide/install/windows.html -REM to install Miniconda before running this script. - - -echo Update conda -call conda update conda --yes - -echo Create conda environment -call conda env create -f tools/environment.yml - -echo Activate conda environment -call conda activate forecasting_env - -echo Install forecasting utility library -call pip install -e fclib - -echo Register conda environment in Jupyter -call python -m ipykernel install --user --name forecasting_env - -echo Environment setup is done! \ No newline at end of file diff --git a/tools/generate_conda_file.py b/tools/generate_conda_file.py deleted file mode 100644 index fdd7349f..00000000 --- a/tools/generate_conda_file.py +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/python - -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -# This script creates yaml files to build conda environments -# For generating a conda file for running only python code: -# $ python generate_conda_file.py -# -# For generating a conda file for running python gpu: -# $ python generate_conda_file.py --gpu - - -import argparse -import textwrap -from sys import platform - - -HELP_MSG = """ -To create the conda environment: -$ conda env create -f {conda_env}.yaml - -To update the conda environment: -$ conda env update -f {conda_env}.yaml - -To register the conda environment in Jupyter: -$ conda activate {conda_env} -$ python -m ipykernel install --user --name {conda_env} \ ---display-name "Python ({conda_env})" -""" - - -CHANNELS = ["defaults", "conda-forge"] - -CONDA_BASE = { - "python": "python==3.6.10", - "pip": "pip>=19.1.1", - "ipykernel": "ipykernel>=4.6.1", - "jupyter": "jupyter>=1.0.0", - "jupyter_nbextensions_configurator": "jupyter_nbextensions_configurator>=0.4.1", - "numpy": "numpy>=1.16.2", - "pandas": "pandas>=0.23.4", - "pytest": "pytest>=3.6.4", - "scipy": "scipy>=1.1.0", - "xlrd": "xlrd>=1.1.0", - "urllib3": "urllib3>=1.21.1", - "scikit-learn": "scikit-learn>=0.20.3", - "tqdm": "tqdm>=4.43.0", - "pylint": "pylint>=2.4.4", - "matplotlib": "matplotlib>=3.1.2", - "r-base": "r-base>=3.3.0", - "papermill": "papermill>=1.0.1", -} - - -CONDA_GPU = {} - -PIP_BASE = { - "azureml-sdk": "azureml-sdk[explain,automl]==1.0.85", - "black": "black>=18.6b4", - "nteract-scrapbook": "nteract-scrapbook>=0.3.1", - "pre-commit": "pre-commit>=1.14.4", - "tensorboard": "tensorboard==2.1.0", - "tensorflow": "tensorflow==2.0", - "flake8": "flake8>=3.3.0", - "jupytext": "jupytext>=1.3.0", - "lightgbm": "lightgbm==2.3.0", - "statsmodels": "statsmodels==0.11.1", - "pmdarima": "pmdarima==1.1.1", - "gitpython": "gitpython==3.0.8", -} - -PIP_GPU = {} - -PIP_DARWIN = {} -PIP_DARWIN_GPU = {} - -PIP_LINUX = {} -PIP_LINUX_GPU = {} - -PIP_WIN32 = {} -PIP_WIN32_GPU = {} - -CONDA_DARWIN = {} -CONDA_DARWIN_GPU = {} - -CONDA_LINUX = {} -CONDA_LINUX_GPU = {} - -CONDA_WIN32 = {} -CONDA_WIN32_GPU = {} - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description=textwrap.dedent( - """ - This script generates a conda file for different environments. - Plain python is the default, - but flags can be used to support GPU functionality.""" - ), - epilog=HELP_MSG, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument("--name", help="specify name of conda environment") - parser.add_argument("--gpu", action="store_true", help="include packages for GPU support") - args = parser.parse_args() - - # set name for environment and output yaml file - conda_env = "forecasting_cpu" - if args.gpu: - conda_env = "forecasting_gpu" - - # overwrite environment name with user input - if args.name is not None: - conda_env = args.name - - # add conda and pip base packages - conda_packages = CONDA_BASE - pip_packages = PIP_BASE - - # update conda and pip packages based on flags provided - if args.gpu: - conda_packages.update(CONDA_GPU) - pip_packages.update(PIP_GPU) - - # update conda and pip packages based on os platform support - if platform == "darwin": - conda_packages.update(CONDA_DARWIN) - pip_packages.update(PIP_DARWIN) - if args.gpu: - conda_packages.update(CONDA_DARWIN_GPU) - pip_packages.update(PIP_DARWIN_GPU) - elif platform.startswith("linux"): - conda_packages.update(CONDA_LINUX) - pip_packages.update(PIP_LINUX) - if args.gpu: - conda_packages.update(CONDA_LINUX_GPU) - pip_packages.update(PIP_LINUX_GPU) - elif platform == "win32": - conda_packages.update(CONDA_WIN32) - pip_packages.update(PIP_WIN32) - if args.gpu: - conda_packages.update(CONDA_WIN32_GPU) - pip_packages.update(PIP_WIN32_GPU) - else: - raise Exception("Unsupported platform. Must be Windows, Linux, or macOS") - - # write out yaml file - conda_file = "{}.yaml".format(conda_env) - with open(conda_file, "w") as f: - for line in HELP_MSG.format(conda_env=conda_env).split("\n"): - f.write("# {}\n".format(line)) - f.write("name: {}\n".format(conda_env)) - f.write("channels:\n") - for channel in CHANNELS: - f.write("- {}\n".format(channel)) - f.write("dependencies:\n") - for conda_package in conda_packages.values(): - f.write("- {}\n".format(conda_package)) - f.write("- pip:\n") - for pip_package in pip_packages.values(): - f.write(" - {}\n".format(pip_package)) - - print("Generated conda file: {}".format(conda_file)) - print(HELP_MSG.format(conda_env=conda_env)) diff --git a/tools/generate_requirements_txt.py b/tools/generate_requirements_txt.py deleted file mode 100644 index 2c6a6e51..00000000 --- a/tools/generate_requirements_txt.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -# This file outputs a requirements.txt based on the libraries defined in generate_conda_file.py -from generate_conda_file import ( - CONDA_BASE, - CONDA_GPU, - PIP_BASE, - PIP_GPU, - PIP_DARWIN, - PIP_LINUX, - PIP_WIN32, - CONDA_DARWIN, - CONDA_LINUX, - CONDA_WIN32, - PIP_DARWIN_GPU, - PIP_LINUX_GPU, - PIP_WIN32_GPU, - CONDA_DARWIN_GPU, - CONDA_LINUX_GPU, - CONDA_WIN32_GPU, -) - - -if __name__ == "__main__": - deps = list(CONDA_BASE.values()) - deps += list(CONDA_GPU.values()) - deps += list(PIP_BASE.values()) - deps += list(PIP_GPU.values()) - deps += list(PIP_DARWIN.values()) - deps += list(PIP_LINUX.values()) - deps += list(PIP_WIN32.values()) - deps += list(CONDA_DARWIN.values()) - deps += list(CONDA_LINUX.values()) - deps += list(CONDA_WIN32.values()) - deps += list(PIP_DARWIN_GPU.values()) - deps += list(PIP_LINUX_GPU.values()) - deps += list(PIP_WIN32_GPU.values()) - deps += list(CONDA_DARWIN_GPU.values()) - deps += list(CONDA_LINUX_GPU.values()) - deps += list(CONDA_WIN32_GPU.values()) - with open("requirements.txt", "w") as f: - f.write("\n".join(set(deps))) diff --git a/contrib/tsperf/readme_generator/Benchmarks.csv b/tools/readme_generator/Benchmarks.csv similarity index 100% rename from contrib/tsperf/readme_generator/Benchmarks.csv rename to tools/readme_generator/Benchmarks.csv diff --git a/contrib/tsperf/readme_generator/TSPerfBoard-Energy.csv b/tools/readme_generator/TSPerfBoard-Energy.csv similarity index 100% rename from contrib/tsperf/readme_generator/TSPerfBoard-Energy.csv rename to tools/readme_generator/TSPerfBoard-Energy.csv diff --git a/contrib/tsperf/readme_generator/TSPerfBoard-Retail.csv b/tools/readme_generator/TSPerfBoard-Retail.csv similarity index 100% rename from contrib/tsperf/readme_generator/TSPerfBoard-Retail.csv rename to tools/readme_generator/TSPerfBoard-Retail.csv diff --git a/tools/readme_generator/readme_generator.py b/tools/readme_generator/readme_generator.py new file mode 100644 index 00000000..136583d3 --- /dev/null +++ b/tools/readme_generator/readme_generator.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# coding: utf-8 + +import csvtomd +import matplotlib.pyplot as plt +import pandas as pd +import numpy as np + + +### Generating performance charts +################################################# + +#Function to plot a performance chart +def plot_perf(x,y,df): + + # extract submission name from submission URL + labels = df.apply(lambda x: x['Submission Name'][1:].split(']')[0], axis=1) + + fig = plt.scatter(x=df[x],y=df[y], label=labels, s=150, alpha = 0.5, + c= ['b', 'g', 'r', 'c', 'm', 'y', 'k']) + plt.xlabel(x) + plt.ylabel(y) + plt.title(y + ' by ' + x) + offset = (max(df[y]) - min(df[y]))/50 + for i,name in enumerate(labels): + ax = df[x][i] + ay = df[y][i] + offset * (-2.5 + i % 5) + plt.text(ax, ay, name, fontsize=10) + + return(fig) + +### Printing the Readme.md file +############################################ +readmefile = '../../Readme.md' +#Write header +#print(file=open(readmefile)) +print('# TSPerf\n', file=open(readmefile, "w")) + +print('TSPerf is a collection of implementations of time-series forecasting algorithms in Azure cloud and comparison of their performance over benchmark datasets. \ +Algorithm implementations are compared by model accuracy, training and scoring time and cost. Each implementation includes all the necessary \ +instructions and tools that ensure its reproducibility.', file=open(readmefile, "a")) + +print('The following table summarizes benchmarks that are currently included in TSPerf.\n', file=open(readmefile, "a")) + +#Read the benchmark table the CSV file and converrt to a table in md format +with open('Benchmarks.csv', 'r') as f: + table = csvtomd.csv_to_table(f, ',') +print(csvtomd.md_table(table), file=open(readmefile, "a")) +print('\n\n\n',file=open(readmefile, "a")) + +print('A complete documentation of TSPerf, along with the instructions for submitting and reviewing implementations, \ +can be found [here](./docs/tsperf_rules.md). The tables below show performance of implementations that are developed so far. Source code of \ +implementations and instructions for reproducing their performance can be found in submission folders, which are linked in the first column.\n', file=open(readmefile, "a")) + +### Write the Energy section +#============================ + +print('## Probabilistic energy forecasting performance board\n\n', file=open(readmefile, "a")) +print('The following table lists the current submision for the energy forecasting and their respective performances.\n\n', file=open(readmefile, "a")) + +#Read the energy perfromane board from the CSV file and converrt to a table in md format +with open('TSPerfBoard-Energy.csv', 'r') as f: + table = csvtomd.csv_to_table(f, ',') +print(csvtomd.md_table(table), file=open(readmefile, "a")) + +#Read Energy Performance Board CSV file +df = pd.read_csv('TSPerfBoard-Energy.csv', engine='python') +#df + +#Plot ,'Pinball Loss' by 'Training and Scoring Cost($)' chart +fig4 = plt.figure(figsize=(12, 8), dpi= 80, facecolor='w', edgecolor='k') #this sets the plotting area size +fig4 = plot_perf('Training and Scoring Cost($)','Pinball Loss',df) +plt.savefig('../../docs/images/Energy-Cost.png') + + +#insetting the performance charts +print('\n\nThe following chart compares the submissions performance on accuracy in Pinball Loss vs. Training and Scoring cost in $:\n\n ', file=open(readmefile, "a")) +print('![EnergyPBLvsTime](./docs/images/Energy-Cost.png)' ,file=open(readmefile, "a")) +print('\n\n\n',file=open(readmefile, "a")) + + +#print the retail sales forcsating section +#======================================== +print('## Retail sales forecasting performance board\n\n', file=open(readmefile, "a")) +print('The following table lists the current submision for the retail forecasting and their respective performances.\n\n', file=open(readmefile, "a")) + +#Read the energy perfromane board from the CSV file and converrt to a table in md format +with open('TSPerfBoard-Retail.csv', 'r') as f: + table = csvtomd.csv_to_table(f, ',') +print(csvtomd.md_table(table), file=open(readmefile, "a")) +print('\n\n\n',file=open(readmefile, "a")) + +#Read Retail Performane Board CSV file +df = pd.read_csv('TSPerfBoard-Retail.csv', engine='python') +#df + +#Plot MAPE (%) by Training and Scoring Cost ($) chart +fig2 = plt.figure(figsize=(12, 8), dpi= 80, facecolor='w', edgecolor='k') #this sets the plotting area size +fig2 = plot_perf('Training and Scoring Cost ($)','MAPE (%)',df) +plt.savefig('../../docs/images/Retail-Cost.png') + + +#insetting the performance charts +print('\n\nThe following chart compares the submissions performance on accuracy in %MAPE vs. Training and Scoring cost in $:\n\n ', file=open(readmefile, "a")) +print('![EnergyPBLvsTime](./docs/images/Retail-Cost.png)' ,file=open(readmefile, "a")) +print('\n\n\n',file=open(readmefile, "a")) + +#insertting build status badge +print('## Build Status\n\n', file=open(readmefile, "a")) +print('| Build Type | Branch | Status | | Branch | Status |' ,file=open(readmefile, "a")) +print('| --- | --- | --- | --- | --- | --- |' ,file=open(readmefile, "a")) +print('| **Python Linux CPU** | master | [![Build Status](https://dev.azure.com/best-practices/forecasting/_apis/build/status/python_unit_tests_base?branchName=master)](https://dev.azure.com/best-practices/forecasting/_build/latest?definitionId=12&branchName=master) | | staging | [![Build Status](https://dev.azure.com/best-practices/forecasting/_apis/build/status/python_unit_tests_base?branchName=chenhui/python_test_pipeline)](https://dev.azure.com/best-practices/forecasting/_build/latest?definitionId=12&branchName=chenhui/python_test_pipeline) |' ,file=open(readmefile, "a")) +print('| **R Linux CPU** | master | [![Build Status](https://dev.azure.com/best-practices/forecasting/_apis/build/status/Forecasting/r_unit_tests_prototype?branchName=master)](https://dev.azure.com/best-practices/forecasting/_build/latest?definitionId=9&branchName=master) | | staging | [![Build Status](https://dev.azure.com/best-practices/forecasting/_apis/build/status/Forecasting/r_unit_tests_prototype?branchName=zhouf/r_test_pipeline)](https://dev.azure.com/best-practices/forecasting/_build/latest?definitionId=9&branchName=zhouf/r_test_pipeline) |' ,file=open(readmefile, "a")) +print('\n\n\n',file=open(readmefile, "a")) + + +print('A new Readme.md file has been generated successfully.') + + diff --git a/tools/repo_metrics/placeholder.txt b/tools/repo_metrics/placeholder.txt new file mode 100644 index 00000000..b3a42524 --- /dev/null +++ b/tools/repo_metrics/placeholder.txt @@ -0,0 +1 @@ +placeholder \ No newline at end of file