From 3e1d8aa1be00aff567ffad0401a8f914b5350041 Mon Sep 17 00:00:00 2001 From: Sun Mingyang Date: Mon, 20 Jun 2022 12:05:19 +0800 Subject: [PATCH 01/10] Update tutorial inference dockerfile - Upgrade azureml-default version to 1.42.0 Closes: #57 --- provider/tutorial/notebooks/inference.dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/provider/tutorial/notebooks/inference.dockerfile b/provider/tutorial/notebooks/inference.dockerfile index 343b442..3ddbe47 100644 --- a/provider/tutorial/notebooks/inference.dockerfile +++ b/provider/tutorial/notebooks/inference.dockerfile @@ -10,8 +10,8 @@ ENV PATH $AZUREML_CONDA_ENVIRONMENT_PATH/bin:$PATH RUN apt-get install -y gcc RUN apt-get install -y unixodbc-dev -RUN pip install 'azureml-defaults==1.35.0' \ - 'feast-azure-provider==0.3.0' \ - 'scikit-learn==0.22.2.post1' \ - 'joblib===1.1.0' \ - 'itsdangerous==2.0.1' \ No newline at end of file +RUN pip install 'azureml-defaults==1.42.0' \ + 'feast-azure-provider==0.3.0' \ + 'scikit-learn==0.22.2.post1' \ + 'joblib===1.1.0' \ + 'itsdangerous==2.0.1' \ No newline at end of file From 2939f49d6808406e931a91d043042db79f2fa4b1 Mon Sep 17 00:00:00 2001 From: Sun Mingyang Date: Mon, 20 Jun 2022 12:09:10 +0800 Subject: [PATCH 02/10] Update tutorial inference dockerfile - Restrict protobuf version less or equal to 3.20 --- provider/tutorial/notebooks/inference.dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/provider/tutorial/notebooks/inference.dockerfile b/provider/tutorial/notebooks/inference.dockerfile index 3ddbe47..3b33d71 100644 --- a/provider/tutorial/notebooks/inference.dockerfile +++ b/provider/tutorial/notebooks/inference.dockerfile @@ -10,7 +10,9 @@ ENV PATH $AZUREML_CONDA_ENVIRONMENT_PATH/bin:$PATH RUN apt-get install -y gcc RUN apt-get install -y unixodbc-dev -RUN pip install 'azureml-defaults==1.42.0' \ +RUN pip install \ + 'protobuf<=3.20' \ + 'azureml-defaults==1.42.0' \ 'feast-azure-provider==0.3.0' \ 'scikit-learn==0.22.2.post1' \ 'joblib===1.1.0' \ From 202ce2ffd2358d8144060ff3b9629e838b71bdd4 Mon Sep 17 00:00:00 2001 From: Sun Mingyang Date: Mon, 20 Jun 2022 12:17:35 +0800 Subject: [PATCH 03/10] Update get user principal id cmd in azuel cli - Update tutorial readme - Update some descriptions in azuel templates - Clean up formatting in azuel templates --- README.md | 2 +- provider/README.md | 2 +- provider/cloud/fs_sqldb_azuredeploy.json | 9 +++------ provider/cloud/fs_synapse_azuredeploy.json | 20 ++++++++++++-------- provider/tutorial/README.md | 2 +- 5 files changed, 18 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index da9d11d..683eb92 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ The only 2 required parameters during the set-up are: - **Principal ID** this is to set the storage permissions for the feast registry store. You can find the value for this by opening **Cloud Shell** and run the following command: ```bash -az ad signed-in-user show --query objectId -o tsv +az ad signed-in-user show --query id -o tsv ``` ## Feast on Azure Kubernetes Service (AKS) The [Feast on AKS](./cluster/README.md) is a secure enterprise deployment on a customer provisioned AKS cluster in an Azure VNET. This deployment does not expose any public endpoints and leverages Azure Private Link over the Feast core and serving so that the traffic flows over the Microsoft Network. This deployment also supports secure connectivity from Azure Synapse Analytics and Azure Machine Learning via private endpoints. diff --git a/provider/README.md b/provider/README.md index 97f7d19..57a1081 100644 --- a/provider/README.md +++ b/provider/README.md @@ -32,7 +32,7 @@ The only 2 required parameters during the set-up are: - **Principal ID** this is to set the storage permissions for the feast registry store. You can find the value for this by opening **Cloud Shell** and run the following command: ```bash -az ad signed-in-user show --query objectId -o tsv +az ad signed-in-user show --query id -o tsv ``` ## Note diff --git a/provider/cloud/fs_sqldb_azuredeploy.json b/provider/cloud/fs_sqldb_azuredeploy.json index 4a3c253..483a408 100644 --- a/provider/cloud/fs_sqldb_azuredeploy.json +++ b/provider/cloud/fs_sqldb_azuredeploy.json @@ -11,7 +11,7 @@ "principalId": { "type": "string", "metadata": { - "description": "Specifies the principal ID assigned to the role. You can find in cloud shell using 'az ad signed-in-user show --query objectId -o tsv'" + "description": "Specifies the principal ID assigned to the role. You can find in cloud shell using 'az ad signed-in-user show --query id -o tsv'" } }, "administratorLogin": { @@ -287,8 +287,7 @@ "dependsOn": [ "[resourceId('Microsoft.Sql/servers', concat(parameters('sqlServerName')))]" ], - "properties": { - } + "properties": {} } ] }, @@ -337,7 +336,5 @@ ] } ], - "outputs": { - - } + "outputs": {} } \ No newline at end of file diff --git a/provider/cloud/fs_synapse_azuredeploy.json b/provider/cloud/fs_synapse_azuredeploy.json index 097fdec..f7f9936 100644 --- a/provider/cloud/fs_synapse_azuredeploy.json +++ b/provider/cloud/fs_synapse_azuredeploy.json @@ -11,7 +11,7 @@ "principalId": { "type": "string", "metadata": { - "description": "Specifies the principal ID assigned to the role. You can find in cloud shell using 'az ad signed-in-user show --query objectId -o tsv'" + "description": "Specifies the principal ID assigned to the role. You can find in cloud shell using 'az ad signed-in-user show --query id -o tsv'" } }, "sku": { @@ -355,7 +355,9 @@ "apiVersion": "2019-06-01-preview", "name": "allowAll", "location": "[variables('location')]", - "dependsOn": [ "[variables('workspaceName')]" ], + "dependsOn": [ + "[variables('workspaceName')]" + ], "properties": { "startIpAddress": "0.0.0.0", "endIpAddress": "255.255.255.255" @@ -366,7 +368,9 @@ "apiVersion": "2019-06-01-preview", "name": "AllowAllWindowsAzureIps", "location": "[variables('location')]", - "dependsOn": [ "[variables('workspaceName')]" ], + "dependsOn": [ + "[variables('workspaceName')]" + ], "properties": { "startIpAddress": "0.0.0.0", "endIpAddress": "0.0.0.0" @@ -377,7 +381,9 @@ "apiVersion": "2019-06-01-preview", "name": "default", "location": "[variables('location')]", - "dependsOn": [ "[variables('workspaceName')]" ], + "dependsOn": [ + "[variables('workspaceName')]" + ], "properties": { "grantSqlControlToManagedIdentity": { "desiredState": "Enabled" @@ -403,7 +409,5 @@ } } ], - "outputs": { - - } -} + "outputs": {} +} \ No newline at end of file diff --git a/provider/tutorial/README.md b/provider/tutorial/README.md index bcdb5f5..6066f24 100644 --- a/provider/tutorial/README.md +++ b/provider/tutorial/README.md @@ -24,7 +24,7 @@ The only 2 required parameters during the set-up are: - **Principal ID** this is to set the storage permissions for the feast registry store. You can find the value for this by opening **Cloud Shell** and run the following command: ```bash -az ad signed-in-user show --query objectId -o tsv +az ad signed-in-user show --query id -o tsv ``` [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Ffeast-azure%2Fmain%2Fprovider%2Fcloud%2Ffs_synapse_azuredeploy.json) From 744985951cfcf19b43fb82e3aed01989b29d672f Mon Sep 17 00:00:00 2001 From: Sun Mingyang Date: Mon, 20 Jun 2022 12:34:41 +0800 Subject: [PATCH 04/10] Add reminder for checking azure providers in readme --- README.md | 2 ++ provider/README.md | 2 ++ provider/tutorial/README.md | 3 +++ 3 files changed, 7 insertions(+) diff --git a/README.md b/README.md index 683eb92..da47f5d 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ We recommend the [getting started tutorial](provider/tutorial/) that walks you t You can deploy the infrastructure for feast using: +> You may want to first make sure your subscription has registered `Microsoft.Synapse`, `Microsoft.SQL` and `Microsoft.Network` providers before running the template below, as some of them may require explicit registration. + [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Ffeast-azure%2Fmain%2Fprovider%2Fcloud%2Ffs_synapse_azuredeploy.json) The only 2 required parameters during the set-up are: diff --git a/provider/README.md b/provider/README.md index 57a1081..3c82690 100644 --- a/provider/README.md +++ b/provider/README.md @@ -24,6 +24,8 @@ We recommend the [getting started tutorial](./tutorial) that walks you through a You can deploy the infrastructure for feast using: +> You may want to first make sure your subscription has registered `Microsoft.Synapse`, `Microsoft.SQL` and `Microsoft.Network` providers before running the template below, as some of them may require explicit registration. + [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Ffeast-azure%2Fmain%2Fprovider%2Fcloud%2Ffs_synapse_azuredeploy.json) The only 2 required parameters during the set-up are: diff --git a/provider/tutorial/README.md b/provider/tutorial/README.md index 6066f24..989e139 100644 --- a/provider/tutorial/README.md +++ b/provider/tutorial/README.md @@ -26,6 +26,9 @@ The only 2 required parameters during the set-up are: ```bash az ad signed-in-user show --query id -o tsv ``` + +> You may want to first make sure your subscription has registered `Microsoft.Synapse`, `Microsoft.SQL` and `Microsoft.Network` providers before running the template below, as some of them may require explicit registration. + [![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Ffeast-azure%2Fmain%2Fprovider%2Fcloud%2Ffs_synapse_azuredeploy.json) ![feast architecture](media/arch.png) From 0d5338fe5458bd2cdc3fffa8d60991b0fe728004 Mon Sep 17 00:00:00 2001 From: Sun Mingyang Date: Mon, 20 Jun 2022 12:37:16 +0800 Subject: [PATCH 05/10] Add clean-up step in tutorial part 3 --- .../part3-train-and-deploy-with-feast.ipynb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/provider/tutorial/notebooks/part3-train-and-deploy-with-feast.ipynb b/provider/tutorial/notebooks/part3-train-and-deploy-with-feast.ipynb index 5f4d1fb..2525ec3 100644 --- a/provider/tutorial/notebooks/part3-train-and-deploy-with-feast.ipynb +++ b/provider/tutorial/notebooks/part3-train-and-deploy-with-feast.ipynb @@ -347,6 +347,22 @@ "\n", "service.run(input_data=input_payload)" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Clean up service" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "service.delete()" + ] } ], "metadata": { From 88887a7edb3b392b9329243f78a0ebaac776dd16 Mon Sep 17 00:00:00 2001 From: Sun Mingyang Date: Mon, 20 Jun 2022 15:18:37 +0800 Subject: [PATCH 06/10] Add hints to fix training issue in tutorial part 3 --- provider/tutorial/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/provider/tutorial/README.md b/provider/tutorial/README.md index 989e139..ed2bcce 100644 --- a/provider/tutorial/README.md +++ b/provider/tutorial/README.md @@ -72,3 +72,5 @@ In the Azure ML Studio, select *Notebooks* from the left-hand menu and then open In the Azure ML Studio, select *Notebooks* from the left-hand menu and then open the [train and deploy a model using feast notebook](notebooks/part3-train-and-deploy-with-feast.ipynb). Work through this notebook. > __💁Ensure the Jupyter kernel is set to Python 3.8 - AzureML__ +> +> If problems are encountered during model training stage, create a new cell and rexecute `!pip install scikit-learn==0.22.1`. Upon completion, restart the Kernel and start over. \ No newline at end of file From 38ea610ad5587581db13c02d49260ee7978ff00a Mon Sep 17 00:00:00 2001 From: Sun Mingyang Date: Mon, 20 Jun 2022 15:53:48 +0800 Subject: [PATCH 07/10] Fix unit test --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7362a27..c7e65d2 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -37,6 +37,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip + python -m pip install protobuf==3.10 python -m pip install db-dtypes python -m pip install -e feast/sdk/python[ci] - name: Run Feast unit tests From 013cfb15fa205fcead9dfda71b9cfe8a81d29ae4 Mon Sep 17 00:00:00 2001 From: Sun Mingyang Date: Mon, 20 Jun 2022 16:15:20 +0800 Subject: [PATCH 08/10] Update protobuf version in inference dockerfile --- provider/tutorial/notebooks/inference.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider/tutorial/notebooks/inference.dockerfile b/provider/tutorial/notebooks/inference.dockerfile index 3b33d71..3a9f823 100644 --- a/provider/tutorial/notebooks/inference.dockerfile +++ b/provider/tutorial/notebooks/inference.dockerfile @@ -11,7 +11,7 @@ RUN apt-get install -y gcc RUN apt-get install -y unixodbc-dev RUN pip install \ - 'protobuf<=3.20' \ + 'protobuf>=3.10,<3.20' \ 'azureml-defaults==1.42.0' \ 'feast-azure-provider==0.3.0' \ 'scikit-learn==0.22.2.post1' \ From d16314bc127b0cdcfe1661de631f4665c2d6a20f Mon Sep 17 00:00:00 2001 From: Sun Mingyang Date: Mon, 20 Jun 2022 16:25:01 +0800 Subject: [PATCH 09/10] Fix unit test --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c7e65d2..414d02f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -37,7 +37,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install protobuf==3.10 + python -m pip install 'protobuf>=3.10,<3.20' python -m pip install db-dtypes python -m pip install -e feast/sdk/python[ci] - name: Run Feast unit tests From 4057d21a5d87dddea6a37102ccfb289a158aa714 Mon Sep 17 00:00:00 2001 From: Sun Mingyang Date: Wed, 22 Jun 2022 13:47:58 +0800 Subject: [PATCH 10/10] Update commend used to get principal id in tutorial --- README.md | 5 +++++ provider/README.md | 4 ++++ provider/tutorial/README.md | 4 ++++ 3 files changed, 13 insertions(+) diff --git a/README.md b/README.md index da47f5d..9cbd8cc 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,13 @@ The only 2 required parameters during the set-up are: - **Principal ID** this is to set the storage permissions for the feast registry store. You can find the value for this by opening **Cloud Shell** and run the following command: ```bash +# If you are using Azure portal CLI or Azure CLI 2.37.0 or above az ad signed-in-user show --query id -o tsv + +# If you are using Azure CLI below 2.37.0 +az ad signed-in-user show --query objectId -o tsv ``` + ## Feast on Azure Kubernetes Service (AKS) The [Feast on AKS](./cluster/README.md) is a secure enterprise deployment on a customer provisioned AKS cluster in an Azure VNET. This deployment does not expose any public endpoints and leverages Azure Private Link over the Feast core and serving so that the traffic flows over the Microsoft Network. This deployment also supports secure connectivity from Azure Synapse Analytics and Azure Machine Learning via private endpoints. ### 📐 Architecture diff --git a/provider/README.md b/provider/README.md index 3c82690..d5f29b3 100644 --- a/provider/README.md +++ b/provider/README.md @@ -34,7 +34,11 @@ The only 2 required parameters during the set-up are: - **Principal ID** this is to set the storage permissions for the feast registry store. You can find the value for this by opening **Cloud Shell** and run the following command: ```bash +# If you are using Azure portal CLI or Azure CLI 2.37.0 or above az ad signed-in-user show --query id -o tsv + +# If you are using Azure CLI below 2.37.0 +az ad signed-in-user show --query objectId -o tsv ``` ## Note diff --git a/provider/tutorial/README.md b/provider/tutorial/README.md index ed2bcce..8c99bce 100644 --- a/provider/tutorial/README.md +++ b/provider/tutorial/README.md @@ -24,7 +24,11 @@ The only 2 required parameters during the set-up are: - **Principal ID** this is to set the storage permissions for the feast registry store. You can find the value for this by opening **Cloud Shell** and run the following command: ```bash +# If you are using Azure portal CLI or Azure CLI 2.37.0 or above az ad signed-in-user show --query id -o tsv + +# If you are using Azure CLI below 2.37.0 +az ad signed-in-user show --query objectId -o tsv ``` > You may want to first make sure your subscription has registered `Microsoft.Synapse`, `Microsoft.SQL` and `Microsoft.Network` providers before running the template below, as some of them may require explicit registration.