Updated custom-vision-model-v2 to custom-vision-model (#86)
* Updated custom-vision-model-v2 to custom-vision-model * Comments fixes
This commit is contained in:
Sushil Kumar
GitHub
Родитель
5979c1c4be
Коммит
73e85ffd54
|
|
@ -69,7 +69,7 @@ jobs:
|
|||
export POSTGRES_ADMIN_LOGIN_PASS=$({ </dev/urandom LC_ALL=C grep -ao '[A-Za-z0-9]' | head -n$((RANDOM % 8 + 9)); echo ${chars:$((RANDOM % ${#chars})):1}; } | shuf | tr -d '\n')
|
||||
|
||||
# Setup the infrastructure for testing the pipeline
|
||||
DEPLOY_PGSQL=${USE_POSTGRE_SQL} $(System.DefaultWorkingDirectory)/deploy/setup.sh ${ENV_CODE} eastus custom-vision-model-v2 ${ENV_CODE}-environment ${CI_BATCH_ACCOUNT_NAME}
|
||||
DEPLOY_PGSQL=${USE_POSTGRE_SQL} $(System.DefaultWorkingDirectory)/deploy/setup.sh ${ENV_CODE} eastus custom-vision-model ${ENV_CODE}-environment ${CI_BATCH_ACCOUNT_NAME}
|
||||
|
||||
# Grant access on Synapse workspace to resource owner azure account
|
||||
SYNAPSE_WORKSPACE_NAME=$(az synapse workspace list --query "[?tags.workspaceId && tags.workspaceId == 'default'].name" -o tsv -g "${ENV_CODE}-pipeline-rg")
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ Arguments | Required | Type | Sample value
|
|||
----------|-----------|-------|-------
|
||||
environmentCode | yes | string | aoi
|
||||
location | yes | string | westus
|
||||
pipelineName | no | string | Allowed values: custom-vision-model, custom-vision-model-v2
|
||||
pipelineName | no | string | Allowed value: custom-vision-model
|
||||
envTag | no | string | synapse\-\<environmentCode\>
|
||||
preprovisionedBatchAccountName | no | string | aoibatchaccount
|
||||
deployPgsql | no | boolean | false
|
||||
|
|
@ -81,7 +81,7 @@ Script has been written to be executed with minimalistic input, it requires foll
|
|||
- `environmentCode` which serves as the prefix for infrastructure services names. Allows only alpha numeric(no special characters) and must be between 3 and 8 characters.
|
||||
- `location` which suggests which azure region infrastructure is deployed in.
|
||||
- `environmentTag` / `envTag` serves as a simple label / tag to all resources being deployed as part of the bicep template to your subscription.
|
||||
- `pipelineName` refers to the name of the pipeline that is to be package for deployment to your Synapse Workspace. Allowed values are custom-vision-model, custom-vision-model-v2.
|
||||
- `pipelineName` refers to the name of the pipeline that is to be package for deployment to your Synapse Workspace. Allowed value is custom-vision-model.
|
||||
|
||||
## Deployment of Infrastructure using bicep template
|
||||
|
||||
|
|
@ -190,7 +190,7 @@ or
|
|||
Arguments | Required | Type | Sample value
|
||||
----------|-----------|-------|---------------
|
||||
environmentCode | yes | string | aoi
|
||||
pipelineName | yes | string | Allowed Values: custom-vision-model, custom-vision-model-v2
|
||||
pipelineName | no | string | Allowed Value: custom-vision-model
|
||||
batchAccountName | no | string | aoibatchaccount
|
||||
batchAccountRG | no | string | aoibatchaccountrg
|
||||
batchAccountStorageAccountName | no | string | aoibatchstorageaccount
|
||||
|
|
@ -210,7 +210,7 @@ Unzip the contents of the ZIP file generate by running the `package.sh` or `setu
|
|||
A few things to consider prior to integration of Git / GitHub Repository with Synapse Studio:
|
||||
|
||||
* Do not bring the files from [workflow folder](../src/workflow) directly into your repository that you will use to integrate with Synapse Studio. You will need to run `package.sh` or `setup.sh` to replace placeholders in one or more files before checking-in the files to your repository to be used with the Synapse Studio.
|
||||
* You can either create a new repository or use a forked version of the [Azure Orbital Analytics Sample](https://github.com/Azure/Azure-Orbital-Analytics-Samples) repository. If you use a new repository, use the Unzipped contents of the ZIP file to load into your new repository. If you use forked version of the [Azure Orbital Analytics Sample](https://github.com/Azure/Azure-Orbital-Analytics-Samples) repository, overwrite the contents of [Custom vision model v2 workflow folder](../src/workflow/custom-vision-model-v2) or [custom vision model workflow folder](../src/workflow/custom-vision-model) (depending on the pipeline being imported) with the Unzipped contents.
|
||||
* You can either create a new repository or use a forked version of the [Azure Orbital Analytics Sample](https://github.com/Azure/Azure-Orbital-Analytics-Samples) repository. If you use a new repository, use the Unzipped contents of the ZIP file to load into your new repository. If you use forked version of the [Azure Orbital Analytics Sample](https://github.com/Azure/Azure-Orbital-Analytics-Samples) repository, overwrite the contents of [Custom vision model workflow folder](../src/workflow/custom-vision-model) with the Unzipped contents.
|
||||
|
||||
|
||||
To import pipeline into the Synape Studio is through Source Control repository like GitHub or Azure DevOps repository, refer to the document on [Source Control](https://docs.microsoft.com/azure/synapse-analytics/cicd/source-control) to learn about Git Integration for Azure Synapse Analytics and how to setup.
|
||||
|
|
@ -323,7 +323,7 @@ docker push <container-registry-name>.azurecr.io/custom_vision_offline:latest
|
|||
```
|
||||
Update the `algImageName` value in [Specification document](../src/aimodels/custom_vision_object_detection_offline/specs/custom_vision_object_detection.json) to point to the new image location.
|
||||
|
||||
Note: When using a private Container Registry, update `containerSettings` property in your [Custom Vision Object Detection v2](/src/workflow/pipeline/Custom%20Vision%20Object%20Detection%20v2.json) pipeline and add the following sub-property in order to authenticate to Container Registry :
|
||||
Note: When using a private Container Registry, update `containerSettings` property in your [Custom Vision Object Detection](/src/workflow/pipeline/Custom%20Vision%20Object%20Detection.json) pipeline and add the following sub-property in order to authenticate to Container Registry :
|
||||
```json
|
||||
"registry": {
|
||||
"registryServer": "",
|
||||
|
|
@ -332,7 +332,7 @@ Note: When using a private Container Registry, update `containerSettings` proper
|
|||
}
|
||||
```
|
||||
|
||||
The above change will need to be made to the `Custom Vision Model Transform v2` pipeline. Look for activity named `Custom Vision` of type Web activity and update the body property (under Settings tab) for that activity.
|
||||
The above change will need to be made to the `Custom Vision Model Transform` pipeline. Look for activity named `Custom Vision` of type Web activity and update the body property (under Settings tab) for that activity.
|
||||
|
||||
[Specification document](../src/aimodels/custom_vision_object_detection_offline/specs/custom_vision_object_detection.json) and [Configuration file](../src/aimodels/custom_vision_object_detection_offline/config/config.json) required to run the Custom Vision Model.
|
||||
|
||||
|
|
@ -346,55 +346,6 @@ Before starting the pipeline, prepare the storage account in <environmentCode>-d
|
|||
|
||||
- Create a new container for every pipeline run. Make sure the container name does not exceed 8 characters.
|
||||
|
||||
- Under the newly created container, add two folders. One folder named `config` with the following configuration files:
|
||||
|
||||
- [Specification document](../src/aimodels/custom_vision_object_detection_offline/specs/custom_vision_object_detection.json) configuration file that is provided by the AI Model partner.
|
||||
- [Config file](../src/aimodels/custom_vision_object_detection_offline/config/config.json) specific to the AI Model that contains parameters to be passed to the AI Model.
|
||||
- [Config file](../src/transforms/spark-jobs/raster_crop/config/config-aoi.json) for Crop transformation that container the Area of Interest to crop to.
|
||||
- [Config file](../src/transforms/spark-jobs/raster_convert/config/config-img-convert-png.json) for GeoTiff to Png transform.
|
||||
- [Config file](../src/transforms/spark-jobs/pool_geolocation/config/config-pool-geolocation.json) for pool gelocation transform which converts Image coordinates to Geolocation coordinates.
|
||||
|
||||
Another folder named `raw` with sample Geotiff to be processed by the pipeline. You can use this [Geotiff file](https://aoigeospatial.blob.core.windows.net/public/samples/sample_4326.tif) hosted as a sample or any Geotiff with CRS of EPSG 4326.
|
||||
|
||||
When using this sample file, update your [Crop Transform's Config file](../src/transforms/spark-jobs/raster_crop/config/config-aoi.json) with bbox of `[-117.063550, 32.749467, -116.999386, 32.812946]`.
|
||||
|
||||
To run the pipeline, open the Synapse Studio for the Synapse workspace that you have created and follow the below listed steps.
|
||||
|
||||
- Open the `E2E Custom Vision Model Flow` and click on debug button
|
||||
|
||||
- When presented with the parameters, fill out the values. Below table provide the details on that each parameter represents.
|
||||
|
||||
| parameter | description |
|
||||
|--|--|
|
||||
| Prefix | This is the Storage container name created in [Running the pipeline section](#running-the-pipeline) that hosts the Raw data|
|
||||
| StorageAccountName | Name of the Storage Account in <environmentCode>-data-rg resource group that hosts the Raw data |
|
||||
| StorageAccountKey | Access Key of the Storage Account in <environmentCode>-data-rg resource group that hosts the Raw data |
|
||||
| BatchAccountName | Name of the Batch Account in <environmentCode>-orc-rg resource group to run the AI Model |
|
||||
| BatchJobName | Job name within the Batch Account in <environmentCode>-orc-rg resource group that runs the AI Model |
|
||||
| BatchLocation | Location of the Batch Account in <environmentCode>-orc-rg resource group that runs the AI Model |
|
||||
|
||||
- Once the parameters are entered, click ok to submit and kick off the pipeline.
|
||||
|
||||
- Wait for the pipeline to complete.
|
||||
# Reviewing data in Postgres (Custom Vision Model Pipeline)
|
||||
|
||||
- To review the data in Postgres, you can use Azure Data Studio following this [guide](https://docs.microsoft.com/en-us/sql/azure-data-studio/quickstart-postgres?view=sql-server-ver15).
|
||||
- You will need the following information to connect:
|
||||
- The server name: `<environmentCode>-pg-server.postgres.database.azure.com`
|
||||
- user name: `<environmentCode>_admin_user@<environmentCode>-pg-server`
|
||||
- password used at the start of the deployment
|
||||
|
||||
- Example Queries
|
||||
- If you would like to query distances from a given lat, long against data in the db you could use the following query and a third column "dist" will display the distances from the point provided in the query to the data in the location column
|
||||
> SELECT *, ST_Distance(ST_SetSRID(ST_MakePoint(given_long, given_lat), 4326), location) AS dist FROM aioutputmodelschema.cvmodel
|
||||
- More documentation on queries can be found [here](https://postgis.net/workshops/postgis-intro/geography.html)
|
||||
|
||||
# Running the pipeline (Custom Vision Model V2)
|
||||
|
||||
Before starting the pipeline, prepare the storage account in <environmentCode>-data-rg resource group by creating a container for the pipeline run.
|
||||
|
||||
- Create a new container for every pipeline run. Make sure the container name does not exceed 8 characters.
|
||||
|
||||
- Under the newly created container, add two folders. One folder named `config` with the following configuration files:
|
||||
|
||||
- [Specification document](../src/aimodels/custom_vision_object_detection_offline/specs/custom_vision_object_detection.json) configuration file that is provided by the AI Model partner.
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@ fi
|
|||
|
||||
base_dir="$(cd "$(dirname "$0")/../../" && pwd)"
|
||||
|
||||
# Uncompress the custom vision model v2 package
|
||||
if ! unzip "${base_dir}"/custom-vision-model-v2.zip -d "${base_dir}"/custom-vision-model-v2-"${ENV_CODE}";
|
||||
# Uncompress the custom vision model package
|
||||
if ! unzip "${base_dir}"/custom-vision-model.zip -d "${base_dir}"/custom-vision-model-"${ENV_CODE}";
|
||||
then
|
||||
echo "Unzip failed please check .."
|
||||
exit 1
|
||||
|
|
@ -28,7 +28,7 @@ else
|
|||
echo "Unzip completed successfully"
|
||||
fi
|
||||
|
||||
pkg_dir=${base_dir}/custom-vision-model-v2-${ENV_CODE}
|
||||
pkg_dir=${base_dir}/custom-vision-model-${ENV_CODE}
|
||||
|
||||
# The following one's are created on the Synapse workspace to run
|
||||
# the pipeline template in Synapse gallery.
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
If you've landed on this page from Synapse gallery then Congrats! -- you are in the right place.
|
||||
|
||||
The instructions on this page will guide you through configuring the pipeline template to successfully run a custom vision model for object detection (for example, detecting swimming pool objects). You can also use the pipeline template to explore other object detection use cases as well. Currently, the template only supports custom vision model v2.
|
||||
The instructions on this page will guide you through configuring the pipeline template to successfully run a custom vision model for object detection (for example, detecting swimming pool objects). You can also use the pipeline template to explore other object detection use cases as well.
|
||||
|
||||
**Prerequisites**
|
||||
|
||||
* Follow the steps in the [readme](https://github.com/Azure/Azure-Orbital-Analytics-Samples/blob/main/deploy/README.md) to deploy the infrastructure. This includes creation of the custom vision model v2 package as well.
|
||||
* Follow the steps in the [readme](https://github.com/Azure/Azure-Orbital-Analytics-Samples/blob/main/deploy/README.md) to deploy the infrastructure. This includes creation of the custom vision model package as well.
|
||||
|
||||
* Run the following command to create the linked services and spark job definition on the Synapse workspace:
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
PRJ_ROOT="$(cd `dirname "${BASH_SOURCE}"`/..; pwd)"
|
||||
|
||||
ENV_CODE=${1:-$ENV_CODE}
|
||||
PIPELINE_NAME=${2:-$PIPELINE_NAME}
|
||||
PIPELINE_NAME=${2:-${PIPELINE_NAME:-"custom-vision-model"}}
|
||||
|
||||
AI_MODEL_INFRA_TYPE=${3:-${AI_MODEL_INFRA_TYPE:-"batch-account"}} # Currently supported values are aks and batch-account
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
|
||||
ENV_CODE=${1:-${ENV_CODE}}
|
||||
LOCATION=${2:-${LOCATION}}
|
||||
PIPELINE_NAME=${3:-${PIPELINE_NAME}}
|
||||
PIPELINE_NAME=${3:-${PIPELINE_NAME:-"custom-vision-model"}}
|
||||
ENV_TAG=${4:-${ENV_TAG}}
|
||||
PRE_PROVISIONED_AI_MODEL_INFRA_NAME=${5:-$PRE_PROVISIONED_AI_MODEL_INFRA_NAME}
|
||||
AI_MODEL_INFRA_TYPE=${6:-${AI_MODEL_INFRA_TYPE:-"batch-account"}} # Currently supported values are aks and batch-account
|
||||
|
|
|
|||
|
|
@ -31,16 +31,16 @@ No | Folder | Purpose
|
|||
|
||||
Two variants of the pipeline are available.
|
||||
|
||||
a. First variant of the [pipeline](../src/workflow/custom-vision-model-v2/pipeline/E2E%20Custom%20Vision%20Model%20Flow.json) uses [Azure Batch Account]() to run AI Model as a container to process the raster data and provide the output.
|
||||
b. Second variant of the [pipeline](../src/workflow/custom-vision-model-v2/pipeline/E2E%20Custom%20Vision%20Model%20Flow_aks.json) uses [Azure Kubernetes Service (AKS)]() to run AI Model as a container to process the raster data and provide the output.
|
||||
a. First variant of the [pipeline](../src/workflow/custom-vision-model/pipeline/E2E%20Custom%20Vision%20Model%20Flow.json) uses [Azure Batch Account]() to run AI Model as a container to process the raster data and provide the output.
|
||||
b. Second variant of the [pipeline](../src/workflow/custom-vision-model/pipeline/E2E%20Custom%20Vision%20Model%20Flow_aks.json) uses [Azure Kubernetes Service (AKS)]() to run AI Model as a container to process the raster data and provide the output.
|
||||
|
||||
You have the option to pick which Azure service will be used to run the AI Model at the time of deployment. Refer to the [README.md](../deploy/README.md) document to understand the deployment steps and options available during deployment.
|
||||
|
||||
The main pipeline contains two steps.
|
||||
|
||||
- It starts by invoking the [Custom Vision Object Transform v2](../src/workflow/custom-vision-model-v2/pipeline/Custom%20Vision%20Model%20Transforms%20v2.json) pipeline to transform your data before it can be consumed by the AI Model.
|
||||
- It starts by invoking the [Custom Vision Object Transform](../src/workflow/custom-vision-model/pipeline/Custom%20Vision%20Model%20Transforms.json) pipeline to transform your data before it can be consumed by the AI Model.
|
||||
|
||||
- Next, the [Custom Vision Model Transform v2](../src/workflow/custom-vision-model-v2/pipeline/Custom%20Vision%20Object%20Detection%20v2.json) or [Custom Vision Model Transform v2_aks](../src/workflow/custom-vision-model-v2/pipeline/Custom%20Vision%20Object%20Detection%20v2_aks.json) pipeline is triggered to process the data through the AI Model running as container in either Azure Batch Account or Azure Kubernetes Services respectively.
|
||||
- Next, the [Custom Vision Model Transform](../src/workflow/custom-vision-model/pipeline/Custom%20Vision%20Object%20Detection.json) or [Custom Vision Model Transform aks](../src/workflow/custom-vision-model/pipeline/Custom%20Vision%20Object%20Detection%20aks.json) pipeline is triggered to process the data through the AI Model running as container in either Azure Batch Account or Azure Kubernetes Services respectively.
|
||||
|
||||
For Custom Vision model, the output is a set of JSON files that contains the location of the swimming pools detected in the image passed to the AI Model as 512 by 512 images.
|
||||
|
||||
|
|
|
|||
|
|
@ -2,14 +2,14 @@
|
|||
"modes": {
|
||||
"batch-account" : {
|
||||
"exclude" : [
|
||||
"../pipeline/Custom Vision Object Detection v2_aks.json",
|
||||
"../pipeline/Custom Vision Object Detection aks.json",
|
||||
"../pipeline/E2E Custom Vision Model Flow_aks.json",
|
||||
"../linkedService/GenBase64Encoding.json"
|
||||
]
|
||||
},
|
||||
"aks": {
|
||||
"exclude" : [
|
||||
"../pipeline/Custom Vision Object Detection v2.json",
|
||||
"../pipeline/Custom Vision Object Detection.json",
|
||||
"../pipeline/E2E Custom Vision Model Flow.json"
|
||||
],
|
||||
"rename" : {
|
||||
|
|
@ -24,7 +24,7 @@
|
|||
"sparkConfigProperties": {
|
||||
"configurationType": "File",
|
||||
"filename": "config.txt",
|
||||
"content": "spark.storage.synapse.linkedServiceName \"AOI Geospatial v2\"\rfs.azure.account.oauth.provider.type com.microsoft.azure.synapse.tokenlibrary.LinkedServiceBasedTokenProvider",
|
||||
"content": "spark.storage.synapse.linkedServiceName \"AOI Geospatial\"\rfs.azure.account.oauth.provider.type com.microsoft.azure.synapse.tokenlibrary.LinkedServiceBasedTokenProvider",
|
||||
"time": "2022-02-22T00:52:46.8995063Z"
|
||||
},
|
||||
"sessionLevelPackagesEnabled": true,
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
"name": "gld",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AOI Geospatial v2",
|
||||
"referenceName": "AOI Geospatial",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
"name": "gls",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AOI Geospatial v2 FS",
|
||||
"referenceName": "AOI Geospatial FS",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"annotations": [],
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
"name": "rawtifs",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AOI Data Storage Account v2",
|
||||
"referenceName": "AOI Data Storage Account",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
"name": "run_container",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AOI Geospatial v2",
|
||||
"referenceName": "AOI Geospatial",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
"name": "run_fileshare",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AOI Geospatial v2 FS",
|
||||
"referenceName": "AOI Geospatial FS",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
"name": "spec_doc_specification",
|
||||
"properties": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AOI Data Storage Account v2",
|
||||
"referenceName": "AOI Data Storage Account",
|
||||
"type": "LinkedServiceReference"
|
||||
},
|
||||
"parameters": {
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "AOI Data Storage Account v2",
|
||||
"name": "AOI Data Storage Account",
|
||||
"properties": {
|
||||
"annotations": [],
|
||||
"type": "AzureBlobStorage",
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "AOI Geospatial v2 FS",
|
||||
"name": "AOI Geospatial FS",
|
||||
"properties": {
|
||||
"annotations": [],
|
||||
"type": "AzureFileStorage",
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "AOI Geospatial v2",
|
||||
"name": "AOI Geospatial",
|
||||
"type": "Microsoft.Synapse/workspaces/linkedservices",
|
||||
"properties": {
|
||||
"annotations": [],
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "Custom Vision Model Transforms v2",
|
||||
"name": "Custom Vision Model Transforms",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "Custom Vision Object Detection v2_aks",
|
||||
"name": "Custom Vision Object Detection aks",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -611,7 +611,7 @@
|
|||
},
|
||||
"logStorageSettings": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AOI Geospatial v2",
|
||||
"referenceName": "AOI Geospatial",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
},
|
||||
|
|
@ -656,7 +656,7 @@
|
|||
},
|
||||
"logStorageSettings": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AOI Geospatial v2",
|
||||
"referenceName": "AOI Geospatial",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
},
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"name": "Custom Vision Object Detection v2",
|
||||
"name": "Custom Vision Object Detection",
|
||||
"properties": {
|
||||
"activities": [
|
||||
{
|
||||
|
|
@ -783,7 +783,7 @@
|
|||
},
|
||||
"logStorageSettings": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AOI Geospatial v2",
|
||||
"referenceName": "AOI Geospatial",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
},
|
||||
|
|
@ -828,7 +828,7 @@
|
|||
},
|
||||
"logStorageSettings": {
|
||||
"linkedServiceName": {
|
||||
"referenceName": "AOI Geospatial v2",
|
||||
"referenceName": "AOI Geospatial",
|
||||
"type": "LinkedServiceReference"
|
||||
}
|
||||
},
|
||||
|
|
@ -9,7 +9,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "Custom Vision Model Transforms v2",
|
||||
"referenceName": "Custom Vision Model Transforms",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -43,7 +43,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "Custom Vision Object Detection v2",
|
||||
"referenceName": "Custom Vision Object Detection",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -9,7 +9,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "Custom Vision Model Transforms v2",
|
||||
"referenceName": "Custom Vision Model Transforms",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -43,7 +43,7 @@
|
|||
"userProperties": [],
|
||||
"typeProperties": {
|
||||
"pipeline": {
|
||||
"referenceName": "Custom Vision Object Detection v2_aks",
|
||||
"referenceName": "Custom Vision Object Detection aks",
|
||||
"type": "PipelineReference"
|
||||
},
|
||||
"waitOnCompletion": true,
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"Prefix":"$RAW_DATA_STORAGE_ACCOUNT_CONTAINER_NAME",
|
||||
"StorageAccountName":"$RAW_DATA_STORAGE_ACCOUNT_NAME",
|
||||
"StorageAccountKey":"$RAW_DATA_STORAGE_ACCOUNT_KEY",
|
||||
"AOI":"$AOI",
|
||||
"BatchAccountName":"$BATCH_ACCOUNT_NAME",
|
||||
"BatchJobName":"$BATCH_JOB_NAME",
|
||||
"BatchLocation":"$BATCH_ACCOUNT_LOCATION"
|
||||
|
|
|
|||
|
|
@ -1,8 +0,0 @@
|
|||
{
|
||||
"Prefix":"$RAW_DATA_STORAGE_ACCOUNT_CONTAINER_NAME",
|
||||
"StorageAccountName":"$RAW_DATA_STORAGE_ACCOUNT_NAME",
|
||||
"AOI":"$AOI",
|
||||
"BatchAccountName":"$BATCH_ACCOUNT_NAME",
|
||||
"BatchJobName":"$BATCH_JOB_NAME",
|
||||
"BatchLocation":"$BATCH_ACCOUNT_LOCATION"
|
||||
}
|
||||
|
|
@ -8,7 +8,7 @@ PRJ_ROOT="$(cd `dirname "${BASH_SOURCE}"`/..; pwd)"
|
|||
ENV_CODE=${1:-${ENV_CODE}}
|
||||
|
||||
BATCH_ACCOUNT_NAME=${2:-${BATCH_ACCOUNT_NAME:-"pipelinecibatch"}}
|
||||
PIPELINE_NAME=${3:-${PIPELINE_NAME:-"custom-vision-model-v2"}}
|
||||
PIPELINE_NAME=${3:-${PIPELINE_NAME:-"custom-vision-model"}}
|
||||
|
||||
DESTINATION_CONTAINER_NAME=${4:-${DESTINATION_CONTAINER_NAME:-"${PIPELINE_NAME}-${ENV_CODE}"}}
|
||||
DESTINATION_STORAGE_ACCOUNT_NAME=${5:-${DESTINATION_STORAGE_ACCOUNT_NAME}}
|
||||
|
|
@ -49,7 +49,6 @@ fi
|
|||
echo "Creating container ${DESTINATION_CONTAINER_NAME} on storage account ${DESTINATION_STORAGE_ACCOUNT_NAME}"
|
||||
az storage container create --name ${DESTINATION_CONTAINER_NAME} --account-name ${DESTINATION_STORAGE_ACCOUNT_NAME} --account-key ${STORAGE_ACCOUNT_KEY}
|
||||
|
||||
|
||||
# Upload test data to destination storage account container
|
||||
echo "Copying resources to storage container ${DESTINATION_CONTAINER_NAME} on storage account ${DESTINATION_STORAGE_ACCOUNT_NAME}"
|
||||
az storage blob copy start \
|
||||
|
|
@ -82,38 +81,6 @@ az storage blob copy start \
|
|||
--source-account-key ${SOURCE_STORAGE_ACCOUNT_KEY} \
|
||||
--source-blob config/custom_vision_object_detection.json
|
||||
|
||||
if [[ "${PIPELINE_NAME}" == "custom-vision-model" ]]; then
|
||||
az storage blob copy start \
|
||||
--destination-blob config/config-aoi.json \
|
||||
--destination-container ${DESTINATION_CONTAINER_NAME} \
|
||||
--account-name ${DESTINATION_STORAGE_ACCOUNT_NAME} \
|
||||
--account-key ${STORAGE_ACCOUNT_KEY} \
|
||||
--source-container ${SOURCE_CONTAINER_NAME} \
|
||||
--source-account-name ${SOURCE_STORAGE_ACCOUNT_NAME} \
|
||||
--source-account-key ${SOURCE_STORAGE_ACCOUNT_KEY} \
|
||||
--source-blob config/config-aoi.json
|
||||
|
||||
az storage blob copy start \
|
||||
--destination-blob config/config-img-convert-png.json \
|
||||
--destination-container ${DESTINATION_CONTAINER_NAME} \
|
||||
--account-name ${DESTINATION_STORAGE_ACCOUNT_NAME} \
|
||||
--account-key ${STORAGE_ACCOUNT_KEY} \
|
||||
--source-container ${SOURCE_CONTAINER_NAME} \
|
||||
--source-account-name ${SOURCE_STORAGE_ACCOUNT_NAME} \
|
||||
--source-account-key ${SOURCE_STORAGE_ACCOUNT_KEY} \
|
||||
--source-blob config/config-img-convert-png.json
|
||||
|
||||
az storage blob copy start \
|
||||
--destination-blob config/config-pool-geolocation.json \
|
||||
--destination-container ${DESTINATION_CONTAINER_NAME} \
|
||||
--account-name ${DESTINATION_STORAGE_ACCOUNT_NAME} \
|
||||
--account-key ${STORAGE_ACCOUNT_KEY} \
|
||||
--source-container ${SOURCE_CONTAINER_NAME} \
|
||||
--source-account-name ${SOURCE_STORAGE_ACCOUNT_NAME} \
|
||||
--source-account-key ${SOURCE_STORAGE_ACCOUNT_KEY} \
|
||||
--source-blob config/config-pool-geolocation.json
|
||||
fi
|
||||
|
||||
# Install Pipeline
|
||||
echo "Preparing commands to install pipeline components"
|
||||
mkdir /tmp/packaged-pipeline
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ PRJ_ROOT="$(cd `dirname "${BASH_SOURCE}"`/..; pwd)"
|
|||
ENV_CODE=${1:-${ENV_CODE}}
|
||||
DESTINATION_BATCH_ACCOUNT_NAME=${2:-${DESTINATION_BATCH_ACCOUNT_NAME}}
|
||||
|
||||
PIPELINE_NAME=${3:-${PIPELINE_NAME:-"custom-vision-model-v2"}}
|
||||
PIPELINE_NAME=${3:-${PIPELINE_NAME:-"custom-vision-model"}}
|
||||
DESTINATION_BATCH_ACCOUNT_ROLE=${4:-${DESTINATION_BATCH_ACCOUNT_ROLE:-"Contributor"}}
|
||||
DESTINATION_BATCH_ACCOUNT_POOL_NAME=${5:-${DESTINATION_BATCH_ACCOUNT_POOL_NAME:-"${ENV_CODE}-data-cpu-pool"}}
|
||||
DESTINATION_BATCH_ACCOUNT_STORAGE_ACCOUNT_NAME=${6:-${DESTINATION_BATCH_ACCOUNT_STORAGE_ACCOUNT_NAME}}
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче