Updated All Docs (#46)
* Updated All Docs * Update to data product from integration * * updated uris * updated product visio
This commit is contained in:
Marvin Buss
GitHub
Родитель
12ef4b7d52
Коммит
5f735c5f70
|
|
@ -1,4 +1,4 @@
|
|||
name: Data Integration Deployment
|
||||
name: Data Product Deployment
|
||||
|
||||
trigger:
|
||||
branches:
|
||||
|
|
@ -8,7 +8,7 @@ trigger:
|
|||
include:
|
||||
- code/*
|
||||
- infra/*
|
||||
- .ado/workflows/dataIntegrationDeployment.yml
|
||||
- .ado/workflows/dataProductDeployment.yml
|
||||
pr:
|
||||
branches:
|
||||
include:
|
||||
|
|
@ -17,7 +17,7 @@ pr:
|
|||
include:
|
||||
- code/*
|
||||
- infra/*
|
||||
- .ado/workflows/dataIntegrationDeployment.yml
|
||||
- .ado/workflows/dataProductDeployment.yml
|
||||
|
||||
variables:
|
||||
AZURE_RESOURCE_MANAGER_CONNECTION_NAME: "integration-product-service-connection" # Update to '{resourceManagerConnectionName}'
|
||||
|
|
@ -60,10 +60,10 @@ stages:
|
|||
ignoreLASTEXITCODE: false
|
||||
pwsh: true
|
||||
|
||||
# Deploy Data Integration - validation
|
||||
# Deploy Data Product - validation
|
||||
- task: AzureResourceManagerTemplateDeployment@3
|
||||
name: data_integration_validation
|
||||
displayName: Deploy Data Integration - validation
|
||||
name: data_product_validation
|
||||
displayName: Deploy Data Product - validation
|
||||
enabled: true
|
||||
continueOnError: false
|
||||
inputs:
|
||||
|
|
@ -117,10 +117,10 @@ stages:
|
|||
ignoreLASTEXITCODE: false
|
||||
pwsh: true
|
||||
|
||||
# Deploy Data Integration
|
||||
# Deploy Data Product
|
||||
- task: AzureResourceManagerTemplateDeployment@3
|
||||
name: data_integration_deployment
|
||||
displayName: Deploy Data Integration
|
||||
name: data_product_deployment
|
||||
displayName: Deploy Data Product
|
||||
enabled: true
|
||||
continueOnError: false
|
||||
inputs:
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
name: Data Integration Deployment
|
||||
name: Data Product Deployment
|
||||
|
||||
on:
|
||||
push:
|
||||
|
|
@ -6,13 +6,13 @@ on:
|
|||
paths:
|
||||
- "code/**"
|
||||
- "infra/**"
|
||||
- ".github/workflows/dataIntegrationDeployment.yml"
|
||||
- ".github/workflows/dataProductDeployment.yml"
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
paths:
|
||||
- "code/**"
|
||||
- "infra/**"
|
||||
- ".github/workflows/dataIntegrationDeployment.yml"
|
||||
- ".github/workflows/dataProductDeployment.yml"
|
||||
|
||||
env:
|
||||
AZURE_SUBSCRIPTION_ID: "2150d511-458f-43b9-8691-6819ba2e6c7b" # Update to '{dataLandingZoneSubscriptionId}'
|
||||
|
|
@ -45,9 +45,9 @@ jobs:
|
|||
echo "Generating Password"
|
||||
pwsh $GITHUB_WORKSPACE/code/GeneratePassword.ps1 -GitHub
|
||||
|
||||
# Deploy Data Integration - validation
|
||||
- name: Deploy Data Integration - validation
|
||||
id: data_integration_validation
|
||||
# Deploy Data Product - validation
|
||||
- name: Deploy Data Product - validation
|
||||
id: data_product_validation
|
||||
uses: azure/arm-deploy@v1
|
||||
with:
|
||||
scope: resourcegroup
|
||||
|
|
@ -95,9 +95,9 @@ jobs:
|
|||
echo "Generating Password"
|
||||
pwsh $GITHUB_WORKSPACE/code/GeneratePassword.ps1 -GitHub
|
||||
|
||||
# Deploy Data Integration
|
||||
- name: Deploy Data Integration
|
||||
id: data_integration_deployment
|
||||
# Deploy Data Product
|
||||
- name: Deploy Data Product
|
||||
id: data_product_deployment
|
||||
uses: azure/arm-deploy@v1
|
||||
with:
|
||||
scope: resourcegroup
|
||||
417
README.md
417
README.md
|
|
@ -1,405 +1,52 @@
|
|||
# Enterprise Scale Analytics and AI - Data Integration: Stream Processing
|
||||
# Enterprise Scale Analytics - Data Product Streaming
|
||||
|
||||
> **General disclaimer** Please be aware that this template is in private preview. Therefore, expect smaller bugs and issues when working with the solution. Please submit an Issue in GitHub if you come across any issues that you would like us to fix.
|
||||
## Objective
|
||||
|
||||
**DO NOT COPY - UNDER DEVELOPMENT - MS INTERNAL ONLY - Please be aware that this template is in private preview without any SLA.**
|
||||
The [Enterprise-Scale Analytics](https://aka.ms/adopt/datamanagement) architecture provides a prescriptive data platform design coupled with Azure best practices and design principles. These principles serve as a compass for subsequent design decisions across critical technical domains. The architecture will continue to evolve alongside the Azure platform and is ultimately driven by the various design decisions that organizations must make to define their Azure data journey.
|
||||
|
||||
## Description
|
||||
The Enterprise-Scale Analytics architecture consists of two core building blocks:
|
||||
|
||||
[**Enterprise Scale Analytics and AI**](https://github.com/Azure/Enterprise-Scale-Analytics) solution pattern emphasizes self-service and follows the concept of creating landing zones for cross-functional teams. Operation and responsibility of these landing zones is handed over to the responsible teams inside the data node. The teams are free to deploy their own services within the guardrails set by Azure Policy. To scale across the landing zones more quickly and allow a shorter time to market, we use the concept of `Data Integration` and `Data Product` templates. Data Integration and Data Product templates are blueprints, which can be used to quickly spin up environments for these cross-functional teams. The teams can fork these repositories to quickly spin up environments based on their requirements. This Data Integration template deploys a set of services, which can be used for data stream processing. The template includes a set of different services for processing data streams, which allows the teams to choose their tools based on their requirements and preferences.
|
||||
1. *Data Management Zone* which provides all data management and data governance capabilities for the data platform of an organization.
|
||||
1. *Data Landing Zone* which is a logical construct and a unit of scale in the Enterprise-Scale Analytics architecture that enables data retention and execution of data workloads for generating insights and value with data.
|
||||
|
||||
## What will be deployed?
|
||||
The architecture is modular by design and allows organizations to start small with a single Data Management Zone and Data Landing Zone, but also allows to scale to a multi-subscription data platform environment by adding more Data Landing Zones to the architecture. Thereby, the reference design allows to implement different modern data platform patterns like data-mesh, data-fabric as well as traditional datalake architectures. Enterprise-Scale Analytics has been very well aligned with the data-mesh approach, and is ideally suited to help organizations build data products and share these across business units of an organization. If core recommendations are followed, the resulting target architecture will put the customer on a path to sustainable scale.
|
||||
|
||||
By default, all the services which come under Data Integration Streaming are enabled, and you must explicitly disable services that you don't want to be deployed.
|
||||
|
||||
|
||||
> **Note:** Before deploying the resources, we recommend to check registration status of the required resource providers in your subscription. For more information, see [Resource providers for Azure services](https://docs.microsoft.com/azure/azure-resource-manager/management/resource-providers-and-types).
|
||||
---
|
||||
|
||||
|
||||
_The Enterprise-Scale Analytics architecture represents the strategic design path and target technical state for your Azure data platform._
|
||||
|
||||
For each Data Integration Streaming template, the following services are created:
|
||||
---
|
||||
|
||||
- [Key Vault](https://docs.microsoft.com/azure/key-vault/general)
|
||||
- [Event Hub](https://docs.microsoft.com/azure/event-hubs/)
|
||||
- [IoT Hub](https://docs.microsoft.com/azure/iot-hub/about-iot-hub)
|
||||
- [Stream Analytics](https://docs.microsoft.com/azure/stream-analytics/stream-analytics-introduction)
|
||||
- [Cosmos DB](https://docs.microsoft.com/azure/cosmos-db/introduction)
|
||||
- [Synapse Workspace](https://docs.microsoft.com/azure/synapse-analytics/)
|
||||
- [Azure SQL Database](https://docs.microsoft.com/azure/azure-sql/database/)
|
||||
- [SQL Pool](https://docs.microsoft.com/azure/synapse-analytics/sql-data-warehouse/sql-data-warehouse-overview-what-is)
|
||||
- [SQL Server](https://docs.microsoft.com/sql/sql-server/?view=sql-server-ver15)
|
||||
- [SQL Elastic Pool](https://docs.microsoft.com/azure/azure-sql/database/elastic-pool-overview)
|
||||
- [BigData Pool](https://docs.microsoft.com/sql/big-data-cluster/concept-data-pool?view=sql-server-ver15)
|
||||
This respository describes a Data Product template for Data Streaming that can also be used for integrating streaming data into the Azure data platform. Data Products are another unit of scale inside a Data Landing Zone through the means of Resource Groups. Resource Groups inside the Data Landing Zone subscription are created and handed over to cross-functional teams to provide them an environment in which they can work on their own data use-cases. The ownership of this resource group and operation of services within is handed over to the Data Product teams. In order to enable self-service, the owning teams are free to deploy their own services within the guardrails set by Azure Policy. Repository templates can be used for these teams to more quickly scale within an organization and rollout common data analysis patterns not just once but multiple times across various use-cases. The ownership of templates is also handed over, which ultimately gives these teams a starting point while allowing them to enhance the template based on their specific requirements. This Data Product template deploys a set of services, which can be used for real-time data processing and integration. The template includes services such as EventHub, IoTHub, Stream Analytics and Azure Synapse. The Data Product teams can then leverage these tools to generate insights and value with data.
|
||||
|
||||
You have two options for deploying this reference architecture:
|
||||
> **Note:** Before getting started with the deployment, please make sure you are familiar with the [complementary documentation in the Cloud Adoption Framework](https://aka.ms/adopt/datamanagement). Also, before deploying your first Data Product, please make sure that you have deployed a [Data Management Zone](https://github.com/Azure/data-management-zone) and at least one [Data Landing Zone](https://github.com/Azure/data-landing-zone). The minimal recommended setup consists of a single [Data Management Zone](https://github.com/Azure/data-management-zone) and a single [Data Landing Zone](https://github.com/Azure/data-landing-zone).
|
||||
|
||||
1. Use the `Deploy to Azure` button for an immediate deployment
|
||||
2. Use GitHub Actions or Azure DevOps Pipelines for an automated, repeatable deployment
|
||||
## Deploy Enterprise-Scale Analytics
|
||||
|
||||
## Prerequisites
|
||||
The Enterprise-Scale Analytics architecture is modular by design and allows customers to start with a small footprint and grow over time. In order to not end up in a migration project, customers should decide upfront how they want to organize data domains across Data Landing Zones. All Enterprise-Scale Analytics architecture building blocks can be deployed through the Azure Portal as well as through GitHub Actions workflows and Azure DevOps Pipelines. The template repositories contain sample YAML pipelines to more quickly get started with the setup of the environments.
|
||||
|
||||
> **Note:** Please make sure you have successfully deployed a [Data Management Landing Zone](https://github.com/Azure/data-management-zone) and a [Data Landing Zone](https://github.com/Azure/data-landing-zone). The Data Integration relies on the Private DNS Zones that are deployed in the Data Management Template. If you have Private DNS Zones deployed elsewhere, you can also point to these. If you do not have the Private DNS Zones deployed for the respective services, this template deployment will fail. Also, this template requires subnets as specified in the prerequisites. The Data Landing Zone already creates a few subnets, which can be used for this Data Integration.
|
||||
| Reference implementation | Description | Deploy to Azure | Link |
|
||||
|:---------------------------|:------------|:----------------|------|
|
||||
| Enterprise-Scale Analytics | Deploys a [Data Management Zone](https://github.com/Azure/data-management-zone) and one or multiple Data Landing Zones all at once. Provides less options than the the individual Data Management Zone and Data Landing Zone deployment options. Helps you to quickly get started and make yourself familiar with the reference design. For more advanced scenarios, please deploy the artifacts individually. |[](https://portal.azure.com/#blade/Microsoft_Azure_CreateUIDef/CustomDeploymentBlade/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Fdata-management-zone%2Fmain%2Fdocs%2Freference%2FenterpriseScaleAnalytics.json/uiFormDefinitionUri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Fdata-management-zone%2Fmain%2Fdocs%2Freference%2Fportal.enterpriseScaleAnalytics.json) | |
|
||||
| Data Management Zone | Deploys a single Data Management Zone to a subscription. |[](https://portal.azure.com/#blade/Microsoft_Azure_CreateUIDef/CustomDeploymentBlade/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Fdata-management-zone%2Fmain%2Finfra%2Fmain.json/uiFormDefinitionUri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Fdata-management-zone%2Fmain%2Fdocs%2Freference%2Fportal.dataManagementZone.json) | [Repository](https://github.com/Azure/data-management-zone) |
|
||||
| Data Landing Zone | Deploys a single Data Landing Zone to a subscription. Please deploy a [Data Management Zone](https://github.com/Azure/data-management-zone) first. |[](https://portal.azure.com/#blade/Microsoft_Azure_CreateUIDef/CustomDeploymentBlade/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Fdata-landing-zone%2Fmain%2Finfra%2Fmain.json/uiFormDefinitionUri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Fdata-landing-zone%2Fmain%2Fdocs%2Freference%2Fportal.dataLandingZone.json) | [Repository](https://github.com/Azure/data-landing-zone) |
|
||||
| Data Product Batch | Deploys a Data Workload template for Data Batch Analysis to a resource group inside a Data Landing Zone. Please deploy a [Data Management Zone](https://github.com/Azure/data-management-zone) and [Data Landing Zone](https://github.com/Azure/data-landing-zone) first. |[](https://portal.azure.com/#blade/Microsoft_Azure_CreateUIDef/CustomDeploymentBlade/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Fdata-product-batch%2Fmain%2Finfra%2Fmain.json/uiFormDefinitionUri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Fdata-product-batch%2Fmain%2Fdocs%2Freference%2Fportal.dataProduct.json) | [Repository](https://github.com/Azure/data-product-batch) |
|
||||
| Data Product Streaming | Deploys a Data Workload template for Data Streaming Analysis to a resource group inside a Data Landing Zone. Please deploy a [Data Management Zone](https://github.com/Azure/data-management-zone) and [Data Landing Zone](https://github.com/Azure/data-landing-zone) first. |[](https://portal.azure.com/#blade/Microsoft_Azure_CreateUIDef/CustomDeploymentBlade/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Fdata-product-streaming%2Fmain%2Finfra%2Fmain.json/uiFormDefinitionUri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Fdata-product-streaming%2Fmain%2Fdocs%2Freference%2Fportal.dataProduct.json) | [Repository](https://github.com/Azure/data-product-streaming) |
|
||||
| Data Product Analytics | Deploys a Data Workload template for Data Analytics and Data Science to a resource group inside a Data Landing Zone. Please deploy a [Data Management Zone](https://github.com/Azure/data-management-zone) and [Data Landing Zone](https://github.com/Azure/data-landing-zone) first. |[](https://portal.azure.com/#blade/Microsoft_Azure_CreateUIDef/CustomDeploymentBlade/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Fdata-product-analytics%2Fmain%2Finfra%2Fmain.json/uiFormDefinitionUri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Fdata-product-analytics%2Fmain%2Fdocs%2Freference%2Fportal.dataProduct.json) | [Repository](https://github.com/Azure/data-product-analytics) |
|
||||
|
||||
The following prerequisites are required to make this repository work:
|
||||
## Deploy Data Product
|
||||
|
||||
- A **Data Management Landing Zone** deployed. For more information, check the [Data Management Landing Zone](https://github.com/Azure/data-management-zone) repo.
|
||||
- A **Data Landing Zone** deployed. For more information, check the [Data Landing Zone](https://github.com/Azure/data-landing-zone) repo.
|
||||
- A resource group within an Azure subscription
|
||||
- [User Access Administrator](https://docs.microsoft.com/azure/role-based-access-control/built-in-roles#user-access-administrator) or [Owner](https://docs.microsoft.com/azure/role-based-access-control/built-in-roles#owner) access to a resource group to be able to create a service principal and role assignments for it.
|
||||
- Access to a subnet with `privateEndpointNetworkPolicies` and `privateLinkServiceNetworkPolicies` set to disabled. The Data Landing Zone deployment already creates a few subnets with this configuration (subnets with suffix `-privatelink-subnet`).
|
||||
- For deployment, please choose one of the below **Supported Regions** list.
|
||||
To deploy the Data Product into your Data Landing Zone, please follow the step-by-step instructions:
|
||||
|
||||
### **Supported Regions:**
|
||||
|
||||
- Asia Southeast
|
||||
- Europe North
|
||||
- Europe West
|
||||
- France Central
|
||||
- Japan East
|
||||
- South Africa North
|
||||
- UK South
|
||||
- US Central
|
||||
- US East
|
||||
- US East 2
|
||||
- US West 2
|
||||
|
||||
If you don't have an Azure subscription, [create your Azure free account today](https://azure.microsoft.com/free/).
|
||||
|
||||
## Option 1: Deploy to Azure - Quickstart
|
||||
|
||||
|Data Integration Streaming |
|
||||
|:---------------------|
|
||||
<!-- [](https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Fdata-integration-streaming%2Fmain%2Fdocs%2Freference%2Fdeploy.dataIntegration.json) -->
|
||||
|
||||
|
||||
## Option 2: GitHub Actions or Azure DevOps Pipelines
|
||||
|
||||
### 1. Create repository from a template
|
||||
|
||||
1. On GitHub, navigate to the main page of this repository.
|
||||
1. Above the file list, click **Use this template**
|
||||
|
||||
|
||||
|
||||
1. Use the **Owner** drop-down menu and select the account you want to own the repository.
|
||||
|
||||
|
||||
|
||||
1. Type a name for your repository and an optional description.
|
||||
1. Choose a repository visibility. For more information, see "[About repository visibility](https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-repository-visibility)."
|
||||
1. Optionally, to include the directory structure and files from all branches in the template and not just the default branch, select **Include all branches**.
|
||||
1. Click **Create repository from template**.
|
||||
|
||||
### 2. Setting up the required Service Principal and access
|
||||
|
||||
A service principal with *Contributor* role needs to be generated for authentication and authorization from GitHub or Azure DevOps to your Azure **Data Landing Zone** subscription, where the data-integration-streaming services will be deployed. Just go to the Azure Portal to find the ID of your subscription. Then start the Cloud Shell or Azure CLI, login to Azure, set the Azure context and execute the following commands to generate the required credentials:
|
||||
|
||||
> **Note:** The purpose of this new **Service Principal** is to assign least-privilege rights. Therefore, it requires the **Contributor** role at a resource group scope in order to deploy the resources inside the resource group dedicated to a specific data integration. The **Network Contributor** role assignment is required as well in this repository in order to add the private endpoint of resources to the dedicated subnet.
|
||||
|
||||
#### Azure CLI
|
||||
|
||||
```sh
|
||||
# Replace {service-principal-name} and {subscription-id} and {resource-group} with your
|
||||
# Azure subscription id and any name for your service principal.
|
||||
az ad sp create-for-rbac \
|
||||
--name {service-principal-name} \
|
||||
--role contributor \
|
||||
--scopes /subscriptions/{subscription-id}/resourceGroups/{resource-group} \
|
||||
--sdk-auth
|
||||
```
|
||||
|
||||
This will generate the following JSON output:
|
||||
|
||||
```json
|
||||
{
|
||||
"clientId": "<GUID>",
|
||||
"clientSecret": "<GUID>",
|
||||
"subscriptionId": "<GUID>",
|
||||
"tenantId": "<GUID>",
|
||||
(...)
|
||||
}
|
||||
```
|
||||
|
||||
> **Note:** Take note of the output. It will be required for the next steps.
|
||||
|
||||
Now that the new Service Principal is created, as mentioned, role assignments are required for this service principal in order to be able to successfully deploy all services. Required role assignments which will be added on a later step include:
|
||||
|
||||
| Role Name | Description | Scope |
|
||||
|:----------|:------------|:------|
|
||||
| [Private DNS Zone Contributor](https://docs.microsoft.com/azure/role-based-access-control/built-in-roles#private-dns-zone-contributor) | We expect you to deploy all Private DNS Zones for all data services into a single subscription and resource group. Therefor, the service principal needs to be Private DNS Zone Contributor on the global dns resource group which was created during the Data Management Zone deployment. This is required to deploy A-records for the respective private endpoints. | (Resource Group Scope) <div style="width: 36ch">`/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}`</div> |
|
||||
| [Network Contributor](https://docs.microsoft.com/azure/role-based-access-control/built-in-roles#network-contributor) | In order to deploy Private Endpoints to the specified privatelink-subnet which was created during the Data Landing Zone deployment, the service principal requires **Network Contributor** access on that specific subnet. | (Child-Resource Scope) `/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName} /providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}"` |
|
||||
|
||||
To add these role assignments, you can use the [Azure Portal](https://portal.azure.com/) or run the following commands using Azure CLI/Azure Powershell:
|
||||
|
||||
#### Azure CLI - Add role assignments
|
||||
|
||||
```sh
|
||||
# Get Service Principal Object ID
|
||||
az ad sp list --display-name "{servicePrincipalName}" --query "[].{objectId:objectId}" --output tsv
|
||||
|
||||
# Add role assignment
|
||||
# Resource Scope level assignment
|
||||
az role assignment create \
|
||||
--assignee "{servicePrincipalObjectId}" \
|
||||
--role "{roleName}" \
|
||||
--scopes "{scope}"
|
||||
|
||||
# Resource group scope level assignment
|
||||
az role assignment create \
|
||||
--assignee "{servicePrincipalObjectId}" \
|
||||
--role "{roleName}" \
|
||||
--resource-group "{resourceGroupName}"
|
||||
|
||||
# For Child-Resource Scope level assignment
|
||||
# TBD
|
||||
```
|
||||
|
||||
#### Azure Powershell - Add role assignments
|
||||
|
||||
```powershell
|
||||
# Get Service Principal Object ID
|
||||
$spObjectId = (Get-AzADServicePrincipal -DisplayName "{servicePrincipalName}").id
|
||||
|
||||
# Add role assignment
|
||||
# For Resource Scope level assignment
|
||||
New-AzRoleAssignment `
|
||||
-ObjectId $spObjectId `
|
||||
-RoleDefinitionName "{roleName}" `
|
||||
-Scope "{scope}"
|
||||
|
||||
# For Resource group scope level assignment
|
||||
New-AzRoleAssignment `
|
||||
-ObjectId $spObjectId `
|
||||
-RoleDefinitionName "{roleName}" `
|
||||
-ResourceGroupName "{resourceGroupName}"
|
||||
|
||||
# For Child-Resource Scope level assignment
|
||||
New-AzRoleAssignment `
|
||||
-ObjectId $spObjectId `
|
||||
-RoleDefinitionName "{roleName}" `
|
||||
-ResourceName "{subnetName}" `
|
||||
-ResourceType "Microsoft.Network/virtualNetworks/subnets" `
|
||||
-ParentResource "virtualNetworks/{virtualNetworkName}" `
|
||||
-ResourceGroupName "{resourceGroupName}
|
||||
```
|
||||
|
||||
### 3. Resource Deployment
|
||||
|
||||
Now that you have set up the Service Principal, you need to choose how would you like to deploy the resources.
|
||||
Deployment options:
|
||||
|
||||
1. [GitHub Actions](#github-actions)
|
||||
1. [Azure DevOps](#azure-devops)
|
||||
|
||||
#### GitHub Actions
|
||||
|
||||
If you want to use GitHub Actions for deploying the resources, add the previous JSON output as a [repository secret](https://docs.github.com/en/actions/reference/encrypted-secrets#creating-encrypted-secrets-for-a-repository) with the name `AZURE_CREDENTIALS` in your GitHub repository:
|
||||
|
||||
|
||||
|
||||
To do so, execute the following steps:
|
||||
|
||||
1. On GitHub, navigate to the main page of the repository.
|
||||
2. Under your repository name, click on the **Settings** tab.
|
||||
3. In the left sidebar, click **Secrets**.
|
||||
4. Click **New repository secret**.
|
||||
5. Type the name `AZURE_CREDENTIALS` for your secret in the Name input box.
|
||||
6. Enter the JSON output from above as value for your secret.
|
||||
7. Click **Add secret**.
|
||||
|
||||
#### Azure DevOps
|
||||
|
||||
If you want to use Azure DevOps Pipelines for deploying the resources, you need to create an Azure Resource Manager service connection. To do so, execute the following steps:
|
||||
|
||||
1. First, you need to create an Azure DevOps Project. Instructions can be found [here](https://docs.microsoft.com/azure/devops/organizations/projects/create-project?view=azure-devops&tabs=preview-page).
|
||||
1. In Azure DevOps, open the **Project settings**.
|
||||
1. Now, select the **Service connections** page from the project settings page.
|
||||
1. Choose **New service connection** and select **Azure Resource Manager**.
|
||||
|
||||
|
||||
|
||||
1. On the next page select **Service principal (manual)**.
|
||||
1. Select the appropriate environment to which you would like to deploy the templates. Only the default option **Azure Cloud** is currently supported.
|
||||
1. For the **Scope Level**, select **Subscription** and enter your `subscription Id` and `name`.
|
||||
1. Enter the details of the service principal that we have generated in step 3. (**Service Principal Id** = **clientId**, **Service Principal Key** = **clientSecret**, **Tenant ID** = **tenantId**) and click on **Verify** to make sure that the connection works.
|
||||
1. Enter a user-friendly **Connection name** to use when referring to this service connection. Take note of the name because this will be required in the parameter update process.
|
||||
1. Optionally, enter a **Description**.
|
||||
1. Click on **Verify and save**.
|
||||
|
||||
|
||||
|
||||
More information can be found [here](https://docs.microsoft.com/azure/devops/pipelines/library/connect-to-azure?view=azure-devops#create-an-azure-resource-manager-service-connection-with-an-existing-service-principal).
|
||||
|
||||
### 4. Parameter Updates
|
||||
|
||||
> **Note:** This section applies for both **Azure DevOps** and **GitHub** Deployment
|
||||
|
||||
In order to deploy the Infrastructure as Code (IaC) templates to the desired Azure subscription, you will need to modify some parameters in the forked repository. Therefore, **this step should not be skipped for neither Azure DevOps/GitHub options**. There are two files that require updates:
|
||||
|
||||
- `.github/workflows/dataIntegrationDeployment.yml` for GitHub Actions,
|
||||
- `.ado/workflows/dataIntegrationDeployment.yml` for Azure DevOps and
|
||||
- `infra/params.dev.json`.
|
||||
|
||||
Update these files in a seperate branch and then merge via Pull Request to trigger the initial deployment.
|
||||
|
||||
#### Configure `dataIntegrationDeployment.yml`
|
||||
|
||||
##### For GitHub Actions
|
||||
|
||||
To begin, please open the [.github/workflows/dataIntegrationDeployment.yml](/.github/workflows/dataIntegrationDeployment.yml). In this file you need to update the environment variables section. Just click on [.github/workflows/dataIntegrationDeployment.yml](/.github/workflows/dataIntegrationDeployment.yml) and edit the following section:
|
||||
|
||||
```yaml
|
||||
env:
|
||||
AZURE_SUBSCRIPTION_ID: "2150d511-458f-43b9-8691-6819ba2e6c7b" # Update to '{dataLandingZoneSubscriptionId}'
|
||||
AZURE_RESOURCE_GROUP_NAME: "dlz01-dev-dd002" # Update to '{dataLandingZoneName}-rg'
|
||||
AZURE_LOCATION: "northeurope" # Update to '{regionName}'
|
||||
```
|
||||
|
||||
Further details about these parameters are provided in a table below.
|
||||
|
||||
##### For Azure DevOps
|
||||
|
||||
To begin, please open the [.ado/workflows/dataIntegrationDeployment.yml](/.ado/workflows/dataIntegrationDeployment.yml). In this file you need to update the variables section. Just click on [.ado/workflows/dataIntegrationDeployment.yml](/.ado/workflows/dataIntegrationDeployment.yml) and edit the following section:
|
||||
|
||||
```yaml
|
||||
variables:
|
||||
AZURE_RESOURCE_MANAGER_CONNECTION_NAME: "integration-product-service-connection" # Update to '{resourceManagerConnectionName}'
|
||||
AZURE_SUBSCRIPTION_ID: "2150d511-458f-43b9-8691-6819ba2e6c7b" # Update to '{dataLandingZoneSubscriptionId}'
|
||||
AZURE_RESOURCE_GROUP_NAME: "dlz01-dev-dd002" # Update to '{dataLandingZoneName}-rg'
|
||||
AZURE_LOCATION: "North Europe" # Update to '{regionName}'
|
||||
```
|
||||
|
||||
The following table explains each of the parameters:
|
||||
|
||||
| Parameter | Description | Sample value |
|
||||
|:-----------------------------------------|:-------------|:-------------|
|
||||
| **AZURE_SUBSCRIPTION_ID** | Specifies the subscription ID of the Data Management Zone where all the resources will be deployed | <div style="width: 36ch">`xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`</div> |
|
||||
| **AZURE_LOCATION** | Specifies the region where you want the resources to be deployed. Please check [Supported Regions](#supported-regions) | `northeurope` |
|
||||
| **AZURE_RESOURCE_GROUP_NAME** | Specifies the name of an existing resource group in your data landing zone, where the resources will be deployed. | `my-rg-name` |
|
||||
| **AZURE_RESOURCE_MANAGER _CONNECTION_NAME** | Specifies the resource manager connection name in Azure DevOps. You can leave the default value if you want to use GitHub Actions for your deployment. More details on how to create the resource manager connection in Azure DevOps can be found in step 4. b) or [here](https://docs.microsoft.com/azure/devops/pipelines/library/connect-to-azure?view=azure-devops#create-an-azure-resource-manager-service-connection-with-an-existing-service-principal). | `my-connection-name` |
|
||||
|
||||
#### Configure `params.dev.json`
|
||||
|
||||
To begin, please open the [infra/params.dev.json](/infra/params.dev.json). In this file you need to update the variable values. Just click on [infra/params.dev.json](/infra/params.dev.json) and edit the values. An explanation of the values is given in the table below:
|
||||
|
||||
| Parameter | Description | Sample value |
|
||||
|:-----------------------------------------|:-------------|:-------------|
|
||||
| location | Specifies the location for all resources. | `northeurope` |
|
||||
| environment | Specifies the environment of the deployment. | `dev`, `tst` or `prd` |
|
||||
| prefix | Specifies the prefix for all resources created in this deployment. | `prefi` |
|
||||
| administratorPassword | Specifies the administrator password of the sql servers. Will be automatically set in the workflow. **Leave this value as is.** | `<your-secure-password>` |
|
||||
| synapseDefaultStorageAccountFileSystemId | Specifies the resource ID of the default storage account file system for synapse. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Storage/storageAccounts/{storage-name}/blobServices/default/containers/{container-name}` |
|
||||
| streamanalyticsDefaultStorageAccountId | Specifies the resource ID of the default storage account for strea analytics. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Storage/storageAccounts/{storage-name}` |
|
||||
| subnetId | Specifies the resource ID of the subnet to which all services will connect. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/virtualNetworks/{vnet-name}/subnets/{subnet-name}` |
|
||||
| purviewId | Specifies the resource ID of the central purview instance. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Purview/accounts/{purview-name}` |
|
||||
| enableRoleAssignments | Specifies whether role assignments should be enabled. **Leave this value as is.** | `true` or `false` |
|
||||
| privateDnsZoneIdKeyVault | Specifies the resource ID of the private DNS zone for KeyVault. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.vaultcore.azure.net` |
|
||||
| privateDnsZoneIdSynapseDev | Specifies the resource ID of the private DNS zone for Synapse Dev. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.dev.azuresynapse.net` |
|
||||
| privateDnsZoneIdSynapseSql | Specifies the resource ID of the private DNS zone for Synapse Sql. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.sql.azuresynapse.net` |
|
||||
| privateDnsZoneIdEventhubNamespace | Specifies the resource ID of the private DNS zone for EventHub Namespace. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.servicebus.windows.net` |
|
||||
| privateDnsZoneIdCosmosdbSql | Specifies the resource ID of the private DNS zone for Cosmos Sql. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.documents.azure.com` |
|
||||
| privateDnsZoneIdSqlServer | Specifies the resource ID of the private DNS zone for Sql Server. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.database.windows.net` |
|
||||
| privateDnsZoneIdIothub | Specifies the resource ID of the private DNS zone for IoT Hub. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.azure-devices.net` |
|
||||
|
||||
### 5. (not applicable for GH Actions) Reference pipeline from GitHub repository in Azure DevOps Pipelines
|
||||
|
||||
#### Install Azure DevOps Pipelines GitHub Application
|
||||
|
||||
First you need to add and install the Azure Pipelines GitHub App to your GitHub account. To do so, execute the following steps:
|
||||
|
||||
1. Click on **Marketplace** in the top navigation bar on GitHub.
|
||||
1. In the Marketplace, search for **Azure Pipelines**. The Azure Pipelines offering is free for anyone to use for public repositories and free for a single build queue if you're using a private repository.
|
||||
|
||||
|
||||
|
||||
1. Select it and click on **Install it for free**.
|
||||
|
||||
|
||||
|
||||
1. If you are part of multiple **GitHub** organizations, you may need to use the **Switch billing account** dropdown to select the one into which you forked this repository.
|
||||
1. You may be prompted to confirm your GitHub password to continue.
|
||||
1. You may be prompted to log in to your Microsoft account. Make sure you log in with the one that is associated with your Azure DevOps account.
|
||||
|
||||
#### Configuring the Azure Pipelines project
|
||||
|
||||
As a last step, you need to create an Azure DevOps pipeline in your project based on the pipeline definition YAML file that is stored in your GitHub repository. To do so, execute the following steps:
|
||||
|
||||
1. Select the Azure DevOps project where you have setup your `Resource Manager Connection`.
|
||||
1. Select **Pipelines** and then **New Pipeline** in order to create a new pipeline.
|
||||
|
||||
|
||||
|
||||
1. Choose **GitHub YAML** and search for your repository (e.g. "`GitHubUserName/RepositoryName`").
|
||||
|
||||
|
||||
|
||||
1. Select your repository.
|
||||
1. Click on **Existing Azure Pipelines in YAML file**
|
||||
1. Select `main` as branch and `/.ado/workflows/dataIntegrationDeployment.yml` as path.
|
||||
|
||||
|
||||
|
||||
1. Click on **Continue** and then on **Run**.
|
||||
|
||||
### 6. Merge these changes back to the `main` branch of your repo
|
||||
|
||||
After following the instructions and updating the parameters and variables in your repository in a separate branch and opening the pull request, you can merge the pull request back into the `main` branch of your repository by clicking on **Merge pull request**. Finally, you can click on **Delete branch** to clean up your repository. By doing this, you trigger the deployment workflow.
|
||||
|
||||
### 7. Follow the workflow deployment
|
||||
|
||||
**Congratulations!** You have successfully executed all steps to deploy the template into your environment through GitHub Actions or Azure DevOps.
|
||||
|
||||
If you are using GitHub Actions, you can navigate to the **Actions** tab of the main page of the repository where you will see a workflow with the name `Data Integration Deployment` running. Click on it to see how it deploys one service after another. If you run into any issues, please open an issue [here](https://github.com/Azure/data-integration-streaming/issues).
|
||||
|
||||
If you are using Azure DevOps Pipelines, you can navigate to the pipeline that you have created as part of step 6 and monitor it as each service is deployed. If you run into any issues, please open an issue [here](https://github.com/Azure/data-integration-streaming/issues).
|
||||
|
||||
### Documentation
|
||||
|
||||
### Code Structure
|
||||
|
||||
| File/folder | Description |
|
||||
| ----------------------------- | ------------------------------------------ |
|
||||
| `.ado/workflows` | Folder for ADO workflows. The `dataIntegrationDeployment.yml` workflow shows the steps for an end-to-end deployment of the architecture. |
|
||||
| `.github/workflows` | Folder for GitHub workflows. The `dataIntegrationDeployment.yml` workflow shows the steps for an end-to-end deployment of the architecture. |
|
||||
| `code` | Sample password generation script that will be run in the deployment workflow for resources that require a password during the deployment. |
|
||||
| `docs` | Resources for this README. |
|
||||
| `infra` | Folder containing all the ARM templates for each of the resources that will be deployed (`deploy.{resource}.json`) together with their parameter files (`params.{resource}.json`). |
|
||||
| `CODE_OF_CONDUCT.md` | Microsoft Open Source Code of Conduct. |
|
||||
| `LICENSE` | The license for the sample. |
|
||||
| `README.md` | This README file. |
|
||||
| `SECURITY.md` | Microsoft Security README. |
|
||||
|
||||
### Enterprise Scale Analytics and AI - Documentation and Implementation
|
||||
|
||||
- [Documentation](https://github.com/Azure/Enterprise-Scale-Analytics)
|
||||
- [Implementation - Data Management](https://github.com/Azure/data-management-zone)
|
||||
- [Implementation - Data Landing Zone](https://github.com/Azure/data-landing-zone)
|
||||
- [Implementation - Data Integration - Batch](https://github.com/Azure/data-integration-batch)
|
||||
- [Implementation - Data Integration - Streaming](https://github.com/Azure/data-integration-streaming)
|
||||
- [Implementation - Data Product - Reporting](https://github.com/Azure/data-product-reporting)
|
||||
- [Implementation - Data Product - Analytics & Data Science](https://github.com/Azure/data-product-analytics)
|
||||
|
||||
## Known issues
|
||||
|
||||
### Error: MissingSubscriptionRegistration
|
||||
|
||||
**Error Message:**
|
||||
|
||||
```text
|
||||
ERROR: Deployment failed. Correlation ID: ***
|
||||
"error": ***
|
||||
"code": "MissingSubscriptionRegistration",
|
||||
"message": "The subscription is not registered to use namespace 'Microsoft.DocumentDB'. See https://aka.ms/rps-not-found for how to register subscriptions.",
|
||||
"details": [
|
||||
***
|
||||
"code": "MissingSubscriptionRegistration",
|
||||
"target": "Microsoft.DocumentDB",
|
||||
"message": "The subscription is not registered to use namespace 'Microsoft.DocumentDB'. See https://aka.ms/rps-not-found for how to register subscriptions."
|
||||
|
||||
```
|
||||
|
||||
**Solution:**
|
||||
|
||||
This error message appears, in case during the deployment it tries to create a type of resource which has never been deployed before inside the subscription. We recommend to check prior the deployment whether the required resource providers are registered for your subscription and if needed, register them through the `Azure Portal`, `Azure Powershell` or `Azure CLI` as mentioned [here](https://docs.microsoft.com/azure/azure-resource-manager/management/resource-providers-and-types).
|
||||
1. [Prerequisites](/docs/EnterpriseScaleAnalytics-Prerequisites.md)
|
||||
2. [Create repository](/docs/EnterpriseScaleAnalytics-CreateRepository.md)
|
||||
3. [Setting up Service Principal](/docs/EnterpriseScaleAnalytics-ServicePrincipal.md)
|
||||
4. Template Deployment
|
||||
1. [GitHub Action Deployment](/docs/EnterpriseScaleAnalytics-GitHubActionsDeployment.md)
|
||||
2. [Azure DevOps Deployment](/docs/EnterpriseScaleAnalytics-AzureDevOpsDeployment.md)
|
||||
5. [Known Issues](/docs/EnterpriseScaleAnalytics-KnownIssues.md)
|
||||
|
||||
## Contributing
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,141 @@
|
|||
# Data Product Streaming - Azure DevOps Deployment
|
||||
|
||||
In the previous step we have generated a JSON output similar to the following, which will be required in the next steps:
|
||||
|
||||
```json
|
||||
{
|
||||
"clientId": "<GUID>",
|
||||
"clientSecret": "<GUID>",
|
||||
"subscriptionId": "<GUID>",
|
||||
"tenantId": "<GUID>",
|
||||
(...)
|
||||
}
|
||||
```
|
||||
|
||||
## Create Service Connection
|
||||
|
||||
First, you need to create an Azure Resource Manager service connection. To do so, execute the following steps:
|
||||
|
||||
1. First, you need to create an Azure DevOps Project. Instructions can be found [here](https://docs.microsoft.com/azure/devops/organizations/projects/create-project?view=azure-devops&tabs=preview-page).
|
||||
1. In Azure DevOps, open the **Project settings**.
|
||||
1. Now, select the **Service connections** page from the project settings page.
|
||||
1. Choose **New service connection** and select **Azure Resource Manager**.
|
||||
|
||||
|
||||
|
||||
1. On the next page select **Service principal (manual)**.
|
||||
1. Select the appropriate environment to which you would like to deploy the templates. Only the default option **Azure Cloud** is currently supported.
|
||||
1. For the **Scope Level**, select **Subscription** and enter your `subscription Id` and `name`.
|
||||
1. Enter the details of the service principal that we have generated in step 3. (**Service Principal Id** = **clientId**, **Service Principal Key** = **clientSecret**, **Tenant ID** = **tenantId**) and click on **Verify** to make sure that the connection works.
|
||||
1. Enter a user-friendly **Connection name** to use when referring to this service connection. Take note of the name because this will be required in the parameter update process.
|
||||
1. Optionally, enter a **Description**.
|
||||
1. Click on **Verify and save**.
|
||||
|
||||
|
||||
|
||||
More information can be found [here](https://docs.microsoft.com/azure/devops/pipelines/library/connect-to-azure?view=azure-devops#create-an-azure-resource-manager-service-connection-with-an-existing-service-principal).
|
||||
|
||||
## Update Parameters
|
||||
|
||||
In order to deploy the Infrastructure as Code (IaC) templates to the desired Azure subscription, you will need to modify some parameters in the forked repository. Therefore, **this step should not be skipped for neither Azure DevOps/GitHub options**. There are two files that require updates:
|
||||
|
||||
- `.ado/workflows/dataProductDeployment.yml` and
|
||||
- `infra/params.dev.json`.
|
||||
|
||||
Update these files in a seperate branch and then merge via Pull Request to trigger the initial deployment.
|
||||
|
||||
### Configure `dataProductDeployment.yml`
|
||||
|
||||
To begin, please open the [.ado/workflows/dataProductDeployment.yml](/.ado/workflows/dataProductDeployment.yml). In this file you need to update the variables section. Just click on [.ado/workflows/dataProductDeployment.yml](/.ado/workflows/dataProductDeployment.yml) and edit the following section:
|
||||
|
||||
```yaml
|
||||
variables:
|
||||
AZURE_RESOURCE_MANAGER_CONNECTION_NAME: "integration-product-service-connection" # Update to '{resourceManagerConnectionName}'
|
||||
AZURE_SUBSCRIPTION_ID: "2150d511-458f-43b9-8691-6819ba2e6c7b" # Update to '{dataLandingZoneSubscriptionId}'
|
||||
AZURE_RESOURCE_GROUP_NAME: "dlz01-dev-di001" # Update to '{dataLandingZoneName}-rg'
|
||||
AZURE_LOCATION: "North Europe" # Update to '{regionName}'
|
||||
```
|
||||
|
||||
The following table explains each of the parameters:
|
||||
|
||||
| Parameter | Description | Sample value |
|
||||
|:--------------------------------------------|:------------|:-------------|
|
||||
| **AZURE_SUBSCRIPTION_ID** | Specifies the subscription ID of the Data Management Zone where all the resources will be deployed | <div style="width: 36ch">`xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`</div> |
|
||||
| **AZURE_LOCATION** | Specifies the region where you want the resources to be deployed. Please check [Supported Regions](/docs/EnterpriseScaleAnalytics-Prerequisites.md#supported-regions) | `northeurope` |
|
||||
| **AZURE_RESOURCE_GROUP_NAME** | Specifies the name of an existing resource group in your data landing zone, where the resources will be deployed. | `my-rg-name` |
|
||||
| **AZURE_RESOURCE_MANAGER _CONNECTION_NAME** | Specifies the resource manager connection name in Azure DevOps. You can leave the default value if you want to use GitHub Actions for your deployment. More details on how to create the resource manager connection in Azure DevOps can be found further above or [here](https://docs.microsoft.com/azure/devops/pipelines/library/connect-to-azure?view=azure-devops#create-an-azure-resource-manager-service-connection-with-an-existing-service-principal). | `my-connection-name` |
|
||||
|
||||
### Configure `params.dev.json`
|
||||
|
||||
To begin, please open the [infra/params.dev.json](/infra/params.dev.json). In this file you need to update the variable values. Just click on [infra/params.dev.json](/infra/params.dev.json) and edit the values. An explanation of the values is given in the table below:
|
||||
|
||||
| Parameter | Description | Sample value |
|
||||
|:-----------------------------------------|:-------------|:-------------|
|
||||
| location | Specifies the location for all resources. | `northeurope` |
|
||||
| environment | Specifies the environment of the deployment. | `dev`, `tst` or `prd` |
|
||||
| prefix | Specifies the prefix for all resources created in this deployment. | `prefi` |
|
||||
| tags | Specifies the tags that you want to apply to all resources. | {`key`: `value`} |
|
||||
| administratorPassword | Specifies the administrator password of the sql servers. Will be automatically set in the workflow. **Leave this value as is.** | `<your-secure-password>` |
|
||||
| synapseDefaultStorageAccountFileSystemId | Specifies the resource ID of the default storage account file system for synapse. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Storage/storageAccounts/{storage-name}/blobServices/default/containers/{container-name}` |
|
||||
| streamanalyticsDefaultStorageAccountFileSystemId | Specifies the resource ID of the default storage account file system for stream analytics. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Storage/storageAccounts/{storage-name}/blobServices/default/containers/{container-name}` |
|
||||
| subnetId | Specifies the resource ID of the subnet to which all services will connect. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/virtualNetworks/{vnet-name}/subnets/{subnet-name}` |
|
||||
| purviewId | Specifies the resource ID of the central purview instance. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Purview/accounts/{purview-name}` |
|
||||
| enableRoleAssignments | Specifies whether role assignments should be enabled. **Leave this value as is.** | `true` or `false` |
|
||||
| privateDnsZoneIdKeyVault | Specifies the resource ID of the private DNS zone for KeyVault. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.vaultcore.azure.net` |
|
||||
| privateDnsZoneIdSynapseDev | Specifies the resource ID of the private DNS zone for Synapse Dev. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.dev.azuresynapse.net` |
|
||||
| privateDnsZoneIdSynapseSql | Specifies the resource ID of the private DNS zone for Synapse Sql. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.sql.azuresynapse.net` |
|
||||
| privateDnsZoneIdEventhubNamespace | Specifies the resource ID of the private DNS zone for EventHub Namespace. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.servicebus.windows.net` |
|
||||
| privateDnsZoneIdCosmosdbSql | Specifies the resource ID of the private DNS zone for Cosmos Sql. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.documents.azure.com` |
|
||||
| privateDnsZoneIdSqlServer | Specifies the resource ID of the private DNS zone for Sql Server. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.database.windows.net` |
|
||||
| privateDnsZoneIdIothub | Specifies the resource ID of the private DNS zone for IoT Hub. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.azure-devices.net` |
|
||||
|
||||
### Install Azure DevOps Pipelines GitHub Application
|
||||
|
||||
First you need to add and install the Azure Pipelines GitHub App to your GitHub account. To do so, execute the following steps:
|
||||
|
||||
1. Click on **Marketplace** in the top navigation bar on GitHub.
|
||||
1. In the Marketplace, search for **Azure Pipelines**. The Azure Pipelines offering is free for anyone to use for public repositories and free for a single build queue if you're using a private repository.
|
||||
|
||||
|
||||
|
||||
1. Select it and click on **Install it for free**.
|
||||
|
||||
|
||||
|
||||
1. If you are part of multiple **GitHub** organizations, you may need to use the **Switch billing account** dropdown to select the one into which you forked this repository.
|
||||
1. You may be prompted to confirm your GitHub password to continue.
|
||||
1. You may be prompted to log in to your Microsoft account. Make sure you log in with the one that is associated with your Azure DevOps account.
|
||||
|
||||
### Configuring the Azure Pipelines project
|
||||
|
||||
As a last step, you need to create an Azure DevOps pipeline in your project based on the pipeline definition YAML file that is stored in your GitHub repository. To do so, execute the following steps:
|
||||
|
||||
1. Select the Azure DevOps project where you have setup your `Resource Manager Connection`.
|
||||
1. Select **Pipelines** and then **New Pipeline** in order to create a new pipeline.
|
||||
|
||||
|
||||
|
||||
1. Choose **GitHub YAML** and search for your repository (e.g. "`GitHubUserName/RepositoryName`").
|
||||
|
||||
|
||||
|
||||
1. Select your repository.
|
||||
1. Click on **Existing Azure Pipelines in YAML file**
|
||||
1. Select `main` as branch and `/.ado/workflows/dataHubDeployment.yml` as path.
|
||||
|
||||
|
||||
|
||||
1. Click on **Continue** and then on **Run**.
|
||||
|
||||
## Merge these changes back to the `main` branch of your repo
|
||||
|
||||
After following the instructions and updating the parameters and variables in your repository in a separate branch and opening the pull request, you can merge the pull request back into the `main` branch of your repository by clicking on **Merge pull request**. Finally, you can click on **Delete branch** to clean up your repository. By doing this, you trigger the deployment workflow.
|
||||
|
||||
## Follow the workflow deployment
|
||||
|
||||
**Congratulations!** You have successfully executed all steps to deploy the template into your environment through Azure DevOps.
|
||||
|
||||
Now, you can navigate to the pipeline that you have created as part of step 5 and monitor it as each service is deployed. If you run into any issues, please check the [Known Issues](/docs/EnterpriseScaleAnalytics-KnownIssues.md) first and open an [issue](https://github.com/Azure/data-product-streaming/issues) if you come accross a potential bug in the repository.
|
||||
|
||||
>[Previous](/docs/EnterpriseScaleAnalytics-ServicePrincipal.md)
|
||||
>[Next](/docs/EnterpriseScaleAnalytics-KnownIssues.md)
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
# Data Product Streaming - Create repository from the template
|
||||
|
||||
First, you must generate your own respository based off this template respository. To do so, please follow the steps below:
|
||||
|
||||
1. On GitHub, navigate to the [main page of this repository](https://github.com/Azure/data-management-zone).
|
||||
1. Above the file list, click **Use this template**
|
||||
|
||||
|
||||
|
||||
1. Use the **Owner** drop-down menu and select the account you want to own the repository.
|
||||
|
||||
|
||||
|
||||
1. Type a name for your repository and an optional description.
|
||||
1. Choose a repository visibility. For more information, see "[About repository visibility](https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-repository-visibility)."
|
||||
1. Optionally, to include the directory structure and files from all branches in the template and not just the default branch, select **Include all branches**.
|
||||
1. Click **Create repository from template**.
|
||||
|
||||
>[Previous](/docs/EnterpriseScaleAnalytics-Prerequisites.md)
|
||||
>[Next](/docs/EnterpriseScaleAnalytics-ServicePrincipal.md)
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
# Data Product Batch - GitHub Action Deployment
|
||||
|
||||
In the previous step we have generated a JSON output similar to the following, which will be required in the next steps:
|
||||
|
||||
```json
|
||||
{
|
||||
"clientId": "<GUID>",
|
||||
"clientSecret": "<GUID>",
|
||||
"subscriptionId": "<GUID>",
|
||||
"tenantId": "<GUID>",
|
||||
(...)
|
||||
}
|
||||
```
|
||||
|
||||
## Adding Secrets to GitHub respository
|
||||
|
||||
If you want to use GitHub Actions for deploying the resources, add the JSON output as a [repository secret](https://docs.github.com/en/actions/reference/encrypted-secrets#creating-encrypted-secrets-for-a-repository) with the name `AZURE_CREDENTIALS` in your GitHub repository:
|
||||
|
||||
|
||||
|
||||
To do so, execute the following steps:
|
||||
|
||||
1. On GitHub, navigate to the main page of the repository.
|
||||
2. Under your repository name, click on the **Settings** tab.
|
||||
3. In the left sidebar, click **Secrets**.
|
||||
4. Click **New repository secret**.
|
||||
5. Type the name `AZURE_CREDENTIALS` for your secret in the Name input box.
|
||||
6. Enter the JSON output from above as value for your secret.
|
||||
7. Click **Add secret**.
|
||||
|
||||
## Update Parameters
|
||||
|
||||
In order to deploy the Infrastructure as Code (IaC) templates to the desired Azure subscription, you will need to modify some parameters in the forked repository. Therefore, **this step should not be skipped for neither Azure DevOps/GitHub options**. There are two files that require updates:
|
||||
|
||||
- `.github/workflows/dataProductDeployment.yml` and
|
||||
- `infra/params.dev.json`.
|
||||
|
||||
Update these files in a seperate branch and then merge via Pull Request to trigger the initial deployment.
|
||||
|
||||
### Configure `dataProductDeployment.yml`
|
||||
|
||||
To begin, please open [.github/workflows/dataProductDeployment.yml](/.github/workflows/dataProductDeployment.yml). In this file you need to update the environment variables section. Just click on [.github/workflows/dataProductDeployment.yml](/.github/workflows/dataProductDeployment.yml) and edit the following section:
|
||||
|
||||
```yaml
|
||||
env:
|
||||
AZURE_SUBSCRIPTION_ID: "2150d511-458f-43b9-8691-6819ba2e6c7b" # Update to '{dataLandingZoneSubscriptionId}'
|
||||
AZURE_RESOURCE_GROUP_NAME: "dlz01-dev-di002" # Update to '{dataLandingZoneName}-rg'
|
||||
AZURE_LOCATION: "northeurope" # Update to '{regionName}'
|
||||
```
|
||||
|
||||
The following table explains each of the parameters:
|
||||
|
||||
| Parameter | Description | Sample value |
|
||||
|:------------------------------|:-------------|:-------------|
|
||||
| **AZURE_SUBSCRIPTION_ID** | Specifies the subscription ID of the Data Management Zone where all the resources will be deployed | <div style="width: 36ch">`xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`</div> |
|
||||
| **AZURE_LOCATION** | Specifies the region where you want the resources to be deployed. Please check [Supported Regions](/docs/EnterpriseScaleAnalytics-Prerequisites.md#supported-regions) | `northeurope` |
|
||||
| **AZURE_RESOURCE_GROUP_NAME** | Specifies the name of an existing resource group in your data landing zone, where the resources will be deployed. | `my-rg-name` |
|
||||
|
||||
### Configure `params.dev.json`
|
||||
|
||||
To begin, please open the [infra/params.dev.json](/infra/params.dev.json). In this file you need to update the variable values. Just click on [infra/params.dev.json](/infra/params.dev.json) and edit the values. An explanation of the values is given in the table below:
|
||||
|
||||
| Parameter | Description | Sample value |
|
||||
|:-----------------------------------------|:-------------|:-------------|
|
||||
| location | Specifies the location for all resources. | `northeurope` |
|
||||
| environment | Specifies the environment of the deployment. | `dev`, `tst` or `prd` |
|
||||
| prefix | Specifies the prefix for all resources created in this deployment. | `prefi` |
|
||||
| tags | Specifies the tags that you want to apply to all resources. | {`key`: `value`} |
|
||||
| administratorPassword | Specifies the administrator password of the sql servers. Will be automatically set in the workflow. **Leave this value as is.** | `<your-secure-password>` |
|
||||
| synapseDefaultStorageAccountFileSystemId | Specifies the resource ID of the default storage account file system for synapse. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Storage/storageAccounts/{storage-name}/blobServices/default/containers/{container-name}` |
|
||||
| streamanalyticsDefaultStorageAccountFileSystemId | Specifies the resource ID of the default storage account file system for stream analytics. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Storage/storageAccounts/{storage-name}/blobServices/default/containers/{container-name}` |
|
||||
| subnetId | Specifies the resource ID of the subnet to which all services will connect. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/virtualNetworks/{vnet-name}/subnets/{subnet-name}` |
|
||||
| purviewId | Specifies the resource ID of the central purview instance. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Purview/accounts/{purview-name}` |
|
||||
| enableRoleAssignments | Specifies whether role assignments should be enabled. **Leave this value as is.** | `true` or `false` |
|
||||
| privateDnsZoneIdKeyVault | Specifies the resource ID of the private DNS zone for KeyVault. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.vaultcore.azure.net` |
|
||||
| privateDnsZoneIdSynapseDev | Specifies the resource ID of the private DNS zone for Synapse Dev. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.dev.azuresynapse.net` |
|
||||
| privateDnsZoneIdSynapseSql | Specifies the resource ID of the private DNS zone for Synapse Sql. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.sql.azuresynapse.net` |
|
||||
| privateDnsZoneIdEventhubNamespace | Specifies the resource ID of the private DNS zone for EventHub Namespace. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.servicebus.windows.net` |
|
||||
| privateDnsZoneIdCosmosdbSql | Specifies the resource ID of the private DNS zone for Cosmos Sql. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.documents.azure.com` |
|
||||
| privateDnsZoneIdSqlServer | Specifies the resource ID of the private DNS zone for Sql Server. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.database.windows.net` |
|
||||
| privateDnsZoneIdIothub | Specifies the resource ID of the private DNS zone for IoT Hub. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.azure-devices.net` |
|
||||
|
||||
## Merge these changes back to the `main` branch of your repo
|
||||
|
||||
After following the instructions and updating the parameters and variables in your repository in a separate branch and opening the pull request, you can merge the pull request back into the `main` branch of your repository by clicking on **Merge pull request**. Finally, you can click on **Delete branch** to clean up your repository. By doing this, you trigger the deployment workflow.
|
||||
|
||||
## Follow the workflow deployment
|
||||
|
||||
**Congratulations!** You have successfully executed all steps to deploy the template into your environment through GitHub Actions.
|
||||
|
||||
Now, you can navigate to the **Actions** tab of the main page of the repository, where you will see a workflow with the name `Data Product Deployment` running. Click on it to see how it deploys the environment. If you run into any issues, please check the [Known Issues](/docs/EnterpriseScaleAnalytics-KnownIssues.md) first and open an [issue](https://github.com/Azure/data-product-streaming/issues) if you come accross a potential bug in the repository.
|
||||
|
||||
>[Previous](/docs/EnterpriseScaleAnalytics-ServicePrincipal.md)
|
||||
>[Next](/docs/EnterpriseScaleAnalytics-KnownIssues.md)
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
# Data Product Streaming - Known Issues
|
||||
|
||||
## Error: MissingSubscriptionRegistration
|
||||
|
||||
**Error Message:**
|
||||
|
||||
```text
|
||||
ERROR: Deployment failed. Correlation ID: ***
|
||||
"error": ***
|
||||
"code": "MissingSubscriptionRegistration",
|
||||
"message": "The subscription is not registered to use namespace 'Microsoft.DocumentDB'. See https://aka.ms/rps-not-found for how to register subscriptions.",
|
||||
"details": [
|
||||
***
|
||||
"code": "MissingSubscriptionRegistration",
|
||||
"target": "Microsoft.DocumentDB",
|
||||
"message": "The subscription is not registered to use namespace 'Microsoft.DocumentDB'. See https://aka.ms/rps-not-found for how to register subscriptions."
|
||||
|
||||
```
|
||||
|
||||
**Solution:**
|
||||
|
||||
This error message appears, in case during the deployment it tries to create a type of resource which has never been deployed before inside the subscription. We recommend to check prior the deployment whether the required resource providers are registered for your subscription and if needed, register them through the `Azure Portal`, `Azure Powershell` or `Azure CLI` as mentioned [here](https://docs.microsoft.com/azure/azure-resource-manager/management/resource-providers-and-types).
|
||||
|
||||
>[Previous (Option (a) GitHub Actions)](/docs/EnterpriseScaleAnalytics-GitHubActionsDeployment.md)
|
||||
>[Previous (Option (b) Azure DevOps)](/docs/EnterpriseScaleAnalytics-AzureDevOpsDeployment.md)
|
||||
|
|
@ -0,0 +1,98 @@
|
|||
# Data Product Streaming - Prerequisites
|
||||
|
||||
This template repsitory contains all templates to deploy a Data Product for real-time data processing inside a Data Landing Zone of the Enterprise-Scale Analytics architecture. Data Products are another unit of scale inside a Data Landing Zone and provide environments to cross-functional teams to work on individual data use-cases. This template can also be used for data integration into the platform. The fundamental difference would be that these teams are then connecting to data sources outside of the Data Landing Zones. Hence, this template qualifis for the following usage:
|
||||
|
||||
| Scenario | Applicability |
|
||||
|:-----------------|:-------------------|
|
||||
| Data Product | :heavy_check_mark: |
|
||||
| Data Integration | :heavy_check_mark: |
|
||||
|
||||
## What will be deployed?
|
||||
|
||||
By navigating through the deployment steps, you will deploy the folowing setup in a subscription:
|
||||
|
||||
> **Note:** Before deploying the resources, we recommend to check registration status of the required resource providers in your subscription. For more information, see [Resource providers for Azure services](https://docs.microsoft.com/azure/azure-resource-manager/management/resource-providers-and-types).
|
||||
|
||||
|
||||
|
||||
The deployment and code artifacts include the following services:
|
||||
|
||||
- [Key Vault](https://docs.microsoft.com/azure/key-vault/general)
|
||||
- [Event Hub](https://docs.microsoft.com/azure/event-hubs/)
|
||||
- [IoT Hub](https://docs.microsoft.com/azure/iot-hub/about-iot-hub)
|
||||
- [Stream Analytics](https://docs.microsoft.com/azure/stream-analytics/stream-analytics-introduction)
|
||||
- [Cosmos DB](https://docs.microsoft.com/azure/cosmos-db/introduction)
|
||||
- [Synapse Workspace](https://docs.microsoft.com/azure/synapse-analytics/)
|
||||
- [Azure SQL Database](https://docs.microsoft.com/azure/azure-sql/database/)
|
||||
- [SQL Pool](https://docs.microsoft.com/azure/synapse-analytics/sql-data-warehouse/sql-data-warehouse-overview-what-is)
|
||||
- [SQL Server](https://docs.microsoft.com/sql/sql-server/?view=sql-server-ver15)
|
||||
- [SQL Elastic Pool](https://docs.microsoft.com/azure/azure-sql/database/elastic-pool-overview)
|
||||
- [BigData Pool](https://docs.microsoft.com/sql/big-data-cluster/concept-data-pool?view=sql-server-ver15)
|
||||
|
||||
## Code Structure
|
||||
|
||||
To help you more quickly understand the structure of the repository, here is an overview of what the respective folders contain:
|
||||
|
||||
| File/folder | Description |
|
||||
| ----------------------------- | ------------------------------------------ |
|
||||
| `.ado/workflows` | Folder for ADO workflows. The `dataProductDeployment.yml` workflow shows the steps for an end-to-end deployment of the architecture. |
|
||||
| `.github/workflows` | Folder for GitHub workflows. The `dataProductDeployment.yml` workflow shows the steps for an end-to-end deployment of the architecture. |
|
||||
| `code` | Sample password generation script that will be run in the deployment workflow for resources that require a password during the deployment. |
|
||||
| `docs` | Resources for this README. |
|
||||
| `infra` | Folder containing all the ARM and Bicep templates for each of the resources that will be deployed. |
|
||||
| `CODE_OF_CONDUCT.md` | Microsoft Open Source Code of Conduct. |
|
||||
| `LICENSE` | The license for the sample. |
|
||||
| `README.md` | This README file. |
|
||||
| `SECURITY.md` | Microsoft Security README. |
|
||||
|
||||
## Supported Regions
|
||||
|
||||
For now, we are recommending to select one of the regions mentioned below. The list of regions is limited for now due to the fact that not all services and features are available in all regions. This is mostly related to the fact that we are recommending to leverage at least the zone-redundant storage replication option for all your central Data Lakes in the Data Landing Zones. Since zone-redundant storage is not available in all regions, we are limiting the regions in the Deploy to Azure experience. If you are planning to deploy the Data Management Zone and Data Landing Zone to a region that is not listed below, then please change the setting in the corresponding bicep files in this repository. Officially supported regions are:
|
||||
|
||||
- (Africa) South Africa North
|
||||
- (Asia Pacific) Southeast Asia
|
||||
- (Asia Pacific) Australia East
|
||||
- (Asia Pacific) Japan East
|
||||
- (Canada) Canada Central
|
||||
- (Europe) North Europe
|
||||
- (Europe) West Europe
|
||||
- (Europe) France Central
|
||||
- (Europe) Germany West Central
|
||||
- (Europe) UK South
|
||||
- (South America) Brazil South
|
||||
- (US) Central US
|
||||
- (US) East US
|
||||
- (US) East US 2
|
||||
- (US) South Central US
|
||||
- (US) West US 2
|
||||
|
||||
## Prerequisites
|
||||
|
||||
> **Note:** Please make sure you have successfully deployed a [Data Management Landing Zone](https://github.com/Azure/data-management-zone) and a [Data Landing Zone](https://github.com/Azure/data-landing-zone) beforehand. Also, this template requires subnets as specified in the prerequisites. The Data Landing Zone already creates a few subnets, which can be used for this Data Product.
|
||||
|
||||
Before we start with the deployment, please make sure that you have the following available:
|
||||
|
||||
- A **Data Management Landing Zone** deployed. For more information, check the [Data Management Landing Zone](https://github.com/Azure/data-management-zone) repo.
|
||||
- A **Data Landing Zone** deployed. For more information, check the [Data Landing Zone](https://github.com/Azure/data-landing-zone) repo.
|
||||
- A resource group within an Azure subscription
|
||||
- An Azure subscription. If you don't have an Azure subscription, [create your Azure free account today](https://azure.microsoft.com/free/).
|
||||
- [User Access Administrator](https://docs.microsoft.com/azure/role-based-access-control/built-in-roles#user-access-administrator) or [Owner](https://docs.microsoft.com/azure/role-based-access-control/built-in-roles#owner) access to the subscription to be able to create a service principal and role assignments for it.
|
||||
- Access to a subnet with `privateEndpointNetworkPolicies` and `privateLinkServiceNetworkPolicies` set to disabled. The Data Landing Zone deployment already creates a few subnets with this configuration (subnets with name `DataProduct00{x}Subnet` or `DataIntegration00{x}Subnet`.).
|
||||
- For the deployment, please choose one of the **Supported Regions**.
|
||||
|
||||
## Deployment
|
||||
|
||||
Now you have two options for the deployment of the Data Landing Zone:
|
||||
|
||||
1. Deploy to Azure Button
|
||||
2. GitHub Actions or Azure DevOps Pipelines
|
||||
|
||||
To use the Deploy to Azure Button, please click on the button below:
|
||||
|
||||
| Reference implementation | Description | Deploy to Azure |
|
||||
|:---------------------------|:------------|:----------------|
|
||||
| Data Product Streaming | Deploys a Data Workload template for Data Streaming Analysis to a resource group inside a Data Landing Zone. Please deploy a [Data Management Zone](https://github.com/Azure/data-management-zone) and [Data Landing Zone](https://github.com/Azure/data-landing-zone) first. |[](https://portal.azure.com/#blade/Microsoft_Azure_CreateUIDef/CustomDeploymentBlade/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Fdata-product-streaming%2Fmain%2Finfra%2Fmain.json/uiFormDefinitionUri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Fdata-product-streaming%2Fmain%2Fdocs%2Freference%2Fportal.dataProduct.json) | [Repository](https://github.com/Azure/data-product-streaming) |
|
||||
|
||||
Alternatively, click on `Next` to follow the steps required to successfully deploy the Data Landing Zone through GitHub Actions or Azure DevOps.
|
||||
|
||||
>[Next](/docs/EnterpriseScaleAnalytics-CreateRepository.md)
|
||||
|
|
@ -0,0 +1,100 @@
|
|||
# Data Product Streaming - Setting up Service Principal
|
||||
|
||||
A service principal with *Contributor*, *Private DNS Zone Contributor* and *Network Contributor* rights needs to be generated for authentication and authorization from GitHub or Azure DevOps to your Azure subscription. This is required to deploy resources to your environment.
|
||||
|
||||
> **Note:** The number of role assignments can be further reduced in a production scenario. The **Private DNS Zone Contributor** is not required if the deployment of DNS A-records of the Private Endpoints is automated through Azure Policies with `deployIfNotExists` effect.
|
||||
|
||||
## Create Service Principal
|
||||
|
||||
First, go to the Azure Portal to find the ID of your subscription. Then start the Cloud Shell or Azure CLI, login to Azure, set the Azure context and execute the following commands to generate the required credentials:
|
||||
|
||||
**Azure CLI:**
|
||||
|
||||
```sh
|
||||
# Replace {service-principal-name} and {subscription-id} with your
|
||||
# Azure subscription id and any name for your service principal.
|
||||
az ad sp create-for-rbac \
|
||||
--name "{service-principal-name}" \
|
||||
--role "Contributor" \
|
||||
--scopes "/subscriptions/{subscription-id}" \
|
||||
--sdk-auth
|
||||
```
|
||||
|
||||
This will generate the following JSON output:
|
||||
|
||||
```json
|
||||
{
|
||||
"clientId": "<GUID>",
|
||||
"clientSecret": "<GUID>",
|
||||
"subscriptionId": "<GUID>",
|
||||
"tenantId": "<GUID>",
|
||||
(...)
|
||||
}
|
||||
```
|
||||
|
||||
> **Note:** Take note of the output. It will be required for the next steps.
|
||||
|
||||
## Adding additional role assigments
|
||||
|
||||
For automation purposes, more role assignments are required for the service principal.
|
||||
Additional required role assignments include:
|
||||
|
||||
| Role Name | Description | Scope |
|
||||
|:----------|:------------|:------|
|
||||
| [Private DNS Zone Contributor](https://docs.microsoft.com/azure/role-based-access-control/built-in-roles#private-dns-zone-contributor) | We expect you to deploy all Private DNS Zones for all data services into a single subscription and resource group. Therefor, the service principal needs to be Private DNS Zone Contributor on the global dns resource group which was created during the Data Management Zone deployment. This is required to deploy A-records for the respective private endpoints.| <div style="width: 36ch">(Resource Group Scope) `/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}`</div> |
|
||||
| [Network Contributor](https://docs.microsoft.com/azure/role-based-access-control/built-in-roles#network-contributor) | In order to deploy Private Endpoints to the specified privatelink-subnet which was created during the Data Landing Zone deployment, the service principal requires **Network Contributor** access on that specific subnet.| <div style="width: 36ch">(Child-Resource Scope) `/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName} /providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}"`</div> |
|
||||
|
||||
To add these role assignments, you can use the [Azure Portal](https://portal.azure.com/) or run the following commands using Azure CLI/Azure Powershell:
|
||||
|
||||
**Azure CLI - Add role assignments:**
|
||||
|
||||
```sh
|
||||
# Get Service Principal Object ID
|
||||
az ad sp list --display-name "{servicePrincipalName}" --query "[].{objectId:objectId}" --output tsv
|
||||
|
||||
# Add role assignment
|
||||
# Resource Scope level assignment
|
||||
az role assignment create \
|
||||
--assignee "{servicePrincipalObjectId}" \
|
||||
--role "{roleName}" \
|
||||
--scopes "{scope}"
|
||||
|
||||
# Resource group scope level assignment
|
||||
az role assignment create \
|
||||
--assignee "{servicePrincipalObjectId}" \
|
||||
--role "{roleName}" \
|
||||
--resource-group "{resourceGroupName}"
|
||||
```
|
||||
|
||||
**Azure Powershell - Add role assignments:**
|
||||
|
||||
```powershell
|
||||
# Get Service Principal Object ID
|
||||
$spObjectId = (Get-AzADServicePrincipal -DisplayName "{servicePrincipalName}").id
|
||||
|
||||
# Add role assignment
|
||||
# For Resource Scope level assignment
|
||||
New-AzRoleAssignment `
|
||||
-ObjectId $spObjectId `
|
||||
-RoleDefinitionName "{roleName}" `
|
||||
-Scope "{scope}"
|
||||
|
||||
# For Resource group scope level assignment
|
||||
New-AzRoleAssignment `
|
||||
-ObjectId $spObjectId `
|
||||
-RoleDefinitionName "{roleName}" `
|
||||
-ResourceGroupName "{resourceGroupName}"
|
||||
|
||||
# For Child-Resource Scope level assignment
|
||||
New-AzRoleAssignment `
|
||||
-ObjectId $spObjectId `
|
||||
-RoleDefinitionName "{roleName}" `
|
||||
-ResourceName "{resourceName}" `
|
||||
-ResourceType "{resourceType (e.g. 'Microsoft.Network/virtualNetworks/subnets')}" `
|
||||
-ParentResource "{parentResource (e.g. 'virtualNetworks/{virtualNetworkName}')" `
|
||||
-ResourceGroupName "{resourceGroupName}
|
||||
```
|
||||
|
||||
>[Previous](/docs/EnterpriseScaleAnalytics-Prerequisites.md)
|
||||
>[Next (Option (a) GitHub Actions)](/docs/EnterpriseScaleAnalytics-GitHubActionsDeployment.md)
|
||||
>[Next (Option (b) Azure DevOps)](/docs/EnterpriseScaleAnalytics-AzureDevOpsDeployment.md)
|
||||
Двоичный файл не отображается.
|
После Ширина: | Высота: | Размер: 2.8 MiB |
Двоичные данные
docs/images/IntegrationStreaming.png
Двоичные данные
docs/images/IntegrationStreaming.png
Двоичный файл не отображается.
|
До Ширина: | Высота: | Размер: 28 KiB |
Двоичный файл не отображается.
|
После Ширина: | Высота: | Размер: 28 KiB |
|
|
@ -26,7 +26,7 @@ param tags object = {}
|
|||
param administratorPassword string
|
||||
@description('Specifies the resource ID of the default storage account file system for synapse.')
|
||||
param synapseDefaultStorageAccountFileSystemId string
|
||||
@description('Specifies the resource ID of the default storage account for strea analytics.')
|
||||
@description('Specifies the resource ID of the default storage account file system for stream analytics.')
|
||||
param streamanalyticsDefaultStorageAccountFileSystemId string
|
||||
@description('Specifies the resource ID of the central purview instance.')
|
||||
param purviewId string = ''
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
"_generator": {
|
||||
"name": "bicep",
|
||||
"version": "0.4.451.19169",
|
||||
"templateHash": "9263634847318123758"
|
||||
"templateHash": "10071379043103949328"
|
||||
}
|
||||
},
|
||||
"parameters": {
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
"streamanalyticsDefaultStorageAccountFileSystemId": {
|
||||
"type": "string",
|
||||
"metadata": {
|
||||
"description": "Specifies the resource ID of the default storage account for strea analytics."
|
||||
"description": "Specifies the resource ID of the default storage account file system for stream analytics."
|
||||
}
|
||||
},
|
||||
"purviewId": {
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче