diff --git a/api_app/README.md b/api_app/README.md index 14e934d62..dc9f866c6 100644 --- a/api_app/README.md +++ b/api_app/README.md @@ -17,7 +17,8 @@ The TRE API is a service that users can interact with to request changes to work * [Running API](#running-api) * [Develop and run locally](#develop-and-run-locally) * [Develop and run in dev container](#develop-and-run-in-dev-container) - * [Deploy with docker](#deploy-with-docker) + * [Deploy with Docker](#deploy-with-docker) +* [Unit tests](#unit-tests) * [Implementation](#implementation) * [Auth in code](#auth-in-code) * [Workspace requests](#workspace-requests) @@ -188,9 +189,9 @@ The API endpoints documentation and the Swagger UI will be available at [https:/ The API endpoints documentation and the Swagger UI will be available at [https://localhost:8000/docs](https://localhost:8000/docs). -### Deploy with docker +### Deploy with Docker -You must have docker and docker-compose tools installed, and an Azure Cosmos DB configured in `.env` as described above. +You must have Docker and Docker Compose tools installed, and an Azure Cosmos DB configured in `.env` as described above. Then run: @@ -201,6 +202,16 @@ docker compose up -d app The API will be available at [https://localhost:8000/api](https://localhost:8000/api) in your browser. +## Unit tests + +The unit tests are written with pytest and located in folder `/api_app/tests_ma/`. + +Run all unit tests with the following command in the root folder of the repository: + +```cmd +pytest --ignore=e2e_tests +``` + ## Implementation *API application folder structure.* diff --git a/devops/terraform/main.tf b/devops/terraform/main.tf index cd74ceb25..c1f22ad7d 100644 --- a/devops/terraform/main.tf +++ b/devops/terraform/main.tf @@ -4,9 +4,7 @@ provider "azurerm" { data "azurerm_client_config" "current" {} -# -# Core management resources -# +# Resource group for TRE core management resource "azurerm_resource_group" "mgmt" { name = var.mgmt_resource_group_name location = var.location @@ -14,7 +12,6 @@ resource "azurerm_resource_group" "mgmt" { lifecycle { ignore_changes = [tags] } } - # Holds Terraform shared state (already exists, created by bootstrap.sh) resource "azurerm_storage_account" "state_storage" { name = var.mgmt_storage_account_name @@ -28,9 +25,15 @@ resource "azurerm_storage_account" "state_storage" { lifecycle { ignore_changes = [tags] } } -# +# Storage container for Porter data +# See https://github.com/getporter/azure-plugins#storage +resource "azurerm_storage_container" "porter_container" { + name = "porter" + storage_account_name = azurerm_storage_account.state_storage.name + container_access_type = "private" +} + # Shared container registry -# resource "azurerm_container_registry" "shared_acr" { name = var.acr_name resource_group_name = azurerm_resource_group.mgmt.name @@ -39,4 +42,4 @@ resource "azurerm_container_registry" "shared_acr" { admin_enabled = true lifecycle { ignore_changes = [tags] } -} \ No newline at end of file +} diff --git a/docs/auth.md b/docs/auth.md index f05982067..cdc23b119 100644 --- a/docs/auth.md +++ b/docs/auth.md @@ -22,10 +22,10 @@ The **TRE API** app registration defines the permissions, scopes and app roles f #### API permissions - TRE API -| API/permission name | Type | Description | Admin consent required | TRE usage | -| ------------------- | ---- | ----------- | ---------------------- | --------- | -| Microsoft Graph/Directory.Read.All (`https://graph.microsoft.com/Directory.Read.All`) | Application* | Allows the app to read data in your organization's directory, such as users, groups and apps, without a signed-in user. | Yes | Used e.g., to retrieve app registration details, user associated app roles etc. | -| Microsoft Graph/User.Read.All (`https://graph.microsoft.com/User.Read.All`) | Application* | Allows the app to read user profiles without a signed in user. | Yes | Reading user role assignments to check that the user has permissions to execute an action e.g., to view workspaces. See [`aad_access_service.py`](../api_app/services/aad_access_service.py). | +| API/permission name | Type | Description | Admin consent required | Status | TRE usage | +| ------------------- | ---- | ----------- | ---------------------- | ------ | --------- | +| Microsoft Graph/Directory.Read.All (`https://graph.microsoft.com/Directory.Read.All`) | Application* | Allows the app to read data in your organization's directory, such as users, groups and apps, without a signed-in user. | Yes | Granted for *[directory name]* | Used e.g., to retrieve app registration details, user associated app roles etc. | +| Microsoft Graph/User.Read.All (`https://graph.microsoft.com/User.Read.All`) | Application* | Allows the app to read user profiles without a signed in user. | Yes | Granted for *[directory name]* | Reading user role assignments to check that the user has permissions to execute an action e.g., to view workspaces. See [`aad_access_service.py`](../api_app/services/aad_access_service.py). | *) See the difference between [delegated and application permission](https://docs.microsoft.com/graph/auth/auth-concepts#delegated-and-application-permissions) types. @@ -58,12 +58,12 @@ The **TRE API** app registration requires no redirect URLs defined or anything e #### API permissions - TRE Swagger UI -| API/permission name | Type | Description | Admin consent required | -| ------------------- | ---- | ----------- | ---------------------- | -| Microsoft Graph/offline_access (`https://graph.microsoft.com/offline_access`) | Delegated* | Allows the app to see and update the data you gave it access to, even when users are not currently using the app. | No | -| Microsoft Graph/openid (`https://graph.microsoft.com/openid`) | Delegated* | Allows users to sign in to the app with their work or school accounts and allows the app to see basic user profile information. | No | -| TRE API/Workspace.Read (`api:///Workspace.Read`) | Delegated* | See [TRE API app registration scopes](#scopes---tre-api). | No | -| TRE API/Workspace.Write (`api:///Workspace.Write`) | Delegated* | See [TRE API app registration scopes](#scopes---tre-api). | No | +| API/permission name | Type | Description | Admin consent required | Status | +| ------------------- | ---- | ----------- | ---------------------- | ------ | +| Microsoft Graph/offline_access (`https://graph.microsoft.com/offline_access`) | Delegated* | Allows the app to see and update the data you gave it access to, even when users are not currently using the app. | No | Granted for *[directory name]* | +| Microsoft Graph/openid (`https://graph.microsoft.com/openid`) | Delegated* | Allows users to sign in to the app with their work or school accounts and allows the app to see basic user profile information. | No | Granted for *[directory name]* | +| TRE API/Workspace.Read (`api:///Workspace.Read`) | Delegated* | See [TRE API app registration scopes](#scopes---tre-api). | No | Granted for *[directory name]* | +| TRE API/Workspace.Write (`api:///Workspace.Write`) | Delegated* | See [TRE API app registration scopes](#scopes---tre-api). | No | Granted for *[directory name]* | *) See the difference between [delegated and application permission](https://docs.microsoft.com/graph/auth/auth-concepts#delegated-and-application-permissions) types. diff --git a/docs/index.md b/docs/index.md index 64242a75d..218d8ce95 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,11 +11,11 @@ * [Authentication & authorization](./auth.md) * The two ways of provisioning an instance of Azure TRE: 1. [GitHub Actions workflows (CI/CD)](./workflows.md) - 1. [Manual deployment](./manual-deployment.md) - * [Testing](./testing.md) + 1. [Quickstart](./deployment-quickstart.md)/[Manual deployment](./manual-deployment.md) * Composition Service components * [API](../api_app/README.md) - * [Resource Processor](../resource_processor/vmss_porter/readme.md) + * [Resource Processor](../resource_processor/README.md) + * [End-to-end tests](../e2e_tests/README.md) * Workspaces and workspace services * [Authoring workspace templates](./authoring-workspace-templates.md) * [Registering workspace templates](./registering-workspace-templates.md) diff --git a/docs/testing.md b/docs/testing.md deleted file mode 100644 index 4a1cbd58d..000000000 --- a/docs/testing.md +++ /dev/null @@ -1,40 +0,0 @@ -# Testing - -## Unit tests - -The unit tests are written with pytest and located in folders: - -- [API](../api_app/README.md) unit tests: `/api_app/tests_ma/` - -> The folders containing the unit tests cannot have the same name. Otherwise, pytest will get confused, when trying to run all tests in the root folder. - -Run all unit tests with the following command in the root folder of the repository: - -```cmd -pytest --ignore=e2e_tests -``` - -## End-to-end tests - -To run the E2E tests locally: - -1. Enter the e2e_tests directory: `cd e2e_tests` -1. Define the following environment variables: - -| Environment variable name | Description | -| ------------------------- | ----------- | -| `RESOURCE_LOCATION` | The Azure Tre deployed environment `LOCATION`. Example: eastus | -| `TRE_ID` | The Azure TRE instance name - used for deployment of resources (can be set to anything when debugging locally). Example value: `mytre-dev-3142` | -| `RESOURCE` | The application (client) ID of the [TRE API](auth.md#tre-api) service principal. | -| `AUTH_TENANT_ID` | The tenant ID of the Azure AD. | -| `CLIENT_ID` | The application (client) ID of the [E2E Test app](auth.md#tre-e2e-test) service principal. | -| `USERNAME` | The username of the [E2E User](auth.md#end-to-end-test-user). | -| `PASSWORD` | The password of the [E2E User](auth.md#end-to-end-test-user). | -| `AUTH_APP_CLIENT_ID` | The application (client) ID of the [Workspaces app](auth.md#workspaces). | -| `ACR_NAME` | The Azure Tre ACR. | - -1. Run the e2e tests: - - ```bash - PYTHONPATH=. python -m pytest --junit-xml pytest_e2e.xml - ``` diff --git a/e2e_tests/README.md b/e2e_tests/README.md new file mode 100644 index 000000000..9cdff65a4 --- /dev/null +++ b/e2e_tests/README.md @@ -0,0 +1,25 @@ +# End-to-end (E2E) tests + +To run the E2E tests locally: + +1. Navigate to the `e2e_tests` folder: `cd e2e_tests` +1. Define the following environment variables: + + | Environment variable name | Description | Example value | + | ------------------------- | ----------- | ------------- | + | `RESOURCE_LOCATION` | The Azure Tre deployed environment `LOCATION`. | `eastus` | + | `TRE_ID` | The Azure TRE instance name - used for deployment of resources (can be set to anything when debugging locally). | `mytre-dev-3142` | + | `RESOURCE` | The application (client) ID of the [TRE API](../docs/auth.md#tre-api) service principal. | | + | `AUTH_TENANT_ID` | The tenant ID of the Azure AD. | | + | `CLIENT_ID` | The application (client) ID of the [E2E Test app](../docs/auth.md#tre-e2e-test) service principal. | | + | `SCOPE` | Scope(s) for the token. | `api:///Workspace.Read api:///Workspace.Write` | + | `USERNAME` | The username of the [E2E User](../docs/auth.md#end-to-end-test-user). | | + | `PASSWORD` | The password of the [E2E User](../docs/auth.md#end-to-end-test-user). | | + | `AUTH_APP_CLIENT_ID` | The application (client) ID of the [workspaces app](auth.md#workspaces). | | + | `ACR_NAME` | The name of the TRE container registry. | | + +1. Run the E2E tests: + + ```bash + PYTHONPATH=. python -m pytest --junit-xml pytest_e2e.xml + ``` diff --git a/e2e_tests/conftest.py b/e2e_tests/conftest.py index a0abed8d5..e60ca1d7e 100644 --- a/e2e_tests/conftest.py +++ b/e2e_tests/conftest.py @@ -28,8 +28,7 @@ async def token(verify) -> str: payload = f"grant_type=password&resource={config.RESOURCE}&username={config.USERNAME}&password={config.PASSWORD}&scope={config.SCOPE}&client_id={config.CLIENT_ID}" url = f"https://login.microsoftonline.com/{config.AUTH_TENANT_ID}/oauth2/token" - response = await client.post(url, - headers=headers, data=payload) + response = await client.post(url, headers=headers, data=payload) token = response.json()["access_token"] assert token is not None, "Token not returned" diff --git a/resource_processor/vmss_porter/readme.md b/resource_processor/README.md similarity index 79% rename from resource_processor/vmss_porter/readme.md rename to resource_processor/README.md index dbd92b1dd..ea86ebd2a 100644 --- a/resource_processor/vmss_porter/readme.md +++ b/resource_processor/README.md @@ -1,16 +1,26 @@ -# VMSS Processor +# Resource Processor (VMSS) -## Build docker container +## Build and run the container -docker build -f ./vmss_porter/Dockerfile -t rp . +1. Navigate to `resource_processor/` folder and run `docker build` command: -docker run -it -v /var/run/docker.sock:/var/run/docker.sock --env-file .env rp + ```cmd + docker build -t resource-processor-vm-porter -f ./vmss_porter/Dockerfile . + ``` + +1. Run the image: + + ```cmd + docker run -it -v /var/run/docker.sock:/var/run/docker.sock --env-file .env resource-processor-vm-porter + ``` ## Local development -To work locally checkout the source code and run +To work locally checkout the source code and run: -``pip install -r requirements.txt`` +```cmd +pip install -r ./vmss_porter/requirements.txt +``` If you use visual studio code you can set up your launch.json to include the follwing block which will enable launching and debugging. @@ -43,7 +53,7 @@ If you use visual studio code you can set up your launch.json to include the fol } ``` -When working locally we use a service principal (SP). This SP needs enough permissions to be able to talk to service bus and to deploy resources into the subscription. That means the service principal needs Owner access to subscription(ARM_SUBSCRIPTION_ID) and also needs **Azure Service Bus Data Sender** and **Azure Service Bus Data Receiver** on the service bus namespace defined above(SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE). +When working locally we use a service principal (SP). This SP needs enough permissions to be able to talk to service bus and to deploy resources into the subscription. That means the service principal needs Owner access to subscription(ARM_SUBSCRIPTION_ID) and also needs **Azure Service Bus Data Sender** and **Azure Service Bus Data Receiver** on the service bus namespace defined above (SERVICE_BUS_FULLY_QUALIFIED_NAMESPACE). Once the above is setup you can simulate receiving messages from service bus by going to service bus explorer on the portal and using a message payload for SERVICE_BUS_RESOURCE_REQUEST_QUEUE as follows @@ -53,6 +63,10 @@ Once the above is setup you can simulate receiving messages from service bus by This will trigger receiving of messages and you can freely debug the code by setting breakpoints as desired. +## Porter Azure plugin + +Resource Processor uses [Porter Azure plugin](https://github.com/getporter/azure-plugins) to store Porter data in TRE management storage account. The storage container, named `porter`, is created during the bootstrapping phase of TRE deployment. [`run.sh`](./run.sh) script generates `config.toml` file in Porter home folder to enable the Azure plugin when the image is started. + ## Debugging deployed processor on Azure Check the section **Checking the Virtual Machine Scale Set(VMSS) instance running resource processor** in [debugging and troubleshooting guide](../../docs/ops_debugging_troubleshooting.md) diff --git a/resource_processor/run.sh b/resource_processor/run.sh new file mode 100644 index 000000000..5aff91bc3 --- /dev/null +++ b/resource_processor/run.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Generate required configuration for Porter Azure plugin +if [[ -z "${MGMT_RESOURCE_GROUP_NAME}" ]]; then + >&2 echo "Environment variable for TRE management resource group name missing" +fi + +if [[ -z "${MGMT_STORAGE_ACCOUNT_NAME}" ]]; then + >&2 echo "Environment variable for TRE management storage account name missing" +fi + +cat > /root/.porter/config.toml << EOF +default-storage = "azurestorage" + +[[storage]] +name = "azurestorage" +plugin = "azure.blob" + +[storage.config] +account="${MGMT_STORAGE_ACCOUNT_NAME}" +resource-group="${MGMT_RESOURCE_GROUP_NAME}" +EOF + +# Launch the runner +python -u vmss_porter/runner.py diff --git a/resource_processor/vmss_porter/Dockerfile b/resource_processor/vmss_porter/Dockerfile index 12eb01d53..b09a0d624 100644 --- a/resource_processor/vmss_porter/Dockerfile +++ b/resource_processor/vmss_porter/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.8-slim-buster -# Azure CLI +# Install Azure CLI RUN apt-get update \ && apt-get -y install ca-certificates curl apt-transport-https lsb-release gnupg \ && curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/microsoft.gpg > /dev/null \ @@ -8,7 +8,7 @@ RUN apt-get update \ && echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $AZ_REPO main" | tee /etc/apt/sources.list.d/azure-cli.list \ && apt-get update && apt-get -y install azure-cli -# Install terraform +# Install Terraform RUN apt-get update && apt-get install -y gnupg software-properties-common curl \ && curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - \ && apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" \ @@ -17,6 +17,9 @@ RUN apt-get update && apt-get install -y gnupg software-properties-common curl \ # Install Porter RUN export PORTER_HOME=/root/.porter && curl -L https://cdn.porter.sh/latest/install-linux.sh | bash +# Install Porter Azure plugin, see https://porter.sh/plugins/azure/ +RUN /root/.porter/porter plugin install azure --url https://cdn.porter.sh/plugins/azure + # Install Docker RUN apt-get update && apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release \ && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg \ @@ -38,4 +41,4 @@ COPY . /app WORKDIR /app/ -CMD ["python","-u", "vmss_porter/runner.py"] +CMD ["./run.sh"]