diff --git a/README.md b/README.md index c395999..fb49bee 100644 --- a/README.md +++ b/README.md @@ -32,20 +32,23 @@ Azure IPAM is a lightweight solution developed on top of the Azure platform desi | `.github/` | Bug Report, Issue Templates and GitHub Actions | | `.vscode/` | VSCode Configuration | | `deploy/` | Deployment Bicep Templates & PowerShell Deployment Scripts | +| `assets/` | Compiled ZIP Archive | | `docs/` | Documentation Folder | | `engine/` | Engine Application Code | | `examples/` | Example Templates, Scripts and Code Snippets for Azure IPAM | | `lb/` | Load Balancer (NGINX) Configs | -| `tests/` | Engine and UI Testing Scripts | +| `tests/` | Testing Scripts | +| `tools/` | Lifecycle Scripts (Build/Version/Update) | | `ui/` | UI Application Code | | `.dockerignore` | Untracked Docker Files to Ignore | | `.env.example` | Example ENV File to be Used with Docker Compose | | `.gitattributes` | Git File and Path Attributes | | `.gitignore` | Untracked Git Files to Ignore | | `CODE_OF_CONDUCT.md` | Microsoft Code of Conduct | -| `docker-compose.prod.yml` | Production Docker Compose File | | `docker-compose.yml` | Development Docker Compose File | -| `Dockerfile` | Single Container Dockerfile | +| `Dockerfile.deb` | Single Container Dockerfile (Debian) | +| `Dockerfile.func` | Single Container Dockerfile (Function) | +| `Dockerfile.rhel` | Single Container Dockerfile (Red Hat) | | `init.sh` | Single Container Init Script | | `LICENSE` | Microsoft MIT License | | `README.md` | This README File | diff --git a/assets/ipam.zip b/assets/ipam.zip index 4cc9dad..af98cc2 100644 Binary files a/assets/ipam.zip and b/assets/ipam.zip differ diff --git a/deploy/appService.bicep b/deploy/appService.bicep index 7924246..5dc958d 100644 --- a/deploy/appService.bicep +++ b/deploy/appService.bicep @@ -78,7 +78,7 @@ resource appService 'Microsoft.Web/sites@2021-02-01' = { alwaysOn: true linuxFxVersion: deployAsContainer ? 'DOCKER|${acrUri}/ipam:latest' : 'PYTHON|3.9' appCommandLine: !deployAsContainer ? 'init.sh 8000' : null - healthCheckPath: '/api/docs' + healthCheckPath: '/api/status' appSettings: concat( [ { diff --git a/deploy/deploy.ps1 b/deploy/deploy.ps1 index a0f7262..7be5aae 100644 --- a/deploy/deploy.ps1 +++ b/deploy/deploy.ps1 @@ -253,7 +253,8 @@ DynamicParam { if ($invalidFields -or $missingFields) { $deploymentType = $PrivateAcr ? "'$($PSCmdlet.ParameterSetName) w/ Private ACR'" : $PSCmdlet.ParameterSetName - Write-Host "ERROR: Missing or improperly formatted field(s) in 'ResourceNames' parameter for deploment type '$deploymentType'" -ForegroundColor Red + Write-Host + Write-Host "ERROR: Missing or improperly formatted field(s) in 'ResourceNames' parameter for deploment type $deploymentType" -ForegroundColor Red foreach ($field in $invalidFields) { Write-Host "ERROR: Invalid Field ->" $field -ForegroundColor Red @@ -266,7 +267,7 @@ DynamicParam { Write-Host "ERROR: Please refer to the 'Naming Rules and Restrictions for Azure Resources'" -ForegroundColor Red Write-Host "ERROR: " -ForegroundColor Red -NoNewline Write-Host "https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-name-rules" -ForegroundColor Yellow - Write-Host "" + Write-Host throw [System.ArgumentException]::New("One of the required resource names is missing or invalid.") } @@ -294,7 +295,7 @@ process { $AZURE_ENV_MAP = @{ AzureCloud = "AZURE_PUBLIC" AzureUSGovernment = "AZURE_US_GOV" - AzureUSSecret = "AZURE_US_GOV_SECRET" + USSec = "AZURE_US_GOV_SECRET" AzureGermanCloud = "AZURE_GERMANY" AzureChinaCloud = "AZURE_CHINA" } @@ -613,7 +614,8 @@ process { $accesstoken = (Get-AzAccessToken -Resource "https://$($msGraphMap[$AzureCloud].Endpoint)/").Token # Switch Access Token to SecureString if Graph Version is 2.x - $graphVersion = [System.Version](Get-InstalledModule -Name Microsoft.Graph).Version + $graphVersion = [System.Version](Get-InstalledModule -Name Microsoft.Graph | Sort-Object -Property Version | Select-Object -Last 1).Version ` + ?? (Get-Module -Name Microsoft.Graph | Sort-Object -Property Version | Select-Object -Last 1).Version if ($graphVersion.Major -gt 1) { $accesstoken = ConvertTo-SecureString $accesstoken -AsPlainText -Force @@ -842,6 +844,72 @@ process { return $deployment } + Function Publish-ZipFile { + Param( + [Parameter(Mandatory=$true)] + [string]$AppName, + [Parameter(Mandatory=$true)] + [string]$ResourceGroupName, + [Parameter(Mandatory=$false)] + [switch]$UseAPI + ) + + if ($UseAPI) { + Write-Host "INFO: Using Kudu API for ZIP Deploy" -ForegroundColor Green + } + + $zipPath = Join-Path -Path $ROOT_DIR -ChildPath 'assets' -AdditionalChildPath "ipam.zip" + + $publishRetries = 3 + $publishSuccess = $False + + if ($UseAPI) { + $accessToken = (Get-AzAccessToken).Token + $zipContents = Get-Item -Path $zipPath + + $publishProfile = Get-AzWebAppPublishingProfile -Name $AppName -ResourceGroupName $ResourceGroupName + $zipUrl = ([System.uri]($publishProfile | Select-Xml -XPath "//publishProfile[@publishMethod='ZipDeploy']" | Select-Object -ExpandProperty Node).publishUrl).Scheme + } + + do { + try { + if (-not $UseAPI) { + Publish-AzWebApp ` + -Name $AppName ` + -ResourceGroupName $ResourceGroupName ` + -ArchivePath $zipPath ` + -Restart ` + -Force ` + | Out-Null + } else { + Invoke-RestMethod ` + -Uri "https://${zipUrl}/api/zipdeploy" ` + -Method Post ` + -ContentType "multipart/form-data" ` + -Headers @{ "Authorization" = "Bearer $accessToken" } ` + -Form @{ file = $zipContents } ` + -StatusCodeVariable statusCode ` + | Out-Null + + if ($statusCode -ne 200) { + throw [System.Exception]::New("Error while uploading ZIP Deploy via Kudu API! ($statusCode)") + } + } + + $publishSuccess = $True + Write-Host "INFO: ZIP Deploy archive successfully uploaded" -ForegroundColor Green + } catch { + if($publishRetries -gt 0) { + Write-Host "WARNING: Problem while uploading ZIP Deploy archive! Retrying..." -ForegroundColor Yellow + $publishRetries-- + } else { + Write-Host "ERROR: Unable to upload ZIP Deploy archive!" -ForegroundColor Red + throw $_ + } + } + } while ($publishSuccess -eq $False -and $publishRetries -ge 0) + } + Function Update-UIApplication { Param( [Parameter(Mandatory=$true)] @@ -983,26 +1051,12 @@ process { if ($PSCmdlet.ParameterSetName -in ('App', 'Function')) { Write-Host "INFO: Uploading ZIP Deploy archive..." -ForegroundColor Green - $zipPath = Join-Path -Path $ROOT_DIR -ChildPath 'assets' -AdditionalChildPath "ipam.zip" - - $publishRetries = 5 - $publishSuccess = $False - - do { - try { - Publish-AzWebApp -ResourceGroupName $deployment.Outputs["resourceGroupName"].Value -Name $deployment.Outputs["appServiceName"].Value -ArchivePath $zipPath -Restart -Force | Out-Null - $publishSuccess = $True - Write-Host "INFO: ZIP Deploy archive successfully uploaded" -ForegroundColor Green - } catch { - if($publishRetries -gt 0) { - Write-Host "WARNING: Problem while uploading ZIP Deploy archive! Retrying..." -ForegroundColor Yellow - $publishRetries-- - } else { - Write-Host "ERROR: Unable to upload ZIP Deploy archive!" -ForegroundColor Red - throw $_ - } - } - } while ($publishSuccess -eq $False -and $publishRetries -gt 0) + try { + Publish-ZipFile -AppName $deployment.Outputs["appServiceName"].Value -ResourceGroupName $deployment.Outputs["resourceGroupName"].Value + } catch { + Write-Host "SWITCH: Retrying ZIP Deploy with Kudu API..." -ForegroundColor Blue + Publish-ZipFile -AppName $deployment.Outputs["appServiceName"].Value -ResourceGroupName $deployment.Outputs["resourceGroupName"].Value -UseAPI + } } if ($PSCmdlet.ParameterSetName -in ('AppContainer', 'FunctionContainer') -and $PrivateAcr) { diff --git a/deploy/functionApp.bicep b/deploy/functionApp.bicep index 4d08e58..514b814 100644 --- a/deploy/functionApp.bicep +++ b/deploy/functionApp.bicep @@ -81,7 +81,7 @@ resource functionApp 'Microsoft.Web/sites@2021-03-01' = { acrUseManagedIdentityCreds: privateAcr ? true : false acrUserManagedIdentityID: privateAcr ? managedIdentityClientId : null linuxFxVersion: deployAsContainer ? 'DOCKER|${acrUri}/ipamfunc:latest' : 'Python|3.9' - healthCheckPath: '/api/docs' + healthCheckPath: '/api/status' appSettings: concat( [ { diff --git a/engine/app/dependencies.py b/engine/app/dependencies.py index 923e05e..c1e85bf 100644 --- a/engine/app/dependencies.py +++ b/engine/app/dependencies.py @@ -14,6 +14,8 @@ from app.routers.common.helper import ( from app.globals import globals +from app.logs.logs import ipam_logger as logger + _session = None async def fetch_jwks_keys(): @@ -76,6 +78,16 @@ async def validate_token(request: Request): except Exception: raise HTTPException(status_code=401, detail="Unable to parse authorization token.") + try: + token_version = int(jwt.decode(token, options={"verify_signature": False})["ver"].split(".")[0]) + except Exception: + raise HTTPException(status_code=401, detail="Unable to decode token version.") + + if token_version == 1: + logger.error("Microsoft Identity v1.0 access tokens are not supported!") + logger.error("https://learn.microsoft.com/en-us/entra/identity-platform/access-tokens#token-formats") + raise HTTPException(status_code=401, detail="Microsoft Identity v1.0 access tokens are not supported.") + if rsa_key: rsa_pem_key = jwt.algorithms.RSAAlgorithm.from_jwk(json.dumps(rsa_key)) rsa_pem_key_bytes = rsa_pem_key.public_bytes( @@ -99,10 +111,10 @@ async def validate_token(request: Request): except jwt.InvalidSignatureError: raise HTTPException(status_code=401, detail="Invalid token signature.") except Exception: - raise HTTPException(status_code=401, detail="Unable to parse authorization token.") + raise HTTPException(status_code=401, detail="Unable to decode authorization token.") else: raise HTTPException(status_code=401, detail="Unable to find appropriate signing key.") - + request.state.tenant_id = payload['tid'] return payload diff --git a/engine/app/main.py b/engine/app/main.py index 6a28da7..fb3ca04 100644 --- a/engine/app/main.py +++ b/engine/app/main.py @@ -470,9 +470,8 @@ async def ipam_startup(): await db_upgrade() -# https://github.com/yuval9313/FastApi-RESTful/issues/138 @app.on_event("startup") -@repeat_every(seconds = 60, wait_first = True) # , wait_first=True +@repeat_every(seconds = 60, wait_first = True) async def find_reservations() -> None: if not os.environ.get("FUNCTIONS_WORKER_RUNTIME"): try: @@ -480,7 +479,7 @@ async def find_reservations() -> None: except Exception as e: logger.error('Error running network check loop!') tb = traceback.format_exc() - logger.debug(tb); + logger.debug(tb) raise e @app.exception_handler(StarletteHTTPException) diff --git a/engine/function_app.py b/engine/function_app.py new file mode 100644 index 0000000..d5fd0de --- /dev/null +++ b/engine/function_app.py @@ -0,0 +1,33 @@ +import logging +import traceback +from datetime import datetime, timezone + +import azure.functions as func + +from app.main import app as ipam +from app.logs.logs import ipam_logger as logger +from app.routers.azure import match_resv_to_vnets + +azureLogger = logging.getLogger('azure') +azureLogger.setLevel(logging.ERROR) + +app = func.AsgiFunctionApp(app=ipam, http_auth_level=func.AuthLevel.ANONYMOUS) + +# @app.function_name(name="ipam-sentinel") +# @app.schedule(schedule="0 * * * * *", arg_name="mytimer", run_on_startup=True, use_monitor=False) +@app.timer_trigger(schedule="0 * * * * *", arg_name="mytimer", run_on_startup=True, use_monitor=False) +async def ipam_sentinel(mytimer: func.TimerRequest) -> None: + utc_timestamp = datetime.utcnow().replace(tzinfo=timezone.utc).isoformat() + + logger.info('Azure IPAM Sentinel function was triggered') + + if mytimer.past_due: + logger.debug('The timer is past due ({})!'.format(utc_timestamp)) + + try: + await match_resv_to_vnets() + except Exception as e: + logger.error('Error running network check loop!') + tb = traceback.format_exc() + logger.debug(tb) + raise e diff --git a/engine/host.json b/engine/host.json index 6d10d82..95ba23d 100644 --- a/engine/host.json +++ b/engine/host.json @@ -6,6 +6,9 @@ } }, "logging": { + "logLevel": { + "default": "Information" + }, "applicationInsights": { "samplingSettings": { "isEnabled": true, diff --git a/engine/ipam-func/__init__.py b/engine/ipam-func/__init__.py deleted file mode 100644 index 36bbfd9..0000000 --- a/engine/ipam-func/__init__.py +++ /dev/null @@ -1,58 +0,0 @@ -import azure.functions as func -from azure.functions._http_asgi import AsgiResponse, AsgiRequest - -# https://github.com/Azure-Samples/fastapi-on-azure-functions/issues/4 -# https://github.com/Azure/azure-functions-python-library/pull/143 -# import nest_asyncio - -import sys -import logging - -from app.main import app as ipam - -# https://github.com/Azure-Samples/fastapi-on-azure-functions/issues/4 -# https://github.com/Azure/azure-functions-python-library/pull/143 -# nest_asyncio.apply() - -logger = logging.getLogger('azure') -logger.setLevel(logging.ERROR) - -IS_INITED = False - -async def run_setup(app): - """Workaround to run Starlette startup events on Azure Function Workers.""" - - global IS_INITED - - if not IS_INITED: - await app.router.startup() - IS_INITED = True - -async def handle_asgi_request(req: func.HttpRequest, context: func.Context) -> func.HttpResponse: - asgi_request = AsgiRequest(req, context) - scope = asgi_request.to_asgi_http_scope() - asgi_response = await AsgiResponse.from_app(ipam, scope, req.get_body()) - - return asgi_response.to_func_response() - -async def main(req: func.HttpRequest, context: func.Context) -> func.HttpResponse: - await run_setup(ipam) - - return await handle_asgi_request(req, context) - -# Current issue where FastAPI startup is ignored: -# https://github.com/Azure/azure-functions-python-worker/issues/911 -# -# Target function format after fix: -# -# async def main(req: func.HttpRequest, context: func.Context) -> func.HttpResponse: -# """Each request is redirected to the ASGI handler.""" -# return await func.AsgiMiddleware(ipam).handle_async(req, context) - -# Keeping an eye on this fix as well: -# https://github.com/Azure/azure-functions-python-library/pull/148 - -# Log Stream Flood Issue(s) -# https://github.com/Azure/azure-functions-dotnet-worker/issues/796 -# https://github.com/Azure/azure-functions-host/issues/8973 -# See logging.getLogger('azure') workaround above... diff --git a/engine/ipam-func/function.json b/engine/ipam-func/function.json deleted file mode 100644 index 01051a4..0000000 --- a/engine/ipam-func/function.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "scriptFile": "__init__.py", - "bindings": [ - { - "authLevel": "anonymous", - "type": "httpTrigger", - "direction": "in", - "name": "req", - "methods": [ - "get", - "post", - "patch", - "put", - "delete" - ], - "route": "/{*route}" - }, - { - "type": "http", - "direction": "out", - "name": "$return" - } - ] -} diff --git a/engine/ipam-sentinel/__init__.py b/engine/ipam-sentinel/__init__.py deleted file mode 100644 index bab1c8a..0000000 --- a/engine/ipam-sentinel/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -import datetime - -import azure.functions as func - -from app.logs.logs import ipam_logger as logger - -from app.routers.azure import match_resv_to_vnets - -async def main(mytimer: func.TimerRequest) -> None: - utc_timestamp = datetime.datetime.utcnow().replace( - tzinfo=datetime.timezone.utc).isoformat() - - if mytimer.past_due: - logger.info('The timer is past due!') - - logger.info('Python timer trigger function ran at %s', utc_timestamp) - - await match_resv_to_vnets() diff --git a/engine/ipam-sentinel/function.json b/engine/ipam-sentinel/function.json deleted file mode 100644 index 76d99ba..0000000 --- a/engine/ipam-sentinel/function.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "scriptFile": "__init__.py", - "bindings": [ - { - "name": "mytimer", - "type": "timerTrigger", - "direction": "in", - "schedule": "0 * * * * *" - } - ] -} diff --git a/build/build.ps1 b/tools/build.ps1 similarity index 96% rename from build/build.ps1 rename to tools/build.ps1 index 4da691f..f326760 100644 --- a/build/build.ps1 +++ b/tools/build.ps1 @@ -114,8 +114,7 @@ try { $FilePath = Join-Path -Path $Path -ChildPath $FileName Compress-Archive -Path ..\engine\app -DestinationPath $FilePath -Force - Compress-Archive -Path ..\engine\ipam-func -DestinationPath $FilePath -Update - Compress-Archive -Path ..\engine\ipam-sentinel -DestinationPath $FilePath -Update + Compress-Archive -Path ..\engine\function_app.py -DestinationPath $FilePath -Update Compress-Archive -Path ..\engine\requirements.txt -DestinationPath $FilePath -Update Compress-Archive -Path ..\engine\host.json -DestinationPath $FilePath -Update Compress-Archive -Path ..\ui\dist -DestinationPath $FilePath -Update diff --git a/build/update.ps1 b/tools/update.ps1 similarity index 81% rename from build/update.ps1 rename to tools/update.ps1 index de90db2..5e74cdd 100644 --- a/build/update.ps1 +++ b/tools/update.ps1 @@ -89,6 +89,72 @@ Function Restart-IpamApp { } while ($restartSuccess -eq $False -and $restartRetries -gt 0) } +Function Publish-ZipFile { + Param( + [Parameter(Mandatory=$true)] + [string]$AppName, + [Parameter(Mandatory=$true)] + [string]$ResourceGroupName, + [Parameter(Mandatory=$false)] + [switch]$UseAPI + ) + + if ($UseAPI) { + Write-Host "INFO: Using Kudu API for ZIP Deploy" -ForegroundColor Green + } + + $zipPath = Join-Path -Path $ROOT_DIR -ChildPath 'assets' -AdditionalChildPath "ipam.zip" + + $publishRetries = 3 + $publishSuccess = $False + + if ($UseAPI) { + $accessToken = (Get-AzAccessToken).Token + $zipContents = Get-Item -Path $zipPath + + $publishProfile = Get-AzWebAppPublishingProfile -Name $AppName -ResourceGroupName $ResourceGroupName + $zipUrl = ([System.uri]($publishProfile | Select-Xml -XPath "//publishProfile[@publishMethod='ZipDeploy']" | Select-Object -ExpandProperty Node).publishUrl).Scheme + } + + do { + try { + if (-not $UseAPI) { + Publish-AzWebApp ` + -Name $AppName ` + -ResourceGroupName $ResourceGroupName ` + -ArchivePath $zipPath ` + -Restart ` + -Force ` + | Out-Null + } else { + Invoke-RestMethod ` + -Uri "https://${zipUrl}/api/zipdeploy" ` + -Method Post ` + -ContentType "multipart/form-data" ` + -Headers @{ "Authorization" = "Bearer $accessToken" } ` + -Form @{ file = $zipContents } ` + -StatusCodeVariable statusCode ` + | Out-Null + + if ($statusCode -ne 200) { + throw [System.Exception]::New("Error while uploading ZIP Deploy via Kudu API! ($statusCode)") + } + } + + $publishSuccess = $True + Write-Host "INFO: ZIP Deploy archive successfully uploaded" -ForegroundColor Green + } catch { + if($publishRetries -gt 0) { + Write-Host "WARNING: Problem while uploading ZIP Deploy archive! Retrying..." -ForegroundColor Yellow + $publishRetries-- + } else { + Write-Host "ERROR: Unable to upload ZIP Deploy archive!" -ForegroundColor Red + throw $_ + } + } + } while ($publishSuccess -eq $False -and $publishRetries -ge 0) +} + Start-Transcript -Path $updateLog | Out-Null try { @@ -267,33 +333,12 @@ try { } else { Write-Host "INFO: Uploading ZIP Deploy archive..." -ForegroundColor Green - $zipPath = Join-Path -Path $ROOT_DIR -ChildPath 'assets' -AdditionalChildPath "ipam.zip" - - $publishRetries = 5 - $publishSuccess = $False - - do { - try { - Publish-AzWebApp ` - -Name $AppName ` - -ResourceGroupName $ResourceGroupName ` - -ArchivePath $zipPath ` - -Restart ` - -Force ` - | Out-Null - - $publishSuccess = $True - Write-Host "INFO: ZIP Deploy archive successfully uploaded" -ForegroundColor Green - } catch { - if($publishRetries -gt 0) { - Write-Host "WARNING: Problem while uploading ZIP Deploy archive! Retrying..." -ForegroundColor Yellow - $publishRetries-- - } else { - Write-Host "ERROR: Unable to upload ZIP Deploy archive!" -ForegroundColor Red - throw $_ - } - } - } while ($publishSuccess -eq $False -and $publishRetries -gt 0) + try { + Publish-ZipFile -AppName $AppName -ResourceGroupName $ResourceGroupName + } catch { + Write-Host "SWITCH: Retrying ZIP Deploy with Kudu API..." -ForegroundColor Blue + Publish-ZipFile -AppName $AppName -ResourceGroupName $ResourceGroupName -UseAPI + } Write-Host Write-Host "NOTE: Please allow ~5 minutes for the ZIP Deploy process to complete" -ForegroundColor Yellow diff --git a/build/version.ps1 b/tools/version.ps1 similarity index 100% rename from build/version.ps1 rename to tools/version.ps1