Added 2 hour delay before cleanup (#2937)

* Added 2 hour delay before cleanup to prevent resource from being deleted when they are still in use

* Removed duplicate cleanup script

.github/workflows/cleanup-cli.yml

@@ -1,29 +0,0 @@
-name: cleanup-cli
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: "0 8 * * *"
-  pull_request:
-    branches:
-      - main
-    paths: 
-      - .github/workflows/cleanup-cli.yml
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-    - name: check out repo
-      uses: actions/checkout@v2
-    - name: azure login
-      uses: azure/login@v1
-      with:
-        creds: ${{secrets.AZUREML_CREDENTIALS}}
-    - name: install ml cli
-      run: az extension add -n ml -y
-    - name: setup workspace
-      run: bash setup.sh
-      working-directory: cli
-      continue-on-error: true
-    - name: run cleanup script
-      run: set -e; bash -x cleanup.sh
-      working-directory: cli

cli/cleanup.sh

@@ -1,67 +0,0 @@
-# delete online endpoints
-ENDPOINT_LIST=$(az ml online-endpoint list --query "[*].[name]" -o tsv)
-echo $ENDPOINT_LIST
-for val in $ENDPOINT_LIST; do
-    echo deleting $val
-    `az ml online-endpoint delete -n "$val" --yes --no-wait`
-done
-
-# delete batch endpoints
-ENDPOINT_LIST=$(az ml batch-endpoint list --query "[*].[name]" -o tsv)
-echo $ENDPOINT_LIST
-for val in $ENDPOINT_LIST; do
-    echo deleting $val
-    `az ml batch-endpoint delete -n "$val" --yes --no-wait`
-done
-
-# delete storage accounts
-STORAGE_ACCOUNT_LIST=$(az storage account list --query "[*].[name]" -o tsv)
-echo $STORAGE_ACCOUNT_LIST
-for val in $STORAGE_ACCOUNT_LIST; do
-    if [[ $val == *"oepstorage"* ]]; then
-        echo deleting $val
-        `az storage account delete -n "$val" --yes`
-    fi
-done
-
-# delete compute instances
-CI_LIST=$(az ml compute list --type ComputeInstance --query "[*].[name]" -o tsv)
-echo $CI_LIST
-for val in $CI_LIST; do
-    echo deleting $val
-    `az ml compute delete -n "$val" --yes --no-wait`
-done
-
-# delete UAI
-NAME_LIST=$(az identity list --query "[].{name:name}" -o tsv | sort -u)
-echo $NAME_LIST
-for name in $NAME_LIST; do
-    if [[ $name == *"oep-user-identity"* ]]; then
-        echo deleting $name
-        `az identity delete --name "$name"`
-    fi
-done
-
-# delete left over autoscale settings created for online endpoints
-AUTOSCALE_SETTINGS_LIST=$(az monitor autoscale list  --query "[*].[name]" -o tsv)
-for val in $AUTOSCALE_SETTINGS_LIST; do
-    if [[ $val == autoscale-* ]]; then
-        echo deleting $val
-        `az monitor autoscale delete -n "$val"`
-    fi
-done
-
-# delete workspaces created via testing
-WORKSPACES_LIST=$(az ml workspace list --query "[*].[name]" -o tsv)
-for val in $WORKSPACES_LIST; do
-    if [[ $val == "mlw-"* ]]; then
-        if [[ $val == "mlw-mevnet" ]]; then
-            echo skipping $val
-        else
-            echo deleting $val
-            `az ml workspace delete -n "$val" --yes --no-wait --all-resources`
-        fi
-    else
-        echo $val not a match
-    fi
-done

infra/bootstrapping/cleanup.sh

@@ -31,6 +31,10 @@ echo "AUTOSCALE_SETTINGS_LIST: $AUTOSCALE_SETTINGS_LIST"
 SAMPLES_WORKSPACE_LIST=$(az ml workspace list --query "[*].[name]" -o tsv | grep -E "mlw-basic-prod-|mlw-basicex-prod-" | sort -u)
 echo "SAMPLES_WORKSPACE_LIST: $SAMPLES_WORKSPACE_LIST"
 
+# Wait for 2 hours so that we don't delete entities that are still in use.
+echo waiting
+sleep 2h
+
 # Delete online endpoints
 for i in $ONLINE_ENDPOINT_LIST; do 
     echo "Deleting online-endpoint:$i" && az ml online-endpoint delete --name $i --yes --no-wait && echo "online-endpoint delete initiated for $i" ;