ci: refactor bootstrapping to avoid needing to invoke `apt get` (#2336)

* refactor: Stop invoking jq in infra scripts * refactor: Remove some commented out code from infra * refactor: Don't install jq Ubuntu runners come with it pre-installed: 4fe7f6bc86/images/linux/Ubuntu2204-Readme.md * refactor: Replace `az command | jq '.QUERY'` with `az command --query QUERY` * refactor: Collapse `jq | jq | jq` into a single jq invocation * refactor: Do not install xmlstarlet Seem to be entirely unused * refactor: Don't install uuid-runtime and remove install_packages function
2023-05-30 16:13:54 -04:00 · 2023-05-30 16:13:54 -04:00 · 57ff370d0f
--- a/cli/batch-score-rest.sh
+++ b/cli/batch-score-rest.sh
@ -1,9 +1,9 @@
 # <create_variables>
-SUBSCRIPTION_ID=$(az account show --query id | tr -d '\r"')
-RESOURCE_GROUP=$(az group show --query name | tr -d '\r"')
-WORKSPACE=$(az configure -l | jq -r '.[] | select(.name=="workspace") | .value')
+SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+RESOURCE_GROUP=$(az group show --query name -o tsv)
+WORKSPACE=$(az configure -l --query "[?name=='workspace'].value" -o tsv)

-LOCATION=$(az ml workspace show| jq -r '.location')
+LOCATION=$(az ml workspace show --query location -o tsv)

 API_VERSION="2022-05-01"

@ -149,7 +149,7 @@ response=$(curl --location --request PUT "https://management.azure.com/subscript
 }")
 #</create_endpoint>

-operation_id=$(echo $response | jq -r '.properties' | jq -r '.properties' | jq -r '.AzureAsyncOperationUri')
+operation_id=$(echo $response | jq -r '.properties.properties.AzureAsyncOperationUri')
 wait_for_completion $operation_id $TOKEN

 # <create_deployment>
@ -201,7 +201,7 @@ response=$(curl --location --request PUT "https://management.azure.com/subscript
    \"location\": \"$LOCATION\"
 }")

-operation_id=$(echo $response | jq -r '.properties' | jq -r '.properties' | jq -r '.AzureAsyncOperationUri')
+operation_id=$(echo $response | jq -r '.properties.properties.AzureAsyncOperationUri')
 wait_for_completion $operation_id $TOKEN
 #</set_endpoint_defaults>

--- a/cli/deploy-rest.sh
+++ b/cli/deploy-rest.sh
@ -1,10 +1,10 @@
 set -x

 # <create_variables>
-SUBSCRIPTION_ID=$(az account show --query id | tr -d '\r"')
-LOCATION=$(az ml workspace show --query location | tr -d '\r"')
-RESOURCE_GROUP=$(az group show --query name | tr -d '\r"')
-WORKSPACE=$(az configure -l | jq -r '.[] | select(.name=="workspace") | .value')
+SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+LOCATION=$(az ml workspace show --query location -o tsv)
+RESOURCE_GROUP=$(az group show --query name -o tsv)
+WORKSPACE=$(az configure -l --query "[?name=='workspace'].value" -o tsv)

 #</create_variables>

@ -124,7 +124,7 @@ response=$(curl --location --request PUT "https://management.azure.com/subscript
 #</create_endpoint>

 echo "Endpoint response: $response"
-operation_id=$(echo $response | jq -r '.properties' | jq -r '.properties' | jq -r '.AzureAsyncOperationUri')
+operation_id=$(echo $response | jq -r '.properties.properties.AzureAsyncOperationUri')
 wait_for_completion $operation_id

 # <create_deployment>
@ -153,7 +153,7 @@ response=$(curl --location --request PUT "https://management.azure.com/subscript
 #</create_deployment>

 echo "Endpoint response: $response"
-operation_id=$(echo $response | jq -r '.properties' | jq -r '.properties' | jq -r '.AzureAsyncOperationUri')
+operation_id=$(echo $response | jq -r '.properties.properties.AzureAsyncOperationUri')
 wait_for_completion $operation_id

 # <get_endpoint>
@ -161,7 +161,7 @@ response=$(curl --location --request GET "https://management.azure.com/subscript
 --header "Content-Type: application/json" \
 --header "Authorization: Bearer $TOKEN")

-scoringUri=$(echo $response | jq -r '.properties' | jq -r '.scoringUri')
+scoringUri=$(echo $response | jq -r '.properties.scoringUri')
 # </get_endpoint>

 # <get_access_token>
--- a/cli/endpoints/batch/deploy-models/custom-outputs-parquet/deploy-and-run.sh
+++ b/cli/endpoints/batch/deploy-models/custom-outputs-parquet/deploy-and-run.sh
@ -50,7 +50,7 @@ az ml batch-deployment show --name $DEPLOYMENT_NAME --endpoint-name $ENDPOINT_NA

 echo "Invoking batch endpoint"
 # <start_batch_scoring_job>
-JOB_NAME = $(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input https://azuremlexampledata.blob.core.windows.net/data/heart-disease-uci/data | jq -r '.name')
+JOB_NAME = $(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input https://azuremlexampledata.blob.core.windows.net/data/heart-disease-uci/data --query name -o tsv)
 # </start_batch_scoring_job>

 echo "Showing job detail"
--- a/cli/endpoints/batch/deploy-models/heart-classifier-mlflow/deploy-and-run.sh
+++ b/cli/endpoints/batch/deploy-models/heart-classifier-mlflow/deploy-and-run.sh
@ -54,7 +54,7 @@ az ml batch-deployment show --name $DEPLOYMENT_NAME --endpoint-name $ENDPOINT_NA

 echo "Invoking batch endpoint"
 # <start_batch_scoring_job>
-JOB_NAME = $(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input azureml:heart-dataset-unlabeled@latest | jq -r '.name')
+JOB_NAME = $(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input azureml:heart-dataset-unlabeled@latest --query name -o tsv)
 # </start_batch_scoring_job>

 echo "Showing job detail"
--- a/cli/endpoints/batch/deploy-pipelines/batch-scoring-with-preprocessing/deploy-and-run.sh
+++ b/cli/endpoints/batch/deploy-pipelines/batch-scoring-with-preprocessing/deploy-and-run.sh
@ -53,7 +53,7 @@ az ml batch-deployment create --endpoint $ENDPOINT_NAME -f deployment.yml --set-
 #</create_deployment>

 #<invoke_deployment_file>
-JOB_NAME=$(az ml batch-endpoint invoke -n $ENDPOINT_NAME --f inputs.yml | jq -r ".name")
+JOB_NAME=$(az ml batch-endpoint invoke -n $ENDPOINT_NAME --f inputs.yml --query name -o tsv)
 #</invoke_deployment_file>

 #<stream_job_logs>
@ -61,7 +61,7 @@ az ml job stream -n $JOB_NAME
 #</stream_job_logs>

 #<get_child_jobs>
-az ml job list --parent-job-name $JOB_NAME | jq -r ".[].name"
+az ml job list --parent-job-name $JOB_NAME --query "[].name" -o tsv
 #</get_child_jobs>

 #<download_outputs>
--- a/cli/endpoints/batch/deploy-pipelines/hello-batch/deploy-and-run.sh
+++ b/cli/endpoints/batch/deploy-pipelines/hello-batch/deploy-and-run.sh
@ -33,7 +33,7 @@ az ml batch-deployment create --endpoint $ENDPOINT_NAME -f deployment.yml --set-
 #</create_deployment>

 #<invoke_deployment_inline>
-JOB_NAME=$(az ml batch-endpoint invoke -n $ENDPOINT_NAME | jq -r ".name")
+JOB_NAME=$(az ml batch-endpoint invoke -n $ENDPOINT_NAME --query name -o tsv)
 #</invoke_deployment_inline>

 #<stream_job_logs>
@ -41,7 +41,7 @@ az ml job stream -n $JOB_NAME
 #</stream_job_logs>

 #<run_pipeline_job_deployment>
-JOB_NAME=$(az ml job create -f pipeline-job.yml | jq -r ".name")
+JOB_NAME=$(az ml job create -f pipeline-job.yml --query name -o tsv)
 #</run_pipeline_job_deployment>

 #<create_deployment_from_job>
--- a/cli/endpoints/batch/deploy-pipelines/training-with-components/deploy-and-run.sh
+++ b/cli/endpoints/batch/deploy-pipelines/training-with-components/deploy-and-run.sh
@ -41,7 +41,7 @@ az ml batch-deployment create --endpoint $ENDPOINT_NAME -f deployment-ordinal/de
 #</create_deployment>

 #<invoke_deployment_file>
-JOB_NAME=$(az ml batch-endpoint invoke -n $ENDPOINT_NAME --f inputs.yml | jq -r ".name")
+JOB_NAME=$(az ml batch-endpoint invoke -n $ENDPOINT_NAME --f inputs.yml --query name -o tsv)
 #</invoke_deployment_file>

 #<stream_job_logs>
@ -49,7 +49,7 @@ az ml job stream -n $JOB_NAME
 #</stream_job_logs>

 #<get_child_jobs>
-az ml job list --parent-job-name $JOB_NAME | jq -r ".[].name"
+az ml job list --parent-job-name $JOB_NAME --query "[].name" -o tsv
 #</get_child_jobs>

 #<download_outputs>
@ -64,7 +64,7 @@ az ml batch-deployment create --endpoint $ENDPOINT_NAME -f deployment-onehot/dep

 #<invoke_nondefault_deployment_file>
 DEPLOYMENT_NAME="uci-classifier-train-onehot"
-JOB_NAME=$(az ml batch-endpoint invoke -n $ENDPOINT_NAME -d $DEPLOYMENT_NAME --f inputs.yml | jq -r ".name")
+JOB_NAME=$(az ml batch-endpoint invoke -n $ENDPOINT_NAME -d $DEPLOYMENT_NAME --f inputs.yml --query name -o tsv)
 #</invoke_nondefault_deployment_file>

 #<stream_nondefault_job_logs>
--- a/cli/train-rest.sh
+++ b/cli/train-rest.sh
@ -1,11 +1,11 @@
 exit 0 # TODO - update script to new API

 # <create_variables>
-SUBSCRIPTION_ID=$(az account show --query id | tr -d '\r"')
-LOCATION=$(az group show --query location | tr -d '\r"')
-RESOURCE_GROUP=$(az group show --query name | tr -d '\r"')
+SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+LOCATION=$(az group show --query location -o tsv)
+RESOURCE_GROUP=$(az group show --query name -o tsv)

-WORKSPACE=$(az configure -l | jq -r '.[] | select(.name=="workspace") | .value')
+WORKSPACE=$(az configure -l --query "[?name=='workspace'].value" -o tsv)
 API_VERSION="2022-02-01-preview"
 COMPUTE_NAME="cpu-cluster"
 TOKEN=$(az account get-access-token --query accessToken -o tsv)
@ -24,7 +24,7 @@ wait_for_completion () {
        job=$(curl --location --request GET "https://management.azure.com/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP/providers/Microsoft.MachineLearningServices/workspaces/$WORKSPACE/jobs/$1?api-version=$API_VERSION" \
            --header "Authorization: Bearer $TOKEN")
        # TODO error handling here
-        job_status=$(echo $job | jq -r '.properties' | jq -r '.status')
+        job_status=$(echo $job | jq -r '.properties.status')
        echo "Current job status: $job_status"
        sleep 5
    done
--- a/deploy-arm-templates-az-cli.sh
+++ b/deploy-arm-templates-az-cli.sh
@ -5,10 +5,10 @@ TOKEN=$(az account get-access-token --query accessToken -o tsv)
 #</get_access_token>

 # <create_variables>
-SUBSCRIPTION_ID=$(az account show --query id | tr -d '\r"')
-LOCATION=$(az ml workspace show --query location | tr -d '\r"')
-RESOURCE_GROUP=$(az group show --query name | tr -d '\r"')
-WORKSPACE=$(az configure -l | jq -r '.[] | select(.name=="workspace") | .value')
+SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+LOCATION=$(az ml workspace show --query location -o tsv)
+RESOURCE_GROUP=$(az group show --query name -o tsv)
+WORKSPACE=$(az configure -l --query "[?name=='workspace'].value" -o tsv)
 #</create_variables>

 # <set_endpoint_name>
@ -115,7 +115,7 @@ response=$(curl --location --request GET "https://management.azure.com/subscript
 --header "Content-Type: application/json" \
 --header "Authorization: Bearer $TOKEN")

-operation_id=$(echo $response | jq -r '.properties' | jq -r '.properties' | jq -r '.AzureAsyncOperationUri')
+operation_id=$(echo $response | jq -r '.properties.properties.AzureAsyncOperationUri')
 wait_for_completion $operation_id
 # </get_endpoint>

@ -142,10 +142,10 @@ response=$(curl --location --request GET "https://management.azure.com/subscript
 --header "Content-Type: application/json" \
 --header "Authorization: Bearer $TOKEN")

-operation_id=$(echo $response | jq -r '.properties' | jq -r '.properties' | jq -r '.AzureAsyncOperationUri')
+operation_id=$(echo $response | jq -r '.properties.properties.AzureAsyncOperationUri')
 wait_for_completion $operation_id

-scoringUri=$(echo $response | jq -r '.properties' | jq -r '.scoringUri')
+scoringUri=$(echo $response | jq -r '.properties.scoringUri')
 # </get_endpoint>

 # <get_endpoint_access_token>
--- a/deploy-arm-templates-rest.sh
+++ b/deploy-arm-templates-rest.sh
@ -1,10 +1,10 @@
 set -x

 # <create_variables>
-SUBSCRIPTION_ID=$(az account show --query id | tr -d '\r"')
-LOCATION=$(az ml workspace show --query location | tr -d '\r"')
-RESOURCE_GROUP=$(az group show --query name | tr -d '\r"')
-WORKSPACE=$(az configure -l | jq -r '.[] | select(.name=="workspace") | .value')
+SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+LOCATION=$(az ml workspace show --query location -o tsv)
+RESOURCE_GROUP=$(az group show --query name -o tsv)
+WORKSPACE=$(az configure -l --query "[?name=='workspace'].value" -o tsv)
 schema='$schema'
 #</create_variables>

@ -35,7 +35,7 @@ wait_for_completion () {
    do
        sleep 5
        response=$($1)
-        operation_id=$(echo $response | jq -r '.properties' | jq -r '.properties' | jq -r '.AzureAsyncOperationUri')
+        operation_id=$(echo $response | jq -r '.properties.properties.AzureAsyncOperationUri')
    done

  while [[ $status != "Succeeded" && $status != "Failed" ]]
--- a/infra/bootstrapping/bootstrap.sh
+++ b/infra/bootstrapping/bootstrap.sh
@ -50,9 +50,6 @@ else
  echo "---------------------------------------------------------"
 fi

-echo_title "Ensuring dependent packages"
-"$SCRIPT_DIR"/sdk_helpers.sh install_packages
-
 echo_title "Installing tools"
 "$SCRIPT_DIR"/sdk_helpers.sh install_tools

@ -61,7 +58,6 @@ echo_title "Installing tools"
 ###################

 "$SCRIPT_DIR"/sdk_helpers.sh validate_tool az || exit 1
-"$SCRIPT_DIR"/sdk_helpers.sh validate_tool jq || exit 1
 "$SCRIPT_DIR"/sdk_helpers.sh validate_tool sed || exit 1

 #login to azure using your credentials
--- a/infra/bootstrapping/sdk_helpers.sh
+++ b/infra/bootstrapping/sdk_helpers.sh
@ -260,43 +260,6 @@ function IsInstalled {
    sudo dpkg -S "$1" &> /dev/null
 }

-function install_packages() {
-    echo_info "------------------------------------------------"
-    echo_info ">>> Updating packages index"
-    echo_info "------------------------------------------------"
-
-    echo_info ">>> Executing: apt-get update"
-    # https://github.com/orgs/community/discussions/47863
-    sudo apt-mark hold grub-efi-amd64-signed
-    sudo apt-get update --fix-missing
-    echo_info ">>> Executing: apt-get upgrade"
-    sudo apt-get upgrade -y
-    echo_info ">>> Executing: apt-get dist-upgrade"
-    sudo apt-get dist-upgrade -y
-
-    echo_info ">>> Installing packages"
-
-    # jq                - Required for running filters on a stream of JSON data from az
-    # uuid-runtime      - Required for containers
-    # uuid-runtime      - Required for aks/arc
-    packages_to_install=(
-      jq
-      uuid-runtime
-      xmlstarlet
-    )
-    for package in "${packages_to_install[@]}"; do
-      echo_info "Installing '$package'"
-      if ! IsInstalled "$package"; then
-          sudo apt-get install -y --no-install-recommends "${package}" > /dev/null 2>&1
-      else
-          echo_info "$package is already installed"
-      fi
-    done
-    echo_info ">>> Clean local cache for packages"
-
-    sudo apt-get autoclean && sudo apt-get autoremove > /dev/null 2>&1
-}
-
 function add_extension() {
    echo_info "az extension add -n $1 "
    az extension add -n "$1" -y
@ -489,7 +452,7 @@ connect_arc(){
    get_kubeconfig "$AKS_CLUSTER_NAME"

    if
-        [[ $(az connectedk8s show --resource-group "${RESOURCE_GROUP_NAME}" --name "${ARC_CLUSTER_NAME}" | jq -r .name) == ${ARC_CLUSTER_NAME} ]]
+        [[ $(az connectedk8s show --resource-group "${RESOURCE_GROUP_NAME}" --name "${ARC_CLUSTER_NAME}" --query name --output tsv) == ${ARC_CLUSTER_NAME} ]]
    then
        echo_info "Cluster: ${ARC_CLUSTER_NAME} is already connected..."
        clusterState=$(az connectedk8s show --resource-group "${RESOURCE_GROUP_NAME}" --name "${ARC_CLUSTER_NAME}" --query connectivityStatus -o json)
@ -527,7 +490,7 @@ function setup_compute() {
    fi

    if
-        [[ $(az ml compute show --resource-group "${RESOURCE_GROUP_NAME}" --name "${COMPUTE_NAME}" | jq -r .provisioning_state) == "Succeeded" ]]
+        [[ $(az ml compute show --resource-group "${RESOURCE_GROUP_NAME}" --name "${COMPUTE_NAME}" --query provisioning_state --output tsv) == "Succeeded" ]]
    then
        echo_info "Cluster is already attached to workspace for the cluster: ${CLUSTER_NAME} as ${COMPUTE_NAME} in workspace:${WORKSPACE_NAME} under namespace: ${COMPUTE_NS}..."
    else
@ -633,7 +596,7 @@ install_k8s_extension(){
    fi

    if
-        [[ $(az k8s-extension show --cluster-type "${CLUSTER_TYPE}" -c "${ARC_CLUSTER_NAME}" -g "${RESOURCE_GROUP_NAME}" --name "${EXTENSION_NAME}" | jq -r .provisioningState) == "Succeeded" ]]
+        [[ $(az k8s-extension show --cluster-type "${CLUSTER_TYPE}" -c "${ARC_CLUSTER_NAME}" -g "${RESOURCE_GROUP_NAME}" --name "${EXTENSION_NAME}" --output tsv --query provisioningState) == "Succeeded" ]]
    then
        echo "Extension:${EXTENSION_NAME} already installed on cluster: ${ARC_CLUSTER_NAME}"
    else
@ -745,18 +708,16 @@ function vmss_upgrade_policy_automatic() {
    local LOCAL_RESOURCE_GROUP_NAME=${1:-testrg}
    printf "Update VMSS upgrade policy in resource group %s\n" ${LOCAL_RESOURCE_GROUP_NAME}
    # get list of all scale sets
-    # VM_SCALE_SETS_JSON=$(az vmss list --resource-group ${LOCAL_RESOURCE_GROUP_NAME} -o json)
-    # VM_SCALE_SETS_LIST=$(echo $VM_SCALE_SETS_JSON | jq -r '.[] | .name')
-    VM_SCALE_SETS=$(az vmss list --subscription "${SUBSCRIPTION_ID}" --resource-group ${LOCAL_RESOURCE_GROUP_NAME} | jq -r '.[].name')
+    VM_SCALE_SETS=$(az vmss list --subscription "${SUBSCRIPTION_ID}" --resource-group ${LOCAL_RESOURCE_GROUP_NAME} --query '[].name' --output tsv)

    printf "Checking scalesets %s in resource-group %s\n" "${VM_SCALE_SETS}" "${LOCAL_RESOURCE_GROUP_NAME}"
    # temporarily disable the flag
    set +e
    for VMSS in ${VM_SCALE_SETS}; do
-        VMSS_PROPERTIES=$(az vmss show --subscription "${SUBSCRIPTION_ID}" --resource-group ${LOCAL_RESOURCE_GROUP_NAME} --name $VMSS)
-        # echo SKU_TEMP $VMSS_PROPERTIES
+        VMSS_UPGRADE_POLICY_MODE=$(az vmss show --subscription "${SUBSCRIPTION_ID}" --resource-group ${LOCAL_RESOURCE_GROUP_NAME} --name $VMSS --query upgradePolicy.mode --output tsv)
+
        # az vmss show -g "${LOCAL_RESOURCE_GROUP_NAME}" -n "${VMSS}" -o json
-        if [[ $(echo $VMSS_PROPERTIES | jq -r '.upgradePolicy.mode') == "Automatic" ]]; then
+        if [[ "$VMSS_UPGRADE_POLICY_MODE" == "Automatic" ]]; then
            echo_info "Skipping to update upgradePolicy for VMSS $VMSS in resource-group ${LOCAL_RESOURCE_GROUP_NAME}..."
            continue
        else
@ -778,7 +739,7 @@ function vmss_upgrade_policy_all_rg() {
    # checking Resource group name to ensure we're in a managed cluster RG
    echo "Number of Resource groups starting with ${RG_PREFIX}:" $(az group list --subscription "${SUBSCRIPTION_ID}" --query "[? starts_with(@.name, '${RG_PREFIX}')] | length(@)")
    # az group list --query "[? starts_with(@.name, '${RG_PREFIX}')].name" -o tsv | xargs -i "$SCRIPT_DIR"/sdk_helpers.sh check_vmss "{}"
-    for LOCAL_RESOURCE_GROUP_NAME in $(az group list --subscription "${SUBSCRIPTION_ID}" --query "[? starts_with(@.name, '${RG_PREFIX}')].name" --output json | jq .[] -r); do
+    for LOCAL_RESOURCE_GROUP_NAME in $(az group list --subscription "${SUBSCRIPTION_ID}" --query "[? starts_with(@.name, '${RG_PREFIX}')].name" --output tsv); do
        # resource_id=$(az resource list --resource-group "${LOCAL_RESOURCE_GROUP_NAME}" --query [].id --output tsv)
        RESOURCE_GROUP_ID=$(az group show --subscription "${SUBSCRIPTION_ID}" --name "${LOCAL_RESOURCE_GROUP_NAME}" --query id -o tsv | tail -n1 | tr -d "[:cntrl:]")
        echo "Current tags for resource-group ${LOCAL_RESOURCE_GROUP_NAME}"