From 033bb4d6b4d6b9d96583f07d9a655e64ece90ad4 Mon Sep 17 00:00:00 2001 From: Mariana Mihova <10135329+marianan@users.noreply.github.com> Date: Tue, 18 May 2021 20:55:37 -0700 Subject: [PATCH 01/12] Fix E2E pipeline and increment the version (#17) * Add parameters to script call * fix e2e yml and some output * fix source paths * fix source paths * line endings to Unix * add unit test output + remove setup step for unit test pipeline * Test local files * Add copyrights * Remove empty unit test files from yml * Fix psot install validation bug * Fix e2e cd location * Quote angle brackets * Allow for changes to propagate * Set test result correctly on success * Publish logs as artifacts * Address PR feedback * Remove log publishing * Fix unit test source * Fix unit test source * Ubuntu 20.04 is not supported by this script --- src/azure-iot-edge-installer.sh | 6 +- src/validate-post-install.sh | 4 +- tests/e2e-tests/test-devicestate.sh | 20 ++-- tests/perf_tests.sh | 3 + tests/send_one_message_to_iot_hub_device.py | 3 + tests/test_utils.sh | 3 + tests/track_duration.sh | 3 + tests/unit-tests/test-cmd-parser.sh | 3 + .../test-install-container-management.sh | 2 - tests/unit-tests/test-install-edge-runtime.sh | 2 - tests/unit-tests/test-logger.sh | 3 + .../unit-tests/test-validate-post-install.sh | 13 ++- tests/unit-tests/test-validate-tier1-os.sh | 8 +- vsts_ci/azure-pipelines-e2e.yml | 95 ++++++++-------- vsts_ci/azure-pipelines-perf.yml | 95 ++++++++-------- vsts_ci/azure-pipelines.yml | 102 +++++++++--------- vsts_ci/continuous-e2e.yml | 26 ++--- vsts_ci/continuous-perf.yml | 51 ++++----- vsts_ci/linux/continuous-linux.yml | 7 +- vsts_ci/raspi/continuous-raspi.yml | 6 +- 20 files changed, 250 insertions(+), 205 deletions(-) delete mode 100644 tests/unit-tests/test-install-container-management.sh delete mode 100644 tests/unit-tests/test-install-edge-runtime.sh diff --git a/src/azure-iot-edge-installer.sh b/src/azure-iot-edge-installer.sh index c7672a3..8f5bf5b 100644 --- a/src/azure-iot-edge-installer.sh +++ b/src/azure-iot-edge-installer.sh @@ -63,7 +63,7 @@ function download_bash_script() { } # script -printf "Running azure-iot-edge-installer.sh\n" > /dev/stdout +printf "Welcome to azure-iot-edge-installer\n" > /dev/stdout # if helper scripts dont exist, fetch via wget if [ -d "iot-edge-installer" ]; @@ -102,7 +102,7 @@ if [[ ${#@} > 0 && ${#parsed_cmds[*]} == 0 ]]; then array=("$*") echo Unknown argument "${array[*]}" - echo Usage + echo "Usage: sudo ./azure-iot-edge-installer.sh -s -r -k " exit 1 fi @@ -111,7 +111,7 @@ then echo Missing argument echo defined: "'"${!parsed_cmds[@]}"'" echo given: "'"${parsed_cmds[@]}"'" - echo Usage + echo "Usage: sudo ./azure-iot-edge-installer.sh -s -r -k " exit 1 fi diff --git a/src/validate-post-install.sh b/src/validate-post-install.sh index b77ac9b..f79dc21 100644 --- a/src/validate-post-install.sh +++ b/src/validate-post-install.sh @@ -44,9 +44,9 @@ function validate_post_install() { "aziot-certd" "aziot-tpmd") - for i in "${iotedge_services[@]}" + for service_name in "${iotedge_services[@]}" do - is_service_running ${iotedge_services[$i]} "$status" + is_service_running $service_name "$status" done log_info "Post install validation completed." diff --git a/tests/e2e-tests/test-devicestate.sh b/tests/e2e-tests/test-devicestate.sh index f7a2d32..4b2f074 100644 --- a/tests/e2e-tests/test-devicestate.sh +++ b/tests/e2e-tests/test-devicestate.sh @@ -1,4 +1,8 @@ #!/usr/bin/env bash + +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + ###################################### # test-devicestate # @@ -127,10 +131,14 @@ scope_id=$(jq -r '.idScope' <<< "$creds") primary_key=$(jq -r '.symmetricKey.primaryKey' <<< "$creds") echo Run the Azure IoT Edge Installer -wget https://github.com/Azure/iot-edge-config/releases/latest/download/azure-iot-edge-installer.sh \ -&& chmod +x azure-iot-edge-installer.sh \ -&& ./azure-iot-edge-installer.sh \ -&& rm -rf azure-iot-edge-installer.sh +#wget -O azure-iot-edge-installer.sh https://github.com/Azure/iot-edge-config/releases/latest/download/azure-iot-edge-installer.sh \ +cd ./../../src +chmod +x azure-iot-edge-installer.sh +sudo LOCAL_E2E=1 ./azure-iot-edge-installer.sh --scope-id "$scope_id" --registration-id "$device_id" --symmetric-key "$primary_key" +rm -rf azure-iot-edge-installer.sh + +# give 2 mins for changes to propagate to central app +sleep 120 # device state should be provisioned after running the script out=$(curl -X GET -H "Authorization:$apiToken" https://${centralapp_name}.azureiotcentral.com/api/preview/devices/${device_id}) @@ -143,10 +151,10 @@ then echo "Error: Device must be provisioned. Exit."; else echo "Device is provisioned as expected. Success."; - $test_result=0 # success + test_result=0 # success fi; # Clean up cleanup "$armToken" "$apiToken" "$device_id" "$token_id" "$rg" "$centralapp_name" -echo test_result: $test_result + exit $test_result diff --git a/tests/perf_tests.sh b/tests/perf_tests.sh index 0599e1c..90d40a1 100644 --- a/tests/perf_tests.sh +++ b/tests/perf_tests.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + ###################################### # perf_tests # diff --git a/tests/send_one_message_to_iot_hub_device.py b/tests/send_one_message_to_iot_hub_device.py index c1ed405..94305b2 100644 --- a/tests/send_one_message_to_iot_hub_device.py +++ b/tests/send_one_message_to_iot_hub_device.py @@ -1,5 +1,8 @@ #!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + import argparse import sys diff --git a/tests/test_utils.sh b/tests/test_utils.sh index 6515e11..cea2f2a 100644 --- a/tests/test_utils.sh +++ b/tests/test_utils.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + if command -v tput &>/dev/null && tty -s; then RED=$(tput setaf 1) GREEN=$(tput setaf 2) diff --git a/tests/track_duration.sh b/tests/track_duration.sh index 738caf5..c7a6b38 100644 --- a/tests/track_duration.sh +++ b/tests/track_duration.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + ###################################### # track_duration # diff --git a/tests/unit-tests/test-cmd-parser.sh b/tests/unit-tests/test-cmd-parser.sh index aa67f17..e551c08 100644 --- a/tests/unit-tests/test-cmd-parser.sh +++ b/tests/unit-tests/test-cmd-parser.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + # import utils.sh source ../../src/utils.sh diff --git a/tests/unit-tests/test-install-container-management.sh b/tests/unit-tests/test-install-container-management.sh deleted file mode 100644 index fa2fcac..0000000 --- a/tests/unit-tests/test-install-container-management.sh +++ /dev/null @@ -1,2 +0,0 @@ -#script -echo "Running test-install-container-management.sh" \ No newline at end of file diff --git a/tests/unit-tests/test-install-edge-runtime.sh b/tests/unit-tests/test-install-edge-runtime.sh deleted file mode 100644 index 472f327..0000000 --- a/tests/unit-tests/test-install-edge-runtime.sh +++ /dev/null @@ -1,2 +0,0 @@ -#script to install edge runtime 1.2 -echo "Running test-install-edge-runtime.sh" \ No newline at end of file diff --git a/tests/unit-tests/test-logger.sh b/tests/unit-tests/test-logger.sh index aa47e8e..7eb92ff 100644 --- a/tests/unit-tests/test-logger.sh +++ b/tests/unit-tests/test-logger.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + exec 3>&1 # bring in the utils library diff --git a/tests/unit-tests/test-validate-post-install.sh b/tests/unit-tests/test-validate-post-install.sh index 341a25e..19f4fe1 100644 --- a/tests/unit-tests/test-validate-post-install.sh +++ b/tests/unit-tests/test-validate-post-install.sh @@ -1,7 +1,11 @@ #!/usr/bin/env bash -source src/validate-post-install.sh -source tests/test_utils.sh +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +source ../../src/utils.sh +source ../../src/validate-post-install.sh +source ../test_utils.sh test_service_running() { is_service_running "servicenameA" "servicenameA Running" @@ -28,6 +32,11 @@ test_service_casesensitive() { assert_eq 0 $? } +test_service_casesensitive() { + is_service_running "servicenameA" "servicenameA ruNNing" + assert_eq 0 $? +} + # run tests test_service_running test_service_ready diff --git a/tests/unit-tests/test-validate-tier1-os.sh b/tests/unit-tests/test-validate-tier1-os.sh index 1aa880a..3e10440 100644 --- a/tests/unit-tests/test-validate-tier1-os.sh +++ b/tests/unit-tests/test-validate-tier1-os.sh @@ -1,7 +1,11 @@ #!/usr/bin/env bash -source src/validate-tier1-os.sh -source tests/test_utils.sh +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +source ../../src/utils.sh +source ../../src/validate-tier1-os.sh +source ../test_utils.sh test_ubuntu1804() { ID="ubuntu" diff --git a/vsts_ci/azure-pipelines-e2e.yml b/vsts_ci/azure-pipelines-e2e.yml index 0a86011..72734c9 100644 --- a/vsts_ci/azure-pipelines-e2e.yml +++ b/vsts_ci/azure-pipelines-e2e.yml @@ -1,46 +1,49 @@ -# Starter pipeline -# Start with a minimal pipeline that you can customize to build and deploy your code. -# Add steps that build, run tests, deploy, and more: -# https://aka.ms/yaml - -trigger: -- main - -stages: -- stage: E2ETest - displayName: E2E Tests - jobs: - - job: SetupScript - steps: - - script: | - sudo apt update - sudo apt install jq - displayName: 'Install jq' - - task: AzureCLI@2 - inputs: - azureSubscription: 'Connection to Pipeline resources for IoT Edge Config' - scriptType: 'bash' - scriptLocation: 'inlineScript' - inlineScript: | - az upgrade - az extension add --name azure-iot - az --version - az account set -s $(AzureSubscriptionId) - displayName: 'Set Azure resources' - - - job: LinuxE2ETests - pool: - vmImage: ubuntu-18.04 - steps: - - template: continuous-e2e.yml - dependsOn: SetupScript - condition: succeeded() - - - job: RaspberryPiE2ETests - pool: - vmImage: ubuntu-latest - steps: - - template: continuous-e2e.yml - dependsOn: SetupScript - condition: succeeded() - +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Starter pipeline +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml + +trigger: +- main + +stages: +- stage: E2ETest + displayName: E2E Tests + jobs: + - job: SetupScript + steps: + - script: | + sudo apt update + sudo apt install jq + displayName: 'Install jq' + - task: AzureCLI@2 + inputs: + azureSubscription: 'Connection to Pipeline resources for IoT Edge Config' + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + az upgrade + az extension add --name azure-iot + az --version + az account set -s $(AzureSubscriptionId) + displayName: 'Set Azure resources' + + - job: LinuxE2ETests + pool: + vmImage: ubuntu-18.04 + steps: + - template: continuous-e2e.yml + dependsOn: SetupScript + condition: succeeded() + + - job: RaspberryPiE2ETests + pool: + vmImage: ubuntu-18.04 + steps: + - template: continuous-e2e.yml + dependsOn: SetupScript + condition: succeeded() + diff --git a/vsts_ci/azure-pipelines-perf.yml b/vsts_ci/azure-pipelines-perf.yml index 8ab8c2b..91aa7d4 100644 --- a/vsts_ci/azure-pipelines-perf.yml +++ b/vsts_ci/azure-pipelines-perf.yml @@ -1,46 +1,49 @@ -# Starter pipeline -# Start with a minimal pipeline that you can customize to build and deploy your code. -# Add steps that build, run tests, deploy, and more: -# https://aka.ms/yaml - -trigger: -- main - -stages: -- stage: PERFTests - displayName: Performance Tests - jobs: - - job: SetupScript - steps: - - script: | - sudo apt update - sudo apt install jq - displayName: 'Install jq' - - task: AzureCLI@2 - inputs: - azureSubscription: 'Connection to Pipeline resources for IoT Edge Config' - scriptType: 'bash' - scriptLocation: 'inlineScript' - inlineScript: | - az config set extension.use_dynamic_install=yes_without_prompt - az extension add --name azure-iot - az --version - az account set -s $(AzureSubscriptionId) - displayName: 'Set Azure resources' - - - job: LinuxPerfTests - pool: - vmImage: ubuntu-18.04 - steps: - - template: continuous-perf.yml - dependsOn: SetupScript - condition: succeeded() - - - job: RaspberryPiPerfTests - pool: - vmImage: ubuntu-latest - steps: - - template: continuous-perf.yml - dependsOn: SetupScript - condition: succeeded() - +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Starter pipeline +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml + +trigger: +- main + +stages: +- stage: PERFTests + displayName: Performance Tests + jobs: + - job: SetupScript + steps: + - script: | + sudo apt update + sudo apt install jq + displayName: 'Install jq' + - task: AzureCLI@2 + inputs: + azureSubscription: 'Connection to Pipeline resources for IoT Edge Config' + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + az config set extension.use_dynamic_install=yes_without_prompt + az extension add --name azure-iot + az --version + az account set -s $(AzureSubscriptionId) + displayName: 'Set Azure resources' + + - job: LinuxPerfTests + pool: + vmImage: ubuntu-18.04 + steps: + - template: continuous-perf.yml + dependsOn: SetupScript + condition: succeeded() + + - job: RaspberryPiPerfTests + pool: + vmImage: ubuntu-latest + steps: + - template: continuous-perf.yml + dependsOn: SetupScript + condition: succeeded() + diff --git a/vsts_ci/azure-pipelines.yml b/vsts_ci/azure-pipelines.yml index 58c5981..92da572 100644 --- a/vsts_ci/azure-pipelines.yml +++ b/vsts_ci/azure-pipelines.yml @@ -1,53 +1,51 @@ -# Starter pipeline -# Start with a minimal pipeline that you can customize to build and deploy your code. -# Add steps that build, run tests, deploy, and more: -# https://aka.ms/yaml - -trigger: -- main - -stages: -- stage: UnitTest - displayName: Unit Tests - jobs: - - job: SetupScript - steps: - - script: echo Hello, world! - displayName: 'Run a one-line script' - - - job: LinuxUnitTests - pool: - vmImage: ubuntu-18.04 - steps: - - template: linux/continuous-linux.yml - - - job: RaspberryPiUnitTests - pool: - vmImage: ubuntu-latest - steps: - - template: raspi/continuous-raspi.yml - - - job: GithubRelease - dependsOn: - - LinuxUnitTests - - RaspberryPiUnitTests - condition: and(succeeded('LinuxUnitTests'), succeeded('RaspberryPiUnitTests')) - steps: - - script: | - mkdir dest - cp src/*.sh dest/ - displayName: Create dest/ directory - - task: GitHubRelease@1 - displayName: 'GitHub release (create) RC' - inputs: - gitHubConnection: 'github.com_cindydeng1998' - tagPattern: '^v?[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+$' - assets: 'dest/*.sh' - isDraft: true - - task: GitHubRelease@1 - displayName: 'GitHub release (create)' - inputs: - gitHubConnection: 'github.com_cindydeng1998' - tagPattern: '^v?[0-9]+\.[0-9]+\.[0-9]+$' - assets: 'dest/*.sh' +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Starter pipeline +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml + +trigger: +- main + +stages: +- stage: UnitTest + displayName: Unit Tests + jobs: + - job: LinuxUnitTests + pool: + vmImage: ubuntu-18.04 + steps: + - template: linux/continuous-linux.yml + + - job: RaspberryPiUnitTests + pool: + vmImage: ubuntu-latest + steps: + - template: raspi/continuous-raspi.yml + + - job: GithubRelease + dependsOn: + - LinuxUnitTests + - RaspberryPiUnitTests + condition: and(succeeded('LinuxUnitTests'), succeeded('RaspberryPiUnitTests')) + steps: + - script: | + mkdir dest + cp src/*.sh dest/ + displayName: Create dest/ directory + - task: GitHubRelease@1 + displayName: 'GitHub release (create) RC' + inputs: + gitHubConnection: 'github.com_cindydeng1998' + tagPattern: '^v?[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+$' + assets: 'dest/*.sh' + isDraft: true + - task: GitHubRelease@1 + displayName: 'GitHub release (create)' + inputs: + gitHubConnection: 'github.com_cindydeng1998' + tagPattern: '^v?[0-9]+\.[0-9]+\.[0-9]+$' + assets: 'dest/*.sh' isDraft: true \ No newline at end of file diff --git a/vsts_ci/continuous-e2e.yml b/vsts_ci/continuous-e2e.yml index f1acfef..0534f7d 100644 --- a/vsts_ci/continuous-e2e.yml +++ b/vsts_ci/continuous-e2e.yml @@ -1,12 +1,14 @@ -steps: -- task: AzureCLI@2 - inputs: - azureSubscription: 'Connection to Pipeline resources for IoT Edge Config' - scriptType: 'bash' - scriptLocation: 'inlineScript' - inlineScript: | - cd tests/e2e-tests - chmod +x test-devicestate.sh - ./test-devicestate.sh $(AzureSubscriptionId) - displayName: 'Run All E2E Tests' - +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +steps: +- task: AzureCLI@2 + inputs: + azureSubscription: 'Connection to Pipeline resources for IoT Edge Config' + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + cd tests/e2e-tests + chmod +x test-devicestate.sh + ./test-devicestate.sh $(AzureSubscriptionId) + displayName: 'Run All E2E Tests' diff --git a/vsts_ci/continuous-perf.yml b/vsts_ci/continuous-perf.yml index a66fbf6..b3787bc 100644 --- a/vsts_ci/continuous-perf.yml +++ b/vsts_ci/continuous-perf.yml @@ -1,25 +1,28 @@ -steps: -- task: UsePythonVersion@0 - displayName: "Use Python 3.8" - inputs: - versionSpec: 3.8 - -- task: AzureCLI@2 - displayName: 'Run All Performance Tests' - inputs: - azureSubscription: 'Connection to Pipeline resources for IoT Edge Config' - scriptType: 'bash' - scriptLocation: 'inlineScript' - inlineScript: | - cd tests - python3 --version - pip3 --version - pip3 install -r requirements.txt - chmod +x ./track_duration.sh - chmod +x ./perf_tests.sh - chmod +x ./e2e-tests/test-devicestate.sh - az config set extension.use_dynamic_install=yes_without_prompt - az extension add --name azure-iot - az --version - az account set -s $(AzureSubscriptionId) +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +steps: +- task: UsePythonVersion@0 + displayName: "Use Python 3.8" + inputs: + versionSpec: 3.8 + +- task: AzureCLI@2 + displayName: 'Run All Performance Tests' + inputs: + azureSubscription: 'Connection to Pipeline resources for IoT Edge Config' + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + cd tests + python3 --version + pip3 --version + pip3 install -r requirements.txt + chmod +x ./track_duration.sh + chmod +x ./perf_tests.sh + chmod +x ./e2e-tests/test-devicestate.sh + az config set extension.use_dynamic_install=yes_without_prompt + az extension add --name azure-iot + az --version + az account set -s $(AzureSubscriptionId) ./perf_tests.sh $(NumberOfRuns) \ No newline at end of file diff --git a/vsts_ci/linux/continuous-linux.yml b/vsts_ci/linux/continuous-linux.yml index c0e9aa0..1ea8ef5 100644 --- a/vsts_ci/linux/continuous-linux.yml +++ b/vsts_ci/linux/continuous-linux.yml @@ -1,12 +1,13 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + steps: - script: echo Hello Linux! displayName: 'Echo Hello Linux' - script: | cd tests/unit-tests - chmod +x ./*.sh - ./test-install-container-management.sh - ./test-install-edge-runtime.sh + chmod +x ./*.sh ./test-validate-post-install.sh ./test-validate-tier1-os.sh ./test-cmd-parser.sh diff --git a/vsts_ci/raspi/continuous-raspi.yml b/vsts_ci/raspi/continuous-raspi.yml index 411ea54..84b3d1d 100644 --- a/vsts_ci/raspi/continuous-raspi.yml +++ b/vsts_ci/raspi/continuous-raspi.yml @@ -1,3 +1,6 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + steps: - script: echo Hello Raspi! displayName: 'Echo Hello Raspi!' @@ -5,11 +8,8 @@ steps: - script: | cd tests/unit-tests chmod +x ./*.sh - ./test-install-container-management.sh - ./test-install-edge-runtime.sh ./test-validate-post-install.sh ./test-validate-tier1-os.sh ./test-cmd-parser.sh ./test-logger.sh displayName: 'Run All Raspberry Pi Unit Tests' - From f822049d6b181ca3d0bfe693c5915b0b97ab4c7b Mon Sep 17 00:00:00 2001 From: Haitham Shami Date: Wed, 19 May 2021 09:43:11 -0700 Subject: [PATCH 02/12] echo green (#23) * echo green * PR comment --- src/azure-iot-edge-installer.sh | 4 ++- src/utils.sh | 31 +++++++++++++++-- tests/test_utils.sh | 34 +++++++++---------- .../unit-tests/test-validate-post-install.sh | 4 ++- tests/unit-tests/test-validate-tier1-os.sh | 4 ++- 5 files changed, 55 insertions(+), 22 deletions(-) diff --git a/src/azure-iot-edge-installer.sh b/src/azure-iot-edge-installer.sh index 8f5bf5b..8800762 100644 --- a/src/azure-iot-edge-installer.sh +++ b/src/azure-iot-edge-installer.sh @@ -5,7 +5,7 @@ if [[ $EUID -ne 0 ]]; then - echo "ERROR: $0 requires elevated privileges.. " + echo "$(echo -en "\e[31m")ERROR: $(echo -en "\e[00m")$0 requires elevated privileges.. " exit 1 fi @@ -145,3 +145,5 @@ then rm -rf iot-edge-installer log_info "Removed temporary directory files for iot-edge-installer." fi + +announce_my_log_file "All logs were appended to" $OUTPUT_FILE diff --git a/src/utils.sh b/src/utils.sh index aaeaf5e..1f9ab53 100644 --- a/src/utils.sh +++ b/src/utils.sh @@ -6,6 +6,14 @@ # create flag:variable_name dictionary declare -A flag_to_variable_dict +RED=$(echo -en "\e[31m") +GREEN=$(echo -en "\e[32m") +MAGENTA=$(echo -en "\e[35m") +DEFAULT=$(echo -en "\e[00m") +BOLD=$(echo -en "\e[01m") +BLINK=$(echo -en "\e[5m") + + ###################################### # add_option_args # @@ -49,6 +57,20 @@ function clear_option_args() { flag_to_variable_dict=() } +###################################### +# cmd_parser +# +# populates a dictionary of arguments and values from +# a given command line +# +# ARGUMENTS: +# command line +# OUTPUTS: +# +# RETURN: +# dictionary +###################################### + function cmd_parser() { # create flag:variable_name dictionary and initialize to empty string declare -A parsed_cmd @@ -124,6 +146,11 @@ log() { fi } +# +function announce_my_log_file() { + printf '\n------\n%s%s%s%s%s\n------\n\n' "$GREEN" "$BOLD" "$BLINK" "$1 '$2'" "$DEFAULT" +} + # # logger log_init() { @@ -136,7 +163,7 @@ log_init() { OUTPUT_FILE=$TD"/"$(echo ${BASE_NAME%.*})-$(echo `date '+%Y-%m-%d'`).log touch $OUTPUT_FILE - echo "All logs will be appended to this file '"$OUTPUT_FILE"'" + announce_my_log_file "All logs will be appended to file" $OUTPUT_FILE } # @@ -159,7 +186,7 @@ log_debug() { log "DEBUG" "$@" } -export -f log_init log_error log_info log_warn log_debug +export -f announce_my_log_file log_init log_error log_info log_warn log_debug ###################################### # prepare_apt diff --git a/tests/test_utils.sh b/tests/test_utils.sh index cea2f2a..5685ee5 100644 --- a/tests/test_utils.sh +++ b/tests/test_utils.sh @@ -3,27 +3,13 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -if command -v tput &>/dev/null && tty -s; then - RED=$(tput setaf 1) - GREEN=$(tput setaf 2) - MAGENTA=$(tput setaf 5) - NORMAL=$(tput sgr0) - BOLD=$(tput bold) -else - RED=$(echo -en "\e[31m") - GREEN=$(echo -en "\e[32m") - MAGENTA=$(echo -en "\e[35m") - NORMAL=$(echo -en "\e[00m") - BOLD=$(echo -en "\e[01m") -fi - # error_output() { - printf "%b\n" "${RED:-}Error: $1${NORMAL:-}" >&2 + printf "%b\n" "${RED:-}Error: $1${DEFAULT:-}" >&2 } output() { - printf "%b\n" "${BOLD:-}${NORMAL:-} $1" >&2 + printf "%b\n" "${BOLD:-}${DEFAULT:-} $1" >&2 } verbose_output() { @@ -52,6 +38,20 @@ assert_eq() { fi } +assert_not_eq() { + local expected=$1; shift + local actual=$1; shift + + NR_TOTALS=$(bc <<< $NR_TOTALS+1) + if [ "$expected" != "$actual" ]; + then + NR_PASSING=$(bc <<< $NR_PASSING+1) + else + NR_FAILING=$(bc <<< $NR_FAILING+1) + error_output "expected: $expected; actual: $actual" + fi +} + assert_file() { local file_name=$1; shift @@ -74,4 +74,4 @@ show_test_totals() { printf "\n\n" } -export -f assert_eq assert_file show_test_totals \ No newline at end of file +export -f assert_eq assert_not_eq assert_file show_test_totals \ No newline at end of file diff --git a/tests/unit-tests/test-validate-post-install.sh b/tests/unit-tests/test-validate-post-install.sh index 19f4fe1..fcfd151 100644 --- a/tests/unit-tests/test-validate-post-install.sh +++ b/tests/unit-tests/test-validate-post-install.sh @@ -42,4 +42,6 @@ test_service_running test_service_ready test_service_not_running test_service_missing -test_service_casesensitive \ No newline at end of file +test_service_casesensitive + +show_test_totals diff --git a/tests/unit-tests/test-validate-tier1-os.sh b/tests/unit-tests/test-validate-tier1-os.sh index 3e10440..48a0553 100644 --- a/tests/unit-tests/test-validate-tier1-os.sh +++ b/tests/unit-tests/test-validate-tier1-os.sh @@ -31,4 +31,6 @@ test_tier2() { # run tests test_ubuntu1804 test_raspbian -test_tier2 \ No newline at end of file +test_tier2 + +show_test_totals From 22caead206c99b72d2915e7e4761b80e5edc188f Mon Sep 17 00:00:00 2001 From: Haitham Shami Date: Wed, 19 May 2021 12:49:51 -0700 Subject: [PATCH 03/12] add telemetry flag machinery (#24) * add telemetry flag machinery * ut * PR feedback * update pipeline to include new ut * perf pipeline issue? * 1-reduce noise caused by Azure VM, and 2-address PR feedback * wget failed? --- src/azure-iot-edge-installer.sh | 31 ++++++++++----- src/utils.sh | 51 ++++++++++++++++++++++++- tests/perf_tests.sh | 4 +- tests/track_duration.sh | 5 ++- tests/unit-tests/test-telemetry-flag.sh | 24 ++++++++++++ vsts_ci/linux/continuous-linux.yml | 1 + vsts_ci/raspi/continuous-raspi.yml | 1 + 7 files changed, 102 insertions(+), 15 deletions(-) create mode 100644 tests/unit-tests/test-telemetry-flag.sh diff --git a/src/azure-iot-edge-installer.sh b/src/azure-iot-edge-installer.sh index 8800762..63a1a36 100644 --- a/src/azure-iot-edge-installer.sh +++ b/src/azure-iot-edge-installer.sh @@ -32,28 +32,28 @@ function download_bash_script() { local url_text=https://github.com/Azure/iot-edge-config/releases/download/${VERSION_TAG}/$file_name local tmp_file=$(echo `mktemp -u`) - printf "attempting to download '%s'.\n" $file_name > /dev/stdout + printf "attempting to download '%s'.\n" $file_name # attempt to download to a temporary file. # use 'sudo LOCAL_E2E=1 ./azure-iot-edge-installer.sh {}' to validate local source... if [ "$LOCAL_E2E" == "1" ]; then - printf "Testing local file '%s'\n" "../$TOPDIR/$file_name" > /dev/stdout + printf "Testing local file '%s'\n" "../$TOPDIR/$file_name" cp ../$TOPDIR/$file_name . else - printf "wget '%s' -q -O '%s'\n" $url_text $tmp_file > /dev/stdout + printf "wget '%s' -q -O '%s'\n" $url_text $tmp_file wget $url_text -q -O $tmp_file # validate request exit_code=$? if [[ $exit_code != 0 ]]; then - printf "ERROR: Failed to download '%s'; error: %d\n" $file_name $exit_code > /dev/stdout + printf "ERROR: Failed to download '%s'; error: %d\n" $file_name $exit_code rm $tmp_file exit $exit_code else - printf "downloaded '%s'\n" $file_name > /dev/stdout + printf "downloaded '%s'\n" $file_name mv -f $tmp_file $file_name chmod +x $file_name @@ -63,32 +63,41 @@ function download_bash_script() { } # script -printf "Welcome to azure-iot-edge-installer\n" > /dev/stdout +printf "Welcome to azure-iot-edge-installer\n" +printf "\n%s\n" "-------------------------" +printf "Telemetry\n" +printf "%s\n" "---------" +printf "The azure-iot-edge-installer collects usage data in order to improve your experience.\n" +printf "The data is anonymous and does not include commandline argument values.\n" +printf "The data is collected by Microsoft.\n" +printf "You can change your telemetry settings by adding -nt or --telemetry-opt-out to the command line.\n" +printf "\n" # if helper scripts dont exist, fetch via wget if [ -d "iot-edge-installer" ]; then - printf "Directory iot-edge-installer already exists.\n" > /dev/stdout + printf "Directory iot-edge-installer already exists.\n" else - printf "Preparing install directory.\n" > /dev/stdout + printf "Preparing install directory.\n" mkdir iot-edge-installer fi cd iot-edge-installer -printf "Downloading helper files to temporary directory ./iot-edge-installer\n" > /dev/stdout +printf "Downloading helper files to temporary directory ./iot-edge-installer\n" download_bash_script validate-tier1-os.sh download_bash_script install-container-management.sh download_bash_script install-edge-runtime.sh download_bash_script validate-post-install.sh download_bash_script utils.sh -printf "Downloaded helper files to temporary directory ./iot-edge-installer\n" > /dev/stdout +printf "Downloaded helper files to temporary directory ./iot-edge-installer\n" # import utils source utils.sh log_init # add flag:variable_name dictionary entries +add_option_args "TELEMETRY_OPT_OUT" -nt --telemetry-opt-out add_option_args "VERBOSE_LOGGING" -v --verbose add_option_args "SCOPE_ID" -s --scope-id add_option_args "REGISTRATION_ID" -r --registration-id @@ -97,6 +106,8 @@ add_option_args "SYMMETRIC_KEY" -k --symmetric-key # parse command line inputs and fetch output from parser declare -A parsed_cmds="$(cmd_parser $@)" +set_opt_out_selection ${parsed_cmds["TELEMETRY_OPT_OUT"]} + # validate that all arguments are acceptable / known if [[ ${#@} > 0 && ${#parsed_cmds[*]} == 0 ]]; then diff --git a/src/utils.sh b/src/utils.sh index 1f9ab53..21dc910 100644 --- a/src/utils.sh +++ b/src/utils.sh @@ -13,6 +13,38 @@ DEFAULT=$(echo -en "\e[00m") BOLD=$(echo -en "\e[01m") BLINK=$(echo -en "\e[5m") +OPT_IN=false + +###################################### +# set_opt_out_selection +# +# records the user's choice of opting out of telemetry +# +# ARGUMENTS: +# does_the_user_NOT_consent_to_sending_telemetry +# +# OUTPUTS: +# Write output to stdout +# RETURN: +# +###################################### + +function set_opt_out_selection() { + if [[ $# == 1 && $1 == true ]]; + then + OPT_IN=false + log_info "The user has opted out of sending usage telemetry." + else + OPT_IN=true + log_info "The user has opted in for sending usage telemetry." + fi +} + +function get_opt_in_selection() { + echo "$OPT_IN" +} + +export -f set_opt_out_selection get_opt_in_selection ###################################### # add_option_args @@ -76,7 +108,7 @@ function cmd_parser() { declare -A parsed_cmd for key in ${!flag_to_variable_dict[*]}; do - parsed_cmd[${flag_to_variable_dict[$key]}]="" + parsed_cmd[${flag_to_variable_dict[$key]}]=false done while [ $# -ne 0 ]; @@ -142,7 +174,7 @@ log() { then printf "$LP$FS\n" $@ >> "$OUTPUT_FILE" fi - printf "$LP$FS\n" $@ > /dev/stdout + printf "$LP$FS\n" $@ fi } @@ -217,12 +249,27 @@ function prepare_apt() { # sources list log_info "Adding'%s' to repository lists." $sources wget $sources -q -O /etc/apt/sources.list.d/microsoft-prod.list + exit_code=$? + if [[ $exit_code != 0 ]]; + then + log_error "prepare_apt() step 1 failed with error: %d\n" exit_code + exit 3 + fi # the key wget https://packages.microsoft.com/keys/microsoft.asc -q -O /dev/stdout | gpg --dearmor > /etc/apt/trusted.gpg.d/microsoft.gpg + exit_code=$? + if [[ $exit_code != 0 ]]; + then + log_error "prepare_apt() step 2 failed with error %d\n" exit_code + rm -f /etc/apt/sources.list.d/microsoft-prod.list &> /dev/null + exit 4 + fi # update apt update + exit_code=$? + log_info "'apt update' returned %d\n" exit_code fi fi } diff --git a/tests/perf_tests.sh b/tests/perf_tests.sh index 90d40a1..96d881c 100644 --- a/tests/perf_tests.sh +++ b/tests/perf_tests.sh @@ -19,6 +19,8 @@ subscription=$(az account show | awk '/"id/ { print substr($2,2,36) }') # for each test ... echo ./track_duration.sh -c $1 -t ./e2e-tests/test-devicestate.sh $subscription -./track_duration.sh -c $1 -t ./e2e-tests/test-devicestate.sh $subscription +cd e2e-tests +../track_duration.sh -v -c $1 -t ./test-devicestate.sh $subscription +cd .. exit 0 \ No newline at end of file diff --git a/tests/track_duration.sh b/tests/track_duration.sh index c7a6b38..284d7c1 100644 --- a/tests/track_duration.sh +++ b/tests/track_duration.sh @@ -26,7 +26,8 @@ set -e exec 3>&1 # bring in the library -source test_utils.sh +MY_LOCATION=$(dirname $0) +source ${MY_LOCATION}/test_utils.sh # verbose=false @@ -101,7 +102,7 @@ do runs[$curr-1]=$(bc <<< $end-$start) total=$(bc <<< $total+${runs[$curr-1]}) verbose_output "run $curr took $(bc <<< $end-$start) seconds" - python3 send_one_message_to_iot_hub_device.py "$IH_CONN_STR" "{\"OSName\": \"$os_name\", \"Kernel\": \"$os_kernel\", \"TestName\": \"$test_name\", \"TimeStamp\": \"$time_stamp\", \"Duration\": ${runs[$iter]}}" + python3 ${MY_LOCATION}/send_one_message_to_iot_hub_device.py "$IH_CONN_STR" "{\"OSName\": \"$os_name\", \"Kernel\": \"$os_kernel\", \"TestName\": \"$test_name\", \"TimeStamp\": \"$time_stamp\", \"Duration\": ${runs[$iter]}}" done verbose_output "" diff --git a/tests/unit-tests/test-telemetry-flag.sh b/tests/unit-tests/test-telemetry-flag.sh new file mode 100644 index 0000000..bd83eac --- /dev/null +++ b/tests/unit-tests/test-telemetry-flag.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +exec 3>&1 + +# bring in the utils library +source ../../src/utils.sh +source ../test_utils.sh + +function test_flag_on() { + set_opt_out_selection true + assert_eq $(get_opt_in_selection) false +} + +function test_flag_off() { + set_opt_out_selection false + assert_eq $(get_opt_in_selection) true +} + +test_flag_on +test_flag_off +show_test_totals diff --git a/vsts_ci/linux/continuous-linux.yml b/vsts_ci/linux/continuous-linux.yml index 1ea8ef5..f3c0997 100644 --- a/vsts_ci/linux/continuous-linux.yml +++ b/vsts_ci/linux/continuous-linux.yml @@ -12,4 +12,5 @@ steps: ./test-validate-tier1-os.sh ./test-cmd-parser.sh ./test-logger.sh + ./test-telemetry-flag.sh displayName: 'Run All Linux Unit Tests' diff --git a/vsts_ci/raspi/continuous-raspi.yml b/vsts_ci/raspi/continuous-raspi.yml index 84b3d1d..501c44a 100644 --- a/vsts_ci/raspi/continuous-raspi.yml +++ b/vsts_ci/raspi/continuous-raspi.yml @@ -12,4 +12,5 @@ steps: ./test-validate-tier1-os.sh ./test-cmd-parser.sh ./test-logger.sh + ./test-telemetry-flag.sh displayName: 'Run All Raspberry Pi Unit Tests' From 236507aafb80e870960a4b850dfd8648f961f676 Mon Sep 17 00:00:00 2001 From: Haitham Shami Date: Fri, 21 May 2021 15:07:47 -0700 Subject: [PATCH 04/12] User/hshami/fix check runtime existing (#25) * handle case where edge runtime is already installed * break down the key steps * fix exit code log_info --- src/azure-iot-edge-installer.sh | 16 +++++++++---- src/install-container-management.sh | 10 +++++++-- src/install-edge-runtime.sh | 25 ++++++++++++++++----- src/utils.sh | 35 +++++++++++++++++++++++------ 4 files changed, 67 insertions(+), 19 deletions(-) diff --git a/src/azure-iot-edge-installer.sh b/src/azure-iot-edge-installer.sh index 63a1a36..7ed6e1c 100644 --- a/src/azure-iot-edge-installer.sh +++ b/src/azure-iot-edge-installer.sh @@ -138,14 +138,22 @@ else platform=$(get_platform) prepare_apt $platform + OK_TO_CONTINUE=true source install-container-management.sh install_container_management - source install-edge-runtime.sh - install_edge_runtime ${parsed_cmds["SCOPE_ID"]} ${parsed_cmds["REGISTRATION_ID"]} ${parsed_cmds["SYMMETRIC_KEY"]} + if [[ $OK_TO_CONTINUE == true ]]; + then + OK_TO_CONTINUE=false + source install-edge-runtime.sh + install_edge_runtime ${parsed_cmds["SCOPE_ID"]} ${parsed_cmds["REGISTRATION_ID"]} ${parsed_cmds["SYMMETRIC_KEY"]} - source validate-post-install.sh - validate_post_install + if [[ $OK_TO_CONTINUE == true ]]; + then + source validate-post-install.sh + validate_post_install + fi + fi fi # cleanup, always diff --git a/src/install-container-management.sh b/src/install-container-management.sh index d7642a1..19cb855 100644 --- a/src/install-container-management.sh +++ b/src/install-container-management.sh @@ -12,7 +12,7 @@ # OUTPUTS: # Write output to stdout # RETURN: -# +# updates the global variable OK_TO_CONTINUE in case of failure to false. ###################################### install_container_management() { @@ -22,6 +22,12 @@ install_container_management() { else log_info "Running install-container-management.sh" - apt install moby-engine -y + apt-get install moby-engine -y + exit_code=$? + if [[ $exit_code != 0 ]]; + then + OK_TO_CONTINUE=false + log_info "'apt-get install moby-engine' returned %d\n" $exit_code + fi fi } diff --git a/src/install-edge-runtime.sh b/src/install-edge-runtime.sh index 60cd184..7f2c7db 100644 --- a/src/install-edge-runtime.sh +++ b/src/install-edge-runtime.sh @@ -8,9 +8,9 @@ ###################################### # install_edge_runtime # -# installs Azure IoT Edge Runtime 1.2 -# generates the edge's configuration file from template and -# fills in the DPS provisioning section from provided parameters +# - installs Azure IoT Edge Runtime 1.2 +# - generates the edge's configuration file from template and +# fills in the DPS provisioning section from provided parameters # # ARGUMENTS: # SCOPE_ID @@ -19,10 +19,9 @@ # OUTPUTS: # Write output to stdout # RETURN: -# +# updates the global variable OK_TO_CONTINUE in case of success to true. ###################################### - function install_edge_runtime() { if [[ $# != 3 || "$1" == "" || "$2" == "" || "$3" == "" ]]; then @@ -37,12 +36,24 @@ function install_edge_runtime() { else log_info "install_edge_runtime..." fi - + apt-get install aziot-edge -y + exit_code=$? + if [[ $exit_code != 0 ]]; + then + log_info "'apt-get install aziot-edge' returned %d\n" $exit_code + return + fi # create .toml from template log_info "create .toml from template." cp /etc/aziot/config.toml.edge.template /etc/aziot/config.toml + exit_code=$? + if [[ $exit_code != 0 ]]; + then + log_info "'cp /etc/aziot/config.toml.edge.template /etc/aziot/config.toml' returned %d\n" $exit_code + return + fi local SCOPE_ID=$1 local REGISTRATION_ID=$2 @@ -69,4 +80,6 @@ symmetric_key = { value = \"'$SYMMETRIC_KEY'\" } log_info "Apply settings - this will restart the edge" iotedge config apply + + OK_TO_CONTINUE=true } diff --git a/src/utils.sh b/src/utils.sh index 21dc910..a6c3cc1 100644 --- a/src/utils.sh +++ b/src/utils.sh @@ -14,6 +14,7 @@ BOLD=$(echo -en "\e[01m") BLINK=$(echo -en "\e[5m") OPT_IN=false +OK_TO_CONTINUE=false ###################################### # set_opt_out_selection @@ -108,7 +109,7 @@ function cmd_parser() { declare -A parsed_cmd for key in ${!flag_to_variable_dict[*]}; do - parsed_cmd[${flag_to_variable_dict[$key]}]=false + parsed_cmd[${flag_to_variable_dict[$key]}]="" done while [ $# -ne 0 ]; @@ -249,15 +250,16 @@ function prepare_apt() { # sources list log_info "Adding'%s' to repository lists." $sources wget $sources -q -O /etc/apt/sources.list.d/microsoft-prod.list - exit_code=$? + local exit_code=$? if [[ $exit_code != 0 ]]; then log_error "prepare_apt() step 1 failed with error: %d\n" exit_code exit 3 fi - # the key - wget https://packages.microsoft.com/keys/microsoft.asc -q -O /dev/stdout | gpg --dearmor > /etc/apt/trusted.gpg.d/microsoft.gpg + log_info "Downloading key\n" + local tmp_file=$(echo `mktemp -u`) + wget https://packages.microsoft.com/keys/microsoft.asc -q -O $tmp_file exit_code=$? if [[ $exit_code != 0 ]]; then @@ -266,10 +268,29 @@ function prepare_apt() { exit 4 fi - # update - apt update + # unpack the key + local gpg_file=/etc/apt/trusted.gpg.d/microsoft.gpg + if [[ -f $gpg_file ]]; + then + rm -f $gpg_file &> /dev/null + fi + gpg --dearmor --output $gpg_file $tmp_file exit_code=$? - log_info "'apt update' returned %d\n" exit_code + + rm -f $tmp_file &> /dev/null + + if [[ $exit_code != 0 ]]; + then + log_error "prepare_apt() step 2 failed with error %d\n" $exit_code + rm -f /etc/apt/sources.list.d/microsoft-prod.list &> /dev/null + exit 4 + fi + log_info "Downloaded key\n" + + # update + apt-get update + exit_code=$? + log_info "'apt-get update' returned %d\n" $exit_code fi fi } From 93a358a44c592f7d768f5caf9b19277b14e8054c Mon Sep 17 00:00:00 2001 From: Haitham Shami Date: Tue, 25 May 2021 13:40:33 -0700 Subject: [PATCH 05/12] User/hshami/error cdes etc (#26) * 1- error codes. 2- long running process is protected from ctrl-c, 3- interrupt handlers, 4- capture std output and std error * added missing brackets * two more long running commands * never mind * append, do not replace * apt-get * true==1 * improved logg details * improved logg details * fix spacing per PR --- README.md | 2 +- src/azure-iot-edge-installer.sh | 49 +++----- src/install-container-management.sh | 10 +- src/install-edge-runtime.sh | 44 ++++--- src/utils.sh | 170 ++++++++++++++++++++++++++-- src/validate-post-install.sh | 3 +- 6 files changed, 212 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index f3e5d8d..55664be 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Azure IoT Edge configuration tool +# Azure IoT Edge configuration tool [![Build Status](https://dev.azure.com/mseng/VSIoT/_apis/build/status/Azure%20IoT%20Edge/iotedgehubdev?branchName=master)](https://dev.azure.com/Azure-IoT-DDE-EdgeExperience/IoTEdgeConfig/_build?definitionId=28&branchName=main) Azure IoT Edge configuration tool (the Tool) is a command-line tool that installs and configures Azure IoT Edge on a device. The Tool greatly simplifies the configuration of IoT Edge by automating several steps into single command. diff --git a/src/azure-iot-edge-installer.sh b/src/azure-iot-edge-installer.sh index 7ed6e1c..8817f13 100644 --- a/src/azure-iot-edge-installer.sh +++ b/src/azure-iot-edge-installer.sh @@ -95,6 +95,7 @@ printf "Downloaded helper files to temporary directory ./iot-edge-installer\n" # import utils source utils.sh log_init +handlers_init # add flag:variable_name dictionary entries add_option_args "TELEMETRY_OPT_OUT" -nt --telemetry-opt-out @@ -114,7 +115,7 @@ then array=("$*") echo Unknown argument "${array[*]}" echo "Usage: sudo ./azure-iot-edge-installer.sh -s -r -k " - exit 1 + exit ${EXIT_CODES[1]} fi if [[ "${parsed_cmds["SCOPE_ID"]}" == "" || "${parsed_cmds["REGISTRATION_ID"]}" == "" || "${parsed_cmds["SYMMETRIC_KEY"]}" == "" ]]; @@ -123,7 +124,7 @@ then echo defined: "'"${!parsed_cmds[@]}"'" echo given: "'"${parsed_cmds[@]}"'" echo "Usage: sudo ./azure-iot-edge-installer.sh -s -r -k " - exit 1 + exit ${EXIT_CODES[2]} fi # check if current OS is Tier 1 @@ -133,36 +134,20 @@ is_os_tier1 if [ "$?" != "0" ]; then log_error "This OS is not supported. Please visit this link for more information https://docs.microsoft.com/en-us/azure/iot-edge/support?view=iotedge-2020-11#tier-1." -else - # run scripts in order, can take parsed input from above - platform=$(get_platform) - prepare_apt $platform - - OK_TO_CONTINUE=true - source install-container-management.sh - install_container_management - - if [[ $OK_TO_CONTINUE == true ]]; - then - OK_TO_CONTINUE=false - source install-edge-runtime.sh - install_edge_runtime ${parsed_cmds["SCOPE_ID"]} ${parsed_cmds["REGISTRATION_ID"]} ${parsed_cmds["SYMMETRIC_KEY"]} - - if [[ $OK_TO_CONTINUE == true ]]; - then - source validate-post-install.sh - validate_post_install - fi - fi + exit ${EXIT_CODES[3]} fi -# cleanup, always -cd .. -if [ -d "iot-edge-installer" ] -then - log_info "Removing temporary directory files for iot-edge-installer." - rm -rf iot-edge-installer - log_info "Removed temporary directory files for iot-edge-installer." -fi +# run scripts in order, can take parsed input from above +platform=$(get_platform) +prepare_apt $platform -announce_my_log_file "All logs were appended to" $OUTPUT_FILE +source install-container-management.sh +install_container_management + +source install-edge-runtime.sh +install_edge_runtime ${parsed_cmds["SCOPE_ID"]} ${parsed_cmds["REGISTRATION_ID"]} ${parsed_cmds["SYMMETRIC_KEY"]} + +source validate-post-install.sh +validate_post_install + +exit ${EXIT_CODES[0]} diff --git a/src/install-container-management.sh b/src/install-container-management.sh index 19cb855..f45efcb 100644 --- a/src/install-container-management.sh +++ b/src/install-container-management.sh @@ -20,14 +20,16 @@ install_container_management() { then log_info "docker command is already available." else - log_info "Running install-container-management.sh" + log_info "Installing moby-engine container management" - apt-get install moby-engine -y + apt-get install moby-engine -y 2>>$STDERR_REDIRECT 1>>$STDOUT_REDIRECT & + long_running_command $! exit_code=$? if [[ $exit_code != 0 ]]; then - OK_TO_CONTINUE=false - log_info "'apt-get install moby-engine' returned %d\n" $exit_code + log_info "moby-engine installation failed with code: %d" $exit_code + exit ${EXIT_CODES[8]} fi + log_info "Installed moby-engine container management" fi } diff --git a/src/install-edge-runtime.sh b/src/install-edge-runtime.sh index 7f2c7db..7cc8f8d 100644 --- a/src/install-edge-runtime.sh +++ b/src/install-edge-runtime.sh @@ -26,40 +26,42 @@ function install_edge_runtime() { if [[ $# != 3 || "$1" == "" || "$2" == "" || "$3" == "" ]]; then log_error "Scope ID, Registration ID, and the Symmetric Key are required" - return + exit ${EXIT_CODES[2]} fi if [ -x "$(command -v iotedge)" ]; then log_error "Edge runtime is already available." - return - else - log_info "install_edge_runtime..." + exit ${EXIT_CODES[9]} fi - apt-get install aziot-edge -y + log_info "Installing edge runtime..." + + apt-get install aziot-edge -y 2>>$STDERR_REDIRECT 1>>$STDOUT_REDIRECT & + long_running_command $! exit_code=$? if [[ $exit_code != 0 ]]; then - log_info "'apt-get install aziot-edge' returned %d\n" $exit_code - return + log_info "aziot-edged installation failed with exit code: %d" $exit_code + exit ${EXIT_CODES[10]} fi + log_info "Installed edge runtime..." # create .toml from template - log_info "create .toml from template." - cp /etc/aziot/config.toml.edge.template /etc/aziot/config.toml + log_info "Create instanance configuration .toml from template." + cp /etc/aziot/config.toml.edge.template /etc/aziot/config.toml &>/dev/null exit_code=$? if [[ $exit_code != 0 ]]; then - log_info "'cp /etc/aziot/config.toml.edge.template /etc/aziot/config.toml' returned %d\n" $exit_code - return + log_info "'cp /etc/aziot/config.toml.edge.template /etc/aziot/config.toml' returned %d" $exit_code + exit ${EXIT_CODES[11]} fi local SCOPE_ID=$1 local REGISTRATION_ID=$2 local SYMMETRIC_KEY=$3 - log_info "set '%s'; '%s'; '%s'" $SCOPE_ID $REGISTRATION_ID $SYMMETRIC_KEY + log_info "Set DPS provisioning parameters." sed -i '/## DPS provisioning with symmetric key/,/## DPS provisioning with X.509 certificate/c\ ## DPS provisioning with symmetric key\ [provisioning]\ @@ -76,10 +78,20 @@ symmetric_key = { value = \"'$SYMMETRIC_KEY'\" } # symmetric_key = { uri = "pkcs11:slot-id=0;object=device%20id?pin-value=1234" } # PKCS#11 URI\ \ ## DPS provisioning with X.509 certificate\ - ' /etc/aziot/config.toml + ' /etc/aziot/config.toml 2>>$STDERR_REDIRECT 1>>$STDOUT_REDIRECT + + exit_code=$? + if [[ $exit_code != 0 ]]; + then + log_info "'sed ....'" $exit_code + exit ${EXIT_CODES[12]} + fi log_info "Apply settings - this will restart the edge" - iotedge config apply - - OK_TO_CONTINUE=true + iotedge config apply 2>>$STDERR_REDIRECT 1>>$STDOUT_REDIRECT + exit_code=$? + if [[ $exit_code == 0 ]]; + then + log_info "IotEdge has been configured successfully" + fi } diff --git a/src/utils.sh b/src/utils.sh index a6c3cc1..c8303e7 100644 --- a/src/utils.sh +++ b/src/utils.sh @@ -6,6 +6,7 @@ # create flag:variable_name dictionary declare -A flag_to_variable_dict +# output color indicators RED=$(echo -en "\e[31m") GREEN=$(echo -en "\e[32m") MAGENTA=$(echo -en "\e[35m") @@ -13,8 +14,26 @@ DEFAULT=$(echo -en "\e[00m") BOLD=$(echo -en "\e[01m") BLINK=$(echo -en "\e[5m") +# OPT_IN - set to true if the user opts in for sending telemetry information to us. OPT_IN=false -OK_TO_CONTINUE=false + +# EXIT_CODES - an array of intigers indicating an exit status +declare -a EXIT_CODES=(0 # success + 1 # invalid argument is given at the command line + 2 # a required argument is missing + 3 # os is not supported + 4 # step 1 of apt-get failed + 5 # step 2 of apt-get failed + 6 # step 3 of apt-get failed + 7 # step 4 of apt-get failed + 8 # failed to install moby-engine + 9 # a version of edge-runtime is already installed + 10 # step 1 of installing edge-runtime failed + 11 # step 2 of installing edge-runtime failed + 12 # step 3 of installing edge-runtime failed + 13 # step 4 of installing edge-runtime failed + 14 # ctrl-c or kill + ) ###################################### # set_opt_out_selection @@ -197,6 +216,12 @@ log_init() { touch $OUTPUT_FILE announce_my_log_file "All logs will be appended to file" $OUTPUT_FILE + + STDOUT_REDIRECT=$TD"/"$(echo ${BASE_NAME%.*})-$(echo `date '+%Y-%m-%d'`).out + echo "-----------------------------------" `date '+%H:%M:%S.%N'` "-----------------------------------" >> $STDOUT_REDIRECT + + STDERR_REDIRECT=$TD"/"$(echo ${BASE_NAME%.*})-$(echo `date '+%Y-%m-%d'`).err + echo "-----------------------------------" `date '+%H:%M:%S.%N'` "-----------------------------------" >> $STDERR_REDIRECT } # @@ -237,35 +262,36 @@ export -f announce_my_log_file log_init log_error log_info log_warn log_debug function prepare_apt() { if [ $# != 1 ]; then - exit 1 + exit ${EXIT_CODES[2]} else local platform=$1 if [[ "$platform" == "" ]]; then log_error "Unsupported platform." - exit 2 + exit ${EXIT_CODES[3]} else sources="https://packages.microsoft.com/config/"$platform"/multiarch/prod.list" # sources list log_info "Adding'%s' to repository lists." $sources - wget $sources -q -O /etc/apt/sources.list.d/microsoft-prod.list + wget $sources -q -O /etc/apt/sources.list.d/microsoft-prod.list 2>>$STDERR_REDIRECT 1>>$STDOUT_REDIRECT local exit_code=$? if [[ $exit_code != 0 ]]; then log_error "prepare_apt() step 1 failed with error: %d\n" exit_code - exit 3 + exit ${EXIT_CODES[4]} fi + log_info "Added'%s' to repository lists." $sources - log_info "Downloading key\n" + log_info "Downloading key" local tmp_file=$(echo `mktemp -u`) - wget https://packages.microsoft.com/keys/microsoft.asc -q -O $tmp_file + wget https://packages.microsoft.com/keys/microsoft.asc -q -O $tmp_file 2>>$STDERR_REDIRECT 1>>$STDOUT_REDIRECT exit_code=$? if [[ $exit_code != 0 ]]; then log_error "prepare_apt() step 2 failed with error %d\n" exit_code rm -f /etc/apt/sources.list.d/microsoft-prod.list &> /dev/null - exit 4 + exit ${EXIT_CODES[5]} fi # unpack the key @@ -283,14 +309,134 @@ function prepare_apt() { then log_error "prepare_apt() step 2 failed with error %d\n" $exit_code rm -f /etc/apt/sources.list.d/microsoft-prod.list &> /dev/null - exit 4 + exit ${EXIT_CODES[6]} fi - log_info "Downloaded key\n" + log_info "Downloaded key" # update - apt-get update + log_info "update - Retrieve new lists of packages" + apt-get update 2>>$STDERR_REDIRECT 1>>$STDOUT_REDIRECT & + long_running_command $! exit_code=$? - log_info "'apt-get update' returned %d\n" $exit_code + log_info "update step completed with exit code: %d\n" $exit_code fi fi } + +BG_PROCESS_ACTIVE=false +BG_PROCESS_ID=-1 +###################################### +# long_running_process +# +# while a long-running process executes, shows 'busy' waiting feedback +# +# ARGUMENTS: +# the ID of a bg process +# +# OUTPUTS: +# RETURN: +# +###################################### + +function long_running_command() { + if [[ $# == 1 ]]; + then + BG_PROCESS_ID=$1 + BG_PROCESS_ACTIVE=true + + while [ $BG_PROCESS_ACTIVE == true ]; + do + for next_symbol in '-' '\\' '|' '/'; + do + echo -en "$next_symbol\b" + sleep 0.2 + local MYPS=$(ps -a | awk '/'$BG_PROCESS_ID'/ {print $1}') + if [ "$MYPS" == "" ]; + then + BG_PROCESS_ID=-1 + BG_PROCESS_ACTIVE=false + break + fi + done + echo -e "\b" + done + fi +} + +###################################### +# handle_ctrl_c +# +# will be called when 'ctrl-c' or kill-9 is issued by the user. +# if a bg process has been launched, wait for it to finish. +# +# ARGUMENTS: +# +# OUTPUTS: +# +# RETURN: +# +###################################### + +function handle_ctrl_c() { + log_info "ctrl C\n" + if [[ "$BG_PROCESS_ID" != "-1" ]]; + then + while kill -0 $BG_PROCESS_ID &> /dev/null; + do + wait $BG_PROCESS_ID; + done + + BG_PROCESS_ACTIVE=false + BG_PROCESS_ID=-1 + fi + + exit ${EXIT_CODES[14]} +} + +###################################### +# handle_exit +# +# will report telemetry if user has opted in +# will be called whenever an exit is encountered +# +# ARGUMENTS: +# exit code +# +# OUTPUTS: +# RETURN: +# +###################################### + +function handle_exit() { + local e_code=$? + log_info "Exit %d\n" $e_code + + # cleanup, always + cd .. + if [ -d "iot-edge-installer" ] + then + log_info "Removing temporary directory files for iot-edge-installer." + rm -rf iot-edge-installer + log_info "Removed temporary directory files for iot-edge-installer." + fi + + announce_my_log_file "All logs were appended to" $OUTPUT_FILE +} + +###################################### +# handlers_init +# +# ARGUMENTS: +# initialize handlers +# +# OUTPUTS: +# RETURN: +# +###################################### + +function handlers_init() { + trap handle_ctrl_c SIGINT + trap handle_exit EXIT +} + +export -f handlers_init diff --git a/src/validate-post-install.sh b/src/validate-post-install.sh index f79dc21..211e9a7 100644 --- a/src/validate-post-install.sh +++ b/src/validate-post-install.sh @@ -30,8 +30,9 @@ function is_service_running() { log_error "sudo iotedge check" return 1 fi + log_info "'%s' is running." $service_name - return 0 + return 0 } function validate_post_install() { From 953897d55834f0dd6851c90e263b433cddecb34d Mon Sep 17 00:00:00 2001 From: Cindy Deng Date: Tue, 25 May 2021 16:22:34 -0700 Subject: [PATCH 06/12] add appinsight telemetry function --- src/azure-iot-edge-installer.sh | 3 +- src/utils.sh | 223 ++++++++++++++++++++++++++++++-- 2 files changed, 212 insertions(+), 14 deletions(-) diff --git a/src/azure-iot-edge-installer.sh b/src/azure-iot-edge-installer.sh index 7ed6e1c..a6dcb04 100644 --- a/src/azure-iot-edge-installer.sh +++ b/src/azure-iot-edge-installer.sh @@ -102,11 +102,12 @@ add_option_args "VERBOSE_LOGGING" -v --verbose add_option_args "SCOPE_ID" -s --scope-id add_option_args "REGISTRATION_ID" -r --registration-id add_option_args "SYMMETRIC_KEY" -k --symmetric-key +add_option_args "SYMMETRIC_KEY" -cv --correlation-vector # parse command line inputs and fetch output from parser declare -A parsed_cmds="$(cmd_parser $@)" -set_opt_out_selection ${parsed_cmds["TELEMETRY_OPT_OUT"]} +set_opt_out_selection ${parsed_cmds["TELEMETRY_OPT_OUT"]} $parsed_cmds["CORRELATION_VECTOR"] # validate that all arguments are acceptable / known if [[ ${#@} > 0 && ${#parsed_cmds[*]} == 0 ]]; diff --git a/src/utils.sh b/src/utils.sh index a6c3cc1..b0bc8e3 100644 --- a/src/utils.sh +++ b/src/utils.sh @@ -6,6 +6,7 @@ # create flag:variable_name dictionary declare -A flag_to_variable_dict +# output color indicators RED=$(echo -en "\e[31m") GREEN=$(echo -en "\e[32m") MAGENTA=$(echo -en "\e[35m") @@ -13,8 +14,28 @@ DEFAULT=$(echo -en "\e[00m") BOLD=$(echo -en "\e[01m") BLINK=$(echo -en "\e[5m") +# OPT_IN - set to true if the user opts in for sending telemetry information to us. OPT_IN=false -OK_TO_CONTINUE=false + +# EXIT_CODES - an array of intigers indicating an exit status +declare -a EXIT_CODES=(0 # success + 1 # invalid argument is given at the command line + 2 # a required argument is missing + 3 # os is not supported + 4 # step 1 of apt-get failed + 5 # step 2 of apt-get failed + 6 # step 3 of apt-get failed + 7 # step 4 of apt-get failed + 8 # failed to install moby-engine + 9 # a version of edge-runtime is already installed + 10 # step 1 of installing edge-runtime failed + 11 # step 2 of installing edge-runtime failed + 12 # step 3 of installing edge-runtime failed + 13 # step 4 of installing edge-runtime failed + 14 # ctrl-c or kill + ) + +CORRELATION_VECTOR="" ###################################### # set_opt_out_selection @@ -31,7 +52,7 @@ OK_TO_CONTINUE=false ###################################### function set_opt_out_selection() { - if [[ $# == 1 && $1 == true ]]; + if [ $1 == true ]; then OPT_IN=false log_info "The user has opted out of sending usage telemetry." @@ -39,6 +60,15 @@ function set_opt_out_selection() { OPT_IN=true log_info "The user has opted in for sending usage telemetry." fi + + # handle correlation vector + if [ -z $2 ]; + then + CORRELATION_VECTOR=$(uuidgen) + else + CORRELATION_VECTOR=$2 + fi + } function get_opt_in_selection() { @@ -197,6 +227,12 @@ log_init() { touch $OUTPUT_FILE announce_my_log_file "All logs will be appended to file" $OUTPUT_FILE + + STDOUT_REDIRECT=$TD"/"$(echo ${BASE_NAME%.*})-$(echo `date '+%Y-%m-%d'`).out + echo "-----------------------------------" `date '+%H:%M:%S.%N'` "-----------------------------------" >> $STDOUT_REDIRECT + + STDERR_REDIRECT=$TD"/"$(echo ${BASE_NAME%.*})-$(echo `date '+%Y-%m-%d'`).err + echo "-----------------------------------" `date '+%H:%M:%S.%N'` "-----------------------------------" >> $STDERR_REDIRECT } # @@ -237,35 +273,36 @@ export -f announce_my_log_file log_init log_error log_info log_warn log_debug function prepare_apt() { if [ $# != 1 ]; then - exit 1 + exit ${EXIT_CODES[2]} else local platform=$1 if [[ "$platform" == "" ]]; then log_error "Unsupported platform." - exit 2 + exit ${EXIT_CODES[3]} else sources="https://packages.microsoft.com/config/"$platform"/multiarch/prod.list" # sources list log_info "Adding'%s' to repository lists." $sources - wget $sources -q -O /etc/apt/sources.list.d/microsoft-prod.list + wget $sources -q -O /etc/apt/sources.list.d/microsoft-prod.list 2>>$STDERR_REDIRECT 1>>$STDOUT_REDIRECT local exit_code=$? if [[ $exit_code != 0 ]]; then log_error "prepare_apt() step 1 failed with error: %d\n" exit_code - exit 3 + exit ${EXIT_CODES[4]} fi + log_info "Added'%s' to repository lists." $sources - log_info "Downloading key\n" + log_info "Downloading key" local tmp_file=$(echo `mktemp -u`) - wget https://packages.microsoft.com/keys/microsoft.asc -q -O $tmp_file + wget https://packages.microsoft.com/keys/microsoft.asc -q -O $tmp_file 2>>$STDERR_REDIRECT 1>>$STDOUT_REDIRECT exit_code=$? if [[ $exit_code != 0 ]]; then log_error "prepare_apt() step 2 failed with error %d\n" exit_code rm -f /etc/apt/sources.list.d/microsoft-prod.list &> /dev/null - exit 4 + exit ${EXIT_CODES[5]} fi # unpack the key @@ -283,14 +320,174 @@ function prepare_apt() { then log_error "prepare_apt() step 2 failed with error %d\n" $exit_code rm -f /etc/apt/sources.list.d/microsoft-prod.list &> /dev/null - exit 4 + exit ${EXIT_CODES[6]} fi - log_info "Downloaded key\n" + log_info "Downloaded key" # update - apt-get update + log_info "update - Retrieve new lists of packages" + apt-get update 2>>$STDERR_REDIRECT 1>>$STDOUT_REDIRECT & + long_running_command $! exit_code=$? - log_info "'apt-get update' returned %d\n" $exit_code + log_info "update step completed with exit code: %d\n" $exit_code fi fi } + +BG_PROCESS_ACTIVE=false +BG_PROCESS_ID=-1 +###################################### +# long_running_process +# +# while a long-running process executes, shows 'busy' waiting feedback +# +# ARGUMENTS: +# the ID of a bg process +# +# OUTPUTS: +# RETURN: +# +###################################### + +function long_running_command() { + if [[ $# == 1 ]]; + then + BG_PROCESS_ID=$1 + BG_PROCESS_ACTIVE=true + + while [ $BG_PROCESS_ACTIVE == true ]; + do + for next_symbol in '-' '\\' '|' '/'; + do + echo -en "$next_symbol\b" + sleep 0.2 + local MYPS=$(ps -a | awk '/'$BG_PROCESS_ID'/ {print $1}') + if [ "$MYPS" == "" ]; + then + BG_PROCESS_ID=-1 + BG_PROCESS_ACTIVE=false + break + fi + done + echo -e "\b" + done + fi +} + +###################################### +# handle_ctrl_c +# +# will be called when 'ctrl-c' or kill-9 is issued by the user. +# if a bg process has been launched, wait for it to finish. +# +# ARGUMENTS: +# +# OUTPUTS: +# +# RETURN: +# +###################################### + +function handle_ctrl_c() { + log_info "ctrl C\n" + if [[ "$BG_PROCESS_ID" != "-1" ]]; + then + while kill -0 $BG_PROCESS_ID &> /dev/null; + do + wait $BG_PROCESS_ID; + done + + BG_PROCESS_ACTIVE=false + BG_PROCESS_ID=-1 + fi + + exit ${EXIT_CODES[14]} +} + +###################################### +# handle_exit +# +# will report telemetry if user has opted in +# will be called whenever an exit is encountered +# +# ARGUMENTS: +# exit code +# +# OUTPUTS: +# RETURN: +# +###################################### + +function handle_exit() { + local e_code=$? + log_info "Exit %d\n" $e_code + + send_appinsight_telemetry e_code + + # cleanup, always + cd .. + if [ -d "iot-edge-installer" ] + then + log_info "Removing temporary directory files for iot-edge-installer." + rm -rf iot-edge-installer + log_info "Removed temporary directory files for iot-edge-installer." + fi + + announce_my_log_file "All logs were appended to" $OUTPUT_FILE +} + +###################################### +# handlers_init +# +# ARGUMENTS: +# initialize handlers +# +# OUTPUTS: +# RETURN: +# +###################################### + +function handlers_init() { + trap handle_ctrl_c SIGINT + trap handle_exit EXIT +} + +export -f handlers_init + +# Constants +InstrumentationKey="e83c2a5c-35d1-4a37-a5fe-b4f0f6e8e2a5" +IngestionEndpoint="https://dc.services.visualstudio.com/v2/track" +EventName="Azure-IoT-Edge-Installer-Summary" +DeviceUniqueID="xinzedPC" +CurrentTime=$(echo `date '+%Y-%m-%dT%H:%M:%S.%N'`) +SchemaVersion="1.0" + +###################################### +# send_appinsight_telemetry +# +# Send Application Insights event as a REST POST request with JSON body containing telemetry +# +# ARGUMENTS: +# CustomProperties: "status":[EXIT_CODE] +# CustomMeasurements: "duration": +# +# OUTPUTS: +# Write output to stdout +# RETURN: +# +###################################### + +function send_appinsight_telemetry () +{ + local customPropertiesObj='"status":'$1'' + + # validate that the user has opted in for telemetry collection + local optin=$(get_opt_in_selection ) + if [[ $optin == true ]]; + then + wget --header='Content-Type: application/json' --header='Accept-Charset: UTF-8' --post-data '{"name":"Microsoft.ApplicationInsights.'$InstrumentationKey'.Event","time": "'$CurrentTime'","iKey": "'$InstrumentationKey'","tags":{"ai.cloud.roleInstance": "'$DeviceUniqueID'"},"data":{"baseType": "EventData","baseData": {"ver": "'$SchemaVersion'","name": "'$EventName'","cv": "'$CORRELATION_VECTOR'","properties":{'$customPropertiesObj'},"measurements":{"duration":57}}}}' $IngestionEndpoint + fi +} + +export -f send_appinsight_telemetry + From d894d67306a63e5e7655731fd30f4c737291d043 Mon Sep 17 00:00:00 2001 From: Haitham Shami Date: Wed, 26 May 2021 10:02:16 -0700 Subject: [PATCH 07/12] fix opt-in flag (#28) * fix opt-in flag * no eol in log_xyz --- src/utils.sh | 16 ++++++++-------- src/validate-post-install.sh | 1 - 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/utils.sh b/src/utils.sh index c8303e7..32bce29 100644 --- a/src/utils.sh +++ b/src/utils.sh @@ -273,15 +273,15 @@ function prepare_apt() { sources="https://packages.microsoft.com/config/"$platform"/multiarch/prod.list" # sources list - log_info "Adding'%s' to repository lists." $sources + log_info "Adding'%s' to package sources lists." $sources wget $sources -q -O /etc/apt/sources.list.d/microsoft-prod.list 2>>$STDERR_REDIRECT 1>>$STDOUT_REDIRECT local exit_code=$? if [[ $exit_code != 0 ]]; then - log_error "prepare_apt() step 1 failed with error: %d\n" exit_code + log_error "prepare_apt() step 1 failed with error: %d" exit_code exit ${EXIT_CODES[4]} fi - log_info "Added'%s' to repository lists." $sources + log_info "Added'%s' to package sources lists." $sources log_info "Downloading key" local tmp_file=$(echo `mktemp -u`) @@ -289,7 +289,7 @@ function prepare_apt() { exit_code=$? if [[ $exit_code != 0 ]]; then - log_error "prepare_apt() step 2 failed with error %d\n" exit_code + log_error "prepare_apt() step 2 failed with error %d" exit_code rm -f /etc/apt/sources.list.d/microsoft-prod.list &> /dev/null exit ${EXIT_CODES[5]} fi @@ -307,7 +307,7 @@ function prepare_apt() { if [[ $exit_code != 0 ]]; then - log_error "prepare_apt() step 2 failed with error %d\n" $exit_code + log_error "prepare_apt() step 2 failed with error %d" $exit_code rm -f /etc/apt/sources.list.d/microsoft-prod.list &> /dev/null exit ${EXIT_CODES[6]} fi @@ -318,7 +318,7 @@ function prepare_apt() { apt-get update 2>>$STDERR_REDIRECT 1>>$STDOUT_REDIRECT & long_running_command $! exit_code=$? - log_info "update step completed with exit code: %d\n" $exit_code + log_info "update step completed with exit code: %d" $exit_code fi fi } @@ -349,7 +349,7 @@ function long_running_command() { for next_symbol in '-' '\\' '|' '/'; do echo -en "$next_symbol\b" - sleep 0.2 + sleep 0.15 local MYPS=$(ps -a | awk '/'$BG_PROCESS_ID'/ {print $1}') if [ "$MYPS" == "" ]; then @@ -358,8 +358,8 @@ function long_running_command() { break fi done - echo -e "\b" done + echo -en " \b" fi } diff --git a/src/validate-post-install.sh b/src/validate-post-install.sh index 211e9a7..5d70d56 100644 --- a/src/validate-post-install.sh +++ b/src/validate-post-install.sh @@ -15,7 +15,6 @@ # RETURN: # 0 if service is running, 1 otherwise ###################################### -source utils.sh function is_service_running() { local service_name=${1,,} From c0dca0e13e27af2bb92247b046c71d380c4fd7d2 Mon Sep 17 00:00:00 2001 From: Cindy Deng Date: Wed, 26 May 2021 10:05:10 -0700 Subject: [PATCH 08/12] some PR changes --- src/azure-iot-edge-installer.sh | 4 ++-- src/utils.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/azure-iot-edge-installer.sh b/src/azure-iot-edge-installer.sh index 3969966..fed106f 100644 --- a/src/azure-iot-edge-installer.sh +++ b/src/azure-iot-edge-installer.sh @@ -11,7 +11,7 @@ fi VERSION_TAG="v0.0.1" -# where am i +# where am I TOPDIR=$(dirname $0) ###################################### @@ -103,7 +103,7 @@ add_option_args "VERBOSE_LOGGING" -v --verbose add_option_args "SCOPE_ID" -s --scope-id add_option_args "REGISTRATION_ID" -r --registration-id add_option_args "SYMMETRIC_KEY" -k --symmetric-key -add_option_args "SYMMETRIC_KEY" -cv --correlation-vector +add_option_args "CORRELATION_VECTOR" -cv --correlation-vector # parse command line inputs and fetch output from parser declare -A parsed_cmds="$(cmd_parser $@)" diff --git a/src/utils.sh b/src/utils.sh index b0bc8e3..d9b848d 100644 --- a/src/utils.sh +++ b/src/utils.sh @@ -485,7 +485,7 @@ function send_appinsight_telemetry () local optin=$(get_opt_in_selection ) if [[ $optin == true ]]; then - wget --header='Content-Type: application/json' --header='Accept-Charset: UTF-8' --post-data '{"name":"Microsoft.ApplicationInsights.'$InstrumentationKey'.Event","time": "'$CurrentTime'","iKey": "'$InstrumentationKey'","tags":{"ai.cloud.roleInstance": "'$DeviceUniqueID'"},"data":{"baseType": "EventData","baseData": {"ver": "'$SchemaVersion'","name": "'$EventName'","cv": "'$CORRELATION_VECTOR'","properties":{'$customPropertiesObj'},"measurements":{"duration":57}}}}' $IngestionEndpoint + wget --header='Content-Type: application/json' --header='Accept-Charset: UTF-8' --post-data '{"name":"Microsoft.ApplicationInsights.'$InstrumentationKey'.Event","time": "'$CurrentTime'","iKey": "'$InstrumentationKey'","tags":{"ai.cloud.roleInstance": "'$DeviceUniqueID'"},"data":{"baseType": "EventData","baseData": {"ver": "'$SchemaVersion'","name": "'$EventName'","cv": "'$CORRELATION_VECTOR'","properties":{'$customPropertiesObj'},"measurements":{"duration":57}}}}' $IngestionEndpoint -O $tmp_file 2>>$STDERR_REDIRECT 1>>$STDOUT_REDIRECT fi } From f40ef60b30dfc54fed252a97bd1d7e3a8490d7f2 Mon Sep 17 00:00:00 2001 From: Cindy Deng Date: Wed, 26 May 2021 11:44:26 -0700 Subject: [PATCH 09/12] address PR comments --- src/azure-iot-edge-installer.sh | 1 - src/utils.sh | 45 ++++++++++++++++++++++++++------- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/src/azure-iot-edge-installer.sh b/src/azure-iot-edge-installer.sh index fed106f..5a64990 100644 --- a/src/azure-iot-edge-installer.sh +++ b/src/azure-iot-edge-installer.sh @@ -129,7 +129,6 @@ then fi # check if current OS is Tier 1 -source /etc/os-release source validate-tier1-os.sh is_os_tier1 if [ "$?" != "0" ]; diff --git a/src/utils.sh b/src/utils.sh index ce04856..b100124 100644 --- a/src/utils.sh +++ b/src/utils.sh @@ -64,7 +64,7 @@ function set_opt_out_selection() { # handle correlation vector if [ -z $2 ]; then - CORRELATION_VECTOR=$(uuidgen) + CORRELATION_VECTOR=$(generate_uuid) else CORRELATION_VECTOR=$2 fi @@ -422,7 +422,7 @@ function handle_exit() { local e_code=$? log_info "Exit %d\n" $e_code - send_appinsight_telemetry e_code + send_appinsight_event_telemetry $e_code # cleanup, always cd .. @@ -454,16 +454,39 @@ function handlers_init() { export -f handlers_init +function generate_uuid() { + source /etc/os-release + case $ID in + ubuntu) + if [ "$VERSION_ID" == "18.04" ]; + then + uuidgen + fi + ;; + + raspbian) + if [ "$VERSION_CODENAME" == "stretch" ] || [ "$VERSION_ID" == "9" ]; + then + uuid + fi + ;; + + *) + log_error "OS is not Tier 1" + ;; + esac + +} + # Constants InstrumentationKey="e83c2a5c-35d1-4a37-a5fe-b4f0f6e8e2a5" IngestionEndpoint="https://dc.services.visualstudio.com/v2/track" EventName="Azure-IoT-Edge-Installer-Summary" DeviceUniqueID="xinzedPC" -CurrentTime=$(echo `date '+%Y-%m-%dT%H:%M:%S.%N'`) SchemaVersion="1.0" ###################################### -# send_appinsight_telemetry +# send_appinsight_event_telemetry # # Send Application Insights event as a REST POST request with JSON body containing telemetry # @@ -477,17 +500,21 @@ SchemaVersion="1.0" # ###################################### -function send_appinsight_telemetry () +function send_appinsight_event_telemetry () { local customPropertiesObj='"status":'$1'' + local customMeasurementsObj='"duration":'$2'' # validate that the user has opted in for telemetry collection - local optin=$(get_opt_in_selection ) + local optin=$(get_opt_in_selection) if [[ $optin == true ]]; then - wget --header='Content-Type: application/json' --header='Accept-Charset: UTF-8' --post-data '{"name":"Microsoft.ApplicationInsights.'$InstrumentationKey'.Event","time": "'$CurrentTime'","iKey": "'$InstrumentationKey'","tags":{"ai.cloud.roleInstance": "'$DeviceUniqueID'"},"data":{"baseType": "EventData","baseData": {"ver": "'$SchemaVersion'","name": "'$EventName'","cv": "'$CORRELATION_VECTOR'","properties":{'$customPropertiesObj'},"measurements":{"duration":57}}}}' $IngestionEndpoint -O $tmp_file 2>>$STDERR_REDIRECT 1>>$STDOUT_REDIRECT + log_info "Ready to send telemetry to AppInsights endpoint with wget" + local CurrentTime=$(echo `date --utc '+%Y-%m-%dT%H:%M:%S.%N'`) + echo '{"name":"Microsoft.ApplicationInsights.'$InstrumentationKey'.Event","time": "'$CurrentTime'","iKey": "'$InstrumentationKey'","tags":{"ai.cloud.roleInstance": "'$DeviceUniqueID'"},"data":{"baseType": "EventData","baseData": {"ver": "'$SchemaVersion'","name": "'$EventName'","cv": "'$CORRELATION_VECTOR'","properties":{'$customPropertiesObj'},"measurements":{'$customMeasurementsObj'}}}}' + + wget --header='Content-Type: application/json' --header='Accept-Charset: UTF-8' --post-data '{"name":"Microsoft.ApplicationInsights.'$InstrumentationKey'.Event","time": "'$CurrentTime'","iKey": "'$InstrumentationKey'","tags":{"ai.cloud.roleInstance": "'$DeviceUniqueID'"},"data":{"baseType": "EventData","baseData": {"ver": "'$SchemaVersion'","name": "'$EventName'","cv": "'$CORRELATION_VECTOR'","properties":{'$customPropertiesObj'},"measurements":{'$customMeasurementsObj'}}}}' $IngestionEndpoint 2>>$STDERR_REDIRECT 1>>$STDOUT_REDIRECT + log_info "Finished sending telemetry to AppInsights endpoint with wget" fi } -export -f send_appinsight_telemetry - From c8652aa6beee5a870ec80634d68acf24e449cd40 Mon Sep 17 00:00:00 2001 From: Cindy Deng Date: Wed, 26 May 2021 11:53:08 -0700 Subject: [PATCH 10/12] forgot to remove echo and changed instrument key to team app insight --- src/utils.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils.sh b/src/utils.sh index b100124..f76190b 100644 --- a/src/utils.sh +++ b/src/utils.sh @@ -479,7 +479,7 @@ function generate_uuid() { } # Constants -InstrumentationKey="e83c2a5c-35d1-4a37-a5fe-b4f0f6e8e2a5" +InstrumentationKey="d403f627-57b8-4fb0-8001-c51b7466682d" IngestionEndpoint="https://dc.services.visualstudio.com/v2/track" EventName="Azure-IoT-Edge-Installer-Summary" DeviceUniqueID="xinzedPC" @@ -511,9 +511,9 @@ function send_appinsight_event_telemetry () then log_info "Ready to send telemetry to AppInsights endpoint with wget" local CurrentTime=$(echo `date --utc '+%Y-%m-%dT%H:%M:%S.%N'`) - echo '{"name":"Microsoft.ApplicationInsights.'$InstrumentationKey'.Event","time": "'$CurrentTime'","iKey": "'$InstrumentationKey'","tags":{"ai.cloud.roleInstance": "'$DeviceUniqueID'"},"data":{"baseType": "EventData","baseData": {"ver": "'$SchemaVersion'","name": "'$EventName'","cv": "'$CORRELATION_VECTOR'","properties":{'$customPropertiesObj'},"measurements":{'$customMeasurementsObj'}}}}' wget --header='Content-Type: application/json' --header='Accept-Charset: UTF-8' --post-data '{"name":"Microsoft.ApplicationInsights.'$InstrumentationKey'.Event","time": "'$CurrentTime'","iKey": "'$InstrumentationKey'","tags":{"ai.cloud.roleInstance": "'$DeviceUniqueID'"},"data":{"baseType": "EventData","baseData": {"ver": "'$SchemaVersion'","name": "'$EventName'","cv": "'$CORRELATION_VECTOR'","properties":{'$customPropertiesObj'},"measurements":{'$customMeasurementsObj'}}}}' $IngestionEndpoint 2>>$STDERR_REDIRECT 1>>$STDOUT_REDIRECT + log_info "Finished sending telemetry to AppInsights endpoint with wget" fi } From f38b1e3dd3fa476050afd9fd1d4562d9e0d77dac Mon Sep 17 00:00:00 2001 From: Cindy Deng Date: Wed, 26 May 2021 12:07:28 -0700 Subject: [PATCH 11/12] more PR comments --- src/utils.sh | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/utils.sh b/src/utils.sh index f76190b..990f438 100644 --- a/src/utils.sh +++ b/src/utils.sh @@ -454,28 +454,32 @@ function handlers_init() { export -f handlers_init +###################################### +# generate_uuid +# +# ARGUMENTS: +# generate UUIDs +# +# OUTPUTS: UUID +# RETURN: +# +###################################### + +source /etc/os-release function generate_uuid() { - source /etc/os-release case $ID in ubuntu) - if [ "$VERSION_ID" == "18.04" ]; - then - uuidgen - fi + uuidgen ;; raspbian) - if [ "$VERSION_CODENAME" == "stretch" ] || [ "$VERSION_ID" == "9" ]; - then - uuid - fi + uuid ;; *) log_error "OS is not Tier 1" ;; esac - } # Constants @@ -513,7 +517,7 @@ function send_appinsight_event_telemetry () local CurrentTime=$(echo `date --utc '+%Y-%m-%dT%H:%M:%S.%N'`) wget --header='Content-Type: application/json' --header='Accept-Charset: UTF-8' --post-data '{"name":"Microsoft.ApplicationInsights.'$InstrumentationKey'.Event","time": "'$CurrentTime'","iKey": "'$InstrumentationKey'","tags":{"ai.cloud.roleInstance": "'$DeviceUniqueID'"},"data":{"baseType": "EventData","baseData": {"ver": "'$SchemaVersion'","name": "'$EventName'","cv": "'$CORRELATION_VECTOR'","properties":{'$customPropertiesObj'},"measurements":{'$customMeasurementsObj'}}}}' $IngestionEndpoint 2>>$STDERR_REDIRECT 1>>$STDOUT_REDIRECT - + log_info "Finished sending telemetry to AppInsights endpoint with wget" fi } From a65f9c5c046e1b9c91f18855f7ed5d8e38ee82d1 Mon Sep 17 00:00:00 2001 From: Mariana Mihova <10135329+marianan@users.noreply.github.com> Date: Wed, 26 May 2021 14:30:22 -0700 Subject: [PATCH 12/12] Calculate runtime duration (#29) * Calculate runtime duration * Pass duration as parameter --- src/azure-iot-edge-installer.sh | 3 +++ src/utils.sh | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/azure-iot-edge-installer.sh b/src/azure-iot-edge-installer.sh index 5a64990..b1f6e52 100644 --- a/src/azure-iot-edge-installer.sh +++ b/src/azure-iot-edge-installer.sh @@ -3,6 +3,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +# Record start time, this value is used for calculating execution time +start=`date +%s` + if [[ $EUID -ne 0 ]]; then echo "$(echo -en "\e[31m")ERROR: $(echo -en "\e[00m")$0 requires elevated privileges.. " diff --git a/src/utils.sh b/src/utils.sh index 990f438..59430c4 100644 --- a/src/utils.sh +++ b/src/utils.sh @@ -420,9 +420,13 @@ function handle_ctrl_c() { function handle_exit() { local e_code=$? - log_info "Exit %d\n" $e_code + log_info "Exit %d" $e_code - send_appinsight_event_telemetry $e_code + local end=`date +%s` + runtime=$((end-start)) + log_info "Runtime duration %d" $runtime + + send_appinsight_event_telemetry $e_code $runtime # cleanup, always cd ..