Remove cleanup for CI Builds (#20395)

* Remove cleanup for CI Builds * - added scripts * Update ios.cake * Update clean-bot.sh * add clean bot to yaml * - fix yml * - fix yaml * - try harder * - chmod * - update * - chmod * Update ios.cake * - add grep logging * - grep * - logs * Update device-tests-steps.yml * Update clean-bot.sh * Update device-tests-steps.yml * - zip up logs * - fix condition * - params * - test * - fix * - again * - test * - zippity * - zip * - fix condition * - log dir * - fix tabs * - diag reports * Try resetting simulators before test runs * Fix cmdline arguments for reset-simulator command --------- Co-authored-by: Ivan Povazan <ivan.povazan@gmail.com>
2024-02-12 06:47:38 -06:00 · 2024-02-12 06:47:38 -06:00 · a195e8da8f
--- a/eng/devices/ios.cake
+++ b/eng/devices/ios.cake
@ -38,7 +38,7 @@ string DOTNET_PLATFORM = TEST_DEVICE.ToLower().Contains("simulator") ?
 	$"iossimulator-{System.Runtime.InteropServices.RuntimeInformation.OSArchitecture.ToString().ToLower()}"
  : $"ios-arm64";
 string CONFIGURATION = Argument("configuration", "Debug");
-bool DEVICE_CLEANUP = Argument("cleanup", true);
+bool DEVICE_CLEANUP = Argument("cleanup", !IsCIBuild());
 string TEST_FRAMEWORK = "net472";

 Information("Project File: {0}", PROJECT);
@ -77,6 +77,7 @@ Setup(context =>
 	if (TEST_DEVICE.IndexOf("_") != -1) 
 	{
 		GetSimulators(TEST_DEVICE);
+		ResetSimulators(TEST_DEVICE);
 	}
 });

@ -480,6 +481,19 @@ void GetSimulators(string version)
 		});
 }

+void ResetSimulators(string version)
+{
+	var logDirectory = GetLogDirectory();
+	DotNetTool("tool", new DotNetToolSettings {
+			ToolPath = DOTNET_TOOL_PATH,
+			DiagnosticOutput = true,
+			ArgumentCustomization = args => args.Append("run xharness apple simulators reset-simulator " +
+				$"--output-directory=\"{logDirectory}\" " +
+				$"--target=\"{version}\" " +
+				$"--verbosity=\"Debug\" ")
+		});
+}
+
 void GetDevices(string version)
 {
 	var deviceUdid = "";
--- a/eng/pipelines/common/device-tests-steps.yml
+++ b/eng/pipelines/common/device-tests-steps.yml
@ -13,6 +13,14 @@ parameters:
  rebootAgent: true

 steps:
+  - ${{ if eq(parameters.platform, 'ios')}}:
+    - bash: |
+        chmod +x $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh
+        chmod +x $(System.DefaultWorkingDirectory)/eng/scripts/clean-simulator-runtime.sh
+        $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh
+      displayName: 'Clean bot'
+      continueOnError: true
+      timeoutInMinutes: 60
  - template: provision.yml
    parameters:
      ${{ if eq(parameters.platform, 'windows')}}:
@ -86,6 +94,15 @@ steps:
    condition: and(succeeded(), eq(variables['Platform.Name'], 'Mac'))
    retryCountOnTaskFailure: 2

+
+  - ${{ if eq(parameters.platform, 'ios')}}:
+    - bash: |
+        zip -9r "$(LogDirectory)/CoreSimulatorLog.zip" "$HOME/Library/Logs/CoreSimulator/"
+        zip -9r "$(LogDirectory)/DiagnosticReports.zip" "$HOME/Library/Logs/DiagnosticReports/"
+      displayName: Zip Simulator Logs
+      condition: always()
+      continueOnError: true
+
  - task: PublishTestResults@2
    displayName: Publish the $(Agent.JobName) test results
    condition: always()
--- a/eng/pipelines/common/ui-tests-compatibility-steps.yml
+++ b/eng/pipelines/common/ui-tests-compatibility-steps.yml
@ -13,6 +13,14 @@ parameters:
  provisionPlatform: "windows"

 steps:
+  - bash: |
+      chmod +x $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh
+      chmod +x $(System.DefaultWorkingDirectory)/eng/scripts/clean-simulator-runtime.sh
+      $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh
+    displayName: 'Clean bot'
+    continueOnError: true
+    condition: ${{ eq(parameters.platform, 'ios') }}
+    timeoutInMinutes: 60

  - template: provision.yml
    parameters:
@ -44,7 +52,14 @@ steps:
  - pwsh: ./build.ps1 -Script eng/devices/${{ parameters.platform }}.cake --target=cg-uitest --project="${{ parameters.path }}" --appproject="${{ parameters.app }}" --device="${{ parameters.device }}" --apiversion="${{ parameters.version }}" --configuration="${{ parameters.configuration }}" --results="$(TestResultsDirectory)" --binlog="$(LogDirectory)" ${{ parameters.cakeArgs }} --verbosity=diagnostic --where="${{ parameters.where }}"
    displayName: $(Agent.JobName)
    # retryCountOnTaskFailure: 2
-  
+
+  - bash: |
+      zip -9r "$(LogDirectory)/artifacts/CoreSimulatorLog_$(new-guid).zip" "$HOME/Library/Logs/CoreSimulator/"
+      zip -9r "$(LogDirectory)/DiagnosticReports_$(new-guid).zip" "$HOME/Library/Logs/DiagnosticReports/"
+    displayName: Zip Simulator Logs
+    condition: ${{ eq(parameters.platform, 'ios') }}
+    continueOnError: true
+
  - task: PublishTestResults@2
    displayName: Publish the $(System.PhaseName) test results
    condition: always()
--- a/eng/pipelines/common/ui-tests-steps.yml
+++ b/eng/pipelines/common/ui-tests-steps.yml
@ -10,6 +10,13 @@ parameters:
  configuration : "Release"

 steps:
+  - ${{ if eq(parameters.platform, 'ios')}}:
+    - bash: |
+        chmod +x $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh
+        $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh
+      displayName: 'Clean bot'
+      continueOnError: true
+      timeoutInMinutes: 60

  - template: provision.yml
    parameters:
@ -102,7 +109,15 @@ steps:
  - pwsh: ./build.ps1 -Script eng/devices/${{ parameters.platform }}.cake --target=uitest --project="${{ parameters.path }}" --appproject="${{ parameters.app }}" --device="${{ parameters.device }}" --apiversion="${{ parameters.version }}" --configuration="${{ parameters.configuration }}" --results="$(TestResultsDirectory)" --binlog="$(LogDirectory)" ${{ parameters.cakeArgs }} --verbosity=diagnostic
    displayName: $(Agent.JobName)
    retryCountOnTaskFailure: 2
-  
+
+  - ${{ if eq(parameters.platform, 'ios')}}:
+    - bash: |
+        zip -9r "$(LogDirectory)/CoreSimulatorLog_$(new-guid).zip" "$HOME/Library/Logs/CoreSimulator/"
+        zip -9r "$(LogDirectory)/DiagnosticReports_$(new-guid).zip" "$HOME/Library/Logs/DiagnosticReports/"
+        displayName: Zip Simulator Logs
+      condition: always()
+      continueOnError: true
+
  - task: PublishTestResults@2
    displayName: Publish the $(System.PhaseName) test results
    condition: always()
--- a/eng/scripts/clean-bot.sh
+++ b/eng/scripts/clean-bot.sh
@ -0,0 +1,129 @@
+#!/bin/bash -ex
+
+# Print disk status before cleaning
+df -h
+
+# We don't care about errors in this section, we just want to clean as much as possible
+set +e
+
+# Delete all the simulator devices. These can take up a lot of space over time (I've seen 100+GB on the bots)
+/Applications/Xcode.app/Contents/Developer/usr/bin/simctl delete all
+
+# Delete old Xcodes.
+ls -lad /Applications/Xcode*.app
+
+oldXcodes=(
+	"/Applications/Xcode44.app"
+	"/Applications/Xcode5.app"
+	"/Applications/Xcode502.app"
+	"/Applications/Xcode511.app"
+	"/Applications/Xcode6.0.1.app"
+	"/Applications/Xcode6.app"
+	"/Applications/Xcode601.app"
+	"/Applications/Xcode61.app"
+	"/Applications/Xcode611.app"
+	"/Applications/Xcode62.app"
+	"/Applications/Xcode63.app"
+	"/Applications/Xcode64.app"
+	"/Applications/Xcode7.app"
+	"/Applications/Xcode701.app"
+	"/Applications/Xcode71.app"
+	"/Applications/Xcode711.app"
+	"/Applications/Xcode72.app"
+	"/Applications/Xcode731.app"
+	"/Applications/Xcode8-GM.app"
+	"/Applications/Xcode8.app"
+	"/Applications/Xcode81-GM.app"
+	"/Applications/Xcode81.app"
+	"/Applications/Xcode82.app"
+	"/Applications/Xcode821.app"
+	"/Applications/Xcode83.app"
+	"/Applications/Xcode833.app"
+	"/Applications/Xcode9-GM.app"
+	"/Applications/Xcode9.app"
+	"/Applications/Xcode91.app"
+	"/Applications/Xcode92.app"
+	"/Applications/Xcode93.app"
+	"/Applications/Xcode94.app"
+	"/Applications/Xcode941.app"
+	"/Applications/Xcode10.app"
+	"/Applications/Xcode101-beta2.app"
+	"/Applications/Xcode101-beta3.app"
+	"/Applications/Xcode101.app"
+	"/Applications/Xcode102-beta1.app"
+	"/Applications/Xcode102.app"
+	"/Applications/Xcode1021.app"
+	"/Applications/Xcode103.app"
+	"/Applications/Xcode10GM.app"
+	"/Applications/Xcode11-beta3.app"
+	"/Applications/Xcode11-GM.app"
+	"/Applications/Xcode11.app"
+	"/Applications/Xcode111.app"
+	"/Applications/Xcode112.app"
+	"/Applications/Xcode1121.app"
+	"/Applications/Xcode113.app"
+	"/Applications/Xcode1131.app"
+	"/Applications/Xcode114-beta1.app"
+	"/Applications/Xcode114-beta2.app"
+	"/Applications/Xcode114-beta3.app"
+	"/Applications/Xcode114.app"
+	"/Applications/Xcode1141.app"
+	"/Applications/Xcode115-beta1.app"
+	"/Applications/Xcode115-beta2.app"
+	"/Applications/Xcode115-GM.app"
+	"/Applications/Xcode_8.0.app"
+	"/Applications/Xcode_8.1.app"
+	"/Applications/Xcode_8.2.1.app"
+	"/Applications/Xcode_8.3.3.app"
+	"/Applications/Xcode_9.0.app"
+	"/Applications/Xcode_9.1.0.app"
+	"/Applications/Xcode_9.2.0.app"
+	"/Applications/Xcode_9.2.app"
+	"/Applications/Xcode_9.4.1.app"
+# Xcode 10.2.1 is currently used by Binding Tools for Swift # /Applications/Xcode_10.2.1.app
+	"/Applications/Xcode_11.3.0.app"
+	"/Applications/Xcode_11.5.0.app"
+	"/Applications/Xcode_11.6.0-beta1.app"
+	"/Applications/Xcode_12.0.0-beta1.app"
+	"/Applications/Xcode_12.0.0-beta2.app"
+	"/Applications/Xcode_12.0.0-beta3.app"
+	"/Applications/Xcode_12.0.0-beta4.app"
+	"/Applications/Xcode_12.0.0-beta5.app"
+	"/Applications/Xcode_12.0.0-beta6.app"
+	"/Applications/Xcode_12.1.0-GM.app"
+	"/Applications/Xcode_12.0.0-GMb.app"
+	"/Applications/Xcode_12.2.0-beta1.app"
+	"/Applications/Xcode_12.2.0-beta2.app"
+	"/Applications/Xcode_12.2.0-beta3.app"
+	"/Applications/Xcode_12.2.0-beta.3.app"
+	"/Applications/Xcode_12.2.0-rc.app"
+	"/Applications/Xcode_12.5.0-rc.app"
+	"/Applications/Xcode_13.0.0-beta.app"
+	"/Applications/Xcode_13.0.0-beta2.app"
+	"/Applications/Xcode_13.0.0-beta3.app"
+	"/Applications/Xcode_14.3.1.app"
+	"/Applications/Xcode_15.0.0.app"
+	"/Applications/Xcode_15.0.1.app"
+	"/Applications/Xcode_15.1.0.app"
+)
+
+# remove wrongly added .xip files under /Applications, confuses provisionator and 
+# are not needed and wrong
+sudo rm -Rf /Applications/*.xip
+
+# pick the current selected xcode to make sure we do not remove it.
+XCODE_SELECT=$(xcode-select -p)
+
+for oldXcode in "${oldXcodes[@]}"; do
+	if [ "$XCODE_SELECT" != "$oldXcode/Contents/Developer" ]; then
+		sudo rm -Rf "$oldXcode"
+	else
+		echo "Not removing $oldXcode because is the currently selected one."
+	fi
+done
+
+DIR="$(dirname "${BASH_SOURCE[0]}")"
+"$DIR"/clean-simulator-runtime.sh
+
+# Print disk status after cleaning
+df -h
--- a/eng/scripts/clean-simulator-runtime.sh
+++ b/eng/scripts/clean-simulator-runtime.sh
@ -0,0 +1,37 @@
+#!/bin/bash -eux
+
+# find if there are any duplicated simulator runtimes for a given platform
+
+set -o pipefail
+IFS=$'\n\t'
+
+xcrun simctl runtime list -j > simruntime.json
+cat simruntime.json
+
+grep -e '"identifier" : ' -e '"runtimeIdentifier" : ' simruntime.json | tr '\n' ' ' | sed -e 's/,//g' -e 's/"//g' -e 's/runtimeIdentifier : //g' -e $'s/identifier : /@/g' | tr '@' '\n' | awk NF | sed 's/^[[:blank:]]*//' > simruntime-lines.txt
+cat simruntime-lines.txt
+
+sed -e 's/.*com.apple/com.apple/g' simruntime-lines.txt > simruntime-runtimes.txt
+cat simruntime-runtimes.txt
+
+sort simruntime-runtimes.txt | uniq -c | sort -n | sed 's/^[[:blank:]]*//' > simruntime-runtimes-by-count.txt
+cat simruntime-runtimes-by-count.txt
+
+grep -v '^1 ' simruntime-runtimes-by-count.txt | sed 's/^[0-9 ]*//' > simruntime-duplicated-runtimes.txt
+cat simruntime-duplicated-runtimes.txt
+
+while IFS= read -r simruntime
+do
+  echo "Duplicated: $simruntime"
+  grep "$simruntime" simruntime-lines.txt | sed 's/ .*//' | while IFS= read -r id
+  do
+    echo "    sudo xcrun simctl runtime delete $id"
+    if ! sudo xcrun simctl runtime delete "$id"; then
+      echo "    failed to delete runtime $id"
+    else
+      echo "    deleted runtime $id"
+    fi
+  done
+done < simruntime-duplicated-runtimes.txt
+
+xcrun simctl runtime list -v