Remove cleanup for CI Builds (#20395)

* Remove cleanup for CI Builds

* - added scripts

* Update ios.cake

* Update clean-bot.sh

* add clean bot to yaml

* - fix yml

* - fix yaml

* - try harder

* - chmod

* - update

* - chmod

* Update ios.cake

* - add grep logging

* - grep

* - logs

* Update device-tests-steps.yml

* Update clean-bot.sh

* Update device-tests-steps.yml

* - zip up logs

* - fix condition

* - params

* - test

* - fix

* - again

* - test

* - zippity

* - zip

* - fix condition

* - log dir

* - fix tabs

* - diag reports

* Try resetting simulators before test runs

* Fix cmdline arguments for reset-simulator command

---------

Co-authored-by: Ivan Povazan <ivan.povazan@gmail.com>

eng/devices/ios.cake

@@ -38,7 +38,7 @@ string DOTNET_PLATFORM = TEST_DEVICE.ToLower().Contains("simulator") ?
 	$"iossimulator-{System.Runtime.InteropServices.RuntimeInformation.OSArchitecture.ToString().ToLower()}"
   : $"ios-arm64";
 string CONFIGURATION = Argument("configuration", "Debug");
-bool DEVICE_CLEANUP = Argument("cleanup", true);
+bool DEVICE_CLEANUP = Argument("cleanup", !IsCIBuild());
 string TEST_FRAMEWORK = "net472";
 
 Information("Project File: {0}", PROJECT);
@@ -77,6 +77,7 @@ Setup(context =>
 	if (TEST_DEVICE.IndexOf("_") != -1) 
 	{
 		GetSimulators(TEST_DEVICE);
+		ResetSimulators(TEST_DEVICE);
 	}
 });
 
@@ -480,6 +481,19 @@ void GetSimulators(string version)
 		});
 }
 
+void ResetSimulators(string version)
+{
+	var logDirectory = GetLogDirectory();
+	DotNetTool("tool", new DotNetToolSettings {
+			ToolPath = DOTNET_TOOL_PATH,
+			DiagnosticOutput = true,
+			ArgumentCustomization = args => args.Append("run xharness apple simulators reset-simulator " +
+				$"--output-directory=\"{logDirectory}\" " +
+				$"--target=\"{version}\" " +
+				$"--verbosity=\"Debug\" ")
+		});
+}
+
 void GetDevices(string version)
 {
 	var deviceUdid = "";

eng/pipelines/common/device-tests-steps.yml

@@ -13,6 +13,14 @@ parameters:
   rebootAgent: true
 
 steps:
+  - ${{ if eq(parameters.platform, 'ios')}}:
+    - bash: |
+        chmod +x $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh
+        chmod +x $(System.DefaultWorkingDirectory)/eng/scripts/clean-simulator-runtime.sh
+        $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh
+      displayName: 'Clean bot'
+      continueOnError: true
+      timeoutInMinutes: 60
   - template: provision.yml
     parameters:
       ${{ if eq(parameters.platform, 'windows')}}:
@@ -86,6 +94,15 @@ steps:
     condition: and(succeeded(), eq(variables['Platform.Name'], 'Mac'))
     retryCountOnTaskFailure: 2
 
+
+  - ${{ if eq(parameters.platform, 'ios')}}:
+    - bash: |
+        zip -9r "$(LogDirectory)/CoreSimulatorLog.zip" "$HOME/Library/Logs/CoreSimulator/"
+        zip -9r "$(LogDirectory)/DiagnosticReports.zip" "$HOME/Library/Logs/DiagnosticReports/"
+      displayName: Zip Simulator Logs
+      condition: always()
+      continueOnError: true
+
   - task: PublishTestResults@2
     displayName: Publish the $(Agent.JobName) test results
     condition: always()

eng/pipelines/common/ui-tests-compatibility-steps.yml

@@ -13,6 +13,14 @@ parameters:
   provisionPlatform: "windows"
 
 steps:
+  - bash: |
+      chmod +x $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh
+      chmod +x $(System.DefaultWorkingDirectory)/eng/scripts/clean-simulator-runtime.sh
+      $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh
+    displayName: 'Clean bot'
+    continueOnError: true
+    condition: ${{ eq(parameters.platform, 'ios') }}
+    timeoutInMinutes: 60
 
   - template: provision.yml
     parameters:
@@ -45,6 +53,13 @@ steps:
     displayName: $(Agent.JobName)
     # retryCountOnTaskFailure: 2
 
+  - bash: |
+      zip -9r "$(LogDirectory)/artifacts/CoreSimulatorLog_$(new-guid).zip" "$HOME/Library/Logs/CoreSimulator/"
+      zip -9r "$(LogDirectory)/DiagnosticReports_$(new-guid).zip" "$HOME/Library/Logs/DiagnosticReports/"
+    displayName: Zip Simulator Logs
+    condition: ${{ eq(parameters.platform, 'ios') }}
+    continueOnError: true
+
   - task: PublishTestResults@2
     displayName: Publish the $(System.PhaseName) test results
     condition: always()

eng/pipelines/common/ui-tests-steps.yml

@@ -10,6 +10,13 @@ parameters:
   configuration : "Release"
 
 steps:
+  - ${{ if eq(parameters.platform, 'ios')}}:
+    - bash: |
+        chmod +x $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh
+        $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh
+      displayName: 'Clean bot'
+      continueOnError: true
+      timeoutInMinutes: 60
 
   - template: provision.yml
     parameters:
@@ -103,6 +110,14 @@ steps:
     displayName: $(Agent.JobName)
     retryCountOnTaskFailure: 2
 
+  - ${{ if eq(parameters.platform, 'ios')}}:
+    - bash: |
+        zip -9r "$(LogDirectory)/CoreSimulatorLog_$(new-guid).zip" "$HOME/Library/Logs/CoreSimulator/"
+        zip -9r "$(LogDirectory)/DiagnosticReports_$(new-guid).zip" "$HOME/Library/Logs/DiagnosticReports/"
+        displayName: Zip Simulator Logs
+      condition: always()
+      continueOnError: true
+
   - task: PublishTestResults@2
     displayName: Publish the $(System.PhaseName) test results
     condition: always()

eng/scripts/clean-bot.sh

@@ -0,0 +1,129 @@
+#!/bin/bash -ex
+
+# Print disk status before cleaning
+df -h
+
+# We don't care about errors in this section, we just want to clean as much as possible
+set +e
+
+# Delete all the simulator devices. These can take up a lot of space over time (I've seen 100+GB on the bots)
+/Applications/Xcode.app/Contents/Developer/usr/bin/simctl delete all
+
+# Delete old Xcodes.
+ls -lad /Applications/Xcode*.app
+
+oldXcodes=(
+	"/Applications/Xcode44.app"
+	"/Applications/Xcode5.app"
+	"/Applications/Xcode502.app"
+	"/Applications/Xcode511.app"
+	"/Applications/Xcode6.0.1.app"
+	"/Applications/Xcode6.app"
+	"/Applications/Xcode601.app"
+	"/Applications/Xcode61.app"
+	"/Applications/Xcode611.app"
+	"/Applications/Xcode62.app"
+	"/Applications/Xcode63.app"
+	"/Applications/Xcode64.app"
+	"/Applications/Xcode7.app"
+	"/Applications/Xcode701.app"
+	"/Applications/Xcode71.app"
+	"/Applications/Xcode711.app"
+	"/Applications/Xcode72.app"
+	"/Applications/Xcode731.app"
+	"/Applications/Xcode8-GM.app"
+	"/Applications/Xcode8.app"
+	"/Applications/Xcode81-GM.app"
+	"/Applications/Xcode81.app"
+	"/Applications/Xcode82.app"
+	"/Applications/Xcode821.app"
+	"/Applications/Xcode83.app"
+	"/Applications/Xcode833.app"
+	"/Applications/Xcode9-GM.app"
+	"/Applications/Xcode9.app"
+	"/Applications/Xcode91.app"
+	"/Applications/Xcode92.app"
+	"/Applications/Xcode93.app"
+	"/Applications/Xcode94.app"
+	"/Applications/Xcode941.app"
+	"/Applications/Xcode10.app"
+	"/Applications/Xcode101-beta2.app"
+	"/Applications/Xcode101-beta3.app"
+	"/Applications/Xcode101.app"
+	"/Applications/Xcode102-beta1.app"
+	"/Applications/Xcode102.app"
+	"/Applications/Xcode1021.app"
+	"/Applications/Xcode103.app"
+	"/Applications/Xcode10GM.app"
+	"/Applications/Xcode11-beta3.app"
+	"/Applications/Xcode11-GM.app"
+	"/Applications/Xcode11.app"
+	"/Applications/Xcode111.app"
+	"/Applications/Xcode112.app"
+	"/Applications/Xcode1121.app"
+	"/Applications/Xcode113.app"
+	"/Applications/Xcode1131.app"
+	"/Applications/Xcode114-beta1.app"
+	"/Applications/Xcode114-beta2.app"
+	"/Applications/Xcode114-beta3.app"
+	"/Applications/Xcode114.app"
+	"/Applications/Xcode1141.app"
+	"/Applications/Xcode115-beta1.app"
+	"/Applications/Xcode115-beta2.app"
+	"/Applications/Xcode115-GM.app"
+	"/Applications/Xcode_8.0.app"
+	"/Applications/Xcode_8.1.app"
+	"/Applications/Xcode_8.2.1.app"
+	"/Applications/Xcode_8.3.3.app"
+	"/Applications/Xcode_9.0.app"
+	"/Applications/Xcode_9.1.0.app"
+	"/Applications/Xcode_9.2.0.app"
+	"/Applications/Xcode_9.2.app"
+	"/Applications/Xcode_9.4.1.app"
+# Xcode 10.2.1 is currently used by Binding Tools for Swift # /Applications/Xcode_10.2.1.app
+	"/Applications/Xcode_11.3.0.app"
+	"/Applications/Xcode_11.5.0.app"
+	"/Applications/Xcode_11.6.0-beta1.app"
+	"/Applications/Xcode_12.0.0-beta1.app"
+	"/Applications/Xcode_12.0.0-beta2.app"
+	"/Applications/Xcode_12.0.0-beta3.app"
+	"/Applications/Xcode_12.0.0-beta4.app"
+	"/Applications/Xcode_12.0.0-beta5.app"
+	"/Applications/Xcode_12.0.0-beta6.app"
+	"/Applications/Xcode_12.1.0-GM.app"
+	"/Applications/Xcode_12.0.0-GMb.app"
+	"/Applications/Xcode_12.2.0-beta1.app"
+	"/Applications/Xcode_12.2.0-beta2.app"
+	"/Applications/Xcode_12.2.0-beta3.app"
+	"/Applications/Xcode_12.2.0-beta.3.app"
+	"/Applications/Xcode_12.2.0-rc.app"
+	"/Applications/Xcode_12.5.0-rc.app"
+	"/Applications/Xcode_13.0.0-beta.app"
+	"/Applications/Xcode_13.0.0-beta2.app"
+	"/Applications/Xcode_13.0.0-beta3.app"
+	"/Applications/Xcode_14.3.1.app"
+	"/Applications/Xcode_15.0.0.app"
+	"/Applications/Xcode_15.0.1.app"
+	"/Applications/Xcode_15.1.0.app"
+)
+
+# remove wrongly added .xip files under /Applications, confuses provisionator and 
+# are not needed and wrong
+sudo rm -Rf /Applications/*.xip
+
+# pick the current selected xcode to make sure we do not remove it.
+XCODE_SELECT=$(xcode-select -p)
+
+for oldXcode in "${oldXcodes[@]}"; do
+	if [ "$XCODE_SELECT" != "$oldXcode/Contents/Developer" ]; then
+		sudo rm -Rf "$oldXcode"
+	else
+		echo "Not removing $oldXcode because is the currently selected one."
+	fi
+done
+
+DIR="$(dirname "${BASH_SOURCE[0]}")"
+"$DIR"/clean-simulator-runtime.sh
+
+# Print disk status after cleaning
+df -h

eng/scripts/clean-simulator-runtime.sh

@@ -0,0 +1,37 @@
+#!/bin/bash -eux
+
+# find if there are any duplicated simulator runtimes for a given platform
+
+set -o pipefail
+IFS=$'\n\t'
+
+xcrun simctl runtime list -j > simruntime.json
+cat simruntime.json
+
+grep -e '"identifier" : ' -e '"runtimeIdentifier" : ' simruntime.json | tr '\n' ' ' | sed -e 's/,//g' -e 's/"//g' -e 's/runtimeIdentifier : //g' -e $'s/identifier : /@/g' | tr '@' '\n' | awk NF | sed 's/^[[:blank:]]*//' > simruntime-lines.txt
+cat simruntime-lines.txt
+
+sed -e 's/.*com.apple/com.apple/g' simruntime-lines.txt > simruntime-runtimes.txt
+cat simruntime-runtimes.txt
+
+sort simruntime-runtimes.txt | uniq -c | sort -n | sed 's/^[[:blank:]]*//' > simruntime-runtimes-by-count.txt
+cat simruntime-runtimes-by-count.txt
+
+grep -v '^1 ' simruntime-runtimes-by-count.txt | sed 's/^[0-9 ]*//' > simruntime-duplicated-runtimes.txt
+cat simruntime-duplicated-runtimes.txt
+
+while IFS= read -r simruntime
+do
+  echo "Duplicated: $simruntime"
+  grep "$simruntime" simruntime-lines.txt | sed 's/ .*//' | while IFS= read -r id
+  do
+    echo "    sudo xcrun simctl runtime delete $id"
+    if ! sudo xcrun simctl runtime delete "$id"; then
+      echo "    failed to delete runtime $id"
+    else
+      echo "    deleted runtime $id"
+    fi
+  done
+done < simruntime-duplicated-runtimes.txt
+
+xcrun simctl runtime list -v