[devops] Unify and simplify provisionator provisioning. (#21268)
This should speed up getting bots ready, since now we only run the provisionator once for each bot, and we also don't try to provision the same thing multiple times.
This commit is contained in:
Rolf Bjarne Kvinge
GitHub
Родитель
3790cbda5a
Коммит
fc9d73b761
|
|
@ -34,9 +34,7 @@ tests/bcl-test/SystemCoreXunit*.csproj
|
|||
tests/bcl-test/SystemXunit.csproj
|
||||
global.json
|
||||
.idea
|
||||
device-tests-provisioning.csx
|
||||
build-provisioning.csx
|
||||
provision-xcode.csx
|
||||
provision-shared.csx
|
||||
mono_crash.*.json
|
||||
*.binlog
|
||||
.vscode
|
||||
|
|
|
|||
|
|
@ -1,44 +1,13 @@
|
|||
TOP=../..
|
||||
include $(TOP)/Make.config
|
||||
|
||||
device-tests-provisioning.csx: device-tests-provisioning.csx.in Makefile $(TOP)/Make.config
|
||||
$(Q_GEN) sed \
|
||||
-e 's#@XCODE_XIP_NAME@#$(notdir $(XCODE_URL))#g' \
|
||||
-e 's#@MONO_PACKAGE@#$(MIN_MONO_URL)#g' \
|
||||
-e 's#@MIN_SHARPIE_URL@#$(MIN_SHARPIE_URL)#g' \
|
||||
-e 's#@INCLUDE_MAC@#$(INCLUDE_MAC)#g' \
|
||||
-e 's#@INCLUDE_IOS@#$(INCLUDE_IOS)#g' \
|
||||
-e 's#@INCLUDE_TVOS@#$(INCLUDE_TVOS)#g' \
|
||||
-e 's#@INCLUDE_WATCH@#$(INCLUDE_WATCH)#g' \
|
||||
$< > $@;
|
||||
@echo "Generated $@:"
|
||||
@cat $@
|
||||
|
||||
build-provisioning.csx: build-provisioning.csx.in Makefile $(TOP)/Make.config
|
||||
$(Q_GEN) sed \
|
||||
-e 's#@XCODE_XIP_NAME@#$(notdir $(XCODE_URL))#g' \
|
||||
-e 's#@MONO_PACKAGE@#$(MIN_MONO_URL)#g' \
|
||||
-e 's#@MIN_SHARPIE_URL@#$(MIN_SHARPIE_URL)#g' \
|
||||
$< > $@;
|
||||
@echo "Generated $@:"
|
||||
@cat $@
|
||||
|
||||
mac-tests-provisioning.csx: mac-tests-provisioning.csx.in Makefile $(TOP)/Make.config
|
||||
$(Q_GEN) sed \
|
||||
-e 's#@MONO_PACKAGE@#$(MIN_MONO_URL)#g' \
|
||||
-e 's#@INCLUDE_MAC@#$(INCLUDE_MAC)#g' \
|
||||
-e 's#@INCLUDE_IOS@#$(INCLUDE_IOS)#g' \
|
||||
$< > $@;
|
||||
@echo "Generated $@:"
|
||||
@cat $@
|
||||
|
||||
provision-xcode.csx: provision-xcode.csx.in Makefile $(TOP)/Make.config
|
||||
provision-shared.csx: provision-shared.in.csx Makefile $(TOP)/Make.config
|
||||
$(Q_GEN) sed \
|
||||
-e 's#@XCODE_XIP_NAME@#$(notdir $(XCODE_URL))#g' \
|
||||
-e 's#@XCODE_ROOT_PATH@#$(XCODE_DEVELOPER_ROOT)#g' \
|
||||
$< > $@;
|
||||
@echo "Generated $@:"
|
||||
@cat $@
|
||||
-e 's#@MONO_PACKAGE@#$(MIN_MONO_URL)#g' \
|
||||
-e 's#@MIN_SHARPIE_URL@#$(MIN_SHARPIE_URL)#g' \
|
||||
$< > $@
|
||||
|
||||
LocProject.json: LocProject.json.in Makefile $(TOP)/Make.config
|
||||
$(Q_GEN) sed \
|
||||
|
|
@ -65,4 +34,4 @@ print-variable:
|
|||
print-variable-value-to-file:
|
||||
@echo $($(VARIABLE)) > "$(FILE)"
|
||||
|
||||
provisioning: build-provisioning.csx device-tests-provisioning.csx provision-xcode.csx
|
||||
provisioning: provision-shared.csx
|
||||
|
|
|
|||
|
|
@ -64,18 +64,8 @@ steps:
|
|||
MacDeveloper: $(mac-developer)
|
||||
HostedMacKeychainPassword: ${{ parameters.keyringPass }}
|
||||
|
||||
- task: xamops.azdevex.provisionator-task.provisionator@2
|
||||
displayName: 'Provision Brew components'
|
||||
inputs:
|
||||
provisioning_script: $(Build.SourcesDirectory)/xamarin-macios/tools/devops/provision-brew-packages.csx
|
||||
provisioning_extra_args: '-vvvv'
|
||||
github_token: ${{ parameters.gitHubToken }}
|
||||
timeoutInMinutes: 30
|
||||
enabled: true
|
||||
continueOnError: true # brew installation can be temperamental, and things usually work even if the installation fail.
|
||||
|
||||
- bash: |
|
||||
make -C $(Build.SourcesDirectory)/xamarin-macios/tools/devops build-provisioning.csx
|
||||
make -C $(Build.SourcesDirectory)/xamarin-macios/tools/devops provisioning
|
||||
displayName: 'Generate provisionator files.'
|
||||
|
||||
- task: xamops.azdevex.provisionator-task.provisionator@2
|
||||
|
|
|
|||
|
|
@ -220,7 +220,7 @@ steps:
|
|||
timeoutInMinutes: 5
|
||||
|
||||
- bash: |
|
||||
make -C $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/ device-tests-provisioning.csx
|
||||
make -C $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/ provisioning
|
||||
displayName: 'Generate Provisionator csx file'
|
||||
|
||||
# Executed ONLY if we want to clear the provisionator cache.
|
||||
|
|
@ -234,19 +234,7 @@ steps:
|
|||
- task: xamops.azdevex.provisionator-task.provisionator@2
|
||||
displayName: 'Provision dependencies'
|
||||
inputs:
|
||||
provisioning_script: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/device-tests-provisioning.csx
|
||||
provisioning_extra_args: '-vvvv'
|
||||
github_token: ${{ parameters.gitHubToken }}
|
||||
timeoutInMinutes: 250
|
||||
|
||||
- bash: |
|
||||
make -C $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/ mac-tests-provisioning.csx
|
||||
displayName: 'Generate Provisionator csx file for macOS'
|
||||
|
||||
- task: xamops.azdevex.provisionator-task.provisionator@2
|
||||
displayName: 'Provision Xamarin.Mac'
|
||||
inputs:
|
||||
provisioning_script: $(Build.SourcesDirectory)/xamarin-macios/tools/devops/mac-tests-provisioning.csx
|
||||
provisioning_script: $(System.DefaultWorkingDirectory)/xamarin-macios/tools/devops/build-provisioning.csx
|
||||
provisioning_extra_args: '-vvvv'
|
||||
github_token: ${{ parameters.gitHubToken }}
|
||||
timeoutInMinutes: 250
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ steps:
|
|||
$vsts.Agents.SetEnabled($pool, $agent, $False)
|
||||
displayName: "Disable macOS bot from pool"
|
||||
|
||||
- bash: make -C $(Build.SourcesDirectory)/xamarin-macios/tools/devops provision-xcode.csx
|
||||
- bash: make -C $(Build.SourcesDirectory)/xamarin-macios/tools/devops provisioning
|
||||
displayName: 'Generate Xcode provisioning csx file'
|
||||
|
||||
- task: xamops.azdevex.provisionator-task.provisionator@2
|
||||
|
|
|
|||
|
|
@ -0,0 +1,8 @@
|
|||
#load "provision-shared.csx"
|
||||
|
||||
if (!ProvisionXcode ())
|
||||
return 1;
|
||||
ProvisionMono ();
|
||||
ProvisionSharpie ();
|
||||
ProvisionBrewPackages ();
|
||||
SetDefaultXcodeInVSMac ();
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
#r "_provisionator/provisionator.dll"
|
||||
|
||||
using System.IO;
|
||||
using System.Reflection;
|
||||
using System.Linq;
|
||||
|
||||
using static Xamarin.Provisioning.ProvisioningScript;
|
||||
|
||||
// Provision Xcode using the xip name declared in Make.config
|
||||
Xcode ("@XCODE_XIP_NAME@").XcodeSelect (allowUntrusted: true);
|
||||
|
||||
// provisionator knows how to deal with this items
|
||||
Item ("@MONO_PACKAGE@");
|
||||
Item ("@MIN_SHARPIE_URL@");
|
||||
|
||||
var appleSdkOverride = Path.Combine (Environment.GetFolderPath (Environment.SpecialFolder.Personal), "Library", "Preferences", "Xamarin", "Settings.plist");
|
||||
Item ("Override Apple SDK Settings")
|
||||
.Action (item => {
|
||||
DeleteSafe (appleSdkOverride);
|
||||
CreateSetting (appleSdkOverride, "AppleSdkRoot", GetSelectedXcodePath ());
|
||||
Console.WriteLine ($"New iOS SDK Location: {GetSettingValue (appleSdkOverride, "AppleSdkRoot")}");
|
||||
});
|
||||
|
||||
void DeleteSafe (string file)
|
||||
{
|
||||
if (File.Exists (file))
|
||||
File.Delete (file);
|
||||
}
|
||||
|
||||
void CreateSetting (string settingFile, string key, string value)
|
||||
{
|
||||
Exec ("defaults", "write", settingFile, key, value);
|
||||
}
|
||||
|
||||
string GetSettingValue (string settingFile, string keyName)
|
||||
{
|
||||
return Exec ("defaults", "read", settingFile, keyName).FirstOrDefault ();
|
||||
}
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
#r "_provisionator/provisionator.dll"
|
||||
|
||||
using System.IO;
|
||||
using System.Reflection;
|
||||
using System.Linq;
|
||||
|
||||
using static Xamarin.Provisioning.ProvisioningScript;
|
||||
|
||||
// Provision Xcode using the xip name declared in Make.config
|
||||
Xcode ("@XCODE_XIP_NAME@").XcodeSelect (allowUntrusted: true);
|
||||
|
||||
Item ("@MONO_PACKAGE@");
|
||||
Item ("@MIN_SHARPIE_URL@");
|
||||
|
||||
BrewPackages ("p7zip");
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
#r "_provisionator/provisionator.dll"
|
||||
|
||||
using System.IO;
|
||||
using System.Reflection;
|
||||
using System.Linq;
|
||||
|
||||
using static Xamarin.Provisioning.ProvisioningScript;
|
||||
|
||||
BrewPackages ("p7zip");
|
||||
|
||||
var appleSdkOverride = Path.Combine (Environment.GetFolderPath (Environment.SpecialFolder.UserProfile), "Library", "Preferences", "Xamarin", "Settings.plist");
|
||||
Item ("Override Apple SDK Settings")
|
||||
.Condition (item => !File.Exists (appleSdkOverride) || GetSettingValue (appleSdkOverride, "AppleSdkRoot") != GetSelectedXcodePath ())
|
||||
.Action (item => {
|
||||
DeleteSafe (appleSdkOverride);
|
||||
CreateSetting (appleSdkOverride, "AppleSdkRoot", GetSelectedXcodePath ());
|
||||
Console.WriteLine ($"New VSMac iOS SDK Location: {GetSelectedXcodePath ()}");
|
||||
});
|
||||
|
||||
void DeleteSafe (string file)
|
||||
{
|
||||
if (File.Exists (file))
|
||||
File.Delete (file);
|
||||
}
|
||||
|
||||
void CreateSetting (string settingFile, string key, string value)
|
||||
{
|
||||
Exec ("defaults", "write", settingFile, key, value);
|
||||
}
|
||||
|
||||
string GetSettingValue (string settingFile, string keyName)
|
||||
{
|
||||
return Exec ("defaults", "read", settingFile, keyName).FirstOrDefault ();
|
||||
}
|
||||
|
|
@ -1,9 +1,3 @@
|
|||
BrewPackages (
|
||||
"shellcheck",
|
||||
"yamllint",
|
||||
"cmake",
|
||||
"p7zip",
|
||||
"msitools",
|
||||
"wget",
|
||||
"azure-cli"
|
||||
);
|
||||
#load "provision-shared.csx"
|
||||
|
||||
ProvisionBrewPackages ();
|
||||
|
|
|
|||
|
|
@ -1,103 +0,0 @@
|
|||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Net;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using Newtonsoft.Json.Linq;
|
||||
|
||||
using Xamarin.Provisioning;
|
||||
using Xamarin.Provisioning.Model;
|
||||
using Xamarin.Provisioning.IO;
|
||||
|
||||
var commit = Environment.GetEnvironmentVariable ("BUILD_SOURCEVERSION");
|
||||
var provision_from_commit = Environment.GetEnvironmentVariable ("PROVISION_FROM_COMMIT") ?? commit;
|
||||
|
||||
// Looks for a variable either in the environment, or in current repo's Make.config.
|
||||
// Returns null if the variable couldn't be found.
|
||||
IEnumerable<string> make_config = null;
|
||||
string FindConfigurationVariable (string variable, string hash = "HEAD")
|
||||
{
|
||||
var value = Environment.GetEnvironmentVariable (variable);
|
||||
if (!string.IsNullOrEmpty (value))
|
||||
return value;
|
||||
|
||||
if (make_config == null) {
|
||||
try {
|
||||
make_config = Exec ("git", "show", $"{hash}:Make.config");
|
||||
} catch {
|
||||
Console.WriteLine ("Could not find a Make.config");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
foreach (var line in make_config) {
|
||||
if (line.StartsWith (variable + "=", StringComparison.Ordinal))
|
||||
return line.Substring (variable.Length + 1);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
string FindVariable (string variable)
|
||||
{
|
||||
var value = FindConfigurationVariable (variable, provision_from_commit);
|
||||
if (!string.IsNullOrEmpty (value))
|
||||
return value;
|
||||
|
||||
throw new Exception ($"Could not find {variable} in environment nor in the commit's ({commit}) manifest.");
|
||||
}
|
||||
|
||||
void ExecVerbose (string filename, params string[] args)
|
||||
{
|
||||
Console.WriteLine ($"{filename} {string.Join (" ", args)}");
|
||||
Exec (filename, args);
|
||||
}
|
||||
|
||||
bool IsAtLeastVersion(string actualVer, string minVer)
|
||||
{
|
||||
if (actualVer.Equals(minVer, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
var actualVerChars = actualVer.ToCharArray();
|
||||
var minVerChars = minVer.ToCharArray();
|
||||
|
||||
var length = Math.Min (minVerChars.Length, actualVerChars.Length);
|
||||
|
||||
var i = 0;
|
||||
while (i < length)
|
||||
{
|
||||
if (actualVerChars[i] > minVerChars[i])
|
||||
{
|
||||
return true;
|
||||
}
|
||||
else if (minVerChars[i] > actualVerChars[i])
|
||||
{
|
||||
return false;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
|
||||
if (actualVerChars.Length == minVerChars.Length)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return actualVerChars.Length > minVerChars.Length;
|
||||
}
|
||||
|
||||
void RemoveXcodeSymlinks (string xcodePath)
|
||||
{
|
||||
Console.WriteLine ($"Checkig if '{xcodePath}' is a symlink...");
|
||||
var resolvedPath = Symlink.Resolve (xcodePath);
|
||||
Console.WriteLine ($"Path resolved: '{resolvedPath}'");
|
||||
if (resolvedPath is string) {
|
||||
Console.WriteLine ($"Removing '{xcodePath}' symlink.");
|
||||
Symlink.Delete (xcodePath);
|
||||
Console.WriteLine ($"Renaming '{resolvedPath}' into '{xcodePath}'");
|
||||
ElevatedExec ("/bin/mv", resolvedPath, xcodePath);
|
||||
} else
|
||||
Console.WriteLine ($"'{xcodePath}' is not a symlink.");
|
||||
}
|
||||
|
|
@ -0,0 +1,118 @@
|
|||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Net;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using Newtonsoft.Json.Linq;
|
||||
|
||||
using Xamarin.Provisioning;
|
||||
using Xamarin.Provisioning.Model;
|
||||
using Xamarin.Provisioning.IO;
|
||||
|
||||
var commit = Environment.GetEnvironmentVariable ("BUILD_SOURCEVERSION");
|
||||
var provision_from_commit = Environment.GetEnvironmentVariable ("PROVISION_FROM_COMMIT") ?? commit;
|
||||
|
||||
void RemoveXcodeSymlinks (string xcodePath)
|
||||
{
|
||||
Console.WriteLine ($"Checking if '{xcodePath}' is a symlink...");
|
||||
var resolvedPath = Symlink.Resolve (xcodePath);
|
||||
Console.WriteLine ($"Path resolved: '{resolvedPath}'");
|
||||
if (resolvedPath is string) {
|
||||
Console.WriteLine ($"Removing '{xcodePath}' symlink.");
|
||||
Symlink.Delete (xcodePath);
|
||||
Console.WriteLine ($"Renaming '{resolvedPath}' into '{xcodePath}'");
|
||||
ElevatedExec ("/bin/mv", resolvedPath, xcodePath);
|
||||
} else
|
||||
Console.WriteLine ($"'{xcodePath}' is not a symlink.");
|
||||
}
|
||||
|
||||
void ListXcodes ()
|
||||
{
|
||||
Console.WriteLine ($"Xcodes:");
|
||||
var lines = Exec ("bash", "-c", "ls -lad /Applications/Xcode*");
|
||||
foreach (var line in lines)
|
||||
Console.WriteLine ($"\t{line}");
|
||||
}
|
||||
|
||||
// Provision Xcode using the xip name declared in Make.config
|
||||
//
|
||||
// Overrides:
|
||||
// * The current commit can be overridden by setting the PROVISION_FROM_COMMIT variable.
|
||||
bool ProvisionXcode ()
|
||||
{
|
||||
|
||||
if (string.IsNullOrEmpty (provision_from_commit)) {
|
||||
Console.Error.WriteLine ($"Either BUILD_SOURCEVERSION or PROVISION_FROM_COMMIT must be set.");
|
||||
Environment.Exit (1);
|
||||
return false;
|
||||
}
|
||||
|
||||
ListXcodes ();
|
||||
|
||||
// Provision Xcode
|
||||
Console.WriteLine ($"Provisioning Xcode from {provision_from_commit}...");
|
||||
|
||||
// Let's turn symlink into an actual path
|
||||
var reqXcode = Path.GetDirectoryName (Path.GetDirectoryName ("@XCODE_ROOT_PATH@"));
|
||||
RemoveXcodeSymlinks (reqXcode);
|
||||
|
||||
// Provision Xcode using the xip name declared in Make.config
|
||||
Xcode ("@XCODE_XIP_NAME@").XcodeSelect (allowUntrusted: true);
|
||||
|
||||
LogInstalledXcodes ();
|
||||
return true;
|
||||
}
|
||||
|
||||
void ProvisionBrewPackages ()
|
||||
{
|
||||
BrewPackages (
|
||||
"shellcheck",
|
||||
"yamllint",
|
||||
"cmake",
|
||||
"p7zip",
|
||||
"msitools",
|
||||
"wget",
|
||||
"azure-cli"
|
||||
);
|
||||
}
|
||||
|
||||
void SetDefaultXcodeInVSMac ()
|
||||
{
|
||||
var appleSdkOverride = Path.Combine (Environment.GetFolderPath (Environment.SpecialFolder.UserProfile), "Library", "Preferences", "Xamarin", "Settings.plist");
|
||||
Item ("Override Apple SDK Settings")
|
||||
.Condition (item => !File.Exists (appleSdkOverride) || GetSettingValue (appleSdkOverride, "AppleSdkRoot") != GetSelectedXcodePath ())
|
||||
.Action (item => {
|
||||
DeleteSafe (appleSdkOverride);
|
||||
CreateSetting (appleSdkOverride, "AppleSdkRoot", GetSelectedXcodePath ());
|
||||
Console.WriteLine ($"New VSMac iOS SDK Location: {GetSelectedXcodePath ()}");
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
void DeleteSafe (string file)
|
||||
{
|
||||
if (File.Exists (file))
|
||||
File.Delete (file);
|
||||
}
|
||||
|
||||
void CreateSetting (string settingFile, string key, string value)
|
||||
{
|
||||
Exec ("defaults", "write", settingFile, key, value);
|
||||
}
|
||||
|
||||
string GetSettingValue (string settingFile, string keyName)
|
||||
{
|
||||
return Exec ("defaults", "read", settingFile, keyName).FirstOrDefault ();
|
||||
}
|
||||
|
||||
void ProvisionMono ()
|
||||
{
|
||||
Item ("@MONO_PACKAGE@");
|
||||
}
|
||||
|
||||
void ProvisionSharpie ()
|
||||
{
|
||||
Item ("@MIN_SHARPIE_URL@");
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
#load "provision-shared.csx"
|
||||
|
||||
if (ProvisionXcode ())
|
||||
return 1;
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
#load "provision-shared.csx"
|
||||
|
||||
// Provision Xcode
|
||||
//
|
||||
// Overrides:
|
||||
// * The current commit can be overridden by setting the PROVISION_FROM_COMMIT variable.
|
||||
|
||||
void ListXcodes ()
|
||||
{
|
||||
Console.WriteLine ($"Xcodes:");
|
||||
var lines = Exec ("bash", "-c", "ls -lad /Applications/Xcode*");
|
||||
foreach (var line in lines)
|
||||
Console.WriteLine ($"\t{line}");
|
||||
}
|
||||
|
||||
if (string.IsNullOrEmpty (provision_from_commit)) {
|
||||
Console.Error.WriteLine ($"Either BUILD_SOURCEVERSION or PROVISION_FROM_COMMIT must be set.");
|
||||
Environment.Exit (1);
|
||||
return 1;
|
||||
}
|
||||
|
||||
ListXcodes ();
|
||||
|
||||
// Provision Xcode
|
||||
Console.WriteLine ($"Provisioning Xcode from {provision_from_commit}...");
|
||||
|
||||
// Let's turn symlink into an actual path
|
||||
var reqXcode = Path.GetDirectoryName (Path.GetDirectoryName ("@XCODE_ROOT_PATH@"));
|
||||
RemoveXcodeSymlinks (reqXcode);
|
||||
|
||||
// Provision Xcode using the xip name declared in Make.config
|
||||
Xcode ("@XCODE_XIP_NAME@").XcodeSelect (allowUntrusted: true);
|
||||
|
||||
LogInstalledXcodes ();
|
||||
Загрузка…
Ссылка в новой задаче