diff --git a/.github/workflows/autoformat2.yml b/.github/workflows/autoformat2.yml index 23961cd000..dcfee86620 100644 --- a/.github/workflows/autoformat2.yml +++ b/.github/workflows/autoformat2.yml @@ -20,7 +20,7 @@ jobs: uses: actions/github-script@v6.3.1 with: script: | - var artifacts = await github.actions.listWorkflowRunArtifacts({ + var artifacts = await github.rest.actions.listWorkflowRunArtifacts({ owner: context.repo.owner, repo: context.repo.repo, run_id: ${{github.event.workflow_run.id }}, @@ -28,7 +28,7 @@ jobs: var matchArtifact = artifacts.data.artifacts.filter((artifact) => { return artifact.name == "autoformat" })[0]; - var download = await github.actions.downloadArtifact({ + var download = await github.rest.actions.downloadArtifact({ owner: context.repo.owner, repo: context.repo.repo, artifact_id: matchArtifact.id, diff --git a/.github/workflows/update-single-platform-branches.yml b/.github/workflows/update-single-platform-branches.yml new file mode 100644 index 0000000000..fe34a47309 --- /dev/null +++ b/.github/workflows/update-single-platform-branches.yml @@ -0,0 +1,28 @@ +name: Update single-platform release tests branches + +on: + # allow triggering this action manually + workflow_dispatch: + # run every saturday (at noon UTC), so the builds occur during the weekend during lower CI load + schedule: + - cron: '0 12 * * 6' + +jobs: + updateSinglePlatformBranches: + name: Merge main into single-platform release test branches + runs-on: ubuntu-latest + + steps: + - name: Checkout repo + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: 'Update branches' + run: | + set -ex + for platform in iOS tvOS MacCatalyst macOS; do + git checkout -b release/release-test-only-dotnet-$platform origin/release/release-test-only-dotnet-$platform + git merge origin/main + git push + done diff --git a/Makefile b/Makefile index 3cc118e887..44ef689a53 100644 --- a/Makefile +++ b/Makefile @@ -110,9 +110,11 @@ package release: $(Q) $(MAKE) -C $(TOP)/release release # copy .pkg, .zip and *updateinfo to the packages directory to be uploaded to storage $(Q) mkdir -p ../package - $(Q) $(CP) $(TOP)/release/*.pkg ../package - $(Q) $(CP) $(TOP)/release/*.zip ../package - $(Q) $(CP) $(TOP)/release/*updateinfo ../package + $(Q) echo "Output from 'make release':" + $(Q) ls -la $(TOP)/release | sed 's/^/ /' + $(Q) if test -n "$$(shopt -s nullglob; echo $(TOP)/release/*.pkg)"; then $(CP) $(TOP)/release/*.pkg ../package; fi + $(Q) if test -n "$$(shopt -s nullglob; echo $(TOP)/release/*.zip)"; then $(CP) $(TOP)/release/*.zip ../package; fi + $(Q) if test -n "$$(shopt -s nullglob; echo $(TOP)/release/*updateinfo)"; then $(CP) $(TOP)/release/*updateinfo ../package; fi $(Q) echo "Packages:" $(Q) ls -la ../package | sed 's/^/ /' diff --git a/NuGet.config b/NuGet.config index 414340f2ca..a568114259 100644 --- a/NuGet.config +++ b/NuGet.config @@ -10,6 +10,12 @@ + + + + + + diff --git a/msbuild/Xamarin.Localization.MSBuild/MSBStrings.Designer.cs b/msbuild/Xamarin.Localization.MSBuild/MSBStrings.Designer.cs index e3f50499f1..7872d2fa94 100644 --- a/msbuild/Xamarin.Localization.MSBuild/MSBStrings.Designer.cs +++ b/msbuild/Xamarin.Localization.MSBuild/MSBStrings.Designer.cs @@ -2726,5 +2726,14 @@ namespace Xamarin.Localization.MSBuild { return ResourceManager.GetString("W7100", resourceCulture); } } + + /// + /// Looks up a localized string similar to Unexpected extension '{0}' for native reference '{1}' in manifest '{2}'.. + /// + public static string W7105 { + get { + return ResourceManager.GetString("W7105", resourceCulture); + } + } } } diff --git a/msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx b/msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx index 2b1f5c18ec..de2d760d62 100644 --- a/msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx +++ b/msbuild/Xamarin.Localization.MSBuild/MSBStrings.resx @@ -1450,4 +1450,13 @@ * 'Remove', 'Boolean', 'String', 'StringArray' + + + Unexpected extension '{0}' for native reference '{1}' in manifest '{2}'. + + {0}: file extension + {1}: path to a file + {2}: path to a file + + diff --git a/msbuild/Xamarin.MacDev.Tasks/Tasks/ResolveNativeReferencesBase.cs b/msbuild/Xamarin.MacDev.Tasks/Tasks/ResolveNativeReferencesBase.cs index 7a210e8003..221cc7c7ef 100644 --- a/msbuild/Xamarin.MacDev.Tasks/Tasks/ResolveNativeReferencesBase.cs +++ b/msbuild/Xamarin.MacDev.Tasks/Tasks/ResolveNativeReferencesBase.cs @@ -167,7 +167,12 @@ namespace Xamarin.MacDev.Tasks { t = new TaskItem (Path.Combine (resources, name)); t.SetMetadata ("Kind", "Dynamic"); break; + case ".a": // static library + t = new TaskItem (Path.Combine (resources, name)); + t.SetMetadata ("Kind", "Static"); + break; default: + Log.LogWarning (MSBStrings.W7105 /* Unexpected extension '{0}' for native reference '{1}' in manifest '{2}'. */, Path.GetExtension (name), name, manifest); t = r; break; } diff --git a/src/Foundation/NSArray.cs b/src/Foundation/NSArray.cs index d9b071172b..90108eecd0 100644 --- a/src/Foundation/NSArray.cs +++ b/src/Foundation/NSArray.cs @@ -37,7 +37,7 @@ using NativeHandle = System.IntPtr; namespace Foundation { - public partial class NSArray { + public partial class NSArray : IEnumerable { // // Creates an array with the elements; If the value passed is null, it @@ -429,5 +429,28 @@ namespace Foundation { return null; } } + + public TKey[] ToArray () where TKey: class, INativeObject + { + var rv = new TKey [GetCount (Handle)]; + for (var i = 0; i < rv.Length; i++) + rv [i] = GetItem ((nuint) i); + return rv; + } + + public NSObject[] ToArray () + { + return ToArray (); + } + + IEnumerator IEnumerable.GetEnumerator () + { + return new NSFastEnumerator (this); + } + + IEnumerator IEnumerable.GetEnumerator () + { + return new NSFastEnumerator (this); + } } } diff --git a/src/Foundation/NSArray_1.cs b/src/Foundation/NSArray_1.cs index 5c42361c8e..eb44a1a5cc 100644 --- a/src/Foundation/NSArray_1.cs +++ b/src/Foundation/NSArray_1.cs @@ -89,5 +89,10 @@ namespace Foundation { return GetItem ((nuint)idx); } } + + public new TKey[] ToArray () + { + return base.ToArray (); + } } } diff --git a/tests/monotouch-test/Foundation/ArrayTest.cs b/tests/monotouch-test/Foundation/ArrayTest.cs index 6836eb5403..0f07e80b23 100644 --- a/tests/monotouch-test/Foundation/ArrayTest.cs +++ b/tests/monotouch-test/Foundation/ArrayTest.cs @@ -8,6 +8,7 @@ // using System; +using System.Linq; using Foundation; using ObjCRuntime; using Security; @@ -123,5 +124,37 @@ namespace MonoTouchFixtures.Foundation { Assert.That (a.Handle, Is.Not.EqualTo (IntPtr.Zero), "Handle"); } } + + [Test] + public void ToArray () + { + using (var a = NSArray.FromStrings (new string [1] { "abc" })) { + var arr = a.ToArray (); + Assert.AreEqual (1, arr.Length, "Length"); + Assert.AreEqual ("abc", arr [0].ToString (), "Value"); + } + } + + [Test] + public void ToArray_T () + { + using (var a = NSArray.FromStrings (new string [1] { "abc" })) { + var arr = a.ToArray (); + Assert.AreEqual (1, arr.Length, "Length"); + Assert.AreEqual ("abc", arr [0].ToString (), "Value"); + } + } + + [Test] + public void Enumerator () + { + using (var a = NSArray.FromStrings (new string [1] { "abc" })) { + foreach (var item in a) + Assert.AreEqual ("abc", item.ToString (), "Value"); + var list = a.ToList (); + Assert.AreEqual (1, list.Count (), "Length"); + Assert.AreEqual ("abc", list [0].ToString (), "Value"); + } + } } } diff --git a/tests/monotouch-test/Foundation/NSArray1Test.cs b/tests/monotouch-test/Foundation/NSArray1Test.cs index 7f14655e49..92eafc2bf3 100644 --- a/tests/monotouch-test/Foundation/NSArray1Test.cs +++ b/tests/monotouch-test/Foundation/NSArray1Test.cs @@ -93,5 +93,29 @@ namespace MonoTouchFixtures.Foundation { Assert.AreSame (str3, arr [2], "NSArray indexer"); } } + + [Test] + public void ToArray () + { + using (var a = NSArray.FromNSObjects ((NSString) "abc")) { + var arr = a.ToArray (); + NSString element = arr [0]; + Assert.AreEqual (1, arr.Length, "Length"); + Assert.AreEqual ("abc", arr [0].ToString (), "Value"); + Assert.AreEqual ("abc", (string) element, "Value element"); + } + } + + [Test] + public void ToArray_T () + { + using (var a = NSArray.FromNSObjects ((NSString) "abc")) { + var arr = a.ToArray (); + NSString element = arr [0]; + Assert.AreEqual (1, arr.Length, "Length"); + Assert.AreEqual ("abc", arr [0].ToString (), "Value"); + Assert.AreEqual ("abc", (string) element, "Value element"); + } + } } } diff --git a/tests/xtro-sharpie/Filter.cs b/tests/xtro-sharpie/Filter.cs index 7c72631cf5..44755a2ca7 100644 --- a/tests/xtro-sharpie/Filter.cs +++ b/tests/xtro-sharpie/Filter.cs @@ -85,7 +85,7 @@ namespace Extrospection { case "TWAIN": case "Virtualization": case "vmnet": - // other non-supported frameworks + // other non-supported frameworks case "GSS": // iOS and macOS case "vecLib": // all return true; diff --git a/tests/xtro-sharpie/u2ignore/u2ignore.cs b/tests/xtro-sharpie/u2ignore/u2ignore.cs index f0939d33e7..c22dfb05b4 100644 --- a/tests/xtro-sharpie/u2ignore/u2ignore.cs +++ b/tests/xtro-sharpie/u2ignore/u2ignore.cs @@ -38,7 +38,7 @@ namespace u2ignore { } } - var header = new string [] { "", $"# Initial result from new rule { id }" }; + var header = new string [] { "", $"# Initial result from new rule {id}" }; foreach (var kvp in dict) { var framework = kvp.Key; var entries = kvp.Value; @@ -47,7 +47,7 @@ namespace u2ignore { var failure = kvp2.Key; var platforms = kvp2.Value; - string[] files; + string [] files; if (platforms.Count == 4) { // same failure in all platforms, the result goes into the common file. files = new string [] { "common" }; @@ -61,7 +61,7 @@ namespace u2ignore { File.AppendAllLines (path, new string [] { "" }); File.AppendAllLines (path, header); } - File.AppendAllLines (path, new string [] { failure } ); + File.AppendAllLines (path, new string [] { failure }); } } } diff --git a/tests/xtro-sharpie/u2todo/u2todo.cs b/tests/xtro-sharpie/u2todo/u2todo.cs index 0d682d4132..0c5e59be55 100644 --- a/tests/xtro-sharpie/u2todo/u2todo.cs +++ b/tests/xtro-sharpie/u2todo/u2todo.cs @@ -3,7 +3,7 @@ using System.IO; using Extrospection; class Program { - static void Main (string[] args) + static void Main (string [] args) { var dir = args.Length == 0 ? "." : args [0]; foreach (var file in Directory.GetFiles (dir, "*.unclassified")) { diff --git a/tests/xtro-sharpie/xtro-report/Reporter.cs b/tests/xtro-sharpie/xtro-report/Reporter.cs index bbfa26be3d..61c55c704f 100644 --- a/tests/xtro-sharpie/xtro-report/Reporter.cs +++ b/tests/xtro-sharpie/xtro-report/Reporter.cs @@ -17,7 +17,7 @@ namespace Extrospection { bool data = false; // merge the shared and specialized ignore data into a single html page List ignore = new List (); - ignore.Add ($"

{framework}

"); + ignore.Add ($"

{framework}

"); var filename = Path.Combine (InputDirectory, $"common-{framework}.ignore"); if (File.Exists (filename)) { data = true; @@ -56,7 +56,7 @@ namespace Extrospection { html.Add ($"{name}"); html.Add ($"

{name}

"); foreach (var line in File.ReadAllLines (filename)) { - html.Add (line); + html.Add (line); if ((line.Length > 0) && (line [0] == '!')) count++; } @@ -70,7 +70,7 @@ namespace Extrospection { var filename = Path.GetFileNameWithoutExtension (file); var fx = filename.Substring (filename.IndexOf ('-') + 1); if (!Frameworks.Contains (fx)) - Frameworks.Add (fx); + Frameworks.Add (fx); } public static int Main (string [] args) @@ -98,9 +98,9 @@ namespace Extrospection { log.WriteLine ("<tr>"); log.WriteLine ("<td rowspan='3' bgcolor='lightgrey'>Frameworks</td>"); if (full) - log.WriteLine ($"<td align='center' bgcolor='green' colspan='{Platforms.Length + 1}'>REVIEWED (ignored)</td>"); - log.WriteLine ($"<td align='center' bgcolor='red' colspan='{Platforms.Length}'>FIXME (unclassified)</td>"); - log.WriteLine ($"<td align='center' bgcolor='orange' colspan='{Platforms.Length}'>TODO (milestone)</td>"); + log.WriteLine ($"<td align='center' bgcolor='green' colspan='{Platforms.Length + 1}'>REVIEWED (ignored)</td>"); + log.WriteLine ($"<td align='center' bgcolor='red' colspan='{Platforms.Length}'>FIXME (unclassified)</td>"); + log.WriteLine ($"<td align='center' bgcolor='orange' colspan='{Platforms.Length}'>TODO (milestone)</td>"); log.WriteLine ("</tr>"); log.WriteLine ("<tr>"); diff --git a/tests/xtro-sharpie/xtro-sanity/Sanitizer.cs b/tests/xtro-sharpie/xtro-sanity/Sanitizer.cs index cdb3824741..49d4a810e9 100644 --- a/tests/xtro-sharpie/xtro-sanity/Sanitizer.cs +++ b/tests/xtro-sharpie/xtro-sanity/Sanitizer.cs @@ -155,7 +155,7 @@ namespace Extrospection { foreach (var file in Directory.GetFiles (directory, "*.todo")) { if (!IsIncluded (file)) continue; - if (!(File.ReadLines(file).Count() > 0)) { + if (!(File.ReadLines (file).Count () > 0)) { Log ($"?empty-todo? File '{Path.GetFileName (file)}' is empty. Empty todo files should be removed."); } } @@ -178,10 +178,10 @@ namespace Extrospection { // cache stuff foreach (var file in Directory.GetFiles (directory, "common-*.ignore")) { - var path = Path.GetFileName (file); + var path = Path.GetFileName (file); var fx = path.Substring (7, path.Length - 14); var common = new List<string> (File.ReadAllLines (file)); - commons.Add (fx, common); + commons.Add (fx, common); } // *.ignore validations @@ -226,8 +226,8 @@ namespace Extrospection { var common = kvp.Value; //ExistingCommonEntries (common, $"common-{fx}.ignore"); List<string> [] raws = new List<string> [Platforms.Count]; - for (int i=0; i < raws.Length; i++) { - var fname = Path.Combine (directory, $"{Platforms[i]}-{fx}.raw"); + for (int i = 0; i < raws.Length; i++) { + var fname = Path.Combine (directory, $"{Platforms [i]}-{fx}.raw"); if (File.Exists (fname)) raws [i] = new List<string> (File.ReadAllLines (fname)); else @@ -305,7 +305,7 @@ namespace Extrospection { var sanitizedOrSkippedSanity = !string.IsNullOrEmpty (Environment.GetEnvironmentVariable ("XTRO_SANITY_SKIP")) || !string.IsNullOrEmpty (Environment.GetEnvironmentVariable ("AUTO_SANITIZE")); - return sanitizedOrSkippedSanity ? 0 : count; + return sanitizedOrSkippedSanity ? 0 : count; } } } diff --git a/tools/autoformat.sh b/tools/autoformat.sh index 16ed4b7961..e40dc77f02 100755 --- a/tools/autoformat.sh +++ b/tools/autoformat.sh @@ -17,6 +17,11 @@ dotnet format whitespace "$SRC_DIR/msbuild/Xamarin.MacDev.Tasks/Xamarin.MacDev.T dotnet format whitespace "$SRC_DIR/msbuild/Xamarin.iOS.Tasks.Windows/Xamarin.iOS.Tasks.Windows.csproj" dotnet format whitespace "$SRC_DIR/msbuild/Xamarin.iOS.Tasks/Xamarin.iOS.Tasks.csproj" dotnet format whitespace "$SRC_DIR/tools/dotnet-linker/dotnet-linker.csproj" +dotnet format whitespace "$SRC_DIR/tests/xtro-sharpie/xtro-sharpie.csproj" +dotnet format whitespace "$SRC_DIR/tests/xtro-sharpie/u2ignore/u2ignore.csproj" +dotnet format whitespace "$SRC_DIR/tests/xtro-sharpie/u2todo/u2todo.csproj" +dotnet format whitespace "$SRC_DIR/tests/xtro-sharpie/xtro-report/xtro-report.csproj" +dotnet format whitespace "$SRC_DIR/tests/xtro-sharpie/xtro-sanity/xtro-sanity.csproj" # dotnet format "$SRC_DIR/[...]" # add more projects here... diff --git a/tools/devops/automation/templates/main-stage.yml b/tools/devops/automation/templates/main-stage.yml index 59511ee84f..9e900c13c6 100644 --- a/tools/devops/automation/templates/main-stage.yml +++ b/tools/devops/automation/templates/main-stage.yml @@ -92,6 +92,59 @@ parameters: type: stepList default: [] +- name: packages + type: object + default: [ + { + job: prepare_packages, + displayName: 'Prepare packages', + packages: [ + { + job: 'microsoft_ios_sign_notarize', + name: 'Microsoft.iOS', + pattern: 'Microsoft.iOS.Bundle*.pkg', + conditionVariable: "INCLUDE_DOTNET_IOS", + }, + { + job: 'microsoft_tvos_sign_notarize', + name: 'Microsoft.tvOS', + pattern: 'Microsoft.tvOS.Bundle*.pkg', + conditionVariable: "INCLUDE_DOTNET_TVOS", + }, + { + job: 'microsoft_mac_sign_notarize', + name: 'Microsoft.macOS', + pattern: 'Microsoft.macOS.Bundle*.pkg', + conditionVariable: "INCLUDE_DOTNET_MACOS", + }, + { + job: 'microsoft_maccatalyst_sign_notarize', + name: 'Microsoft.MacCatalyst', + pattern: 'Microsoft.MacCatalyst.Bundle*.pkg', + conditionVariable: "INCLUDE_DOTNET_MACCATALYST", + }, + ], + }, + { + job: prepare_packages_legacy, + displayName: 'Prepare legacy packages', + packages: [ + { + job: 'xamarin_ios_sign_notarize', + name: 'Xamarin.iOS', + pattern: 'xamarin.ios-*', + conditionVariable: "INCLUDE_LEGACY_IOS", + }, + { + job: 'xamarin_mac_sing_notarie', + name: 'Xamarin.Mac', + pattern: 'xamarin.mac-*', + conditionVariable: "INCLUDE_LEGACY_MAC", + }, + ], + } + ] + stages: - ${{ if eq(parameters.runGovernanceTests, true) }}: @@ -150,18 +203,56 @@ stages: skipESRP: ${{ parameters.skipESRP }} pool: ${{ parameters.pool }} -- stage: prepare_packages - displayName: 'Prepare packages' +- ${{ each pkg_obj in parameters.packages }}: + - stage: ${{ pkg_obj.job }} + displayName: ${{ pkg_obj.displayName }} + dependsOn: + - build_packages + jobs: + - template: ./sign-and-notarized/prepare-pkg-stage.yml + parameters: + isPR: ${{ parameters.isPR }} + signingSetupSteps: ${{ parameters.signingSetupSteps }} + keyringPass: $(pass--lab--mac--builder--keychain) + enableDotnet: ${{ parameters.enableDotnet }} + skipESRP: ${{ parameters.skipESRP }} + packages: ${{ pkg_obj.packages }} + +- ${{ if eq(parameters.enableDotnet, true) }}: + - stage: sign_notarize_dotnet + displayName: 'Sign & Notarize Dotnet' + dependsOn: + - build_packages + jobs: + - template: ./sign-and-notarized/dotnet-signing.yml + parameters: + isPR: ${{ parameters.isPR }} + +# .NET Release Prep and VS Insertion Stages, only execute them when the build comes from an official branch and is not a schedule build from OneLoc +# setting the stage at this level makes the graph of the UI look better, else the lines overlap and is not clear. +- ${{ if and(ne(variables['Build.Reason'], 'Schedule'), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), eq(variables['Build.SourceBranch'], 'refs/heads/net7.0'), eq(parameters.forceInsertion, true))) }}: + - template: ./release/vs-insertion-prep.yml + parameters: + dependsOn: + - sign_notarize_dotnet + isPR: ${{ parameters.isPR }} + +- stage: funnel + displayName: 'Collect signed artifacts' dependsOn: - build_packages + - ${{ if eq(parameters.enableDotnet, true) }}: + - sign_notarize_dotnet + - ${{ each pkg_obj in parameters.packages }}: + - ${{ pkg_obj.job }} jobs: - - template: ./sign-and-notarized/prepare-pkg-stage.yml + - template: ./sign-and-notarized/funnel.yml parameters: isPR: ${{ parameters.isPR }} - signingSetupSteps: ${{ parameters.signingSetupSteps }} - keyringPass: $(pass--lab--mac--builder--keychain) - enableDotnet: ${{ parameters.enableDotnet }} - skipESRP: ${{ parameters.skipESRP }} + packages: # flatten the pkgs for the parameter + - ${{ each pkg_obj in parameters.packages }}: + - ${{ each pkg in pkg_obj.packages }}: + - ${{ pkg }} - ${{ if eq(parameters.enableAPIDiff, true) }}: - stage: generate_api_diff @@ -179,12 +270,6 @@ stages: enableDotnet: ${{ parameters.enableDotnet }} pool: ${{ parameters.pool }} -# .NET Release Prep and VS Insertion Stages, only execute them when the build comes from an official branch and is not a schedule build from OneLoc -- ${{ if and(ne(variables['Build.Reason'], 'Schedule'), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), eq(variables['Build.SourceBranch'], 'refs/heads/net7.0'), eq(parameters.forceInsertion, true))) }}: - - template: ./release/vs-insertion-prep.yml - parameters: - isPR: ${{ parameters.isPR }} - # Test stages # always run simulator tests diff --git a/tools/devops/automation/templates/release/vs-insertion-prep.yml b/tools/devops/automation/templates/release/vs-insertion-prep.yml index aec1eebdab..8d9355e24b 100644 --- a/tools/devops/automation/templates/release/vs-insertion-prep.yml +++ b/tools/devops/automation/templates/release/vs-insertion-prep.yml @@ -4,8 +4,8 @@ parameters: default: true - name: dependsOn - type: string - default: prepare_packages + type: object + default: [] - name: isPR type: boolean @@ -14,13 +14,13 @@ stages: - stage: prepare_release displayName: Prepare Release dependsOn: ${{ parameters.dependsOn }} - condition: and(or(eq(dependencies.${{ parameters.dependsOn }}.result, 'Succeeded'), eq(dependencies.${{ parameters.dependsOn }}.result, 'SucceededWithIssues')), eq(${{ parameters.isPR }}, false), eq(${{ parameters.enableDotnet }}, true)) + condition: and(eq(${{ parameters.isPR }}, false), eq(${{ parameters.enableDotnet }}, true)) jobs: # Check - "xamarin-macios (Prepare Release Sign NuGets)" - template: sign-artifacts/jobs/v2.yml@templates parameters: - artifactName: package + artifactName: dotnet-signed signType: Real usePipelineArtifactTasks: true @@ -30,7 +30,7 @@ stages: yamlResourceName: templates dependsOn: signing artifactName: nuget-signed - propsArtifactName: package + propsArtifactName: dotnet-signed signType: Real useDateTimeVersion: true diff --git a/tools/devops/automation/templates/sign-and-notarized/dotnet-signing.yml b/tools/devops/automation/templates/sign-and-notarized/dotnet-signing.yml index 16bf35b89d..5ba692e31e 100644 --- a/tools/devops/automation/templates/sign-and-notarized/dotnet-signing.yml +++ b/tools/devops/automation/templates/sign-and-notarized/dotnet-signing.yml @@ -7,35 +7,93 @@ parameters: - name: isPR type: boolean -steps: +jobs: +- job: configure + displayName: 'Configure build' + pool: + vmImage: windows-latest -- template: setup.yml - parameters: - isPR: ${{ parameters.isPR }} + steps: + - template: ../common/configure.yml -- task: DownloadPipelineArtifact@2 - displayName: Download not notaraized build - inputs: - artifact: 'not-signed-package' - patterns: '!*.pkg' - allowFailedBuilds: true - path: $(Build.SourcesDirectory)/package +- job: sign_notarize_dotnet + dependsOn: + - configure + displayName: 'Sign & Notarize Dotnet' + timeoutInMinutes: 1000 + pool: + vmImage: internal-macos-11 + workspace: + clean: all -- ${{ if eq(parameters.isPR, false) }}: - - pwsh : | - # Get the list of files to sign - $msiFiles = Get-ChildItem -Path $(Build.SourcesDirectory)/package/ -Filter "*.msi" + steps: - # Add those files to an array - $SignFiles = @() - foreach($msi in $msiFiles) { - Write-Host "$($msi.FullName)" - $SignFiles += @{ "SrcPath"="$($msi.FullName)"} + - template: setup.yml + parameters: + isPR: ${{ parameters.isPR }} + + - task: DownloadPipelineArtifact@2 + displayName: Download not notaraized build + inputs: + artifact: 'not-signed-package' + patterns: '!*.pkg' + allowFailedBuilds: true + path: $(Build.SourcesDirectory)/package + + - ${{ if eq(parameters.isPR, false) }}: + - pwsh : | + # Get the list of files to sign + $msiFiles = Get-ChildItem -Path $(Build.SourcesDirectory)/package/ -Filter "*.msi" + + # Add those files to an array + $SignFiles = @() + foreach($msi in $msiFiles) { + Write-Host "$($msi.FullName)" + $SignFiles += @{ "SrcPath"="$($msi.FullName)"} + } + + Write-Host "$msiFiles" + + # array of dicts + $SignFileRecord = @( + @{ + "Certs" = "400"; + "SignFileList" = $SignFiles; + } + ) + + $SignFileList = @{ + "SignFileRecordList" = $SignFileRecord + } + + # Write the json to a file + ConvertTo-Json -InputObject $SignFileList -Depth 5 | Out-File -FilePath $(Build.ArtifactStagingDirectory)/MsiFiles2Notarize.json -Force + dotnet $Env:MBSIGN_APPFOLDER/ddsignfiles.dll /filelist:$(Build.ArtifactStagingDirectory)/MsiFiles2Notarize.json + displayName: 'Sign .msi' + condition: ${{ parameters.condition }} + + - pwsh: | + mv $(Build.SourcesDirectory)/package/bundle.zip $(Build.ArtifactStagingDirectory)/not-signed-bundle.zip + $bundlePath = "$(Build.ArtifactStagingDirectory)/bundle" + unzip $(Build.ArtifactStagingDirectory)/not-signed-bundle.zip -d $bundlePath + $patterns = @( + "*.iOS.dll", + "*.tvOS.dll", + "*.Mac.dll", "*.macOS.dll", "*XamMac.dll", + "*.MacCatalyst.dll", + "*.WatchOS.dll" + ) + $files = @() + foreach ($p in $patterns) { + $files += Get-ChildItem -Path $bundlePath -Recurse -Filter $p } - Write-Host "$msiFiles" + $SignFiles = @() + foreach($f in $files) { + Write-Host "$($f.FullName)" + $SignFiles += @{ "SrcPath"="$($f.FullName)"} + } - # array of dicts $SignFileRecord = @( @{ "Certs" = "400"; @@ -48,61 +106,22 @@ steps: } # Write the json to a file - ConvertTo-Json -InputObject $SignFileList -Depth 5 | Out-File -FilePath $(Build.ArtifactStagingDirectory)/MsiFiles2Notarize.json -Force - dotnet $Env:MBSIGN_APPFOLDER/ddsignfiles.dll /filelist:$(Build.ArtifactStagingDirectory)/MsiFiles2Notarize.json - displayName: 'Sign .msi' - condition: ${{ parameters.condition }} + ConvertTo-Json -InputObject $SignFileList -Depth 100 | Out-File -FilePath $(Build.ArtifactStagingDirectory)/bundle.json -Force + dotnet $Env:MBSIGN_APPFOLDER/ddsignfiles.dll /filelist:$(Build.ArtifactStagingDirectory)/bundle.json + # rezip and move back + ditto -c -k --sequesterRsrc $bundlePath bundle.zip + mv bundle.zip $(Build.SourcesDirectory)/package/bundle.zip + displayName: 'Sign bundle.zip' + workingDirectory: $(Build.ArtifactStagingDirectory) -- pwsh: | - mv $(Build.SourcesDirectory)/package/bundle.zip $(Build.ArtifactStagingDirectory)/not-signed-bundle.zip - $bundlePath = "$(Build.ArtifactStagingDirectory)/bundle" - unzip $(Build.ArtifactStagingDirectory)/not-signed-bundle.zip -d $bundlePath - $patterns = @( - "*.iOS.dll", - "*.tvOS.dll", - "*.Mac.dll", "*.macOS.dll", "*XamMac.dll", - "*.MacCatalyst.dll", - "*.WatchOS.dll" - ) - $files = @() - foreach ($p in $patterns) { - $files += Get-ChildItem -Path $bundlePath -Recurse -Filter $p - } + - template: publish-nugets.yml + parameters: + isPR: ${{ parameters.isPR }} - $SignFiles = @() - foreach($f in $files) { - Write-Host "$($f.FullName)" - $SignFiles += @{ "SrcPath"="$($f.FullName)"} - } - - $SignFileRecord = @( - @{ - "Certs" = "400"; - "SignFileList" = $SignFiles; - } - ) - - $SignFileList = @{ - "SignFileRecordList" = $SignFileRecord - } - - # Write the json to a file - ConvertTo-Json -InputObject $SignFileList -Depth 100 | Out-File -FilePath $(Build.ArtifactStagingDirectory)/bundle.json -Force - dotnet $Env:MBSIGN_APPFOLDER/ddsignfiles.dll /filelist:$(Build.ArtifactStagingDirectory)/bundle.json - # rezip and move back - ditto -c -k --sequesterRsrc $bundlePath bundle.zip - mv bundle.zip $(Build.SourcesDirectory)/package/bundle.zip - displayName: 'Sign bundle.zip' - workingDirectory: $(Build.ArtifactStagingDirectory) - -- template: publish-nugets.yml - parameters: - isPR: ${{ parameters.isPR }} - -# always upload no matter what, since if we are not signing we need the artifact in the pipeline -- task: PublishPipelineArtifact@1 - displayName: 'Publish Notarized Dotnet Artifacts' - inputs: - targetPath: $(Build.SourcesDirectory)/package - artifactName: dotnet-signed - continueOnError: true + # always upload no matter what, since if we are not signing we need the artifact in the pipeline + - task: PublishPipelineArtifact@1 + displayName: 'Publish Notarized Dotnet Artifacts' + inputs: + targetPath: $(Build.SourcesDirectory)/package + artifactName: dotnet-signed + continueOnError: true diff --git a/tools/devops/automation/templates/sign-and-notarized/funnel.yml b/tools/devops/automation/templates/sign-and-notarized/funnel.yml index a657aacc1d..a446606ddc 100644 --- a/tools/devops/automation/templates/sign-and-notarized/funnel.yml +++ b/tools/devops/automation/templates/sign-and-notarized/funnel.yml @@ -3,78 +3,160 @@ parameters: - name: packages type: object -steps: +- name: enableDotnet + type: boolean + default: true -# DO NOT USE THE checkout.yml template. The reason is that the template changes the hash which results in a problem with the artifacts scripts -- checkout: self # https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=azure-devops&tabs=schema#checkout - clean: true # Executes: git clean -ffdx && git reset --hard HEAD - submodules: recursive - path: s/xamarin-macios +- name: isPR + type: boolean -- checkout: maccore - clean: true - persistCredentials: true # hugely important, else there are some scripts that check a single file from maccore that will fail -- checkout: templates - clean: true +jobs: +- job: configure + displayName: 'Configure build' + pool: + vmImage: windows-latest -- checkout: release-scripts - clean: true + variables: + isMain: $[eq(variables['Build.SourceBranch'], 'refs/heads/main')] + isScheduled: $[eq(variables['Build.Reason'], 'Schedule')] -- bash: | - mkdir -p $(Build.SourcesDirectory)/package/notarized - displayName: 'Create target directories.' + steps: + - template: ../common/configure.yml -- task: DownloadPipelineArtifact@2 - displayName: Download notarized build dotnet - inputs: - artifact: 'dotnet-signed' - allowFailedBuilds: true - path: $(Build.SourcesDirectory)/package +- job: funnel_job + dependsOn: + - configure + displayName: 'Collect signed artifacts' + condition: and(not(failed()), not(canceled())) # default is succeded(), but that fails if there are any skipped jobs, so change the condition to !failed && !cancelled + timeoutInMinutes: 1000 + pool: + vmImage: internal-macos-11 + workspace: + clean: all + variables: + ${{ each pkg in parameters.packages }}: + ${{ pkg.conditionVariable }}: $[ dependencies.configure.outputs['configure_platforms.${{ pkg.conditionVariable }}'] ] -- ${{ each pkg in parameters.packages }}: - - task: DownloadPipelineArtifact@2 - displayName: Download notarized build ${{ pkg.name }} - condition: ne('', variables['${{ pkg.conditionVariable }}']) - inputs: - artifact: 'classic-${{ pkg.name }}-signed' - allowFailedBuilds: true - path: '$(Build.ArtifactStagingDirectory)/classic-${{ pkg.name }}-signed' + steps: + + # DO NOT USE THE checkout.yml template. The reason is that the template changes the hash which results in a problem with the artifacts scripts + - checkout: self # https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=azure-devops&tabs=schema#checkout + clean: true # Executes: git clean -ffdx && git reset --hard HEAD + submodules: recursive + path: s/xamarin-macios + + - checkout: maccore + clean: true + persistCredentials: true # hugely important, else there are some scripts that check a single file from maccore that will fail + + - checkout: templates + clean: true + + - checkout: release-scripts + clean: true - bash: | - set -x - set -e + mkdir -p $(Build.SourcesDirectory)/package/notarized + displayName: 'Create target directories.' - FULL_PATH="$(Build.ArtifactStagingDirectory)/classic-${{ pkg.name }}-signed" - ls -lR $FULL_PATH - cp -a "$FULL_PATH/." "$(Build.SourcesDirectory)/package" - displayName: 'Move pkg ${{ pkg.name }} to its final destination' - condition: ne('', variables['${{ pkg.conditionVariable }}']) + - task: DownloadPipelineArtifact@2 + displayName: Download notarized build dotnet + inputs: + artifact: 'dotnet-signed' + allowFailedBuilds: true + path: $(Build.SourcesDirectory)/package -- template: generate-workspace-info.yml@templates - parameters: - GitHubToken: $(GitHub.Token) - ArtifactDirectory: $(Build.SourcesDirectory)/package-internal + - ${{ each pkg in parameters.packages }}: + - task: DownloadPipelineArtifact@2 + displayName: Download notarized build ${{ pkg.name }} + condition: ne('', variables['${{ pkg.conditionVariable }}']) + inputs: + artifact: 'classic-${{ pkg.name }}-signed' + allowFailedBuilds: true + path: '$(Build.ArtifactStagingDirectory)/classic-${{ pkg.name }}-signed' -# download workload json and add it to out package internal dir, this allows the rest of jobs -# not to need several artifacts but just package-internal -- task: DownloadPipelineArtifact@2 - displayName: Download WorkloadRollback.json - inputs: - patterns: '**/WorkloadRollback.json' - allowFailedBuilds: true - path: $(Build.SourcesDirectory)/package-internal + - bash: | + set -x + set -e -- task: PublishPipelineArtifact@1 - displayName: 'Publish Build Internal Artifacts' - inputs: - targetPath: $(Build.SourcesDirectory)/package-internal - artifactName: package-internal - continueOnError: true + FULL_PATH="$(Build.ArtifactStagingDirectory)/classic-${{ pkg.name }}-signed" + ls -lR $FULL_PATH + cp -a "$FULL_PATH/." "$(Build.SourcesDirectory)/package" + displayName: 'Move pkg ${{ pkg.name }} to its final destination' + condition: ne('', variables['${{ pkg.conditionVariable }}']) -- task: PublishPipelineArtifact@1 - displayName: 'Publish Build Artifacts (notarized)' - inputs: - targetPath: $(Build.SourcesDirectory)/package - artifactName: package - continueOnError: true + - template: generate-workspace-info.yml@templates + parameters: + GitHubToken: $(GitHub.Token) + ArtifactDirectory: $(Build.SourcesDirectory)/package-internal + + # download workload json and add it to out package internal dir, this allows the rest of jobs + # not to need several artifacts but just package-internal + - task: DownloadPipelineArtifact@2 + displayName: Download WorkloadRollback.json + inputs: + patterns: '**/WorkloadRollback.json' + allowFailedBuilds: true + path: $(Build.SourcesDirectory)/package-internal + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Build Internal Artifacts' + inputs: + targetPath: $(Build.SourcesDirectory)/package-internal + artifactName: package-internal + continueOnError: true + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Build Artifacts (notarized)' + inputs: + targetPath: $(Build.SourcesDirectory)/package + artifactName: package + continueOnError: true + +# This job uploads the pkgs generated by the build step in the azure blob storage. This has to be done in a different job +# because the azure blob storate tools DO NOT work on mac OS meaning that we need a bot running Windows. build uploads the contents +# to the pipeline artefacts and we download and upload to azure in this job. +- job: upload_azure_blob + displayName: 'Upload packages to Azure & SBOM' + timeoutInMinutes: 1000 + dependsOn: + - funnel_job + condition: and(not(failed()), not(canceled())) # default is succeded(), but that fails if there are any skipped jobs, so change the condition to !failed && !cancelled + + variables: + Parameters.outputStorageUri: '' + NUGETS_PUBLISHED: $[ stageDependencies.sign_notarize_dotnet.sign_notarize_dotnet.outputs['nugetPublishing.NUGETS_PUBLISHED'] ] # not a typo, stage and job have the same name + SKIP_NUGETS: $[ dependencies.configure.outputs['labels.skip-nugets'] ] + + pool: + vmImage: 'windows-latest' + workspace: + clean: all + steps: + - template: upload-azure.yml + parameters: + enableDotnet: ${{ parameters.enableDotnet }} + sbomFilter: '*.nupkg;*.pkg;*.msi' + +# Job that runs on a vm that downloads the artifacts information and adds a github comment pointing to the results of the build. +- job: artifacts_github_comment + displayName: 'Publish GitHub Comment - Artifacts' + timeoutInMinutes: 1000 + dependsOn: + - configure + - upload_azure_blob + condition: succeededOrFailed() + variables: + PR_ID: $[ dependencies.configure.outputs['labels.pr-number'] ] + BUILD_PACKAGE: $[ dependencies.configure.outputs['labels.build-package'] ] + TESTS_BOT: $[ stageDependencies.build_packages.build.outputs['build.TESTS_BOT'] ] # we build in a diff bot than the ones used for the comments + GIT_HASH: $[ stageDependencies.build_packages.build.outputs['fix_commit.GIT_HASH'] ] + pool: + vmImage: 'windows-latest' + workspace: + clean: all + steps: + - template: artifact-github-comment.yml + parameters: + isPR: ${{ parameters.isPR }} diff --git a/tools/devops/automation/templates/sign-and-notarized/prepare-pkg-stage.yml b/tools/devops/automation/templates/sign-and-notarized/prepare-pkg-stage.yml index 035fe087a4..8e2458cbf8 100644 --- a/tools/devops/automation/templates/sign-and-notarized/prepare-pkg-stage.yml +++ b/tools/devops/automation/templates/sign-and-notarized/prepare-pkg-stage.yml @@ -20,44 +20,7 @@ parameters: - name: packages type: object - default: [ - { - job: 'xamarin_ios_sign_notarize', - name: 'Xamarin.iOS', - pattern: 'xamarin.ios-*', - conditionVariable: "INCLUDE_LEGACY_IOS", - }, - { - job: 'xamarin_mac_sing_notarie', - name: 'Xamarin.Mac', - pattern: 'xamarin.mac-*', - conditionVariable: "INCLUDE_LEGACY_MAC", - }, - { - job: 'microsoft_ios_sign_notarize', - name: 'Microsoft.iOS', - pattern: 'Microsoft.iOS.Bundle*.pkg', - conditionVariable: "INCLUDE_DOTNET_IOS", - }, - { - job: 'microsoft_tvos_sign_notarize', - name: 'Microsoft.tvOS', - pattern: 'Microsoft.tvOS.Bundle*.pkg', - conditionVariable: "INCLUDE_DOTNET_TVOS", - }, - { - job: 'microsoft_mac_sign_notarize', - name: 'Microsoft.macOS', - pattern: 'Microsoft.macOS.Bundle*.pkg', - conditionVariable: "INCLUDE_DOTNET_MACOS", - }, - { - job: 'microsoft_maccatalyst_sign_notarize', - name: 'Microsoft.MacCatalyst', - pattern: 'Microsoft.MacCatalyst.Bundle*.pkg', - conditionVariable: "INCLUDE_DOTNET_MACCATALYST", - }, - ] + default: [] jobs: - job: configure @@ -93,90 +56,3 @@ jobs: skipESRP: ${{ parameters.skipESRP }} packageName: ${{ pkg.name }} packagePattern: ${{ pkg.pattern }} - -- ${{ if eq(parameters.enableDotnet, true) }}: - - job: sign_notarize_dotnet - dependsOn: - - configure - displayName: 'Sign & Notarize Dotnet' - timeoutInMinutes: 1000 - pool: - vmImage: internal-macos-11 - workspace: - clean: all - - steps: - - template: dotnet-signing.yml - parameters: - isPR: ${{ parameters.isPR }} - -- job: funnel_job - dependsOn: - - configure - - ${{ if eq(parameters.enableDotnet, true) }}: - - sign_notarize_dotnet - - ${{ each pkg in parameters.packages }}: - - ${{ pkg.job }} - displayName: 'Collect signed artifacts' - condition: and(not(failed()), not(canceled())) # default is succeded(), but that fails if there are any skipped jobs, so change the condition to !failed && !cancelled - timeoutInMinutes: 1000 - pool: - vmImage: internal-macos-11 - workspace: - clean: all - variables: - ${{ each pkg in parameters.packages }}: - ${{ pkg.conditionVariable }}: $[ dependencies.configure.outputs['configure_platforms.${{ pkg.conditionVariable }}'] ] - - steps: - - template: funnel.yml - parameters: - packages: ${{ parameters.packages }} - - -# This job uploads the pkgs generated by the build step in the azure blob storage. This has to be done in a different job -# because the azure blob storate tools DO NOT work on mac OS meaning that we need a bot running Windows. build uploads the contents -# to the pipeline artefacts and we download and upload to azure in this job. -- job: upload_azure_blob - displayName: 'Upload packages to Azure & SBOM' - timeoutInMinutes: 1000 - dependsOn: - - funnel_job - condition: and(not(failed()), not(canceled())) # default is succeded(), but that fails if there are any skipped jobs, so change the condition to !failed && !cancelled - - variables: - Parameters.outputStorageUri: '' - NUGETS_PUBLISHED: $[ dependencies.sign_notarize.outputs['nugetPublishing.NUGETS_PUBLISHED'] ] - SKIP_NUGETS: $[ dependencies.configure.outputs['labels.skip-nugets'] ] - - pool: - vmImage: 'windows-latest' - workspace: - clean: all - steps: - - template: upload-azure.yml - parameters: - enableDotnet: ${{ parameters.enableDotnet }} - sbomFilter: '*.nupkg;*.pkg;*.msi' - -# Job that runs on a vm that downloads the artifacts information and adds a github comment pointing to the results of the build. -- job: artifacts_github_comment - displayName: 'Publish GitHub Comment - Artifacts' - timeoutInMinutes: 1000 - dependsOn: - - configure - - upload_azure_blob - condition: succeededOrFailed() - variables: - PR_ID: $[ dependencies.configure.outputs['labels.pr-number'] ] - BUILD_PACKAGE: $[ dependencies.configure.outputs['labels.build-package'] ] - TESTS_BOT: $[ stageDependencies.build_packages.build.outputs['build.TESTS_BOT'] ] # we build in a diff bot than the ones used for the comments - GIT_HASH: $[ stageDependencies.build_packages.build.outputs['fix_commit.GIT_HASH'] ] - pool: - vmImage: 'windows-latest' - workspace: - clean: all - steps: - - template: artifact-github-comment.yml - parameters: - isPR: ${{ parameters.isPR }} diff --git a/tools/devops/automation/templates/sign-and-notarized/sign-and-notarized.yml b/tools/devops/automation/templates/sign-and-notarized/sign-and-notarized.yml index de7654d3c4..42eb989a50 100644 --- a/tools/devops/automation/templates/sign-and-notarized/sign-and-notarized.yml +++ b/tools/devops/automation/templates/sign-and-notarized/sign-and-notarized.yml @@ -32,7 +32,7 @@ steps: - ${{ each step in parameters.signingSetupSteps }}: - ${{ each pair in step }}: - ${{ pair.key }}: ${{ pair.value }} + ${{ pair.key }}: ${{ pair.value }} - task: DownloadPipelineArtifact@2 displayName: Download not notarized build