From cecc343301bbadb63eaffe6a07b813ff24f20306 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Wed, 19 May 2021 13:32:03 -0400 Subject: [PATCH 01/19] [CI] Fix API diff uploads since they broke when we wanted to continue on failure. (#11609) --- .../templates/build/download-artifacts.yml | 36 ++++++++----------- .../templates/build/publish-html.yml | 5 --- .../automation/templates/build/stage.yml | 6 ++-- .../templates/build/upload-vsdrops.yml | 25 ++++++------- 4 files changed, 28 insertions(+), 44 deletions(-) diff --git a/tools/devops/automation/templates/build/download-artifacts.yml b/tools/devops/automation/templates/build/download-artifacts.yml index f71f1b7d6f..6e120333e4 100644 --- a/tools/devops/automation/templates/build/download-artifacts.yml +++ b/tools/devops/automation/templates/build/download-artifacts.yml @@ -3,10 +3,6 @@ parameters: type: boolean default: true -- name: apiGeneratorDiffBuilt - type: boolean - default: true - steps: - checkout: self @@ -32,32 +28,28 @@ steps: archiveFilePatterns: '$(System.DefaultWorkingDirectory)/Reports/apidiff-stable/apidiff-stable.zip' destinationFolder: '$(System.DefaultWorkingDirectory)/apidiff-stable' -- ${{ if eq(parameters.apiGeneratorDiffBuilt, true) }}: - - task: DownloadPipelineArtifact@2 - displayName: 'Download API & Generator comparison' - inputs: - patterns: 'apicomparison/apicomparison.zip' - allowFailedBuilds: true - path: $(System.DefaultWorkingDirectory)/Reports +- task: DownloadPipelineArtifact@2 + displayName: 'Download API & Generator comparison' + condition: and(succeeded(), contains(variables.API_GENERATOR_DIFF_BUILT, 'True')) + inputs: + patterns: 'apicomparison/apicomparison.zip' + allowFailedBuilds: true + path: $(System.DefaultWorkingDirectory)/Reports - - task: ExtractFiles@1 - displayName: 'Extract API & Generator comparison' - inputs: - archiveFilePatterns: '$(System.DefaultWorkingDirectory)/Reports/apicomparison/apicomparison.zip' - destinationFolder: '$(System.DefaultWorkingDirectory)/apicomparison' +- task: ExtractFiles@1 + displayName: 'Extract API & Generator comparison' + condition: and(succeeded(), contains(variables.API_GENERATOR_DIFF_BUILT, 'True')) + inputs: + archiveFilePatterns: '$(System.DefaultWorkingDirectory)/Reports/apicomparison/apicomparison.zip' + destinationFolder: '$(System.DefaultWorkingDirectory)/apicomparison' - powershell: | Write-Host "##vso[task.setvariable variable=STABLE_APIDDIFF_PATH]$Env:SYSTEM_DEFAULTWORKINGDIRECTORY\apidiff-stable" - if ($Env:apiGeneratorDiffBuilt -eq "True") { + if ($Env:API_GENERATOR_DIFF_BUILT -eq "True") { Write-Host "##vso[task.setvariable variable=STABLE_APID_GENERATOR_DIFF_PATH]$Env:SYSTEM_DEFAULTWORKINGDIRECTORY\apicomparison" } displayName: Publish apidiff paths name: apidiff # not to be confused with the displayName, this is used to later use the name of the step to access the output variables from an other job - env: - ${{ if eq(parameters.apiGeneratorDiffBuilt, true) }}: - apiGeneratorDiffBuilt: 'True' - ${{ if ne(parameters.apiGeneratorDiffBuilt, true) }}: - apiGeneratorDiffBuilt: 'False' # download the artifacts.json, which will use to find the URI of the built pkg to later be given to the user - task: DownloadPipelineArtifact@2 diff --git a/tools/devops/automation/templates/build/publish-html.yml b/tools/devops/automation/templates/build/publish-html.yml index fff5766832..2966d9755c 100644 --- a/tools/devops/automation/templates/build/publish-html.yml +++ b/tools/devops/automation/templates/build/publish-html.yml @@ -29,10 +29,6 @@ parameters: type: boolean default: true -- name: apiGeneratorDiffBuilt - type: string - default: true - steps: - checkout: self @@ -41,7 +37,6 @@ steps: - template: download-artifacts.yml parameters: runTests: ${{ parameters.runTests }} - apiGeneratorDiffBuilt: ${{ contains(parameters.apiGeneratorDiffBuilt, 'True') }} - ${{ if eq(parameters.runTests, true) }}: - pwsh: | diff --git a/tools/devops/automation/templates/build/stage.yml b/tools/devops/automation/templates/build/stage.yml index fc5f67fd97..7263adb038 100644 --- a/tools/devops/automation/templates/build/stage.yml +++ b/tools/devops/automation/templates/build/stage.yml @@ -135,6 +135,9 @@ jobs: dependsOn: build # can start as soon as the tests are done condition: and(succeededOrFailed() , contains (dependencies.build.outputs['runTests.TESTS_RAN'], 'True')) # only run when we did run the tests + variables: + API_GENERATOR_DIFF_BUILT: $[ dependencies.build.outputs['apiGeneratorDiff.API_GENERATOR_BUILT'] ] + pool: vmImage: 'windows-latest' workspace: @@ -144,7 +147,6 @@ jobs: parameters: devicePrefix: sim runTests: ${{ parameters.runTests }} - apiGeneratorDiffBuilt: $[ dependencies.build.outputs['apiGeneratorDiff.API_GENERATOR_BUILT'] ] - job: upload_vsts_tests displayName: 'Upload xml to vsts' @@ -179,6 +181,7 @@ jobs: TESTS_JOBSTATUS: $[ dependencies.build.outputs['runTests.TESTS_JOBSTATUS'] ] APIDIFF_MESSAGE: $[ dependencies.build.outputs['apidiff.APIDIFF_MESSAGE'] ] APIDIFF_BUILT: $[ dependencies.build.outputs['apidiff.APIDIFF_BUILT'] ] + API_GENERATOR_DIFF_BUILT: $[ dependencies.build.outputs['apiGeneratorDiff.API_GENERATOR_BUILT'] ] PR_ID: $[ dependencies.configure.outputs['labels.pr-number'] ] pool: vmImage: 'windows-latest' @@ -190,5 +193,4 @@ jobs: statusContext: "Build" runTests: ${{ parameters.runTests }} vsdropsPrefix: ${{ parameters.vsdropsPrefix }} - apiGeneratorDiffBuilt: $[ dependencies.build.outputs['apiGeneratorDiff.API_GENERATOR_BUILT'] ] devicePrefix: sim diff --git a/tools/devops/automation/templates/build/upload-vsdrops.yml b/tools/devops/automation/templates/build/upload-vsdrops.yml index eb49f111cd..da27f7227e 100644 --- a/tools/devops/automation/templates/build/upload-vsdrops.yml +++ b/tools/devops/automation/templates/build/upload-vsdrops.yml @@ -10,10 +10,6 @@ parameters: type: boolean default: true -- name: apiGeneratorDiffBuilt - type: string - default: true - steps: - checkout: self @@ -22,7 +18,6 @@ steps: - template: download-artifacts.yml parameters: runTests: ${{ parameters.runTests }} - apiGeneratorDiffBuilt: ${{ contains(parameters.apiGeneratorDiffBuilt, 'True') }} # Upload full report to vsdrops using the the build numer and id as uuids. - ${{ if eq(parameters.runTests, true) }}: @@ -47,13 +42,13 @@ steps: detailedLog: true usePat: true -- ${{ if contains(parameters.apiGeneratorDiffBuilt, 'True') }}: - - task: ms-vscs-artifact.build-tasks.artifactDropTask-1.artifactDropTask@0 - displayName: 'Publish API & Generator comparisonn to Artifact Services Drop' - inputs: - dropServiceURI: 'https://devdiv.artifacts.visualstudio.com/DefaultCollection' - dropMetadataContainerName: 'DropMetadata-APIGeneratorDiff' - buildNumber: 'xamarin-macios/device-tests/$(Build.BuildNumber)/$(Build.BuildId)/APIGeneratorDiff' - sourcePath: $(STABLE_APID_GENERATOR_DIFF_PATH) - detailedLog: true - usePat: true +- task: ms-vscs-artifact.build-tasks.artifactDropTask-1.artifactDropTask@0 + displayName: 'Publish API & Generator comparisonn to Artifact Services Drop' + condition: and(succeeded(), contains(variables.API_GENERATOR_DIFF_BUILT, 'True')) + inputs: + dropServiceURI: 'https://devdiv.artifacts.visualstudio.com/DefaultCollection' + dropMetadataContainerName: 'DropMetadata-APIGeneratorDiff' + buildNumber: 'xamarin-macios/device-tests/$(Build.BuildNumber)/$(Build.BuildId)/APIGeneratorDiff' + sourcePath: $(STABLE_APID_GENERATOR_DIFF_PATH) + detailedLog: true + usePat: true From 72496111dfb30c7f8ded17d3f593fc985fc0e4ee Mon Sep 17 00:00:00 2001 From: TJ Lambert <50846373+tj-devel709@users.noreply.github.com> Date: Wed, 19 May 2021 15:36:41 -0500 Subject: [PATCH 02/19] [Localization] Sync Localization branch with Main (#11603) Add changes to sync main into Localization branch Co-authored-by: tj_devel709 Co-authored-by: Connor Adsit --- tools/devops/automation/templates/build/configure.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools/devops/automation/templates/build/configure.yml b/tools/devops/automation/templates/build/configure.yml index 8da12de982..140f73a5b4 100644 --- a/tools/devops/automation/templates/build/configure.yml +++ b/tools/devops/automation/templates/build/configure.yml @@ -11,6 +11,7 @@ steps: - checkout: self # https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=azure-devops&tabs=schema#checkout clean: true # Executes: git clean -ffdx && git reset --hard HEAD submodules: false + persistCredentials: true - pwsh: | Import-Module $Env:SYSTEM_DEFAULTWORKINGDIRECTORY/tools/devops/automation/scripts/GitHub.psm1 @@ -155,6 +156,15 @@ steps: continueOnError: true workingDirectory: $(Build.SourcesDirectory)\tools\devops +- powershell: | + git config user.email "valco@microsoft.com" + git config user.name "vs-mobiletools-engineering-service2" + git checkout -b Localization origin/Localization + git merge origin/main + git push origin Localization + displayName: "Sync Localization branch with main" + condition: and(succeeded(), in(variables['build.reason'], 'Schedule', 'Manual'), eq(variables.isMain, 'True')) + - task: OneLocBuild@2 condition: and(succeeded(), eq(variables.isMain, 'True')) continueOnError: true From a33a86a9b7c1dc6c164dffc5d0346e4a0f667977 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Thu, 20 May 2021 07:33:16 +0200 Subject: [PATCH 03/19] [runtime] Implement mono_get_exception_out_of_memory for CoreCLR. (#11608) --- runtime/coreclr-bridge.m | 8 ++++++++ runtime/exports.t4 | 4 +++- runtime/xamarin/main.h | 1 + src/ObjCRuntime/Runtime.CoreCLR.cs | 4 ++++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/runtime/coreclr-bridge.m b/runtime/coreclr-bridge.m index 6421efb968..ca8c46cecc 100644 --- a/runtime/coreclr-bridge.m +++ b/runtime/coreclr-bridge.m @@ -399,6 +399,14 @@ xamarin_create_system_entry_point_not_found_exception (const char *entrypoint) return rv; } +MonoException * +mono_get_exception_out_of_memory () +{ + MonoException *rv = xamarin_bridge_create_exception (XamarinExceptionTypes_System_OutOfMemoryException, NULL); + LOG_CORECLR (stderr, "%s (%p) => %p\n", __func__, entrypoint, rv); + return rv; +} + MonoMethodSignature * mono_method_signature (MonoMethod* method) { diff --git a/runtime/exports.t4 b/runtime/exports.t4 index cc21d51fb6..246ad5250f 100644 --- a/runtime/exports.t4 +++ b/runtime/exports.t4 @@ -530,7 +530,9 @@ XamarinRuntime = RuntimeMode.MonoVM, }, - new Export ("MonoException *", "mono_get_exception_out_of_memory"), + new Export ("MonoException *", "mono_get_exception_out_of_memory") { + HasCoreCLRBridgeFunction = true, + }, #endregion diff --git a/runtime/xamarin/main.h b/runtime/xamarin/main.h index 3d84cb94aa..724a1ca6f0 100644 --- a/runtime/xamarin/main.h +++ b/runtime/xamarin/main.h @@ -80,6 +80,7 @@ enum XamarinExceptionTypes : int { XamarinExceptionTypes_System_Exception, XamarinExceptionTypes_System_InvalidCastException, XamarinExceptionTypes_System_EntryPointNotFoundException, + XamarinExceptionTypes_System_OutOfMemoryException, }; extern bool mono_use_llvm; // this is defined inside mono diff --git a/src/ObjCRuntime/Runtime.CoreCLR.cs b/src/ObjCRuntime/Runtime.CoreCLR.cs index f5e76ffa39..72aebf3130 100644 --- a/src/ObjCRuntime/Runtime.CoreCLR.cs +++ b/src/ObjCRuntime/Runtime.CoreCLR.cs @@ -42,6 +42,7 @@ namespace ObjCRuntime { System_Exception, System_InvalidCastException, System_EntryPointNotFoundException, + System_OutOfMemoryException, } // This struct must be kept in sync with the _MonoObject struct in coreclr-bridge.h @@ -89,6 +90,9 @@ namespace ObjCRuntime { case ExceptionType.System_EntryPointNotFoundException: rv = new System.EntryPointNotFoundException (str0); break; + case ExceptionType.System_OutOfMemoryException: + rv = new System.OutOfMemoryException (); + break; default: throw new ArgumentOutOfRangeException (nameof (type)); } From d60514b02b4b81e790682c44f831c745bc6923bf Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Thu, 20 May 2021 07:33:26 +0200 Subject: [PATCH 04/19] [runtime] Postpone implementing mono_domain_set_config for CoreCLR. (#11610) --- runtime/exports.t4 | 4 +++- runtime/monotouch-main.m | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/runtime/exports.t4 b/runtime/exports.t4 index 246ad5250f..0bcfc0a33d 100644 --- a/runtime/exports.t4 +++ b/runtime/exports.t4 @@ -408,7 +408,9 @@ "MonoDomain *", "domain", "const char *", "base_dir", "const char *", "config_file_name" - ), + ) { + XamarinRuntime = RuntimeMode.MonoVM, + }, #endregion diff --git a/runtime/monotouch-main.m b/runtime/monotouch-main.m index 0c9ebeb76a..ec8bc09977 100644 --- a/runtime/monotouch-main.m +++ b/runtime/monotouch-main.m @@ -458,7 +458,14 @@ xamarin_main (int argc, char *argv[], enum XamarinLaunchMode launch_mode) snprintf (base_dir, sizeof (base_dir), "%s/" ARCH_SUBDIR, xamarin_get_bundle_path ()); snprintf (config_file_name, sizeof (config_file_name), "%s/%s.config", base_dir, xamarin_executable_name); // xamarin_executable_name should never be NULL for extensions. +#if defined (CORECLR_RUNTIME) + // Need to figure out how to implement the equivalent of mono_domain_set_config for CoreCLR. + // That will need a test case (app extension), which we haven't implemented for CoreCLR yet. + // It's likely to require a completely different implementation, probably a property passed to coreclr_initialize. + xamarin_assertion_message ("Not implemented for CoreCLR: mono_domain_set_config."); +#else mono_domain_set_config (mono_domain_get (), base_dir, config_file_name); +#endif rv = xamarin_extension_main (argc, argv); break; From 551be877bd41d4fddc6dd01552c21134280d0109 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Thu, 20 May 2021 07:33:55 +0200 Subject: [PATCH 05/19] [monotouch-test] Ignore the SslSupportedCiphers test on .NET due to #11498. (#11613) The underlying size of the SslCipherSuite is wrong, so the test fails. Ref: https://github.com/xamarin/xamarin-macios/issues/11498 --- tests/monotouch-test/Security/SecureTransportTest.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/monotouch-test/Security/SecureTransportTest.cs b/tests/monotouch-test/Security/SecureTransportTest.cs index d97b4a2af9..272e778878 100644 --- a/tests/monotouch-test/Security/SecureTransportTest.cs +++ b/tests/monotouch-test/Security/SecureTransportTest.cs @@ -137,6 +137,9 @@ namespace MonoTouchFixtures.Security { } [Test] +#if NET + [Ignore ("Fails on ARM64 due to: https://github.com/xamarin/xamarin-macios/issues/11498)")] +#endif public void SslSupportedCiphers () { TestRuntime.AssertSystemVersion (PlatformName.MacOSX, 10, 8, throwIfOtherPlatform: false); From a6f0dbdd67c9b6c743de7d6f2d2f7dda8c797573 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Thu, 20 May 2021 07:34:17 +0200 Subject: [PATCH 06/19] [runtime] Fix declaration of xamarin_bridge_get_method_declaring_type. (#11614) The native xamarin_bridge_get_method_declaring_type method and the corresponding managed method (GetMethodDeclaringType) takes and returns a MonoObject*, not a GCHandle. Due to the wonders of void pointers, this worked just fine before - there's no actual change to the compiled code - but the code is now more consistent and less confusing. --- runtime/delegates.t4 | 4 ++-- src/ObjCRuntime/Runtime.CoreCLR.cs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/runtime/delegates.t4 b/runtime/delegates.t4 index bde480bf29..ecd81292d9 100644 --- a/runtime/delegates.t4 +++ b/runtime/delegates.t4 @@ -471,8 +471,8 @@ OnlyCoreCLR = true, }, - new XDelegate ("MonoObject *", "IntPtr", "xamarin_bridge_get_method_declaring_type", - "MonoObject *", "IntPtr", "gchandle" + new XDelegate ("MonoObject *", "MonoObject *", "xamarin_bridge_get_method_declaring_type", + "MonoObject *", "MonoObject *", "mobj" ) { WrappedManagedFunction = "GetMethodDeclaringType", OnlyDynamicUsage = false, diff --git a/src/ObjCRuntime/Runtime.CoreCLR.cs b/src/ObjCRuntime/Runtime.CoreCLR.cs index 72aebf3130..2dc64e19dc 100644 --- a/src/ObjCRuntime/Runtime.CoreCLR.cs +++ b/src/ObjCRuntime/Runtime.CoreCLR.cs @@ -341,10 +341,10 @@ namespace ObjCRuntime { return (byte) obj.FlagsInternal; } - static IntPtr GetMethodDeclaringType (MonoObjectPtr mobj) + static unsafe MonoObject* GetMethodDeclaringType (MonoObject *mobj) { var method = (MethodBase) GetMonoObjectTarget (mobj); - return GetMonoObject (method.DeclaringType); + return (MonoObject *) GetMonoObject (method.DeclaringType); } static IntPtr ObjectGetType (MonoObjectPtr mobj) From ee5e2f0b0934be9b02a2feae4cb146aa313ca501 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Thu, 20 May 2021 07:34:30 +0200 Subject: [PATCH 07/19] [monotouch-test] Ignore tests that require exception marshalling on CoreCLR for now, they fail. (#11615) This makes monotouch-test green when using CoreCLR, both when using the dynamic and the static registrar. Ref: https://github.com/xamarin/xamarin-macios/issues/10940 --- tests/api-shared/ObjCRuntime/RegistrarTest.cs | 3 +++ tests/monotouch-test/ObjCRuntime/ExceptionsTest.cs | 6 ++++++ tests/monotouch-test/ObjCRuntime/RegistrarTest.cs | 6 ++++++ tests/monotouch-test/ObjCRuntime/RuntimeTest.cs | 9 +++++++++ 4 files changed, 24 insertions(+) diff --git a/tests/api-shared/ObjCRuntime/RegistrarTest.cs b/tests/api-shared/ObjCRuntime/RegistrarTest.cs index 7f80a65de7..17c42b66ee 100644 --- a/tests/api-shared/ObjCRuntime/RegistrarTest.cs +++ b/tests/api-shared/ObjCRuntime/RegistrarTest.cs @@ -26,6 +26,9 @@ namespace XamarinTests.ObjCRuntime { } [Test] +#if NET + [Ignore ("Ignored on CoreCLR for now due to missing support for marshalling exceptions")] +#endif public void IntPtrCtor () { IntPtr ptr = IntPtr.Zero; diff --git a/tests/monotouch-test/ObjCRuntime/ExceptionsTest.cs b/tests/monotouch-test/ObjCRuntime/ExceptionsTest.cs index be78baa411..6dc4184340 100644 --- a/tests/monotouch-test/ObjCRuntime/ExceptionsTest.cs +++ b/tests/monotouch-test/ObjCRuntime/ExceptionsTest.cs @@ -85,6 +85,9 @@ namespace MonoTouchFixtures.ObjCRuntime { // Simulator/desktop only (except for watchOS, where it works everywhere) [Test] +#if NET + [Ignore ("Ignored on CoreCLR for now due to missing support for marshalling exceptions")] +#endif public void ObjCException () { #if !__WATCHOS__ && !__MACOS__ @@ -131,6 +134,9 @@ namespace MonoTouchFixtures.ObjCRuntime { // Simulator/desktop only test (except for watchOS, where it works everywhere) [Test] +#if NET + [Ignore ("Ignored on CoreCLR for now due to missing support for marshalling exceptions")] +#endif public void ManagedExceptionPassthrough () { Exception thrownException = null; diff --git a/tests/monotouch-test/ObjCRuntime/RegistrarTest.cs b/tests/monotouch-test/ObjCRuntime/RegistrarTest.cs index 2c307447dc..6e4627f999 100644 --- a/tests/monotouch-test/ObjCRuntime/RegistrarTest.cs +++ b/tests/monotouch-test/ObjCRuntime/RegistrarTest.cs @@ -433,6 +433,9 @@ namespace MonoTouchFixtures.ObjCRuntime { } [Test] +#if NET + [Ignore ("Ignored on CoreCLR for now due to missing support for marshalling exceptions")] +#endif public void TestGeneric () { var g1 = new GenericTestClass (); @@ -1240,6 +1243,9 @@ namespace MonoTouchFixtures.ObjCRuntime { } [Test] +#if NET + [Ignore ("Ignored on CoreCLR for now due to missing support for marshalling exceptions")] +#endif public void TestConstrainedGenericType () { IntPtr value; diff --git a/tests/monotouch-test/ObjCRuntime/RuntimeTest.cs b/tests/monotouch-test/ObjCRuntime/RuntimeTest.cs index affa1014b4..1c2b18974f 100644 --- a/tests/monotouch-test/ObjCRuntime/RuntimeTest.cs +++ b/tests/monotouch-test/ObjCRuntime/RuntimeTest.cs @@ -579,6 +579,9 @@ namespace MonoTouchFixtures.ObjCRuntime { } [Test] +#if NET + [Ignore ("Ignored on CoreCLR for now due to missing support for marshalling exceptions")] +#endif public void MX8029 () { var handle = Messaging.IntPtr_objc_msgSend (Messaging.IntPtr_objc_msgSend (Class.GetHandle (typeof (Dummy)), Selector.GetHandle ("alloc")), Selector.GetHandle ("init")); @@ -613,6 +616,9 @@ namespace MonoTouchFixtures.ObjCRuntime { #if DYNAMIC_REGISTRAR [Test] +#if NET + [Ignore ("Ignored on CoreCLR for now due to missing support for marshalling exceptions")] +#endif public void MX8029_b () { try { @@ -634,6 +640,9 @@ Additional information: } [Test] +#if NET + [Ignore ("Ignored on CoreCLR for now due to missing support for marshalling exceptions")] +#endif public void MX8033 () { try { From f38187f05410b39a8d6999d04b027bcf7fe998fd Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Thu, 20 May 2021 07:35:23 +0200 Subject: [PATCH 08/19] [runtime] Refactor the toggle ref code to separate MonoVM-specific code and generic logic. (#11616) This makes it simpler to impement the toggle ref logic for CoreCLR once we can do that. --- runtime/coreclr-bridge.m | 6 ++++ runtime/exports.t4 | 8 +++-- runtime/monovm-bridge.m | 60 +++++++++++++++++++++++++++++++++++ runtime/runtime.m | 67 +++++++-------------------------------- runtime/xamarin/runtime.h | 5 +++ 5 files changed, 89 insertions(+), 57 deletions(-) diff --git a/runtime/coreclr-bridge.m b/runtime/coreclr-bridge.m index ca8c46cecc..b246165366 100644 --- a/runtime/coreclr-bridge.m +++ b/runtime/coreclr-bridge.m @@ -29,6 +29,12 @@ xamarin_bridge_initialize () { } +void +xamarin_enable_new_refcount () +{ + // Nothing to do here. +} + bool xamarin_bridge_vm_initialize (int propertyCount, const char **propertyKeys, const char **propertyValues) { diff --git a/runtime/exports.t4 b/runtime/exports.t4 index 0bcfc0a33d..a0bcf1ebb1 100644 --- a/runtime/exports.t4 +++ b/runtime/exports.t4 @@ -240,7 +240,9 @@ new Export ("void", "mono_profiler_install_gc", "MonoProfileGCFunc", "callback", "MonoProfileGCResizeFunc", "heap_resize_callback" - ), + ) { + XamarinRuntime = RuntimeMode.MonoVM, + }, new Export ("void", "mono_profiler_load", "const char *", "desc" @@ -567,7 +569,9 @@ new Export ("void", "mono_gc_toggleref_add", "MonoObject *", "object", "mono_bool", "strong_ref" - ), + ) { + XamarinRuntime = RuntimeMode.MonoVM, + }, new Export ("void", "mono_gc_toggleref_register_callback", "MonoToggleRefCallback", "process_toggleref" diff --git a/runtime/monovm-bridge.m b/runtime/monovm-bridge.m index d56a818b38..46653b8f35 100644 --- a/runtime/monovm-bridge.m +++ b/runtime/monovm-bridge.m @@ -419,4 +419,64 @@ xamarin_mono_object_retain (MonoObject *mobj) #endif // DOTNET +/* + * ToggleRef support + */ +// #define DEBUG_TOGGLEREF 1 + +static void +gc_register_toggleref (MonoObject *obj, id self, bool isCustomType) +{ + // COOP: This is an icall, at entry we're in unsafe mode. Managed memory is accessed, so we stay in unsafe mode. + MONO_ASSERT_GC_UNSAFE; + +#ifdef DEBUG_TOGGLEREF + id handle = xamarin_get_nsobject_handle (obj); + + PRINT ("**Registering object %p handle %p RC %d flags: %i isCustomType: %i", + obj, + handle, + (int) (handle ? [handle retainCount] : 0), + xamarin_get_nsobject_flags (obj), + isCustomType + ); +#endif + mono_gc_toggleref_add (obj, TRUE); + + // Make sure the GCHandle we have is a weak one for custom types. + if (isCustomType) { + MONO_ENTER_GC_SAFE; + xamarin_switch_gchandle (self, true); + MONO_EXIT_GC_SAFE; + } +} + +static MonoToggleRefStatus +gc_toggleref_callback (MonoObject *object) +{ + // COOP: this is a callback called by the GC, so I assume the mode here doesn't matter + MonoToggleRefStatus res; + uint8_t flags = xamarin_get_nsobject_flags (object); + + res = xamarin_gc_toggleref_callback (flags, NULL, xamarin_get_nsobject_handle, object); + + return res; +} + +static void +gc_event_callback (MonoProfiler *prof, MonoGCEvent event, int generation) +{ + // COOP: this is a callback called by the GC, I believe the mode here doesn't matter. + xamarin_gc_event (event); +} + +void +xamarin_enable_new_refcount () +{ + mono_gc_toggleref_register_callback (gc_toggleref_callback); + + xamarin_add_internal_call ("Foundation.NSObject::RegisterToggleRef", (const void *) gc_register_toggleref); + mono_profiler_install_gc (gc_event_callback, NULL); +} + #endif // !CORECLR_RUNTIME diff --git a/runtime/runtime.m b/runtime/runtime.m index 9d7a694ae8..d3c5771f6a 100644 --- a/runtime/runtime.m +++ b/runtime/runtime.m @@ -796,51 +796,24 @@ xamarin_type_get_full_name (MonoType *type, GCHandle *exception_gchandle) * ToggleRef support */ // #define DEBUG_TOGGLEREF 1 -#if !defined (CORECLR_RUNTIME) -static void -gc_register_toggleref (MonoObject *obj, id self, bool isCustomType) -{ - // COOP: This is an icall, at entry we're in unsafe mode. Managed memory is accessed, so we stay in unsafe mode. - MONO_ASSERT_GC_UNSAFE; -#ifdef DEBUG_TOGGLEREF - id handle = xamarin_get_nsobject_handle (obj); - - PRINT ("**Registering object %p handle %p RC %d flags: %i isCustomType: %i", - obj, - handle, - (int) (handle ? [handle retainCount] : 0), - xamarin_get_nsobject_flags (obj), - isCustomType - ); -#endif - mono_gc_toggleref_add (obj, TRUE); - - // Make sure the GCHandle we have is a weak one for custom types. - if (isCustomType) { - MONO_ENTER_GC_SAFE; - xamarin_switch_gchandle (self, true); - MONO_EXIT_GC_SAFE; - } -} - -static MonoToggleRefStatus -gc_toggleref_callback (MonoObject *object) +MonoToggleRefStatus +xamarin_gc_toggleref_callback (uint8_t flags, id handle, xamarin_get_handle_func get_handle, MonoObject *info) { // COOP: this is a callback called by the GC, so I assume the mode here doesn't matter - id handle = NULL; MonoToggleRefStatus res; - uint8_t flags = xamarin_get_nsobject_flags (object); bool disposed = (flags & NSObjectFlagsDisposed) == NSObjectFlagsDisposed; bool has_managed_ref = (flags & NSObjectFlagsHasManagedRef) == NSObjectFlagsHasManagedRef; if (disposed || !has_managed_ref) { res = MONO_TOGGLE_REF_DROP; /* Already disposed, we don't need the managed object around */ } else { - handle = xamarin_get_nsobject_handle (object); + if (handle == NULL) + handle = get_handle (info); + if (handle == NULL) { /* This shouldn't really happen */ - return MONO_TOGGLE_REF_DROP; + res = MONO_TOGGLE_REF_DROP; } else { if ([handle retainCount] == 1) res = MONO_TOGGLE_REF_WEAK; @@ -861,21 +834,17 @@ gc_toggleref_callback (MonoObject *object) rv = "UNKNOWN"; } const char *cn = NULL; - if (handle == NULL) { - cn = object_getClassName (xamarin_get_nsobject_handle (object)); - } else { - cn = object_getClassName (handle); - } + if (handle == NULL) + handle = get_handle (info); + cn = object_getClassName (handle); PRINT ("\tinspecting %p handle:%p %s flags: %i RC %d -> %s\n", object, handle, cn, (int) flags, (int) (handle ? [handle retainCount] : 0), rv); #endif return res; } -#endif -#if !defined (CORECLR_RUNTIME) -static void -gc_event_callback (MonoProfiler *prof, MonoGCEvent event, int generation) +void +xamarin_gc_event (MonoGCEvent event) { // COOP: this is a callback called by the GC, I believe the mode here doesn't matter. switch (event) { @@ -892,16 +861,6 @@ gc_event_callback (MonoProfiler *prof, MonoGCEvent event, int generation) } } -static void -gc_enable_new_refcount (void) -{ - mono_gc_toggleref_register_callback (gc_toggleref_callback); - - xamarin_add_internal_call ("Foundation.NSObject::RegisterToggleRef", (const void *) gc_register_toggleref); - mono_profiler_install_gc (gc_event_callback, NULL); -} -#endif // !CORECLR_RUNTIME - #if !defined (CORECLR_RUNTIME) struct _MonoProfiler { int dummy; @@ -1309,9 +1268,7 @@ xamarin_initialize () pthread_mutex_init (&framework_peer_release_lock, &attr); pthread_mutexattr_destroy (&attr); -#if !defined (CORECLR_RUNTIME) - gc_enable_new_refcount (); -#endif + xamarin_enable_new_refcount (); MONO_EXIT_GC_UNSAFE; } diff --git a/runtime/xamarin/runtime.h b/runtime/xamarin/runtime.h index 8e92d97f0b..7d234f3642 100644 --- a/runtime/xamarin/runtime.h +++ b/runtime/xamarin/runtime.h @@ -214,6 +214,7 @@ MonoMethod * xamarin_bridge_get_mono_method (MonoReflectionMethod *method); void xamarin_bridge_free_mono_signature (MonoMethodSignature **signature); bool xamarin_register_monoassembly (MonoAssembly *assembly, GCHandle *exception_gchandle); void xamarin_install_nsautoreleasepool_hooks (); +void xamarin_enable_new_refcount (); MonoObject * xamarin_new_nsobject (id self, MonoClass *klass, GCHandle *exception_gchandle); bool xamarin_has_managed_ref (id self); @@ -286,6 +287,10 @@ MonoObject * xamarin_gchandle_get_target (GCHandle handle); void xamarin_gchandle_free (GCHandle handle); MonoObject * xamarin_gchandle_unwrap (GCHandle handle); // Will get the target and free the GCHandle +typedef id (*xamarin_get_handle_func) (MonoObject *info); +MonoToggleRefStatus xamarin_gc_toggleref_callback (uint8_t flags, id handle, xamarin_get_handle_func get_handle, MonoObject *info); +void xamarin_gc_event (MonoGCEvent event); + /* * In MonoVM MonoObjects are tracked in memory/the stack directly by the GC, but that doesn't * work for CoreCLR, so we make it ref-counted. All code must use the functions below to retain/release From f8a19c7cbf524104aa0ef898924210a7982cb3d2 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Thu, 20 May 2021 07:35:38 +0200 Subject: [PATCH 09/19] [runtime] Remove the declarations for mono_jit_parse_options and mono_gc_max_generation. (#11617) We don't used these mono functions anymore. --- runtime/exports.t4 | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/runtime/exports.t4 b/runtime/exports.t4 index a0bcf1ebb1..07fcb408cf 100644 --- a/runtime/exports.t4 +++ b/runtime/exports.t4 @@ -490,12 +490,6 @@ #endregion - #region metadata/mono-gc.h - - new Export ("int", "mono_gc_max_generation"), - - #endregion - #region metadata/mono-hash.h new Export ("MonoGHashTable *", "mono_g_hash_table_new_type", @@ -611,11 +605,6 @@ HasCoreCLRBridgeFunction = true, }, - new Export ("void", "mono_jit_parse_options", - "int", "argc", - "char**", "argv" - ), - new Export ("void", "mono_jit_set_aot_mode", "MonoAotMode", "mode" ), From 1b655344dfffe81818ad83aee9b8d76f7cca71e0 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Thu, 20 May 2021 07:35:52 +0200 Subject: [PATCH 10/19] [mmp] Don't define CORECLR_RUNTIME when compiling the partial static registrar code (#11618) We already define it in the generated code, so the native compiler shows a warning: Microsoft.macOS.registrar.coreclr.x86_64.m:1:9: warning: 'CORECLR_RUNTIME' macro redefined [-Wmacro-redefined] #define CORECLR_RUNTIME ^ :2:9: note: previous definition is here #define CORECLR_RUNTIME 1 ^ --- tools/mmp/Makefile | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/mmp/Makefile b/tools/mmp/Makefile index d61c7923ae..ee54da7051 100644 --- a/tools/mmp/Makefile +++ b/tools/mmp/Makefile @@ -103,9 +103,6 @@ Xamarin.Mac.registrar.full.arm64.m: $(TOP)/src/build/mac/full-64/Xamarin.Mac.d $(GENERATE_PART_REGISTRAR) --arch=arm64 --target-framework Xamarin.Mac,Version=v4.5,Profile=Full -a:$(FULL_BCL_DIR)/mscorlib.dll $(Q) touch Xamarin.Mac.registrar.full.arm64.m Xamarin.Mac.registrar.full.arm64.h -%.coreclr.x86_64.a: export EXTRA_DEFINES=-DCORECLR_RUNTIME -%.coreclr.arm64.a: export EXTRA_DEFINES=-DCORECLR_RUNTIME - %.x86_64.a: %.x86_64.m $(Q_CC) $(MAC_CC) -DDEBUG -g -gdwarf-2 -x objective-c++ -std=c++14 -o $@ -c -arch x86_64 $< -Wall -Wno-unguarded-availability-new -I$(TOP)/runtime -mmacosx-version-min=$(MIN_OSX_SDK_VERSION) -fobjc-runtime=macosx $(EXTRA_DEFINES) From e4ce5bb828776c8a24d8e23b5a21a9943ad971fd Mon Sep 17 00:00:00 2001 From: Sebastien Pouliot Date: Thu, 20 May 2021 09:26:55 -0400 Subject: [PATCH 11/19] [catalyst][authenticationservices] Update xtro - framework is not available (#11626) --- ...onServices.todo => MacCatalyst-AuthenticationServices.ignore} | 1 + 1 file changed, 1 insertion(+) rename tests/xtro-sharpie/{MacCatalyst-AuthenticationServices.todo => MacCatalyst-AuthenticationServices.ignore} (86%) diff --git a/tests/xtro-sharpie/MacCatalyst-AuthenticationServices.todo b/tests/xtro-sharpie/MacCatalyst-AuthenticationServices.ignore similarity index 86% rename from tests/xtro-sharpie/MacCatalyst-AuthenticationServices.todo rename to tests/xtro-sharpie/MacCatalyst-AuthenticationServices.ignore index ed58d479c5..926b1a04f3 100644 --- a/tests/xtro-sharpie/MacCatalyst-AuthenticationServices.todo +++ b/tests/xtro-sharpie/MacCatalyst-AuthenticationServices.ignore @@ -1,3 +1,4 @@ +## macOS only (added in 10.15) without any catalyst annotations (headers or web docs) !missing-protocol! ASWebAuthenticationSessionWebBrowserSessionHandling not bound !missing-selector! +ASWebAuthenticationSessionWebBrowserSessionManager::sharedManager not bound !missing-selector! ASWebAuthenticationSessionWebBrowserSessionManager::sessionHandler not bound From 9c78ba39cdd3311224ba8e3d7ade8000f0ebedb0 Mon Sep 17 00:00:00 2001 From: Sebastien Pouliot Date: Thu, 20 May 2021 09:30:55 -0400 Subject: [PATCH 12/19] [catalyst][adsupport] Update xtro - framework is not available (#11625) * [catalyst][adsupport] Update xtro - framework is not available also found out an API was removed by Apple... turned it into a stub for compatibility. --- src/AdSupport/ASCompat.cs | 22 +++++++++++++++++++ src/adsupport.cs | 1 + src/frameworks.sources | 5 +++++ .../xtro-sharpie/MacCatalyst-AdSupport.ignore | 3 +++ tests/xtro-sharpie/MacCatalyst-AdSupport.todo | 1 - 5 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 src/AdSupport/ASCompat.cs create mode 100644 tests/xtro-sharpie/MacCatalyst-AdSupport.ignore delete mode 100644 tests/xtro-sharpie/MacCatalyst-AdSupport.todo diff --git a/src/AdSupport/ASCompat.cs b/src/AdSupport/ASCompat.cs new file mode 100644 index 0000000000..5b39e65036 --- /dev/null +++ b/src/AdSupport/ASCompat.cs @@ -0,0 +1,22 @@ +#if !XAMCORE_4_0 + +using System; +using System.Runtime.Versioning; + +namespace AdSupport { + + public partial class ASIdentifierManager { + +#if MONOMAC +#if NET + [UnsupportedOSPlatform ("macos")] +#endif + [Obsolete ("Empty stub. This member was retroactively marked as unavailable for macOS.")] + public virtual void ClearAdvertisingIdentifier () + { + } +#endif + } +} + +#endif diff --git a/src/adsupport.cs b/src/adsupport.cs index af41a7d543..ea76ed3301 100644 --- a/src/adsupport.cs +++ b/src/adsupport.cs @@ -32,6 +32,7 @@ namespace AdSupport { NSUuid AdvertisingIdentifier { get; } [NoTV][NoiOS] + [NoMac] // unclear when that was changed (xcode 12 GM allowed it) [Export ("clearAdvertisingIdentifier")] void ClearAdvertisingIdentifier (); } diff --git a/src/frameworks.sources b/src/frameworks.sources index 5fbfa350ca..8ee1466945 100644 --- a/src/frameworks.sources +++ b/src/frameworks.sources @@ -64,6 +64,11 @@ ADDRESSBOOKUI_SOURCES = \ AddressBookUI/ABUnknownPersonViewController.cs \ AddressBookUI/DisplayedPropertiesCollection.cs \ +# AdSupport + +ADSUPPORT_SOURCES = \ + AdSupport/ASCompat.cs \ + # AppKit APPKIT_CORE_SOURCES = \ diff --git a/tests/xtro-sharpie/MacCatalyst-AdSupport.ignore b/tests/xtro-sharpie/MacCatalyst-AdSupport.ignore new file mode 100644 index 0000000000..79bd828714 --- /dev/null +++ b/tests/xtro-sharpie/MacCatalyst-AdSupport.ignore @@ -0,0 +1,3 @@ +## weird case, it was available on macOS (xcode12 GM) but later removed so it's not in iOS, tvOS or macOS +## there's no annotation specific for Catalyst but it does not make sense to have it +!missing-selector! ASIdentifierManager::clearAdvertisingIdentifier not bound diff --git a/tests/xtro-sharpie/MacCatalyst-AdSupport.todo b/tests/xtro-sharpie/MacCatalyst-AdSupport.todo deleted file mode 100644 index 6eae8b73d2..0000000000 --- a/tests/xtro-sharpie/MacCatalyst-AdSupport.todo +++ /dev/null @@ -1 +0,0 @@ -!missing-selector! ASIdentifierManager::clearAdvertisingIdentifier not bound From 4ea2c8b2c0d80b3f8f4c3fa4736b12b5aab2f3cb Mon Sep 17 00:00:00 2001 From: Sebastien Pouliot Date: Thu, 20 May 2021 09:35:09 -0400 Subject: [PATCH 13/19] [catalyst][accounts] Update xtro - framework is not available (#11624) --- .../{MacCatalyst-Accounts.todo => MacCatalyst-Accounts.ignore} | 1 + 1 file changed, 1 insertion(+) rename tests/xtro-sharpie/{MacCatalyst-Accounts.todo => MacCatalyst-Accounts.ignore} (62%) diff --git a/tests/xtro-sharpie/MacCatalyst-Accounts.todo b/tests/xtro-sharpie/MacCatalyst-Accounts.ignore similarity index 62% rename from tests/xtro-sharpie/MacCatalyst-Accounts.todo rename to tests/xtro-sharpie/MacCatalyst-Accounts.ignore index ccea64a0ca..8d32ecd509 100644 --- a/tests/xtro-sharpie/MacCatalyst-Accounts.todo +++ b/tests/xtro-sharpie/MacCatalyst-Accounts.ignore @@ -1,3 +1,4 @@ +## macOS only (deprecated in 10.13) without any catalyst annotations (headers or web docs) !missing-field! ACAccountTypeIdentifierLinkedIn not bound !missing-field! ACLinkedInAppIdKey not bound !missing-field! ACLinkedInPermissionsKey not bound From 52c75d30b453f481db12fce8fe13429af7e5f041 Mon Sep 17 00:00:00 2001 From: Sebastien Pouliot Date: Thu, 20 May 2021 09:36:05 -0400 Subject: [PATCH 14/19] [dotnet][linker] Use substitution files to remove UIButton debug code (#11623) The debug code is removed only on release builds. Unlike the _legacy_ linker the method itself is still present (not removed) from the assembly - but it will be empty (no code). Unit test was adapted to work under all conditions. Fixes https://github.com/xamarin/xamarin-macios/issues/9613 --- src/ILLink.Substitutions.ios.xml | 3 ++ src/ILLink.Substitutions.tvos.xml | 3 ++ .../ios/link sdk/LinkSdkRegressionTest.cs | 29 +++++++++++++++---- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/ILLink.Substitutions.ios.xml b/src/ILLink.Substitutions.ios.xml index 6b49d093f2..951d30ecf6 100644 --- a/src/ILLink.Substitutions.ios.xml +++ b/src/ILLink.Substitutions.ios.xml @@ -3,5 +3,8 @@ + + + diff --git a/src/ILLink.Substitutions.tvos.xml b/src/ILLink.Substitutions.tvos.xml index fef490eeb0..9f66a54dba 100644 --- a/src/ILLink.Substitutions.tvos.xml +++ b/src/ILLink.Substitutions.tvos.xml @@ -3,5 +3,8 @@ + + + diff --git a/tests/linker/ios/link sdk/LinkSdkRegressionTest.cs b/tests/linker/ios/link sdk/LinkSdkRegressionTest.cs index 871180dc00..6b29a0a2b7 100644 --- a/tests/linker/ios/link sdk/LinkSdkRegressionTest.cs +++ b/tests/linker/ios/link sdk/LinkSdkRegressionTest.cs @@ -1025,18 +1025,16 @@ namespace LinkSdk { #endif #if !__WATCHOS__ - static Type type_uibutton = typeof (UIButton); - [Test] -#if NET - [Ignore ("Not implemented yet: https://github.com/xamarin/xamarin-macios/issues/9612")] -#endif public void UIButtonSubclass () { // ensure the linker keeps the .ctor(UIButtonType) around using (var b = new UIButton (UIButtonType.Custom)) { // https://trello.com/c/Nf2B8mIM/484-remove-debug-code-in-the-linker - var m = type_uibutton.GetMethod ("VerifyIsUIButton", BindingFlags.Instance | BindingFlags.NonPublic); + var m = b.GetType ().GetMethod ("VerifyIsUIButton", BindingFlags.Instance | BindingFlags.NonPublic); +#if NET + CheckILLinkStubbedMethod (m); +#else // NET #if DEBUG // kept in debug builds Assert.NotNull (m, "VerifyIsUIButton"); @@ -1044,11 +1042,30 @@ namespace LinkSdk { // removed from release builds Assert.Null (m, "VerifyIsUIButton"); #endif +#endif // NET } } #endif // !__WATCHOS__ +#if NET + static void CheckILLinkStubbedMethod (MethodInfo m) + { + // ILLink does not remove the method, but it can "stub" (empty) it + Assert.NotNull (m, "Method not found (null"); + var mb = m.GetMethodBody (); + Assert.NotNull (m, "GetMethodBody"); + var il = mb.GetILAsByteArray (); +#if DEBUG + // means some stuff in addition to the `ret` instruction + Assert.That (il.Length, Is.GreaterThan (1), "il > 1"); +#else + // empty means a `ret` instruction (and that's true even if IL is stripped) + Assert.That (il.Length, Is.EqualTo (1), "il == 1"); +#endif + } +#endif + [Test] public void MonoRuntime34671 () { From 8ec9182b128aa28bf10e4d0ea6115a4bbbebd844 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Thu, 20 May 2021 09:37:22 -0400 Subject: [PATCH 15/19] [CI] Allow to ignore tests if dotnet is enabled. (#11604) --- tests/Makefile | 1 + tests/common/Configuration.cs | 10 ++++++++++ tests/msbuild/Xamarin.MacDev.Tests/DotnetTest.cs | 2 ++ tests/xharness/Harness.cs | 2 ++ tests/xharness/IHarness.cs | 1 + tests/xharness/Jenkins/NUnitTestTasksEnumerable.cs | 2 +- tests/xharness/Jenkins/TestSelector.cs | 5 +++++ tests/xharness/Jenkins/TestTasks/BuildTool.cs | 2 ++ tests/xharness/Jenkins/TestTasks/BuildToolTask.cs | 5 +++++ tests/xharness/Jenkins/TestTasks/MSBuild.cs | 2 ++ 10 files changed, 31 insertions(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index c02aaf2da4..752f5b3358 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -108,6 +108,7 @@ test.config: Makefile $(TOP)/Make.config $(TOP)/mk/mono.mk @echo "WATCH_SDK_VERSION=$(WATCH_SDK_VERSION)" >> $@ @echo "OSX_SDK_VERSION=$(OSX_SDK_VERSION)" >> $@ @echo "DOTNET6_BCL_DIR=$(DOTNET6_BCL_DIR)" >> $@ + @echo "ENABLE_DOTNET=$(ENABLE_DOTNET)" >> $@ test-system.config: Makefile $(TOP)/Make.config $(TOP)/mk/mono.mk @rm -f $@ diff --git a/tests/common/Configuration.cs b/tests/common/Configuration.cs index 04fa1df00d..22266e2138 100644 --- a/tests/common/Configuration.cs +++ b/tests/common/Configuration.cs @@ -42,6 +42,7 @@ namespace Xamarin.Tests public static bool include_dotnet_watchos; public static bool include_maccatalyst; public static bool include_device; + public static bool include_dotnet; static Version xcode_version; public static Version XcodeVersion { @@ -270,6 +271,7 @@ namespace Xamarin.Tests include_dotnet_watchos = !string.IsNullOrEmpty (GetVariable ("INCLUDE_DOTNET_WATCH", "")); include_maccatalyst = !string.IsNullOrEmpty (GetVariable ("INCLUDE_MACCATALYST", "")); include_device = !string.IsNullOrEmpty (GetVariable ("INCLUDE_DEVICE", "")); + include_dotnet = !string.IsNullOrEmpty (GetVariable ("ENABLE_DOTNET", "")); DotNet6BclDir = GetVariable ("DOTNET6_BCL_DIR", null); XcodeVersionString = GetXcodeVersion (xcode_root); @@ -294,6 +296,7 @@ namespace Xamarin.Tests Console.WriteLine (" INCLUDE_TVOS={0}", include_tvos); Console.WriteLine (" INCLUDE_WATCHOS={0}", include_watchos); Console.WriteLine (" INCLUDE_MACCATALYST={0}", include_maccatalyst); + Console.WriteLine (" ENABLE_DOTNET={0}", include_dotnet); } public static string RootPath { @@ -700,6 +703,13 @@ namespace Xamarin.Tests Assert.Ignore ("This build does not include device support."); } + public static void AssertDotNetAvailable () + { + if (include_dotnet) + return; + Assert.Ignore (".NET tests not enabled"); + } + public static string CloneTestDirectory (string directory) { // Copy the test projects to a temporary directory so that we can run the tests from there without affecting the working directory. diff --git a/tests/msbuild/Xamarin.MacDev.Tests/DotnetTest.cs b/tests/msbuild/Xamarin.MacDev.Tests/DotnetTest.cs index 0976603b02..05caf1275e 100644 --- a/tests/msbuild/Xamarin.MacDev.Tests/DotnetTest.cs +++ b/tests/msbuild/Xamarin.MacDev.Tests/DotnetTest.cs @@ -51,6 +51,8 @@ namespace Xamarin.iOS.Tasks { [TestCase ("MyXamarinFormsApp")] public void CompareBuilds (string project, int expectedErrorCount = 0) { + Configuration.AssertDotNetAvailable (); + tfi = "Xamarin.iOS"; switch (project) { case "MyMetalGame": diff --git a/tests/xharness/Harness.cs b/tests/xharness/Harness.cs index f1a43545e1..8845db7058 100644 --- a/tests/xharness/Harness.cs +++ b/tests/xharness/Harness.cs @@ -132,6 +132,7 @@ namespace Xharness { public string MONO_IOS_SDK_DESTDIR { get; } public string MONO_MAC_SDK_DESTDIR { get; } public bool ENABLE_XAMARIN { get; } + public bool ENABLE_DOTNET { get; } // Run @@ -203,6 +204,7 @@ namespace Xharness { MONO_IOS_SDK_DESTDIR = config ["MONO_IOS_SDK_DESTDIR"]; MONO_MAC_SDK_DESTDIR = config ["MONO_MAC_SDK_DESTDIR"]; ENABLE_XAMARIN = config.ContainsKey ("ENABLE_XAMARIN") && !string.IsNullOrEmpty (config ["ENABLE_XAMARIN"]); + ENABLE_DOTNET = config.ContainsKey ("ENABLE_DOTNET") && !string.IsNullOrEmpty (config ["ENABLE_DOTNET"]); if (string.IsNullOrEmpty (SdkRoot)) SdkRoot = config ["XCODE_DEVELOPER_ROOT"] ?? configuration.SdkRoot; diff --git a/tests/xharness/IHarness.cs b/tests/xharness/IHarness.cs index de45c45a93..1f6bf98ba4 100644 --- a/tests/xharness/IHarness.cs +++ b/tests/xharness/IHarness.cs @@ -43,6 +43,7 @@ namespace Xharness { string MONO_IOS_SDK_DESTDIR { get; } string MONO_MAC_SDK_DESTDIR { get; } bool ENABLE_XAMARIN { get; } + bool ENABLE_DOTNET { get; } string XcodeRoot { get; } string LogDirectory { get; } double Timeout { get; } diff --git a/tests/xharness/Jenkins/NUnitTestTasksEnumerable.cs b/tests/xharness/Jenkins/NUnitTestTasksEnumerable.cs index a0ea4e79b2..4badf3d592 100644 --- a/tests/xharness/Jenkins/NUnitTestTasksEnumerable.cs +++ b/tests/xharness/Jenkins/NUnitTestTasksEnumerable.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections; using System.Collections.Generic; using System.IO; diff --git a/tests/xharness/Jenkins/TestSelector.cs b/tests/xharness/Jenkins/TestSelector.cs index a6e65e21fa..a53cb34777 100644 --- a/tests/xharness/Jenkins/TestSelector.cs +++ b/tests/xharness/Jenkins/TestSelector.cs @@ -327,6 +327,11 @@ namespace Xharness.Jenkins { MainLog.WriteLine ("The macOS build is disabled, so any macOS tests will be disabled as well."); jenkins.IncludeMac = false; } + + if (!Harness.ENABLE_DOTNET) { + MainLog.WriteLine ("The .NET build is disabled, so any .NET tests will be disabled as well."); + jenkins.IncludeDotNet = false; + } } } } diff --git a/tests/xharness/Jenkins/TestTasks/BuildTool.cs b/tests/xharness/Jenkins/TestTasks/BuildTool.cs index 80a4f7e8ac..32385e5ee5 100644 --- a/tests/xharness/Jenkins/TestTasks/BuildTool.cs +++ b/tests/xharness/Jenkins/TestTasks/BuildTool.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Threading.Tasks; using Microsoft.DotNet.XHarness.Common.Execution; using Microsoft.DotNet.XHarness.Common.Logging; @@ -14,6 +15,7 @@ namespace Xharness.Jenkins.TestTasks { public bool SpecifyPlatform { get; set; } = true; public bool SpecifyConfiguration { get; set; } = true; + public List Constants { get; } = new List (); public BuildTool (IProcessManager processManager) { diff --git a/tests/xharness/Jenkins/TestTasks/BuildToolTask.cs b/tests/xharness/Jenkins/TestTasks/BuildToolTask.cs index e4e914648c..f8921270e1 100644 --- a/tests/xharness/Jenkins/TestTasks/BuildToolTask.cs +++ b/tests/xharness/Jenkins/TestTasks/BuildToolTask.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Threading.Tasks; using Microsoft.DotNet.XHarness.Common.Execution; using Microsoft.DotNet.XHarness.Common.Logging; @@ -33,6 +34,10 @@ namespace Xharness.Jenkins.TestTasks { set => buildToolTask.SpecifyConfiguration = value; } + public List Constants { + get => buildToolTask.Constants; + } + public override TestProject TestProject { get => base.TestProject; set { diff --git a/tests/xharness/Jenkins/TestTasks/MSBuild.cs b/tests/xharness/Jenkins/TestTasks/MSBuild.cs index a8ae46b40d..d7cf3d8880 100644 --- a/tests/xharness/Jenkins/TestTasks/MSBuild.cs +++ b/tests/xharness/Jenkins/TestTasks/MSBuild.cs @@ -32,6 +32,8 @@ namespace Xharness.Jenkins.TestTasks { if (Platform == TestPlatform.MacCatalyst) args.Add ("/r"); args.Add (projectFile); + if (Constants.Count > 0) + args.Add($"/p:DefineConstants=\"{string.Join(";", Constants)}\""); return args; } From b4264689c8ac212f904086d411b229a4780cf828 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Thu, 20 May 2021 09:41:23 -0400 Subject: [PATCH 16/19] [Make] Use stable for completeness. (#11505) --- Make.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Make.config b/Make.config index f6feb078d7..9fab156933 100644 --- a/Make.config +++ b/Make.config @@ -127,8 +127,8 @@ MACCATALYST_NUGET_VERSION_FULL=$(MACCATALYST_NUGET_VERSION_NO_METADATA)+$(NUGET_ # Xcode version should have both a major and a minor version (even if the minor version is 0) XCODE_VERSION=12.5 -XCODE_URL=https://dl.internalx.com/internal-files/xcodes/Xcode_12.5_Release_Candidate.xip -XCODE_DEVELOPER_ROOT=/Applications/Xcode_12.5.0-rc.app/Contents/Developer +XCODE_URL=https://dl.internalx.com/internal-files/xcodes/Xcode_12.5.xip +XCODE_DEVELOPER_ROOT=/Applications/Xcode_12.5.0.app/Contents/Developer XCODE_PRODUCT_BUILD_VERSION:=$(shell /usr/libexec/PlistBuddy -c 'Print :ProductBuildVersion' $(XCODE_DEVELOPER_ROOT)/../version.plist) # Mono version embedded in XI/XM (NEEDED_MONO_VERSION/BRANCH) are specified in mk/mono.mk From e30179c81930fa3e9b2166f1feb670a7831f9d92 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Thu, 20 May 2021 09:42:03 -0400 Subject: [PATCH 17/19] [CI] Create branch in cascade pipeline for the html of a given build. (#11621) The branch name will be: * pr/branchname/branch_commit/buildid - for builds triggered by a pr * ci/branchname/branch_commit/buildid - for all others. Commit and build id have been added to ensure we have no collisions. --- .../templates/publish-html-result.yml | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/tools/devops/automation/templates/publish-html-result.yml b/tools/devops/automation/templates/publish-html-result.yml index 12904bdc0c..7a7d416d91 100644 --- a/tools/devops/automation/templates/publish-html-result.yml +++ b/tools/devops/automation/templates/publish-html-result.yml @@ -19,9 +19,10 @@ steps: path: $(Build.SourcesDirectory)/build-configuration - pwsh: | - $configFile = Join-Path $(Build.SourcesDirectory) build-configuration + $configFile = Join-Path $(Build.SourcesDirectory) "build-configuration" "configuration.json" $config = Get-Content $configFile | ConvertFrom-Json # export variables to be present in the othe steps + Write-Host "##vso[task.setvariable variable=BuildReason;isOutput=true]$($config.BuildReason)" Write-Host "##vso[task.setvariable variable=BuildSourceBranchName;isOutput=true]$($config.BuildSourceBranchName)" Write-Host "##vso[task.setvariable variable=BuildSourceBranch;isOutput=true]$($config.BuildSourceBranch)" Write-Host "##vso[task.setvariable variable=BuildId;isOutput=true]$($config.BuildId)" @@ -31,6 +32,26 @@ steps: timeoutInMinutes: 1 - pwsh: | - Write-Host "Work in progress" + git config user.email "valco@microsoft.com" + git config user.name "vs-mobiletools-engineering-service2" + + $branchName = "$Env:OriginalSourceBranch/$Env:OriginalCommit/$Env:OriginalBuildId" + if (Env:OriginalBuildReason -eq "PullRequest") { + $branchName = "pr/$branchName" + } else { + $branchName = "ci/$branchName" + } + + Write-Host "New branch name: $branchName" + $htmlRepo = Join-Path $(Build.SourcesDirectory) "macios.ci" + cd $htmlRepo + git checkout -b $branchName displayName: 'Create remote branch' timeoutInMinutes: 1 + env: + OriginalBuildReason: $(configuration.BuildReason) + OriginalSourceBranch: $(configuration.BuildSourceBranchName) + OriginalBuildId: $(configuration.BuildId) + OriginalCommit: $(configuration.Commit) + + From dc3cf56cad4ef025c7f938fb317d529d2633bb15 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Thu, 20 May 2021 16:02:25 +0200 Subject: [PATCH 18/19] [src] Fix revision part of version string to have matching parenthesis. (#11632) Before: $ grep Revision build/*/Constants.cs build/ios/Constants.cs: internal const string Revision = "170 (main: cdef4823199"; build/mac/Constants.cs: internal const string Revision = "170 (main: cdef4823199"; build/maccatalyst/Constants.cs: internal const string Revision = "170 (main: cdef4823199"; build/tvos/Constants.cs: internal const string Revision = "170 (main: cdef4823199"; build/watch/Constants.cs: internal const string Revision = "170 (main: cdef4823199"; After: $ grep Revision build/*/Constants.cs build/ios/Constants.cs: internal const string Revision = "170 (main: 746748bec2d)"; build/mac/Constants.cs: internal const string Revision = "170 (main: 746748bec2d)"; build/maccatalyst/Constants.cs: internal const string Revision = "170 (main: 746748bec2d)"; build/tvos/Constants.cs: internal const string Revision = "170 (main: 746748bec2d)"; build/watch/Constants.cs: internal const string Revision = "170 (main: 746748bec2d)"; There's a closing parenthesis at the end now. --- src/Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Makefile b/src/Makefile index 7093cc90bc..cb6fb893d3 100644 --- a/src/Makefile +++ b/src/Makefile @@ -140,7 +140,7 @@ IOS_CORE_DEFINES=-define:COREBUILD $(IOS_DEFINES) $(IOS_BUILD_DIR)/Constants.cs: Constants.iOS.cs.in Makefile $(TOP)/Make.config.inc | $(IOS_BUILD_DIR) $(call Q_PROF_GEN,ios) sed \ -e "s/@VERSION@/$(IOS_PACKAGE_VERSION_MAJOR).$(IOS_PACKAGE_VERSION_MINOR).$(IOS_PACKAGE_VERSION_REV)/g" \ - -e 's/@REVISION@/$(IOS_COMMIT_DISTANCE) ($(CURRENT_BRANCH_SED_ESCAPED): $(shell git log -1 --pretty=%h)/g' \ + -e 's/@REVISION@/$(IOS_COMMIT_DISTANCE) ($(CURRENT_BRANCH_SED_ESCAPED): $(shell git log -1 --pretty=%h))/g' \ -e "s/@IOS_SDK_VERSION@/$(IOS_SDK_VERSION)/g" \ $< > $@ @@ -479,7 +479,7 @@ MAC_HTTP_SOURCES = \ $(MAC_BUILD_DIR)/Constants.cs: Constants.mac.cs.in Makefile $(TOP)/Make.config.inc | $(MAC_BUILD_DIR) $(Q) sed \ -e "s/@VERSION@/$(MAC_PACKAGE_VERSION_MAJOR).$(MAC_PACKAGE_VERSION_MINOR).$(MAC_PACKAGE_VERSION_REV)/g" \ - -e 's/@REVISION@/$(MAC_COMMIT_DISTANCE) ($(CURRENT_BRANCH_SED_ESCAPED): $(shell git log -1 --pretty=%h)/g' \ + -e 's/@REVISION@/$(MAC_COMMIT_DISTANCE) ($(CURRENT_BRANCH_SED_ESCAPED): $(shell git log -1 --pretty=%h))/g' \ -e "s/@OSX_SDK_VERSION@/$(OSX_SDK_VERSION)/g" \ -e "s/@MIN_XM_MONO_VERSION@/$(MIN_XM_MONO_VERSION)/g" \ $< > $@ @@ -769,7 +769,7 @@ WATCHOS_SOURCES += \ $(WATCH_BUILD_DIR)/Constants.cs: $(TOP)/src/Constants.watch.cs.in Makefile $(TOP)/Make.config.inc | $(WATCH_BUILD_DIR) $(call Q_PROF_GEN,watch) sed \ -e "s/@VERSION@/$(IOS_PACKAGE_VERSION_MAJOR).$(IOS_PACKAGE_VERSION_MINOR).$(IOS_PACKAGE_VERSION_REV)/g" \ - -e 's/@REVISION@/$(IOS_COMMIT_DISTANCE) ($(CURRENT_BRANCH_SED_ESCAPED): $(shell git log -1 --pretty=%h)/g' \ + -e 's/@REVISION@/$(IOS_COMMIT_DISTANCE) ($(CURRENT_BRANCH_SED_ESCAPED): $(shell git log -1 --pretty=%h))/g' \ -e "s/@WATCH_SDK_VERSION@/$(WATCH_SDK_VERSION)/g" \ $< > $@ @@ -1058,7 +1058,7 @@ TVOS_SOURCES += \ $(TVOS_BUILD_DIR)/Constants.cs: $(TOP)/src/Constants.tvos.cs.in Makefile $(TOP)/Make.config.inc | $(TVOS_BUILD_DIR) $(call Q_PROF_GEN,tvos) sed \ -e "s/@VERSION@/$(IOS_PACKAGE_VERSION_MAJOR).$(IOS_PACKAGE_VERSION_MINOR).$(IOS_PACKAGE_VERSION_REV)/g" \ - -e 's/@REVISION@/$(IOS_COMMIT_DISTANCE) ($(CURRENT_BRANCH_SED_ESCAPED): $(shell git log -1 --pretty=%h)/g' \ + -e 's/@REVISION@/$(IOS_COMMIT_DISTANCE) ($(CURRENT_BRANCH_SED_ESCAPED): $(shell git log -1 --pretty=%h))/g' \ -e "s/@TVOS_SDK_VERSION@/$(TVOS_SDK_VERSION)/g" \ $< > $@ @@ -1268,7 +1268,7 @@ MACCATALYST_SOURCES += \ $(MACCATALYST_BUILD_DIR)/Constants.cs: $(TOP)/src/Constants.maccatalyst.cs.in Makefile $(TOP)/Make.config.inc | $(MACCATALYST_BUILD_DIR) $(call Q_PROF_GEN,maccatalyst) sed \ -e "s/@VERSION@/$(IOS_PACKAGE_VERSION_MAJOR).$(IOS_PACKAGE_VERSION_MINOR).$(IOS_PACKAGE_VERSION_REV)/g" \ - -e 's/@REVISION@/$(IOS_COMMIT_DISTANCE) ($(CURRENT_BRANCH_SED_ESCAPED): $(shell git log -1 --pretty=%h)/g' \ + -e 's/@REVISION@/$(IOS_COMMIT_DISTANCE) ($(CURRENT_BRANCH_SED_ESCAPED): $(shell git log -1 --pretty=%h))/g' \ -e "s/@MACCATALYST_SDK_VERSION@/$(MACCATALYST_SDK_VERSION)/g" \ $< > $@ From 1762888e1dc0105e9b17adb504bbeda78cee89b7 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Thu, 20 May 2021 16:12:21 +0200 Subject: [PATCH 19/19] [tests] Build the custom-type-assembly separately for .NET (#11629) It's a bit weird to build it twice, but it's the most straight forward way of making the test work if either the .NET build or the legacy build is disabled. --- .../monotouch-test/ObjCRuntime/RegistrarTest.cs | 4 ++++ .../test-libraries/custom-type-assembly/Makefile | 16 ++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/monotouch-test/ObjCRuntime/RegistrarTest.cs b/tests/monotouch-test/ObjCRuntime/RegistrarTest.cs index 6e4627f999..9745de6971 100644 --- a/tests/monotouch-test/ObjCRuntime/RegistrarTest.cs +++ b/tests/monotouch-test/ObjCRuntime/RegistrarTest.cs @@ -2154,7 +2154,11 @@ namespace MonoTouchFixtures.ObjCRuntime { [Test] public void CustomUserTypeWithDynamicallyLoadedAssembly () { +#if NET + var customTypeAssemblyPath = global::System.IO.Path.Combine (global::Xamarin.Tests.Configuration.RootPath, "tests", "test-libraries", "custom-type-assembly", ".libs", "dotnet", "macos", "custom-type-assembly.dll"); +#else var customTypeAssemblyPath = global::System.IO.Path.Combine (global::Xamarin.Tests.Configuration.RootPath, "tests", "test-libraries", "custom-type-assembly", ".libs", "macos", "custom-type-assembly.dll"); +#endif Assert.That (customTypeAssemblyPath, Does.Exist, "existence"); var size = 10; diff --git a/tests/test-libraries/custom-type-assembly/Makefile b/tests/test-libraries/custom-type-assembly/Makefile index a80139e344..c4f54c15c6 100644 --- a/tests/test-libraries/custom-type-assembly/Makefile +++ b/tests/test-libraries/custom-type-assembly/Makefile @@ -3,12 +3,24 @@ TOP=../../.. include $(TOP)/Make.config .libs/macos/custom-type-assembly.dll: custom-type-assembly.cs Makefile | .libs/macos - $(MAC_mobile_CSC) $< -out:$@ -r:$(TOP)/_mac-build/Library/Frameworks/Xamarin.Mac.framework/Versions/Current/lib/mono/Xamarin.Mac/Xamarin.Mac.dll -target:library + $(Q_CSC) $(MAC_mobile_CSC) $< -out:$@ -r:$(TOP)/_mac-build/Library/Frameworks/Xamarin.Mac.framework/Versions/Current/lib/mono/Xamarin.Mac/Xamarin.Mac.dll -target:library -.libs/macos: +.libs/dotnet/macos/custom-type-assembly.dll: custom-type-assembly.cs Makefile | .libs/dotnet/macos + $(Q_CSC) $(DOTNET6_CSC) $< -out:$@ -r:$(TOP)/_build/Microsoft.macOS.Ref/ref/net6.0/Xamarin.Mac.dll -target:library -r:$(DOTNET6_BCL_DIR)/System.Runtime.dll /nologo + +.libs/macos .libs/dotnet/macos: $(Q) mkdir -p $@ +ifdef INCLUDE_XAMARIN_LEGACY TARGETS += \ .libs/macos/custom-type-assembly.dll \ +endif + +ifdef ENABLE_DOTNET +TARGETS += \ + .libs/dotnet/macos/custom-type-assembly.dll \ + +endif + build-assembly: $(TARGETS)