From 85a50c8099731971f8dd7c927e683e83f5462ea8 Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Thu, 28 Oct 2021 16:25:25 -0400 Subject: [PATCH] [ci] Improve build and test result packaging (#6411) Attempt to improve our ability to analyze build and test failures with a handful of changes. The Xamarin.Android.Build.Tests and MSBuildDeviceIntegration test suites will now generate and build test projects in a [temporary directory][0] when running on CI. A timestamped directory has been appended to this new test output path, to serve as an extra precaution for identifying any potentially old results. However, the docs mention that the output folder should always be clean at the start of each job: Note: Build.ArtifactStagingDirectory and Build.StagingDirectory are interchangeable. This directory is purged before each new build, so you don't have to clean it up yourself. The result packaging MSBuild targets have been replaced with a step in the xaprepare tool. We will now upload an unzipped directory structure that can be expanded and viewed in the pipelines artifact tab. This should make it easier to download an individual log file for a failing test without having to download zip (or in some cases nested zip) files. Finally, we will now only upload results when a job has a failing step. This should make it easier to quickly find the artifact that corresponds to a failing job, as we will no longer have to search through dozens of similarly named artifacts for a single test failure. [0]: https://docs.microsoft.com/en-us/azure/devops/pipelines/build/variables?view=azure-devops&tabs=yaml#build-variables-devops-services --- ...amarin.Android.Tools.BootstrapTasks.csproj | 2 - .../result-packaging.targets | 102 ------------- .../automation/azure-pipelines-nightly.yaml | 3 +- .../automation/azure-pipelines-oss.yaml | 11 +- build-tools/automation/azure-pipelines.yaml | 12 +- .../yaml-templates/upload-results.yaml | 43 ++++-- build-tools/scripts/Packaging.mk | 11 -- .../Application/GeneratedMakeRulesFile.cs | 10 -- .../xaprepare/ConfigAndData/Configurables.cs | 56 ------- .../Scenario_CopyExtraResultFilesForCI.cs | 26 ++++ .../Steps/Step_CopyExtraResultFilesForCI.cs | 139 ++++++++++++++++++ .../CodeBehindTests.cs | 2 +- .../EmbeddedDSOTests.cs | 2 +- .../MakeBundleTests.cs | 4 +- .../Tasks/GeneratePackageManagerJavaTests.cs | 3 +- .../Tasks/LinkerTests.cs | 4 +- .../Tasks/ManagedResourceParserTests.cs | 8 +- .../Utilities/BaseTest.cs | 11 +- .../Common/XamarinProject.cs | 2 +- .../Utilities/FileSystemUtils.cs | 2 +- .../Xamarin.ProjectTools/XABuildPaths.cs | 23 ++- .../Xamarin.Android.MakeBundle-Tests.csproj | 4 +- .../EmbeddedDSO/EmbeddedDSO.csproj | 6 +- .../Tests/PerformanceTest.cs | 1 + 24 files changed, 263 insertions(+), 224 deletions(-) delete mode 100644 build-tools/Xamarin.Android.Tools.BootstrapTasks/result-packaging.targets create mode 100644 build-tools/xaprepare/xaprepare/Scenarios/Scenario_CopyExtraResultFilesForCI.cs create mode 100644 build-tools/xaprepare/xaprepare/Steps/Step_CopyExtraResultFilesForCI.cs diff --git a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj index fce090164..f9dbf9a49 100644 --- a/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj +++ b/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj @@ -19,6 +19,4 @@ - - diff --git a/build-tools/Xamarin.Android.Tools.BootstrapTasks/result-packaging.targets b/build-tools/Xamarin.Android.Tools.BootstrapTasks/result-packaging.targets deleted file mode 100644 index 1bfc641e7..000000000 --- a/build-tools/Xamarin.Android.Tools.BootstrapTasks/result-packaging.targets +++ /dev/null @@ -1,102 +0,0 @@ - - - - - - <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)Configuration.OperatingSystem.props" Condition="Exists('$(XamarinAndroidSourcePath)Configuration.OperatingSystem.props')" /> - <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)Configuration.Override.props" Condition="Exists('$(XamarinAndroidSourcePath)Configuration.Override.props')" /> - <_BuildStatusFiles Include="$(BootstrapOutputDirectory)XABuildConfig.cs" Condition="Exists('$(BootstrapOutputDirectory)XABuildConfig.cs')" /> - <_BuildStatusFiles Include="$(BootstrapOutputDirectory)*.binlog" /> - <_BuildStatusFiles Include="$(BootstrapOutputDirectory)prepare*.log" /> - <_BuildStatusFiles Include="$(BootstrapOutputDirectory)*.json" /> - <_BuildStatusFiles Include="$(BootstrapOutputDirectory)*.mk" /> - <_BuildStatusFiles Include="$(BootstrapOutputDirectory)*.projitems" /> - <_BuildStatusFiles Include="$(BootstrapOutputDirectory)*.cmake" /> - <_BuildStatusFiles Include="$(BootstrapOutputDirectory)*.targets" /> - <_BuildStatusFiles Include="$(TestOutputDirectory)XABuildConfig.cs" Condition="Exists('$(TestOutputDirectory)XABuildConfig.cs')" /> - <_BuildStatusFiles Include="$(TestOutputDirectory)*.binlog" /> - <_BuildStatusFiles Include="$(TestOutputDirectory)prepare*.log" /> - <_BuildStatusFiles Include="$(TestOutputDirectory)*.mk" /> - <_BuildStatusFiles Include="$(TestOutputDirectory)*.projitems" /> - <_BuildStatusFiles Include="$(TestOutputDirectory)*.cmake" /> - <_BuildStatusFiles Include="$(TestOutputDirectory)*.targets" /> - <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)external\Java.Interop\bin\Build$(Configuration)\*.props" /> - <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)**\ThirdPartyNotices.txt" /> - <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)**\config.log" /> - <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)**\config.status" /> - <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)**\config.h" /> - <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)**\CMakeCache.txt" /> - <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)**\CMakeFiles\*.log" /> - <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)**\.ninja_log" /> - <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)**\android-*.config.cache" /> - <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)\bin\Build$(Configuration)\clang-tidy*.log" /> - <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)\src\monodroid\jni\*.include.generated" /> - <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)\src\monodroid\jni\*.include.diff" /> - - - <_TestResultFiles Include="$(XamarinAndroidSourcePath)TestResult-*.xml" /> - <_TestResultFiles Include="$(TestOutputDirectory)*.apkdesc" /> - <_TestResultFiles Include="$(TestOutputDirectory)*.aabdesc" /> - <_TestResultFiles Include="$(TestOutputDirectory)TestResult-*.xml" /> - <_TestResultFiles Include="$(TestOutputDirectory)compatibility\*" /> - <_TestResultFiles Include="$(TestOutputDirectory)logcat*" /> - <_TestResultFiles Include="$(TestOutputDirectory)*log" /> - <_TestResultFiles Include="$(TestOutputDirectory)temp\**\*" Exclude="$(TestOutputDirectory)temp\packages\**"> - temp\ - - <_TestResultFiles Include="$(TestOutputDirectory)TestOutput-*.txt" /> - <_TestResultFiles Include="$(TestOutputDirectory)Timing_*" /> - <_TestResultFiles Include="$(XamarinAndroidSourcePath)*.csv" /> - <_TestResultFiles Include="$(TEMP)\llc.exe-*" /> - - - - <_ResultSuffix>$(ProductVersion).$(XAVersionCommitCount)_$(XAVersionHash)-$(HostOS)-$(Configuration) - $(XamarinAndroidSourcePath)bin\Build$(Configuration) - $(XamarinAndroidSourcePath)bin\Test$(Configuration) - xa-build-status-$(_ResultSuffix) - xa-test-results-$(_ResultSuffix) - - - - - - - <_BuildStatusFilesToZip Include="$(BuildStatusZipOutputPath)\$(BuildStatusZipName)\**\*" /> - - - - - - - - - <_TestResultFilesToZip Include="$(TestResultZipOutputPath)\$(TestResultZipName)\**\*" /> - - - - - diff --git a/build-tools/automation/azure-pipelines-nightly.yaml b/build-tools/automation/azure-pipelines-nightly.yaml index 31edc90cf..78f4fb21e 100644 --- a/build-tools/automation/azure-pipelines-nightly.yaml +++ b/build-tools/automation/azure-pipelines-nightly.yaml @@ -64,8 +64,9 @@ stages: - template: yaml-templates/upload-results.yaml parameters: - solution: $(System.DefaultWorkingDirectory)/xamarin-android/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj + xaSourcePath: $(System.DefaultWorkingDirectory)/xamarin-android artifactName: Build Results - Nightly macOS + includeBuildResults: true - stage: test displayName: Test diff --git a/build-tools/automation/azure-pipelines-oss.yaml b/build-tools/automation/azure-pipelines-oss.yaml index 1ab69ebae..1c7814fce 100644 --- a/build-tools/automation/azure-pipelines-oss.yaml +++ b/build-tools/automation/azure-pipelines-oss.yaml @@ -130,12 +130,6 @@ stages: workingDirectory: $(Build.SourcesDirectory) displayName: make all-tests - - script: > - echo "make package-build-status CONFIGURATION=$(XA.Build.Configuration)" && - make package-build-status CONFIGURATION=$(XA.Build.Configuration) - workingDirectory: $(Build.SourcesDirectory) - displayName: package build status - - script: > echo "make run-performance-tests CONFIGURATION=$(XA.Build.Configuration)" && make run-performance-tests CONFIGURATION=$(XA.Build.Configuration) @@ -145,7 +139,8 @@ stages: - template: yaml-templates/upload-results.yaml parameters: - artifactName: Build Results - macOS + includeBuildResults: true + artifactName: OSS Build Results - macOS - stage: linux_stage displayName: Linux @@ -199,5 +194,5 @@ stages: - template: yaml-templates/upload-results.yaml parameters: - configuration: $(XA.Build.Configuration) artifactName: OSS Build Results - Linux + includeBuildResults: true diff --git a/build-tools/automation/azure-pipelines.yaml b/build-tools/automation/azure-pipelines.yaml index 92b936f85..f32b9e8dd 100644 --- a/build-tools/automation/azure-pipelines.yaml +++ b/build-tools/automation/azure-pipelines.yaml @@ -138,8 +138,9 @@ stages: - template: yaml-templates/upload-results.yaml parameters: - solution: $(System.DefaultWorkingDirectory)/xamarin-android/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj + xaSourcePath: $(System.DefaultWorkingDirectory)/xamarin-android artifactName: Build Results - macOS + includeBuildResults: true - template: yaml-templates/run-xaprepare.yaml parameters: @@ -270,6 +271,7 @@ stages: - template: yaml-templates\upload-results.yaml parameters: artifactName: Build Results - Windows + includeBuildResults: true - template: yaml-templates\fail-on-issue.yaml @@ -382,6 +384,7 @@ stages: - template: yaml-templates\upload-results.yaml parameters: artifactName: Build Results - Windows DotNet + includeBuildResults: true - template: yaml-templates\fail-on-issue.yaml @@ -463,8 +466,9 @@ stages: - template: yaml-templates/upload-results.yaml parameters: - solution: $(System.DefaultWorkingDirectory)/xamarin-android/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj + xaSourcePath: $(System.DefaultWorkingDirectory)/xamarin-android artifactName: Build Results - Linux + includeBuildResults: true - template: yaml-templates\fail-on-issue.yaml @@ -819,7 +823,6 @@ stages: - template: yaml-templates/upload-results.yaml parameters: - configuration: $(XA.Build.Configuration) artifactName: Test Results - APKs .NET - macOS - task: MSBuild@1 @@ -1580,8 +1583,9 @@ stages: - template: yaml-templates/upload-results.yaml parameters: - solution: $(System.DefaultWorkingDirectory)/xamarin-android/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj + xaSourcePath: $(System.DefaultWorkingDirectory)/xamarin-android artifactName: Legacy Notarization Results - macOS + includeBuildResults: true # Check - "Xamarin.Android (Finalize Installers Queue Vsix Signing)" - job: queue_vsix_signing diff --git a/build-tools/automation/yaml-templates/upload-results.yaml b/build-tools/automation/yaml-templates/upload-results.yaml index 52df9946c..227794634 100644 --- a/build-tools/automation/yaml-templates/upload-results.yaml +++ b/build-tools/automation/yaml-templates/upload-results.yaml @@ -1,18 +1,37 @@ parameters: - solution: build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj + xaSourcePath: $(System.DefaultWorkingDirectory) configuration: $(XA.Build.Configuration) artifactName: results + includeBuildResults: false steps: -- task: MSBuild@1 - displayName: package build and test results - inputs: - solution: ${{ parameters.solution }} - configuration: ${{ parameters.configuration }} - msbuildArguments: /restore /t:Build,ZipBuildStatus,ZipTestResults /p:BuildStatusZipOutputPath=$(Build.ArtifactStagingDirectory) /p:TestResultZipOutputPath=$(Build.ArtifactStagingDirectory) - condition: always() - -- template: publish-artifact.yaml +- template: run-xaprepare.yaml parameters: - displayName: upload build and test results - artifactName: ${{ parameters.artifactName }} + configuration: ${{ parameters.configuration }} + arguments: --s=CopyExtraResultFilesForCI --verbosity v + xaSourcePath: ${{ parameters.xaSourcePath }} + displayName: Copy extra result files + condition: ne(variables['Agent.JobStatus'], 'Succeeded') + +- ${{ if eq(parameters.includeBuildResults, false) }}: + - template: publish-artifact.yaml + parameters: + displayName: upload test results + artifactName: ${{ parameters.artifactName }} + targetPath: $(Build.StagingDirectory)/Test${{ parameters.configuration }} + condition: ne(variables['Agent.JobStatus'], 'Succeeded') + +# Copy Build$(Configuration) folder into test result root and upload single artifact +- ${{ if eq(parameters.includeBuildResults, true) }}: + - task: CopyFiles@2 + inputs: + sourceFolder: $(Build.StagingDirectory)/Build${{ parameters.configuration }} + targetFolder: $(Build.StagingDirectory)/Test${{ parameters.configuration }}/Build${{ parameters.configuration }} + condition: ne(variables['Agent.JobStatus'], 'Succeeded') + + - template: publish-artifact.yaml + parameters: + displayName: upload build and test results + artifactName: ${{ parameters.artifactName }} + targetPath: $(Build.StagingDirectory)/Test${{ parameters.configuration }} + condition: ne(variables['Agent.JobStatus'], 'Succeeded') diff --git a/build-tools/scripts/Packaging.mk b/build-tools/scripts/Packaging.mk index 24ef8a0e4..4d199b176 100644 --- a/build-tools/scripts/Packaging.mk +++ b/build-tools/scripts/Packaging.mk @@ -78,14 +78,3 @@ package-deb: $(ZIP_OUTPUT) cp LICENSE $(ZIP_OUTPUT_BASENAME)/debian/copyright cd $(ZIP_OUTPUT_BASENAME) && DEBEMAIL="Xamarin Public Jenkins (auto-signing) " dch --create -v $(PRODUCT_VERSION).$(-num-commits-since-version-change) --package xamarin.android-oss --force-distribution --distribution alpha "New release - please see git log for $(GIT_COMMIT)" cd $(ZIP_OUTPUT_BASENAME) && dpkg-buildpackage -us -uc -rfakeroot - - -_RESULT_PACKAGE_SUFFIX = -v$(PRODUCT_VERSION).$(-num-commits-since-version-change)_$(OS_NAME)-$(OS_ARCH)_$(GIT_BRANCH)_$(GIT_COMMIT)-$(CONFIGURATION) - -"xa-test-results$(_RESULT_PACKAGE_SUFFIX).zip" package-test-results: - msbuild build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj /t:ZipTestResults \ - /p:Configuration=$(CONFIGURATION) /p:TestResultZipName="xa-test-results$(_RESULT_PACKAGE_SUFFIX)" - -"xa-build-status$(_RESULT_PACKAGE_SUFFIX).zip" package-build-status: - msbuild build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj /t:ZipBuildStatus \ - /p:Configuration=$(CONFIGURATION) /p:BuildStatusZipName="xa-build-status$(_RESULT_PACKAGE_SUFFIX)" diff --git a/build-tools/xaprepare/xaprepare/Application/GeneratedMakeRulesFile.cs b/build-tools/xaprepare/xaprepare/Application/GeneratedMakeRulesFile.cs index 5f368bb45..832ed29a3 100644 --- a/build-tools/xaprepare/xaprepare/Application/GeneratedMakeRulesFile.cs +++ b/build-tools/xaprepare/xaprepare/Application/GeneratedMakeRulesFile.cs @@ -51,12 +51,7 @@ namespace Xamarin.Android.Prepare WriteVariable ("ZIP_EXTENSION", context.OS.ZipExtension); WriteVariable ("ZIP_OUTPUT_BASENAME", GetOutputFileName (context, "xamarin.android-oss")); - WriteVariable ("_TEST_RESULTS_BASENAME", GetOutputFileName (context, "xa-test-results")); - WriteVariable ("_BUILD_STATUS_BASENAME", GetOutputFileName (context, "xa-build-status")); - WriteVariable ("ZIP_OUTPUT", "$(ZIP_OUTPUT_BASENAME).$(ZIP_EXTENSION)"); - WriteVariable ("_BUILD_STATUS_ZIP_OUTPUT", "$(_BUILD_STATUS_BASENAME).$(ZIP_EXTENSION)"); - WriteVariable ("_TEST_RESULTS_ZIP_OUTPUT", "$(_TEST_RESULTS_BASENAME).$(ZIP_EXTENSION)"); var allApiLevels = new List (); var allPlatformIDs = new List (); @@ -118,11 +113,6 @@ namespace Xamarin.Android.Prepare WriteListVariable ("_BUNDLE_ZIPS_INCLUDE", Configurables.Defaults.BundleZipsInclude); WriteListVariable ("_BUNDLE_ZIPS_EXCLUDE", Configurables.Defaults.BundleZipsExclude); - WriteListVariable ("_TEST_RESULTS_BUNDLE_INCLUDE", Configurables.Defaults.TestResultsBundleInclude); - WriteListVariable ("_TEST_RESULTS_BUNDLE_EXCLUDE", Configurables.Defaults.TestResultsBundleExclude); - WriteListVariable ("_BUILD_STATUS_BUNDLE_INCLUDE", Configurables.Defaults.BuildStatusBundleInclude); - WriteListVariable ("_BUILD_STATUS_BUNDLE_INCLUDE", Configurables.Defaults.BuildStatusBundleIncludeConditional, true); - WriteListVariable ("_BUILD_STATUS_BUNDLE_EXCLUDE", Configurables.Defaults.BuildStatusBundleExclude); sw.WriteLine (); sw.WriteLine (".PHONY: framework-assemblies"); diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs index faafdfe65..588b8db7d 100644 --- a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs +++ b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs @@ -218,62 +218,6 @@ namespace Xamarin.Android.Prepare "$(ZIP_OUTPUT_BASENAME)/bin/*/bundle-*.zip" }; - /// - /// Used in rules.mk generator. Files to include in test results bundle. Must be syntactically - /// correct for GNU Make. - /// - public static readonly List TestResultsBundleInclude = new List { - "$(wildcard TestResult-*.xml)", - "$(wildcard bin/Test$(CONFIGURATION)/compatibility)", - "$(wildcard bin/Test$(CONFIGURATION)/logcat*)", - "$(wildcard bin/Test$(CONFIGURATION)/msbuild*.binlog*)", - "$(wildcard bin/Test$(CONFIGURATION)/temp)", - "$(wildcard bin/Test$(CONFIGURATION)/EmbeddedDSO)", - "$(wildcard bin/Test$(CONFIGURATION)/CodeBehind)", - "$(wildcard bin/Test$(CONFIGURATION)/TestOutput-*.txt)", - "$(wildcard bin/Test$(CONFIGURATION)/Timing_*)", - "$(wildcard *.csv)", - }; - - /// - /// Used in rules.mk generator. Files to exclude from the test results bundle. Must be syntactically - /// correct for GNU Make. - /// - public static readonly List TestResultsBundleExclude = new List { - }; - - /// - /// Used in rules.mk generator. Files to include in build status bundle archive. Must be syntactically - /// correct for GNU Make. - /// - public static readonly List BuildStatusBundleInclude = new List { - "Configuration.OperatingSystem.props", - "$(wildcard bin/Build$(CONFIGURATION)/msbuild*.binlog)", - "$(shell find . -name 'config.log')", - "$(shell find . -name 'config.status')", - "$(shell find . -name 'config.h')", - "$(shell find . -name 'CMakeCache.txt')", - "$(shell find . -name 'config.h')", - "$(shell find . -name '.ninja_log')", - "$(shell find . -name 'android-*.config.cache')", - "bin/Build$(CONFIGURATION)/XABuildConfig.cs", - }; - - /// - /// Used in rules.mk generator. Optional files to include in the build status bundle (included only if - /// they exist). Must be syntactically correct for GNU Make. - /// - public static readonly List BuildStatusBundleIncludeConditional = new List { - "Configuration.Override.props", - }; - - /// - /// Used in rules.mk generator. Files to exclude from the build status bundle. Must be syntactically - /// correct for GNU Make. - /// - public static readonly List BuildStatusBundleExclude = new List { - }; - public static readonly List NDKTools = new List { new NDKTool (name: "as"), new NDKTool (name: "ld"), diff --git a/build-tools/xaprepare/xaprepare/Scenarios/Scenario_CopyExtraResultFilesForCI.cs b/build-tools/xaprepare/xaprepare/Scenarios/Scenario_CopyExtraResultFilesForCI.cs new file mode 100644 index 000000000..fc426dc61 --- /dev/null +++ b/build-tools/xaprepare/xaprepare/Scenarios/Scenario_CopyExtraResultFilesForCI.cs @@ -0,0 +1,26 @@ +using System.IO; + +namespace Xamarin.Android.Prepare +{ + [Scenario (isDefault: false)] + partial class Scenario_CopyExtraResultFilesForCI : ScenarioNoStandardEndSteps + { + public Scenario_CopyExtraResultFilesForCI () + : base ("CopyExtraResultFilesForCI", "Copy extra result files to artifact directory") + { + // Do not create a log that we would attempt to copy during this scenario. + Log.Instance.SetLogFile (Path.Combine (Context.Instance.LogDirectory, $"package-results-{Context.Instance.BuildTimeStamp}.txt")); + } + + protected override void AddSteps (Context context) + { + Steps.Add (new Step_CopyExtraResultFilesForCI ()); + + // disable installation of missing programs... + context.SetCondition (KnownConditions.AllowProgramInstallation, false); + + // ...but do not signal an error when any are missing + context.SetCondition (KnownConditions.IgnoreMissingPrograms, true); + } + } +} diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_CopyExtraResultFilesForCI.cs b/build-tools/xaprepare/xaprepare/Steps/Step_CopyExtraResultFilesForCI.cs new file mode 100644 index 000000000..2a18bfc56 --- /dev/null +++ b/build-tools/xaprepare/xaprepare/Steps/Step_CopyExtraResultFilesForCI.cs @@ -0,0 +1,139 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; + +namespace Xamarin.Android.Prepare +{ + class Step_CopyExtraResultFilesForCI : Step + { + public Step_CopyExtraResultFilesForCI () + : base ("Copying extra result files to artifact directory") + {} + + protected override async Task Execute (Context context) + { + // Set when running on Azure Pipelines https://docs.microsoft.com/en-us/azure/devops/pipelines/build/variables + var rootDir = Environment.GetEnvironmentVariable ("BUILD_STAGINGDIRECTORY"); + if (!Directory.Exists (rootDir)) + return false; + + CopyExtraTestFiles (Path.Combine (rootDir, $"Test{context.Configuration}"), context); + CopyExtraBuildFiles (Path.Combine (rootDir, $"Build{context.Configuration}"), context); + return true; + } + + string[] xaRootDirBuildFiles = { + "Configuration.OperatingSystem.props", + "Configuration.Override.props", + "ThirdPartyNotices.txt", + "config.log", + "config.status", + "config.h", + "android-*.config.cache", + }; + + string [] buildConfigFiles = { + "XABuildConfig.cs", + "*.binlog", + "prepare*log", + "*.json", + "*.mk", + "*.projitems", + "*.cmake", + "*.targets", + "CMakeCache.txt", + ".ninja_log", + "clang-tidy*.log", + }; + + void CopyExtraBuildFiles (string destinationRoot, Context context) + { + Directory.CreateDirectory (destinationRoot); + var filesToCopyPreserveRelative = new List (); + + foreach (var fileMatch in xaRootDirBuildFiles) { + filesToCopyPreserveRelative.AddRange (Directory.GetFiles (BuildPaths.XamarinAndroidSourceRoot, fileMatch, SearchOption.AllDirectories)); + } + + var cmakeFileDirs = Directory.GetDirectories (BuildPaths.XamarinAndroidSourceRoot, "CMakeFiles"); + foreach (var cmakeFileDir in cmakeFileDirs) { + filesToCopyPreserveRelative.AddRange (Directory.GetFiles (cmakeFileDir, "*.log")); + } + + var javaInteropBuildConfigDir = Path.Combine (context.Properties.GetRequiredValue (KnownProperties.JavaInteropFullPath), "bin", $"Build{context.Configuration}"); + if (Directory.Exists (javaInteropBuildConfigDir)) { + filesToCopyPreserveRelative.AddRange (Directory.GetFiles (javaInteropBuildConfigDir, "*.props")); + } + + filesToCopyPreserveRelative.AddRange (Directory.GetFiles (Path.Combine (BuildPaths.XamarinAndroidSourceRoot, "src", "monodroid", "jni"), "*.include.*")); + + var buildConfigDir = Path.Combine (BuildPaths.XamarinAndroidSourceRoot, "bin", $"Build{context.Configuration}"); + if (Directory.Exists (buildConfigDir)) { + foreach (var fileMatch in buildConfigFiles) { + Utilities.CopyFilesSimple (Directory.GetFiles (buildConfigDir, fileMatch), destinationRoot, false); + } + } + + foreach (var file in filesToCopyPreserveRelative) { + Utilities.CopyFile (file, file.Replace (BuildPaths.XamarinAndroidSourceRoot, destinationRoot), false); + } + } + + string [] testConfigFiles = { + "*.apkdesc", + "*.aabdesc", + "logcat*", + "*log", + "TestOutput-*.txt", + "Timing_*", + }; + + void CopyExtraTestFiles (string destinationRoot, Context context) + { + Directory.CreateDirectory (destinationRoot); + var filesToCopy = new List (); + + var testConfigDir = Path.Combine (BuildPaths.XamarinAndroidSourceRoot, "bin", $"Test{context.Configuration}"); + if (Directory.Exists (testConfigDir)) { + foreach (var fileMatch in testConfigFiles) { + filesToCopy.AddRange (Directory.GetFiles (testConfigDir, fileMatch)); + } + } + + var testConfigCompatDir = Path.Combine (testConfigDir, "compatibility"); + if (Directory.Exists (testConfigCompatDir)) { + Utilities.CopyFilesSimple (Directory.GetFiles (testConfigCompatDir, "*"), Path.Combine (destinationRoot, "compatibility")); + } + + var extraFilesToCopy = new List (); + extraFilesToCopy.AddRange (Directory.GetFiles (BuildPaths.XamarinAndroidSourceRoot, "TestResult*.xml")); + extraFilesToCopy.AddRange (Directory.GetFiles (BuildPaths.XamarinAndroidSourceRoot, "*.csv")); + extraFilesToCopy.AddRange (Directory.GetFiles (Path.GetTempPath (), "llc.exe-*")); + if (extraFilesToCopy.Any ()) { + Utilities.CopyFilesSimple (extraFilesToCopy, Path.Combine (destinationRoot, "test-extras"), false); + } + + // Remove NuGet package directories, and any empty directories that may have been left behind before uploading + var packagesDirs = Directory.EnumerateDirectories (destinationRoot, "packages", SearchOption.AllDirectories); + foreach (var packagesDir in packagesDirs) { + Utilities.DeleteDirectory (packagesDir, ignoreErrors: true); + } + + DeleteEmptyDirectories (destinationRoot); + + void DeleteEmptyDirectories (string directory) + { + foreach (var dir in Directory.EnumerateDirectories (directory)) { + DeleteEmptyDirectories (dir); + + if (!Directory.EnumerateFileSystemEntries (dir).Any ()) { + Utilities.DeleteDirectory (dir, ignoreErrors: true, recurse: false); + } + } + } + } + + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/CodeBehindTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/CodeBehindTests.cs index 66106922b..752133464 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/CodeBehindTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/CodeBehindTests.cs @@ -153,7 +153,7 @@ namespace Xamarin.Android.Build.Tests { TestProjectRootDirectory = Path.GetFullPath (Path.Combine (XABuildPaths.TopDirectory, "tests", "CodeBehind", "BuildTests")); CommonSampleLibraryRootDirectory = Path.GetFullPath (Path.Combine (XABuildPaths.TopDirectory, "tests", "CodeBehind", CommonSampleLibraryName)); - TestOutputDir = Path.Combine (XABuildPaths.TestOutputDirectory, "temp", "CodeBehind"); + TestOutputDir = Path.Combine (SetUp.TestDirectoryRoot, "temp", "CodeBehind"); if (Builder.UseDotNet) { ProjectName += ".NET"; } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/EmbeddedDSOTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/EmbeddedDSOTests.cs index 5114b1eae..5d46ded9c 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/EmbeddedDSOTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/EmbeddedDSOTests.cs @@ -58,7 +58,7 @@ namespace Xamarin.Android.Build.Tests static EmbeddedDSOTests () { TestProjectRootDirectory = Path.GetFullPath (Path.Combine (XABuildPaths.TopDirectory, "tests", "EmbeddedDSOs", "EmbeddedDSO")); - TestOutputDir = Path.Combine (XABuildPaths.TestOutputDirectory, "temp", "EmbeddedDSO"); + TestOutputDir = Path.Combine (SetUp.TestDirectoryRoot, "temp", "EmbeddedDSO"); produced_binaries = new List { $"{ProjectAssemblyName}.dll", diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/MakeBundleTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/MakeBundleTests.cs index 7da8b8810..bca641176 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/MakeBundleTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/MakeBundleTests.cs @@ -18,7 +18,7 @@ namespace Xamarin.Android.Build.Tests { [Category ("Node-2"), Category ("MkBundle"), Category ("StaticProject")] [Parallelizable (ParallelScope.Children)] - public class MakeBundleTests + public class MakeBundleTests : BaseTest { sealed class LocalBuilder : Builder { @@ -70,7 +70,7 @@ namespace Xamarin.Android.Build.Tests static MakeBundleTests () { TestProjectRootDirectory = Path.GetFullPath (Path.Combine (XABuildPaths.TopDirectory, "tests", "CodeGen-MkBundle", "Xamarin.Android.MakeBundle-Tests")); - TestOutputDir = Path.Combine (XABuildPaths.TestOutputDirectory, "temp", "CodeGen-MkBundle"); + TestOutputDir = Path.Combine (SetUp.TestDirectoryRoot, "temp", "CodeGen-MkBundle"); } [OneTimeSetUp] diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/GeneratePackageManagerJavaTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/GeneratePackageManagerJavaTests.cs index 7ee5e6db0..71d686dda 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/GeneratePackageManagerJavaTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/GeneratePackageManagerJavaTests.cs @@ -7,6 +7,7 @@ using Xamarin.Android.Tasks; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using Microsoft.Android.Build.Tasks; +using Xamarin.ProjectTools; namespace Xamarin.Android.Build.Tests { @@ -80,7 +81,7 @@ namespace Xamarin.Android.Build.Tests InstantRunEnabled = false, }; Assert.IsTrue (task.Execute (), "Task should have executed."); - AssertFileContentsMatch (Path.Combine (Root, "Expected", "CheckPackageManagerAssemblyOrder.java"), Path.Combine(path, "src", "mono", "MonoPackageManager_Resources.java")); + AssertFileContentsMatch (Path.Combine (XABuildPaths.TestAssemblyOutputDirectory, "Expected", "CheckPackageManagerAssemblyOrder.java"), Path.Combine(path, "src", "mono", "MonoPackageManager_Resources.java")); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs index c16466e64..4ca53fd82 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs @@ -16,7 +16,7 @@ namespace Xamarin.Android.Build.Tests [Test] public void FixAbstractMethodsStep_SkipDimMembers () { - var path = Path.Combine (Path.GetFullPath (XABuildPaths.TestOutputDirectory), "temp", TestName); + var path = Path.Combine (Root, "temp", TestName); var step = new FixAbstractMethodsStep (new TypeDefinitionCache ()); var pipeline = new Pipeline (); @@ -75,7 +75,7 @@ namespace Xamarin.Android.Build.Tests [Test] public void FixAbstractMethodsStep_Explicit () { - var path = Path.Combine (Path.GetFullPath (XABuildPaths.TestOutputDirectory), "temp", TestName); + var path = Path.Combine (Root, "temp", TestName); var step = new FixAbstractMethodsStep (new TypeDefinitionCache ()); var pipeline = new Pipeline (); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/ManagedResourceParserTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/ManagedResourceParserTests.cs index e92b4f48c..085cafde1 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/ManagedResourceParserTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/ManagedResourceParserTests.cs @@ -380,9 +380,11 @@ int xml myxml 0x7f140000 return task; } + static string ExpectedOutputDir = Path.Combine (XABuildPaths.TestAssemblyOutputDirectory, "Expected"); + void AssertResourceDesigner (GenerateResourceDesigner task, string expectedFile) { - var expected = Path.Combine (Root, "Expected", expectedFile); + var expected = Path.Combine (ExpectedOutputDir, expectedFile); FileAssert.Exists (task.NetResgenOutputFile); CompareFilesIgnoreRuntimeInfoString (task.NetResgenOutputFile, expected); @@ -455,7 +457,7 @@ int xml myxml 0x7f140000 task.JavaPlatformJarPath = Path.Combine (AndroidSdkDirectory, "platforms", "android-27", "android.jar"); Assert.IsTrue (task.Execute (), "Task should have executed successfully."); Assert.IsTrue (File.Exists (task.NetResgenOutputFile), $"{task.NetResgenOutputFile} should have been created."); - var expected = Path.Combine (Root, "Expected", "GenerateDesignerFileExpected.cs"); + var expected = Path.Combine (ExpectedOutputDir, "GenerateDesignerFileExpected.cs"); CompareFilesIgnoreRuntimeInfoString (task.NetResgenOutputFile, expected); // Update the id, and force the managed parser to re-parse the output File.WriteAllText (Path.Combine (Root, path, "res", "layout", "main.xml"), Main.Replace ("@+id/textview.withperiod", "@+id/textview.withperiod2")); @@ -737,7 +739,7 @@ int styleable ElevenAttributes_attr10 10"; task.JavaPlatformJarPath = Path.Combine (AndroidSdkDirectory, "platforms", "android-27", "android.jar"); Assert.IsTrue (task.Execute (), "Task should have executed successfully."); Assert.IsTrue (File.Exists (task.NetResgenOutputFile), $"{task.NetResgenOutputFile} should have been created."); - var expected = Path.Combine (Root, "Expected", "GenerateDesignerFileWithElevenStyleableAttributesExpected.cs"); + var expected = Path.Combine (ExpectedOutputDir, "GenerateDesignerFileWithElevenStyleableAttributesExpected.cs"); CompareFilesIgnoreRuntimeInfoString (task.NetResgenOutputFile, expected); Directory.Delete (Path.Combine (Root, path), recursive: true); } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs index daecd898b..93a711dd7 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs @@ -1,4 +1,4 @@ -using NUnit.Framework; +using NUnit.Framework; using System; using System.Collections.Concurrent; using System.Collections.Generic; @@ -47,6 +47,11 @@ namespace Xamarin.Android.Build.Tests private set; } + public static string TestDirectoryRoot { + get; + private set; + } + static SetUp () { using (var builder = new Builder ()) { @@ -68,6 +73,8 @@ namespace Xamarin.Android.Build.Tests [OneTimeSetUp] public void BeforeAllTests () { + TestDirectoryRoot = XABuildPaths.TestOutputDirectory; + try { DeviceSdkVersion = GetSdkVersion (); if (DeviceSdkVersion != -1) { @@ -138,7 +145,7 @@ namespace Xamarin.Android.Build.Tests public string Root { get { - return Path.GetFullPath (XABuildPaths.TestOutputDirectory); + return Path.GetFullPath (SetUp.TestDirectoryRoot); } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/XamarinProject.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/XamarinProject.cs index 91ad67a12..03e0c94da 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/XamarinProject.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/XamarinProject.cs @@ -379,7 +379,7 @@ $@" return; var isWindows = Environment.OSVersion.Platform == PlatformID.Win32NT; - var nuget = Path.Combine (Root, "nuget", "NuGet.exe"); + var nuget = Path.Combine (XABuildPaths.TestAssemblyOutputDirectory, "nuget", "NuGet.exe"); var psi = new ProcessStartInfo (isWindows ? nuget : "mono") { Arguments = $"{(isWindows ? "" : "\"" + nuget + "\"")} restore -Verbosity Detailed -PackagesDirectory \"{Path.Combine (Root, directory, "..", "packages")}\" \"{Path.Combine (Root, directory, "packages.config")}\"", CreateNoWindow = true, diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Utilities/FileSystemUtils.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Utilities/FileSystemUtils.cs index fec24d5c4..071ff759f 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Utilities/FileSystemUtils.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Utilities/FileSystemUtils.cs @@ -46,7 +46,7 @@ namespace Xamarin.ProjectTools } bool isWindows = Environment.OSVersion.Platform == PlatformID.Win32NT; - string nugetPath = Path.Combine (XABuildPaths.TestOutputDirectory, "NuGet.exe"); + string nugetPath = Path.Combine (XABuildPaths.TestAssemblyOutputDirectory, "nuget", "NuGet.exe"); if (File.Exists (nugetPath)) { var psi = new ProcessStartInfo (isWindows ? nugetPath : "mono") { diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/XABuildPaths.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/XABuildPaths.cs index 9eb0f9bd9..e2ebdf721 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/XABuildPaths.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/XABuildPaths.cs @@ -18,7 +18,8 @@ namespace Xamarin.ProjectTools public static readonly string BinDirectory = Path.Combine (PrefixDirectory, "bin"); public static readonly string XABuildScript = Path.Combine (BinDirectory, "xabuild"); public static readonly string XABuildExe = Path.Combine (BinDirectory, "xabuild.exe"); - public static readonly string TestOutputDirectory = Path.Combine (TopDirectory, "bin", $"Test{Configuration}"); + public static readonly string TestAssemblyOutputDirectory = Path.Combine (TopDirectory, "bin", $"Test{Configuration}"); + public static readonly string TestOutputDirectory = GetTestDirectoryRoot (); public static readonly string BuildOutputDirectory = Path.Combine (TopDirectory, "bin", $"Build{Configuration}"); static string GetTopDirRecursive (string searchDirectory, int maxSearchDepth = 5) @@ -31,5 +32,25 @@ namespace Xamarin.ProjectTools return GetTopDirRecursive (Directory.GetParent (searchDirectory).FullName, --maxSearchDepth); } + + static string _testOutputDirectory; + static string GetTestDirectoryRoot () + { + if (Directory.Exists (_testOutputDirectory)) + return _testOutputDirectory; + + // Set when running on Azure Pipelines https://docs.microsoft.com/en-us/azure/devops/pipelines/build/variables + var rootDir = Environment.GetEnvironmentVariable ("BUILD_STAGINGDIRECTORY"); + if (!Directory.Exists (rootDir)) { + _testOutputDirectory = TestAssemblyOutputDirectory; + } else { + var timeStamp = DateTime.UtcNow.ToString ("MM-dd_HH.mm.ss"); + _testOutputDirectory = Path.Combine (rootDir, $"Test{Configuration}", timeStamp); + } + + Directory.CreateDirectory (_testOutputDirectory); + return _testOutputDirectory; + } + } } diff --git a/tests/CodeGen-MkBundle/Xamarin.Android.MakeBundle-Tests/Xamarin.Android.MakeBundle-Tests.csproj b/tests/CodeGen-MkBundle/Xamarin.Android.MakeBundle-Tests/Xamarin.Android.MakeBundle-Tests.csproj index e2fe9fbf1..8151bb8de 100644 --- a/tests/CodeGen-MkBundle/Xamarin.Android.MakeBundle-Tests/Xamarin.Android.MakeBundle-Tests.csproj +++ b/tests/CodeGen-MkBundle/Xamarin.Android.MakeBundle-Tests/Xamarin.Android.MakeBundle-Tests.csproj @@ -21,7 +21,9 @@ - ..\..\..\..\..\..\.. + + $(BUILD_SOURCESDIRECTORY) + ..\..\..\..\..\..\.. diff --git a/tests/EmbeddedDSOs/EmbeddedDSO/EmbeddedDSO.csproj b/tests/EmbeddedDSOs/EmbeddedDSO/EmbeddedDSO.csproj index 47a40b0d0..7593d18a1 100644 --- a/tests/EmbeddedDSOs/EmbeddedDSO/EmbeddedDSO.csproj +++ b/tests/EmbeddedDSOs/EmbeddedDSO/EmbeddedDSO.csproj @@ -18,7 +18,9 @@ - ..\..\..\..\..\..\.. + + $(BUILD_SOURCESDIRECTORY) + ..\..\..\..\..\..\.. @@ -102,7 +104,7 @@ - + diff --git a/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs b/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs index 69dbd3b7e..9bd182ce2 100644 --- a/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs +++ b/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs @@ -326,6 +326,7 @@ namespace Xamarin.Android.Build.Tests app.Sources.Add (new BuildItem.Source ("Foo.cs") { TextContent = () => "public class Foo : Bar { }" }); + app.PackageReferences.Add (KnownPackages.XamarinForms_4_0_0_425677); //NOTE: this will skip a 382ms from the support library app.SetProperty ("XamarinAndroidSupportSkipVerifyVersions", "True");