From 7da04184cf40ff3cdbcf80e9473fca9ca2d01d81 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Mon, 12 Aug 2024 18:58:17 +0200 Subject: [PATCH] [tests] Improve the CustomizedCodeSigning tests to not depend on file order. (#20889) The CustomizedCodeSigning test asserts that a certain condition shows a particular error message from codesign. There are multiple files in the app bundle that can trigger this particular message, so change the logic to not assert on a particular file, instead use assert on the remained of the error message. Fixes this random test failure: Xamarin.Tests.DotNetProjectTest.CustomizedCodeSigning(iOS,"ios-arm64"): Failure when comparing error messages: Unexpected error message #0: Expected: /usr/bin/codesign exited with code 1:\n/Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: replacing existing signature\n/Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: code object is not signed at all\nIn subcomponent: /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app/System.Diagnostics.DiagnosticSource.dll Actual: /usr/bin/codesign exited with code 1:\n/Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: replacing existing signature\n/Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: code object is not signed at all\nIn subcomponent: /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app/System.Collections.NonGeneric.aotdata.arm64 Unexpected error message #1: Expected: Failed to codesign '/Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app': /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: replacing existing signature\n/Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: code object is not signed at all\nIn subcomponent: /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app/System.Diagnostics.DiagnosticSource.dll Actual: Failed to codesign '/Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app': /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: replacing existing signature\n/Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: code object is not signed at all\nIn subcomponent: /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app/System.Collections.NonGeneric.aotdata.arm64 All errors: /usr/bin/codesign exited with code 1: /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: replacing existing signature /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: code object is not signed at all In subcomponent: /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app/System.Collections.NonGeneric.aotdata.arm64 Failed to codesign '/Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app': /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: replacing existing signature /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app: code object is not signed at all In subcomponent: /Users/builder/azdo/_work/4/s/xamarin-macios/tests/dotnet/CustomizedCodeSigning/iOS/bin/Debug/net9.0-ios/ios-arm64/CustomizedCodeSigning.app/System.Collections.NonGeneric.aotdata.arm64 --------- Co-authored-by: Alex Soto Co-authored-by: Manuel de la Pena --- tests/dotnet/UnitTests/ProjectTest.cs | 14 +++++--- tests/dotnet/UnitTests/TestBaseClass.cs | 44 ++++++++++++++++++++----- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/tests/dotnet/UnitTests/ProjectTest.cs b/tests/dotnet/UnitTests/ProjectTest.cs index e886797aa0..f3d9e9247a 100644 --- a/tests/dotnet/UnitTests/ProjectTest.cs +++ b/tests/dotnet/UnitTests/ProjectTest.cs @@ -1508,15 +1508,19 @@ namespace Xamarin.Tests { properties ["CodesignDisallowResourcesSubdirectoryInAppBundle"] = "false"; buildFailure = DotNet.AssertBuildFailure (project_path, properties); errors = BinLog.GetBuildLogErrors (buildFailure.BinLogPath).ToArray (); - AssertErrorMessages (errors, + var errorMessagePrefixes = new string [] + { $"/usr/bin/codesign exited with code 1:\n" + $"{appPath}: replacing existing signature\n" + - $"{appPath}: code object is not signed at all\n" + - $"In subcomponent: {appPath}/System.Diagnostics.DiagnosticSource.dll", + $"{appPath}: code object is not signed at all\n", $"Failed to codesign '{appPath}': {appPath}: replacing existing signature\n" + - $"{appPath}: code object is not signed at all\n" + - $"In subcomponent: {appPath}/System.Diagnostics.DiagnosticSource.dll" + $"{appPath}: code object is not signed at all\n", + }; + + AssertErrorMessages (errors, + errorMessagePrefixes.Select (prefix => new Func ((msg) => msg.StartsWith (prefix))).ToArray (), + errorMessagePrefixes.Select (prefix => new Func (() => prefix)).ToArray () ); // Remove the dir, and now the build should succeed again. diff --git a/tests/dotnet/UnitTests/TestBaseClass.cs b/tests/dotnet/UnitTests/TestBaseClass.cs index 339667db26..41b8c67cc6 100644 --- a/tests/dotnet/UnitTests/TestBaseClass.cs +++ b/tests/dotnet/UnitTests/TestBaseClass.cs @@ -445,20 +445,48 @@ namespace Xamarin.Tests { AssertBuildMessages ("error", actualErrors, expectedErrorMessages); } + public static void AssertErrorMessages (IList actualErrors, Func [] matchesExpectedErrorMessage, Func [] rendersExpectedErrorMessage) + { + AssertBuildMessages ("error", actualErrors, matchesExpectedErrorMessage, rendersExpectedErrorMessage); + } + public static void AssertBuildMessages (string type, IList actualMessages, params string [] expectedMessages) { - if (actualMessages.Count != expectedMessages.Length) { - Assert.Fail ($"Expected {expectedMessages.Length} {type}s, got {actualMessages.Count} {type}s:\n\t{string.Join ("\n\t", actualMessages.Select (v => v.Message?.TrimEnd ()))}"); + AssertBuildMessages (type, actualMessages, + expectedMessages.Select (v => new Func ((msg) => msg == v)).ToArray (), + expectedMessages.Select (v => new Func (() => v)).ToArray () + ); + } + + static string makeSingleLine (string? msg) + { + if (msg is null) + return ""; + return msg.TrimEnd ().Replace ("\n", "\\n").Replace ("\r", "\\r"); + } + + public static void AssertBuildMessages (string type, IList actualMessages, Func [] matchesExpectedMessage, Func [] rendersExpectedMessage) + { + var expectedCount = matchesExpectedMessage.Length; + if (expectedCount != rendersExpectedMessage.Length) + throw new InvalidOperationException ($"Mismatched function count"); + + if (actualMessages.Count != expectedCount) { + Assert.Fail ($"Expected {expectedCount} {type}(s), got {actualMessages.Count} {type}(s)\n" + + $"\tExpected:\n" + + $"\t\t{string.Join ("\n\t\t", rendersExpectedMessage.Select (v => makeSingleLine (v ())))}" + + $"\tActual:\n" + + $"\t\t{string.Join ("\n\t\t", actualMessages.Select (v => makeSingleLine (v.Message)))}"); return; } var failures = new List (); - for (var i = 0; i < expectedMessages.Length; i++) { - var actual = (actualMessages [i].Message ?? string.Empty).Trim ('\n', '\r', ' '); - var expected = expectedMessages [i].Trim ('\n', '\r', ' '); - if (actual != expected) { - actual = actual.Replace ("\n", "\\n").Replace ("\r", "\\r"); - expected = expected.Replace ("\n", "\\n").Replace ("\r", "\\r"); + for (var i = 0; i < expectedCount; i++) { + var actual = actualMessages [i].Message ?? string.Empty; + var isExpected = matchesExpectedMessage [i]; + if (!isExpected (actual)) { + actual = makeSingleLine (actual); + var expected = makeSingleLine (rendersExpectedMessage [i] ()); failures.Add ($"\tUnexpected {type} message #{i}:\n\t\tExpected: {expected}\n\t\tActual: {actual}"); } }