[msbuild] Call the generator using a response file. (#12756)

Fixes this warning:

> Xamarin.Shared.targets(992,3): warning MSB6002: The command-line for the "BTouch" task is too long. Command-lines longer than 32000 characters are likely to fail. Try reducing the length of the command-line by breaking down the call to "BTouch" into multiple calls with fewer parameters per call.

msbuild/Xamarin.Mac.Tasks.Core/Tasks/MmpTaskBase.cs

@@ -114,7 +114,7 @@ namespace Xamarin.Mac.Tasks
 			if (IsXPCService)
 				args.AddQuotedLine ("/xpc");
 
-			return CreateResponseFile (args, ExtraArgs == null ? null : CommandLineArgumentBuilder.Parse (ExtraArgs));
+			return args.CreateResponseFile (this, ResponseFilePath, ExtraArgs == null ? null : CommandLineArgumentBuilder.Parse (ExtraArgs));
 		}
 
 		public override bool Execute ()

msbuild/Xamarin.MacDev.Tasks.Core/CommandLineArgumentBuilder.cs

@@ -1,7 +1,9 @@
 using System;
+using System.IO;
 using System.Text;
 using System.Collections.Generic;
 
+using Microsoft.Build.Utilities;
 using Xamarin.Utils;
 
 namespace Xamarin.MacDev
@@ -49,6 +51,13 @@ namespace Xamarin.MacDev
 			hash.Add (argument);
 		}
 
+		public void AddQuotedSwitchIfNotNull (string name, string value)
+		{
+			if (value == null)
+				return;
+			AddQuoted (name + value);
+		}
+
 		/// <summary>
 		/// Adds an argument without escaping or quoting and goes to the next line
 		/// </summary>
@@ -184,5 +193,40 @@ namespace Xamarin.MacDev
 
 			return argv;
 		}
+
+		// Creates a response file for the given arguments, and returns the command line
+		// with the response file and any arguments that can't go into the response file.
+		public string CreateResponseFile (Task task, string responseFilePath, IList<string> nonResponseArguments)
+		{
+			// Generate a response file
+			var responseFile = Path.GetFullPath (responseFilePath);
+
+			if (File.Exists (responseFile))
+				File.Delete (responseFile);
+
+			try {
+				using (var fs = File.Create (responseFile)) {
+					using (var writer = new StreamWriter (fs))
+						writer.Write (this);
+				}
+			} catch (Exception ex) {
+				task.Log.LogWarning ("Failed to create response file '{0}': {1}", responseFile, ex);
+			}
+
+			// Some arguments can not safely go in the response file and are 
+			// added separately. They must go _after_ the response file
+			// as they may override options passed in the response file
+			var actualArgs = new CommandLineArgumentBuilder ();
+
+			actualArgs.AddQuoted ($"@{responseFile}");
+
+			if (nonResponseArguments != null) {
+				foreach (var arg in nonResponseArguments)
+					actualArgs.AddQuoted (arg);
+			}
+
+			// Generate the command line
+			return actualArgs.ToString ();
+		}
 	}
 }

msbuild/Xamarin.MacDev.Tasks.Core/Tasks/BTouchTaskBase.cs

@@ -3,6 +3,7 @@
 using System;
 using System.IO;
 using System.Collections.Generic;
+using System.Text;
 
 using Microsoft.Build.Framework;
 using Microsoft.Build.Utilities;
@@ -66,6 +67,9 @@ namespace Xamarin.MacDev.Tasks {
 
 		public ITaskItem[] Sources { get; set; }
 
+		[Required]
+		public string ResponseFilePath { get; set; }
+
 		bool IsDotNet {
 			get { return TargetFramework.IsDotNet; }
 		}
@@ -102,119 +106,102 @@ namespace Xamarin.MacDev.Tasks {
 			return Path.Combine (ToolPath, ToolExe);
 		}
 
-		protected virtual void HandleReferences (CommandLineBuilder cmd)
+		protected virtual void HandleReferences (CommandLineArgumentBuilder cmd)
 		{
 			if (References != null) {
 				foreach (var item in References)
-					cmd.AppendSwitchIfNotNull ("-r ", Path.GetFullPath (item.ItemSpec));
+					cmd.AddQuoted ("-r:" + Path.GetFullPath (item.ItemSpec));
 			}
 		}
 
 		protected override string GenerateCommandLineCommands ()
 		{
-			var cmd = new CommandLineBuilder ();
-
-			if (IsDotNet)
-				cmd.AppendFileNameIfNotNull (Path.Combine (BTouchToolPath, BTouchToolExe));
+			var cmd = new CommandLineArgumentBuilder ();
 
 			#if DEBUG
-			cmd.AppendSwitch ("/v");
+			cmd.Add ("/v");
 			#endif
 
-			cmd.AppendSwitch ("/nostdlib");
-			cmd.AppendSwitchIfNotNull ("/baselib:", BaseLibDll);
-			cmd.AppendSwitchIfNotNull ("/out:", OutputAssembly);
+			cmd.Add ("/nostdlib");
+			cmd.AddQuotedSwitchIfNotNull ("/baselib:", BaseLibDll);
+			cmd.AddQuotedSwitchIfNotNull ("/out:", OutputAssembly);
 
-			cmd.AppendSwitchIfNotNull ("/attributelib:", AttributeAssembly);
+			cmd.AddQuotedSwitchIfNotNull ("/attributelib:", AttributeAssembly);
 
 			string dir;
 			if (!string.IsNullOrEmpty (BaseLibDll)) {
 				dir = Path.GetDirectoryName (BaseLibDll);
-				cmd.AppendSwitchIfNotNull ("/lib:", dir);
+				cmd.AddQuotedSwitchIfNotNull ("/lib:", dir);
 			}
 
 			if (ProcessEnums)
-				cmd.AppendSwitch ("/process-enums");
+				cmd.Add ("/process-enums");
 
 			if (EmitDebugInformation)
-				cmd.AppendSwitch ("/debug");
+				cmd.Add ("/debug");
 
 			if (AllowUnsafeBlocks)
-				cmd.AppendSwitch ("/unsafe");
+				cmd.Add ("/unsafe");
 
-			cmd.AppendSwitchIfNotNull ("/ns:", Namespace);
+			cmd.AddQuotedSwitchIfNotNull ("/ns:", Namespace);
 
 			if (!string.IsNullOrEmpty (DefineConstants)) {
-				var strv = DefineConstants.Split (new [] { ';' });
-				var sanitized = new List<string> ();
-
-				foreach (var str in strv) {
-					if (str != string.Empty)
-						sanitized.Add (str);
-				}
-
-				if (sanitized.Count > 0)
-					cmd.AppendSwitchIfNotNull ("/d:", string.Join (";", sanitized.ToArray ()));
+				var strv = DefineConstants.Split (new [] { ';' }, StringSplitOptions.RemoveEmptyEntries);
+				foreach (var str in strv)
+					cmd.AddQuoted ("/d:" + str);
 			}
 
 			//cmd.AppendSwitch ("/e");
 
 			foreach (var item in ApiDefinitions)
-				cmd.AppendFileNameIfNotNull (Path.GetFullPath (item.ItemSpec));
+				cmd.AddQuoted (Path.GetFullPath (item.ItemSpec));
 
 			if (CoreSources != null) {
 				foreach (var item in CoreSources)
-					cmd.AppendSwitchIfNotNull ("/s:", Path.GetFullPath (item.ItemSpec));
+					cmd.AddQuoted ("/s:" + Path.GetFullPath (item.ItemSpec));
 			}
 
 			if (Sources != null) {
 				foreach (var item in Sources)
-					cmd.AppendSwitchIfNotNull ("/x:", Path.GetFullPath (item.ItemSpec));
+					cmd.AddQuoted ("/x:" + Path.GetFullPath (item.ItemSpec));
 			}
 
 			if (AdditionalLibPaths != null) {
 				foreach (var item in AdditionalLibPaths)
-					cmd.AppendSwitchIfNotNull ("/lib:", Path.GetFullPath (item.ItemSpec));
+					cmd.AddQuoted ("/lib:" + Path.GetFullPath (item.ItemSpec));
 			}
 
 			HandleReferences (cmd);
 
 			if (Resources != null) {
 				foreach (var item in Resources) {
-					var args = new List<string> ();
-					string id;
-
-					args.Add (item.ToString ());
-					id = item.GetMetadata ("LogicalName");
+					var argument = item.ToString ();
+					var id = item.GetMetadata ("LogicalName");
 					if (!string.IsNullOrEmpty (id))
-						args.Add (id);
+						argument += "," + id;
 
-					cmd.AppendSwitchIfNotNull ("/res:", args.ToArray (), ",");
+					cmd.AddQuoted ("/res:" + argument);
 				}
 			}
 
 			if (NativeLibraries != null) {
 				foreach (var item in NativeLibraries) {
-					var args = new List<string> ();
-					string id;
-
-					args.Add (item.ToString ());
-					id = item.GetMetadata ("LogicalName");
+					var argument = item.ToString ();
+					var id = item.GetMetadata ("LogicalName");
 					if (string.IsNullOrEmpty (id))
-						id = Path.GetFileName (args[0]);
-					args.Add (id);
+						id = Path.GetFileName (argument);
 
-					cmd.AppendSwitchIfNotNull ("/link-with:", args.ToArray (), ",");
+					cmd.AddQuoted ("/res:" + argument + "," + id);
 				}
 			}
 
 			if (GeneratedSourcesDir != null)
-				cmd.AppendSwitchIfNotNull ("/tmpdir:", Path.GetFullPath (GeneratedSourcesDir));
+				cmd.AddQuoted ("/tmpdir:" + Path.GetFullPath (GeneratedSourcesDir));
 
 			if (GeneratedSourcesFileList != null)
-				cmd.AppendSwitchIfNotNull ("/sourceonly:", Path.GetFullPath (GeneratedSourcesFileList));
+				cmd.AddQuoted ("/sourceonly:" + Path.GetFullPath (GeneratedSourcesFileList));
 
-			cmd.AppendSwitch ($"/target-framework={TargetFrameworkMoniker}");
+			cmd.Add ($"/target-framework={TargetFrameworkMoniker}");
 
 			if (!string.IsNullOrEmpty (ExtraArgs)) {
 				var extraArgs = CommandLineArgumentBuilder.Parse (ExtraArgs);
@@ -248,20 +235,17 @@ namespace Xamarin.MacDev.Tasks {
 
 				for (int i = 0; i < extraArgs.Length; i++) {
 					var argument = extraArgs[i];
-					cmd.AppendTextUnquoted (" ");
-					cmd.AppendTextUnquoted (StringParserService.Parse (argument, customTags));
+					cmd.Add (StringParserService.Parse (argument, customTags));
 				}
 			}
 
-			var v = VerbosityUtils.Merge (ExtraArgs, (LoggerVerbosity) Verbosity);
-			if (v.Length > 0) {
-				foreach (var arg in v) {
-					cmd.AppendTextUnquoted (" ");
-					cmd.AppendTextUnquoted (arg);
-				}
-			}
+			cmd.Add (VerbosityUtils.Merge (ExtraArgs, (LoggerVerbosity) Verbosity));
 
-			return cmd.ToString ();
+			var commandLine = cmd.CreateResponseFile (this, ResponseFilePath, null);
+			if (IsDotNet)
+				commandLine = StringUtils.Quote (Path.Combine (BTouchToolPath, BTouchToolExe)) + " " + commandLine;
+
+			return commandLine.ToString ();
 		}
 
 		public override bool Execute ()

msbuild/Xamarin.MacDev.Tasks.Core/Tasks/BundlerToolTaskBase.cs

@@ -121,40 +121,5 @@ namespace Xamarin.MacDev.Tasks {
 
 			return args;
 		}
-
-		// Creates a response file for the given arguments, and returns the command line
-		// with the response file and any arguments that can't go into the response file.
-		protected string CreateResponseFile (CommandLineArgumentBuilder arguments, IList<string> nonResponseArguments)
-		{
-			// Generate a response file
-			var responseFile = Path.GetFullPath (ResponseFilePath);
-
-			if (File.Exists (responseFile))
-				File.Delete (responseFile);
-
-			try {
-				using (var fs = File.Create (responseFile)) {
-					using (var writer = new StreamWriter (fs))
-						writer.Write (arguments);
-				}
-			} catch (Exception ex) {
-				Log.LogWarning ("Failed to create response file '{0}': {1}", responseFile, ex);
-			}
-
-			// Some arguments can not safely go in the response file and are 
-			// added separately. They must go _after_ the response file
-			// as they may override options passed in the response file
-			var actualArgs = new CommandLineArgumentBuilder ();
-
-			actualArgs.AddQuoted ($"@{responseFile}");
-
-			if (nonResponseArguments != null) {
-				foreach (var arg in nonResponseArguments)
-					actualArgs.AddQuoted (arg);
-			}
-
-			// Generate the command line
-			return actualArgs.ToString ();
-		}
 	}
 }

msbuild/Xamarin.Shared/Xamarin.Shared.targets

@@ -1017,6 +1017,7 @@ Copyright (C) 2018 Microsoft. All rights reserved.
 			ProcessEnums="$(ProcessEnums)"
 			ProjectDir="$(MSBuildProjectDirectory)"
 			References="@(ReferencePath);@(BTouchReferencePath)"
+			ResponseFilePath="$(DeviceSpecificIntermediateOutputPath)response-file.rsp"
 			TargetFrameworkMoniker="$(_ComputedTargetFrameworkMoniker)"
 			BTouchToolPath="$(BTouchToolPath)"
 			BTouchToolExe="$(BTouchToolExe)"

msbuild/Xamarin.iOS.Tasks.Core/Tasks/MTouchTaskBase.cs

@@ -333,7 +333,7 @@ namespace Xamarin.iOS.Tasks
 			if (!string.IsNullOrWhiteSpace (License))
 				args.AddLine ($"--license={License}");
 
-			return CreateResponseFile (args, unescapedArgs);
+			return args.CreateResponseFile (this, ResponseFilePath, unescapedArgs);
 		}
 
 		static bool IsFrameworkItem (ITaskItem item)

tests/mmptest/src/BindingProjectTests.cs

@@ -134,21 +134,29 @@ namespace Xamarin.MMP.Tests
 				Assert.True (logs.BindingBuildResult.BuildOutput.Contains ("csc"), "Bindings project must use csc:\n" + logs.Item1); 
 
 				var bgenInvocation = logs.BindingBuildResult.BuildOutputLines.First (x => x.Contains ("bin/bgen"));
-				var bgenParts = bgenInvocation.Split (new char[] { ' ' });
+				Assert.IsTrue (StringUtils.TryParseArguments (bgenInvocation, out var bgenArguments, out var _), "Parse bgen arguments");
+				// unfurl any response files
+				var bgenParts = bgenArguments.ToList ();
+				var responseFiles = bgenParts.Where (v => v [0] == '@').ToArray ();
+				bgenParts.RemoveAll (v => v [0] == '@');
+				foreach (var rsp in responseFiles) {
+					Assert.IsTrue (StringUtils.TryParseArguments (File.ReadAllText (rsp.Substring (1)).Replace ('\n', ' '), out var args, out var _), "Parse response file");
+					bgenParts.AddRange (args);
+				}
 				var mscorlib = bgenParts.First (x => x.Contains ("mscorlib.dll"));
 				var system = bgenParts.First (x => x.Contains ("System.dll"));
 
 				switch (type) {
 				case BindingProjectType.Modern:
 				case BindingProjectType.ModernNoTag:
-					Assert.True (mscorlib.EndsWith ("lib/mono/Xamarin.Mac/mscorlib.dll", StringComparison.Ordinal), "mscorlib not found in expected Modern location: " + mscorlib);
-					Assert.True (system.EndsWith ("lib/mono/Xamarin.Mac/System.dll", StringComparison.Ordinal), "system not found in expected Modern location: " + system);
+					Assert.That (mscorlib, Does.EndWith ("lib/mono/Xamarin.Mac/mscorlib.dll"), "mscorlib not found in expected Modern location: " + mscorlib);
+					Assert.That (system, Does.EndWith ("lib/mono/Xamarin.Mac/System.dll"), "system not found in expected Modern location: " + system);
 					break;
 				case BindingProjectType.Full:
 				case BindingProjectType.FullTVF:
 				case BindingProjectType.FullXamMacTag:
-					Assert.True (mscorlib.EndsWith ("lib/mono/4.5/mscorlib.dll", StringComparison.Ordinal), "mscorlib not found in expected Full location: " + mscorlib);
-					Assert.True (system.EndsWith ("lib/mono/4.5/System.dll", StringComparison.Ordinal), "system not found in expected Full location: " + system);
+					Assert.That (mscorlib, Does.EndWith ("lib/mono/4.5/mscorlib.dll"), "mscorlib not found in expected Full location: " + mscorlib);
+					Assert.That (system, Does.EndWith ("lib/mono/4.5/System.dll"), "system not found in expected Full location: " + system);
 					break;
 				default:
 					throw new NotImplementedException ();

tests/msbuild/Xamarin.MacDev.Tasks.Tests/TaskTests/BTouchTaskTest.cs

@@ -25,11 +25,12 @@ namespace Xamarin.iOS.Tasks
 
 			task.ApiDefinitions = new[] { new TaskItem ("apidefinition.cs") };
 			task.References = new[] { new TaskItem ("a.dll"), new TaskItem ("b.dll"), new TaskItem ("c.dll") };
+			task.ResponseFilePath = Path.Combine (Cache.CreateTemporaryDirectory (), "response-file.txt");
 
-			var args = task.GetCommandLineCommands ();
-			Assert.IsTrue (args.Contains ("-r " + Path.Combine (Environment.CurrentDirectory, "a.dll")), "#1a");
-			Assert.IsTrue (args.Contains ("-r " + Path.Combine (Environment.CurrentDirectory, "b.dll")), "#1b");
-			Assert.IsTrue (args.Contains ("-r " + Path.Combine (Environment.CurrentDirectory, "c.dll")), "#1c");
+			var args = task.GetCommandLineCommands () + " " + File.ReadAllText (task.ResponseFilePath);
+			Assert.That (args, Does.Contain ("-r:" + Path.Combine (Environment.CurrentDirectory, "a.dll")), "#1a");
+			Assert.That (args, Does.Contain ("-r:" + Path.Combine (Environment.CurrentDirectory, "b.dll")), "#1b");
+			Assert.That (args, Does.Contain ("-r:" + Path.Combine (Environment.CurrentDirectory, "c.dll")), "#1c");
 		}
 
 		[Test]
@@ -40,10 +41,11 @@ namespace Xamarin.iOS.Tasks
 			task.ApiDefinitions = new[] { new TaskItem ("apidefinition.cs") };
 			task.References = new[] { new TaskItem ("a.dll"), new TaskItem ("b.dll"), new TaskItem ("c.dll") };
 			task.ProjectDir = "~/"; // not important, but required (so can't be null)
+			task.ResponseFilePath = Path.Combine (Cache.CreateTemporaryDirectory (), "response-file.txt");
 
 			task.OutputAssembly = null; // default, but important for the bug (in case that default changes)
 			task.ExtraArgs = "-invalid";
-			var args = task.GetCommandLineCommands ();
+			var args = task.GetCommandLineCommands () + " " + File.ReadAllText (task.ResponseFilePath);
 			Assert.That (args.Contains (" -invalid"), "incorrect ExtraArg not causing an exception");
 		}
 	}