[Foundation] Make sure we use the cookies from the cookie storage. Fixes #5148 (#5244)

The response object does not have all the cookie values, instead we must
rust the cookie storage which can be used to retrieve ALL the cookies
for a task.

The header value has to be created manually because the native objects
do not expose a valid way to get the header. Tests have been added to
ensure we return the same as the managed client.

Fixes https://github.com/xamarin/xamarin-macios/issues/5148

src/Foundation/NSUrlSessionHandler.cs

@@ -27,13 +27,16 @@
 
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Net;
 using System.Net.Http;
+using System.Net.Http.Headers;
 using System.Runtime.InteropServices;
 using System.Threading;
 using System.Threading.Tasks;
+using System.Text;
 
 #if UNIFIED
 using CoreFoundation;
@@ -53,8 +56,62 @@ namespace System.Net.Http {
 #else
 namespace Foundation {
 #endif
+
+	// useful extensions for the class in order to set it in a header
+	static class NSHttpCookieExtensions
+	{
+		static void AppendSegment(StringBuilder builder, string name, string value)
+		{
+			if (builder.Length > 0)
+				builder.Append ("; ");
+
+			builder.Append (name);
+			if (value != null)
+				builder.Append ("=").Append (value);
+		}
+
+		// returns the header for a cookie
+		public static string GetHeaderValue (this NSHttpCookie cookie)
+		{
+			var header = new StringBuilder();
+			AppendSegment (header, cookie.Name, cookie.Value);
+			AppendSegment (header, NSHttpCookie.KeyPath.ToString (), cookie.Path.ToString ());
+			AppendSegment (header, NSHttpCookie.KeyDomain.ToString (), cookie.Domain.ToString ());
+			AppendSegment (header, NSHttpCookie.KeyVersion.ToString (), cookie.Version.ToString ());
+
+			if (cookie.Comment != null)
+				AppendSegment (header, NSHttpCookie.KeyComment.ToString (), cookie.Comment.ToString());
+
+			if (cookie.CommentUrl != null)
+				AppendSegment (header, NSHttpCookie.KeyCommentUrl.ToString (), cookie.CommentUrl.ToString());
+
+			if (cookie.Properties.ContainsKey (NSHttpCookie.KeyDiscard))
+				AppendSegment (header, NSHttpCookie.KeyDiscard.ToString (), null);
+
+			if (cookie.ExpiresDate != null) {
+				// Format according to RFC1123; 'r' uses invariant info (DateTimeFormatInfo.InvariantInfo)
+				var dateStr = ((DateTime) cookie.ExpiresDate).ToUniversalTime ().ToString("r", CultureInfo.InvariantCulture);
+				AppendSegment (header, NSHttpCookie.KeyExpires.ToString (), dateStr);
+			}
+
+			if (cookie.Properties.ContainsKey (NSHttpCookie.KeyMaximumAge)) {
+				var timeStampString = (NSString) cookie.Properties[NSHttpCookie.KeyMaximumAge];
+				AppendSegment (header, NSHttpCookie.KeyMaximumAge.ToString (), timeStampString);
+			}
+
+			if (cookie.IsSecure)
+				AppendSegment (header, NSHttpCookie.KeySecure.ToString(), null);
+
+			if (cookie.IsHttpOnly)
+				AppendSegment (header, "httponly", null); // Apple does not show the key for the httponly
+
+			return header.ToString ();
+		}
+	}
+
 	public partial class NSUrlSessionHandler : HttpMessageHandler
 	{
+		private const string SetCookie = "Set-Cookie";
 		readonly Dictionary<string, string> headerSeparators = new Dictionary<string, string> {
 			["User-Agent"] = " ",
 			["Server"] = " "
@@ -265,11 +322,18 @@ namespace Foundation {
 					foreach (var v in urlResponse.AllHeaderFields) {
 						// NB: Cocoa trolling us so hard by giving us back dummy dictionary entries
 						if (v.Key == null || v.Value == null) continue;
+						// NSUrlSession tries to be smart with cookies, we will not use the raw value but the ones provided by the cookie storage
+						if (v.Key.ToString () == SetCookie) continue;
 
 						httpResponse.Headers.TryAddWithoutValidation (v.Key.ToString (), v.Value.ToString ());
 						httpResponse.Content.Headers.TryAddWithoutValidation (v.Key.ToString (), v.Value.ToString ());
 					}
 
+					var cookies = session.Configuration.HttpCookieStorage.CookiesForUrl (response.Url);
+					for (var index = 0; index < cookies.Length; index++) {
+						httpResponse.Headers.TryAddWithoutValidation (SetCookie, cookies [index].GetHeaderValue ());
+					}
+
 					inflight.Response = httpResponse;
 
 					// We don't want to send the response back to the task just yet.  Because we want to mimic .NET behavior

tests/monotouch-test/System.Net.Http/MessageHandlers.cs

@@ -71,4 +71,44 @@ namespace MonoTests.System.Net.Http
 			// The handlers throw different types of exceptions, so we can't assert much more than that something went wrong.			
 		}
 
-	}}
+#if !__WATCHOS__
+		// ensure that we do get the same number of cookies as the managed handler
+		[TestCase]
+		public void TestNSUrlSessionHandlerCookies ()
+		{
+			bool areEqual = false;
+			var manageCount = 0;
+			var nativeCount = 0;
+			Exception ex = null;
+
+			TestRuntime.RunAsync (DateTime.Now.AddSeconds (30), async () =>
+			{
+				try {
+					var managedClient = new HttpClient (new HttpClientHandler ());
+					var managedResponse = await managedClient.GetAsync ("https://google.com");
+					if (managedResponse.Headers.TryGetValues ("Set-Cookie", out var managedCookies)) {
+						var nativeClient = new HttpClient (new NSUrlSessionHandler ());
+						var nativeResponse = await nativeClient.GetAsync ("https://google.com");
+						if (managedResponse.Headers.TryGetValues ("Set-Cookie", out var nativeCookies)) {
+							manageCount = managedCookies.Count ();
+							nativeCount = nativeCookies.Count ();
+							areEqual = manageCount == nativeCount;
+						} else {
+							manageCount = -1;
+							nativeCount = -1;
+							areEqual = false;
+						}
+					}
+					
+				} catch (Exception e) {
+					ex = e;
+				} 
+			}, () => areEqual);
+
+			Assert.IsTrue (areEqual, $"Cookies are different - Managed {manageCount} vs Native {nativeCount}");
+			Assert.IsNull (ex, "Exception");
+		}
+#endif
+
+	}
+}