New setting EnableKernelResponseBuffering to improve response throughtput

src/Microsoft.AspNetCore.Server.HttpSys/HttpSysOptions.cs

@@ -61,6 +61,15 @@ namespace Microsoft.AspNetCore.Server.HttpSys
         /// </summary>
         public bool ThrowWriteExceptions { get; set; }
 
+        /// <summary>
+        /// Enable buffering of response data in the Kernel.
+        /// It should be used by an application doing synchronous I/O or by an application doing asynchronous I/O with
+        /// no more than one outstanding write at a time, and can significantly improve throughput over high-latency connections.
+        /// Applications that use asynchronous I/O and that may have more than one send outstanding at a time should not use this flag.
+        /// Enabling this can results in higher CPU and memory usage by Http.Sys.
+        /// </summary>
+        public bool EnableKernelResponseBuffering { get; set; }
+
         /// <summary>
         /// Gets or sets the maximum number of concurrent connections to accept, -1 for infinite, or null to
         /// use the machine wide setting from the registry. The default value is null.

src/Microsoft.AspNetCore.Server.HttpSys/RequestProcessing/ResponseBody.cs

@@ -43,6 +43,8 @@ namespace Microsoft.AspNetCore.Server.HttpSys
 
         internal bool ThrowWriteExceptions => RequestContext.Server.Options.ThrowWriteExceptions;
 
+        internal bool EnableKernelResponseBuffering => RequestContext.Server.Options.EnableKernelResponseBuffering;
+
         internal bool IsDisposed => _disposed;
 
         public override bool CanSeek
@@ -436,6 +438,10 @@ namespace Microsoft.AspNetCore.Server.HttpSys
             else if (!endOfRequest && _leftToWrite != writeCount)
             {
                 flags |= HttpApiTypes.HTTP_FLAGS.HTTP_SEND_RESPONSE_FLAG_MORE_DATA;
+                if (EnableKernelResponseBuffering)
+                {
+                    flags |= HttpApiTypes.HTTP_FLAGS.HTTP_SEND_RESPONSE_FLAG_BUFFER_DATA;
+                }
             }
 
             // Update _leftToWrite now so we can queue up additional async writes.

test/Microsoft.AspNetCore.Server.HttpSys.FunctionalTests/ResponseBodyTests.cs

@@ -38,6 +38,41 @@ namespace Microsoft.AspNetCore.Server.HttpSys
             }
         }
 
+        [ConditionalTheory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public async Task ResponseBody_WriteNoHeaders_SetsChunked_LargeBody(bool enableKernelBuffering)
+        {
+            const int WriteSize = 1024 * 1024;
+            const int NumWrites = 32;
+
+            string address;
+            using (Utilities.CreateHttpServer(
+                baseAddress: out address,
+                configureOptions: options => { options.EnableKernelResponseBuffering = enableKernelBuffering; },
+                app: async httpContext =>
+                {
+					httpContext.Features.Get<IHttpBodyControlFeature>().AllowSynchronousIO = true;
+                    for (int i = 0; i < NumWrites - 1; i++)
+                    {
+                        httpContext.Response.Body.Write(new byte[WriteSize], 0, WriteSize);
+                    }
+                    await httpContext.Response.Body.WriteAsync(new byte[WriteSize], 0, WriteSize);
+                }))
+            {
+                var response = await SendRequestAsync(address);
+                Assert.Equal(200, (int)response.StatusCode);
+                Assert.Equal(new Version(1, 1), response.Version);
+                IEnumerable<string> ignored;
+                Assert.False(response.Content.Headers.TryGetValues("content-length", out ignored), "Content-Length");
+                Assert.True(response.Headers.TransferEncodingChunked.HasValue, "Chunked");
+
+                var bytes = await response.Content.ReadAsByteArrayAsync();
+                Assert.Equal(WriteSize * NumWrites, bytes.Length);
+                Assert.True(bytes.All(b => b == 0));
+            }
+        }
+
         [ConditionalFact]
         public async Task ResponseBody_WriteNoHeadersAndFlush_DefaultsToChunked()
         {