[c# epoxy] Add connection keep-alive

CHANGELOG.md

@@ -11,6 +11,20 @@ tag versions. The Bond compiler (`gbc`) and
 different versioning scheme, following the Haskell community's
 [package versioning policy](https://wiki.haskell.org/Package_versioning_policy).
 
+## Unreleased #
+
+* `gbc` & compiler library: TBD
+* IDL core version: TBD
+* IDL comm version: TBD
+* C++ version: TBD
+* C# NuGet version: TBD
+* C# Comm NuGet version: minor version bump needed
+
+### C# Comm ###
+* `EpoxyTransport` can be configured to enable TCP keep-alive to help detect
+  dead connections. See `EpoxyTransportBuilder.SetKeepAliveTimes` for
+  details.
+
 ## 5.0.0: 2016-09-12 #
 
 * `gbc` & compiler library: 0.6.0.0

cs/src/comm/epoxy-transport/EpoxyListener.cs

@@ -16,7 +16,8 @@ namespace Bond.Comm.Epoxy
     public class EpoxyListener : Listener
     {
         readonly EpoxyTransport parentTransport;
-	readonly EpoxyServerTlsConfig tlsConfig;
+        readonly EpoxyServerTlsConfig tlsConfig;
+        readonly EpoxyTransport.TimeoutConfig timeoutConfig;
         readonly TcpListener listener;
         readonly ServiceHost serviceHost;
 
@@ -33,6 +34,7 @@ namespace Bond.Comm.Epoxy
             EpoxyTransport parentTransport,
             IPEndPoint listenEndpoint,
             EpoxyServerTlsConfig tlsConfig,
+            EpoxyTransport.TimeoutConfig timeoutConfig,
             Logger logger, Metrics metrics) : base(logger, metrics)
         {
             Debug.Assert(parentTransport != null);
@@ -42,6 +44,7 @@ namespace Bond.Comm.Epoxy
 
             // will be null if not using TLS
             this.tlsConfig = tlsConfig;
+            this.timeoutConfig = timeoutConfig;
 
             listener = new TcpListener(listenEndpoint);
             serviceHost = new ServiceHost(logger);
@@ -110,6 +113,8 @@ namespace Bond.Comm.Epoxy
                     socket = await listener.AcceptSocketAsync();
                     logger.Site().Debug("Accepted connection from {0}.", socket.RemoteEndPoint);
 
+                    EpoxyTransport.ConfigureSocketKeepAlive(socket, timeoutConfig, logger);
+
                     epoxyStream = await EpoxyNetworkStream.MakeServerStreamAsync(socket, tlsConfig, logger);
                     socket = null; // epoxyStream now owns the socket
 
@@ -145,11 +150,6 @@ namespace Bond.Comm.Epoxy
                 catch (ObjectDisposedException)
                 {
                     ShutdownSocketSafe(socket, epoxyStream);
-
-                    // TODO: ignoring this exception is needed during shutdown,
-                    //       but there should be a cleaner way. We should
-                    //       switch to having a proper life-cycle for a
-                    //       connection.
                 }
             }
 

cs/src/comm/epoxy-transport/EpoxyTransport.cs

@@ -4,7 +4,6 @@
 namespace Bond.Comm.Epoxy
 {
     using System;
-    using System.Linq;
     using System.Net;
     using System.Net.Sockets;
     using System.Threading;
@@ -17,6 +16,8 @@ namespace Bond.Comm.Epoxy
         EpoxyServerTlsConfig serverTlsConfig;
         EpoxyClientTlsConfig clientTlsConfig;
 
+        EpoxyTransport.TimeoutConfig timeoutConfig;
+
         /// <summary>
         /// Sets a customer host resolver.
         /// </summary>
@@ -71,6 +72,43 @@ namespace Bond.Comm.Epoxy
             return this;
         }
 
+        /// <summary>
+        /// Sets the transport's per-connection keep-alive timeout and interval.
+        /// </summary>
+        /// <param name="keepAliveTime">
+        /// The amount of time to wait for the connection to be idle before starting to send
+        /// keep-alive probes.
+        /// </param>
+        /// <param name="keepAliveInterval">
+        /// The interval to wait between unsuccessful keep-alive probes.
+        /// </param>
+        /// <returns>The builder.</returns>
+        /// <remarks>
+        /// <para>
+        /// Connection keep-alive can be used to detect if an Epoxy connection has gone down even if
+        /// there are no messages being sent. If both keep-alive parameters are set to non-zero
+        /// values, connection keep-alive will be enabled, otherwise no keep-alive probing will be
+        /// performed and detection of a down connection will rely on attempts to send messages.
+        /// </para>
+        /// <para>
+        /// When keep-alive is being used, some arbitrary, fixed number of probes must fail in a row
+        /// before the connection is declared dead. The number of probes is not configurable.
+        /// </para>
+        /// <para>
+        /// Connection keep-alive utilizes TCP keep-alive if available. If TCP keep-alive is not
+        /// available or cannot be configured, these settings have no effect and detection of a down
+        /// connection will rely on attempts to send messages.
+        /// </para>
+        /// </remarks>
+        public EpoxyTransportBuilder SetKeepAliveTimes(
+            TimeSpan keepAliveTime,
+            TimeSpan keepAliveInterval)
+        {
+            timeoutConfig.KeepAliveTime = keepAliveTime;
+            timeoutConfig.KeepAliveInterval = keepAliveInterval;
+            return this;
+        }
+
         public override EpoxyTransport Construct()
         {
             return new EpoxyTransport(
@@ -78,6 +116,7 @@ namespace Bond.Comm.Epoxy
                 resolver,
                 serverTlsConfig,
                 clientTlsConfig,
+                timeoutConfig,
                 LogSink,
                 EnableDebugLogs,
                 MetricsSink);
@@ -93,6 +132,7 @@ namespace Bond.Comm.Epoxy
         readonly Func<string, Task<IPAddress>> resolver;
         readonly EpoxyServerTlsConfig serverTlsConfig;
         readonly EpoxyClientTlsConfig clientTlsConfig;
+        readonly TimeoutConfig timeoutConfig;
         readonly Logger logger;
         readonly Metrics metrics;
 
@@ -124,11 +164,18 @@ namespace Bond.Comm.Epoxy
             }
         }
 
+        public struct TimeoutConfig
+        {
+            public TimeSpan KeepAliveTime;
+            public TimeSpan KeepAliveInterval;
+        }
+
         public EpoxyTransport(
             ILayerStackProvider layerStackProvider,
             Func<string, Task<IPAddress>> resolver,
             EpoxyServerTlsConfig serverTlsConfig,
             EpoxyClientTlsConfig clientTlsConfig,
+            TimeoutConfig timeoutConfig,
             ILogSink logSink, bool enableDebugLogs,
             IMetricsSink metricsSink)
         {
@@ -146,6 +193,8 @@ namespace Bond.Comm.Epoxy
             // happened to get null
             this.clientTlsConfig = clientTlsConfig ?? EpoxyClientTlsConfig.Default;
 
+            this.timeoutConfig = timeoutConfig;
+
             // Log sink may be null
             logger = new Logger(logSink, enableDebugLogs);
             // Metrics sink may be null
@@ -256,7 +305,7 @@ namespace Bond.Comm.Epoxy
 
         public EpoxyListener MakeListener(IPEndPoint address)
         {
-            return new EpoxyListener(this, address, serverTlsConfig, logger, metrics);
+            return new EpoxyListener(this, address, serverTlsConfig, timeoutConfig, logger, metrics);
         }
 
         public override Task StopAsync()
@@ -347,10 +396,60 @@ namespace Bond.Comm.Epoxy
             logger.Site().Debug("Resolved {0} to {1}.", endpoint.Host, ipAddress);
 
             var socket = new Socket(ipAddress.AddressFamily, SocketType.Stream, ProtocolType.Tcp);
+            ConfigureSocketKeepAlive(socket, timeoutConfig, logger);
             await Task.Factory.FromAsync(socket.BeginConnect, socket.EndConnect, ipAddress, endpoint.Port, state: null);
 
             logger.Site().Information("Established TCP connection to {0} at {1}:{2}", endpoint.Host, ipAddress, endpoint.Port);
             return socket;
         }
+
+        internal static void ConfigureSocketKeepAlive(Socket socket, TimeoutConfig timeoutConfig, Logger logger)
+        {
+            if (timeoutConfig.KeepAliveTime != TimeSpan.Zero && timeoutConfig.KeepAliveInterval != TimeSpan.Zero)
+            {
+                // Socket.IOControl for IOControlCode.KeepAliveValues is expecting a structure like
+                // the following on Windows:
+                //
+                // struct tcp_keepalive
+                // {
+                //     u_long onoff; // 0 for off, non-zero for on
+                //     u_long keepalivetime; // milliseconds
+                //     u_long keepaliveinterval; // milliseconds
+                // };
+                //
+                // On some platforms this gets mapped to the relevant OS structures, but on other
+                // platforms, this may fail with a PlatformNotSupportedException.
+                UInt32 keepAliveTimeMillis = checked((UInt32) timeoutConfig.KeepAliveTime.TotalMilliseconds);
+                UInt32 keepAliveIntervalMillis = checked((UInt32) timeoutConfig.KeepAliveInterval.TotalMilliseconds);
+
+                var keepAliveVals = new byte[sizeof (UInt32)*3];
+                keepAliveVals[0] = 1;
+
+                keepAliveVals[4] = (byte) (keepAliveTimeMillis & 0xff);
+                keepAliveVals[5] = (byte) ((keepAliveTimeMillis >> 8) & 0xff);
+                keepAliveVals[6] = (byte) ((keepAliveTimeMillis >> 16) & 0xff);
+                keepAliveVals[7] = (byte) ((keepAliveTimeMillis >> 24) & 0xff);
+
+                keepAliveVals[8] = (byte) (keepAliveIntervalMillis & 0xff);
+                keepAliveVals[9] = (byte) ((keepAliveIntervalMillis >> 8) & 0xff);
+                keepAliveVals[10] = (byte) ((keepAliveIntervalMillis >> 16) & 0xff);
+                keepAliveVals[11] = (byte) ((keepAliveIntervalMillis >> 24) & 0xff);
+
+                try
+                {
+                    socket.IOControl(IOControlCode.KeepAliveValues, keepAliveVals, null);
+                }
+                catch (ObjectDisposedException)
+                {
+                    // Oh well: the connection went down before we could configure it. Nothing to be
+                    // done, except to wait for the next socket operation to fail and let normal
+                    // clean up take over.
+                }
+                catch (Exception ex) when (ex is SocketException || ex is PlatformNotSupportedException)
+                {
+                    logger.Site().Warning(ex, "Socket keep-alive could not be configured");
+                }
+            }
+        }
     }
 }