Fix spurious fail log messages (#389)

src/clients/tls_client.h

@@ -115,6 +115,9 @@ public:
 
   void disconnect()
   {
+    // Signal the end of the connection
+    mbedtls_ssl_close_notify(&ssl);
+
     mbedtls_net_free(&server_fd);
     mbedtls_ssl_free(&ssl);
     mbedtls_ssl_config_free(&conf);

src/enclave/rpcsessions.h

@@ -98,7 +98,7 @@ namespace enclave
     void remove_session(size_t id)
     {
       std::lock_guard<SpinLock> guard(lock);
-      LOG_DEBUG_FMT("Stopping a session inside the enclave: {}", id);
+      LOG_DEBUG_FMT("Closing a session inside the enclave: {}", id);
       sessions.erase(id);
     }
 
@@ -141,8 +141,8 @@ namespace enclave
         });
 
       DISPATCHER_SET_MESSAGE_HANDLER(
-        disp, tls::tls_stop, [this](const uint8_t* data, size_t size) {
-          auto [id] = ringbuffer::read_message<tls::tls_stop>(data, size);
+        disp, tls::tls_close, [this](const uint8_t* data, size_t size) {
+          auto [id] = ringbuffer::read_message<tls::tls_close>(data, size);
           remove_session(id);
         });
     }

src/enclave/tlsendpoint.h

@@ -47,6 +47,11 @@ namespace enclave
       ctx->set_bio(this, send_callback, recv_callback, dbg_callback);
     }
 
+    ~TLSEndpoint()
+    {
+      RINGBUFFER_WRITE_MESSAGE(tls::tls_closed, to_host, session_id);
+    }
+
     std::string hostname()
     {
       if (status != ready)
@@ -378,14 +383,23 @@ namespace enclave
       {
         case closed:
         {
-          RINGBUFFER_WRITE_MESSAGE(tls::tls_closed, to_host, session_id);
+          RINGBUFFER_WRITE_MESSAGE(
+            tls::tls_stop, to_host, session_id, std::string("Session closed"));
           break;
         }
 
         case authfail:
+        {
+          RINGBUFFER_WRITE_MESSAGE(
+            tls::tls_stop,
+            to_host,
+            session_id,
+            std::string("Authentication failed"));
+        }
         case error:
         {
-          RINGBUFFER_WRITE_MESSAGE(tls::tls_error, to_host, session_id);
+          RINGBUFFER_WRITE_MESSAGE(
+            tls::tls_stop, to_host, session_id, std::string("Error"));
           break;
         }
 

src/host/proxy.h

@@ -41,6 +41,7 @@ namespace asynchost
     proxy_ptr(proxy_ptr<T>& that) : internal(that.internal) {}
     proxy_ptr(const proxy_ptr<T>& that) : internal(that.internal) {}
     proxy_ptr(proxy_ptr<T>&& that) : internal(std::move(that.internal)) {}
+    proxy_ptr(std::nullptr_t that) : internal(that) {}
 
     template <typename... Args>
     proxy_ptr(Args&&... args) :
@@ -51,6 +52,13 @@ namespace asynchost
     {
       return internal.get()->raw;
     }
+
+    proxy_ptr<T>& operator=(const proxy_ptr<T>& that) = default;
+
+    bool is_null()
+    {
+      return internal == nullptr;
+    }
   };
 
   template <typename handle_type>

src/host/rpcconnections.h

@@ -54,8 +54,7 @@ namespace asynchost
 
       void cleanup()
       {
-        parent.sockets.erase(id);
-        RINGBUFFER_WRITE_MESSAGE(tls::tls_stop, parent.to_enclave, (size_t)id);
+        RINGBUFFER_WRITE_MESSAGE(tls::tls_close, parent.to_enclave, (size_t)id);
       }
     };
 
@@ -169,9 +168,22 @@ namespace asynchost
         return false;
       }
 
+      if (s->second.is_null())
+        return false;
+
       return s->second->write(len, data);
     }
 
+    bool stop(int64_t id)
+    {
+      // Invalidating the TCP socket will result in the handle being closed. No
+      // more messages will be read from or written to the TCP socket.
+      sockets[id] = nullptr;
+      RINGBUFFER_WRITE_MESSAGE(tls::tls_close, to_enclave, (size_t)id);
+
+      return true;
+    }
+
     bool close(int64_t id)
     {
       if (sockets.erase(id) < 1)
@@ -209,6 +221,14 @@ namespace asynchost
             connect(connect_id, host, service);
         });
 
+      DISPATCHER_SET_MESSAGE_HANDLER(
+        disp, tls::tls_stop, [this](const uint8_t* data, size_t size) {
+          auto [id, msg] = ringbuffer::read_message<tls::tls_stop>(data, size);
+
+          LOG_DEBUG_FMT("rpc stop from enclave {}, {}", id, msg);
+          stop(id);
+        });
+
       DISPATCHER_SET_MESSAGE_HANDLER(
         disp, tls::tls_closed, [this](const uint8_t* data, size_t size) {
           auto [id] = ringbuffer::read_message<tls::tls_closed>(data, size);
@@ -216,14 +236,6 @@ namespace asynchost
           LOG_DEBUG_FMT("rpc closed from enclave {}", id);
           close(id);
         });
-
-      DISPATCHER_SET_MESSAGE_HANDLER(
-        disp, tls::tls_error, [this](const uint8_t* data, size_t size) {
-          auto [id] = ringbuffer::read_message<tls::tls_error>(data, size);
-
-          LOG_DEBUG_FMT("rpc error from enclave {}", id);
-          close(id);
-        });
     }
 
   private:

src/tls/msg_types.h

@@ -25,16 +25,17 @@ namespace tls
     /// Data sent from the enclave, to be written to socket. Enclave -> Host
     DEFINE_RINGBUFFER_MSG_TYPE(tls_outbound),
 
-    /// While processing data, the enclave decided this connection is closed.
+    /// While processing data, the enclave decided this connection is stopped.
     /// Enclave -> Host
-    DEFINE_RINGBUFFER_MSG_TYPE(tls_closed),
-
-    /// While processing data, the enclave encountered an error. Enclave -> Host
-    DEFINE_RINGBUFFER_MSG_TYPE(tls_error),
-
-    /// Connection has been removed. No more messages will be sent regarding
-    /// this connection. Host -> Enclave
     DEFINE_RINGBUFFER_MSG_TYPE(tls_stop),
+
+    /// Connection has been invalidated. No more messages will be sent regarding
+    /// this connection. Host -> Enclave
+    DEFINE_RINGBUFFER_MSG_TYPE(tls_close),
+
+    /// Enclave session has been deleted. Host can now safely remove the
+    /// corresponding connection. Enclave -> Host
+    DEFINE_RINGBUFFER_MSG_TYPE(tls_closed),
   };
 }
 
@@ -45,6 +46,6 @@ DECLARE_RINGBUFFER_MESSAGE_PAYLOAD(
   tls::tls_inbound, tls::ConnID, serializer::ByteRange);
 DECLARE_RINGBUFFER_MESSAGE_PAYLOAD(
   tls::tls_outbound, tls::ConnID, serializer::ByteRange);
+DECLARE_RINGBUFFER_MESSAGE_PAYLOAD(tls::tls_stop, tls::ConnID, std::string);
+DECLARE_RINGBUFFER_MESSAGE_PAYLOAD(tls::tls_close, tls::ConnID);
 DECLARE_RINGBUFFER_MESSAGE_PAYLOAD(tls::tls_closed, tls::ConnID);
-DECLARE_RINGBUFFER_MESSAGE_PAYLOAD(tls::tls_error, tls::ConnID);
-DECLARE_RINGBUFFER_MESSAGE_PAYLOAD(tls::tls_stop, tls::ConnID);