port: plug memory leak where deleted socket would never be freed

src/port.c

@@ -51,6 +51,7 @@ ep_port_t* ep_port_new(HANDLE* iocp_out) {
 
   port_info->iocp = iocp;
   queue_init(&port_info->sock_update_queue);
+  queue_init(&port_info->sock_deleted_queue);
   tree_init(&port_info->sock_tree);
   reflock_tree_node_init(&port_info->handle_tree_node);
   InitializeCriticalSection(&port_info->lock);
@@ -86,6 +87,7 @@ int ep_port_close(ep_port_t* port_info) {
 
 int ep_port_delete(ep_port_t* port_info) {
   tree_node_t* tree_node;
+  queue_node_t* queue_node;
 
   EnterCriticalSection(&port_info->lock);
 
@@ -97,6 +99,11 @@ int ep_port_delete(ep_port_t* port_info) {
     ep_sock_force_delete(port_info, sock_info);
   }
 
+  while ((queue_node = queue_first(&port_info->sock_deleted_queue)) != NULL) {
+    ep_sock_t* sock_info = container_of(queue_node, ep_sock_t, queue_node);
+    ep_sock_force_delete(port_info, sock_info);
+  }
+
   for (size_t i = 0; i < array_count(port_info->poll_group_allocators); i++) {
     poll_group_allocator_t* pga = port_info->poll_group_allocators[i];
     if (pga != NULL)
@@ -395,3 +402,17 @@ void ep_port_cancel_socket_update(ep_port_t* port_info, ep_sock_t* sock_info) {
     return;
   queue_remove(&sock_info->queue_node);
 }
+
+void ep_port_add_deleted_socket(ep_port_t* port_info, ep_sock_t* sock_info) {
+  if (queue_enqueued(&sock_info->queue_node))
+    return;
+  queue_append(&port_info->sock_deleted_queue, &sock_info->queue_node);
+}
+
+void ep_port_remove_deleted_socket(ep_port_t* port_info,
+                                   ep_sock_t* sock_info) {
+  unused(port_info);
+  if (!queue_enqueued(&sock_info->queue_node))
+    return;
+  queue_remove(&sock_info->queue_node);
+}

src/port.h

@@ -21,6 +21,7 @@ typedef struct ep_port {
       poll_group_allocators[array_count(AFD_PROVIDER_GUID_LIST)];
   tree_t sock_tree;
   queue_t sock_update_queue;
+  queue_t sock_deleted_queue;
   reflock_tree_node_t handle_tree_node;
   CRITICAL_SECTION lock;
   size_t active_poll_count;
@@ -60,4 +61,9 @@ WEPOLL_INTERNAL void ep_port_request_socket_update(ep_port_t* port_info,
 WEPOLL_INTERNAL void ep_port_cancel_socket_update(ep_port_t* port_info,
                                                   ep_sock_t* sock_info);
 
+WEPOLL_INTERNAL void ep_port_add_deleted_socket(ep_port_t* port_info,
+                                                ep_sock_t* sock_info);
+WEPOLL_INTERNAL void ep_port_remove_deleted_socket(ep_port_t* port_info,
+                                                   ep_sock_t* sock_info);
+
 #endif /* WEPOLL_PORT_H_ */

src/sock.c

@@ -34,7 +34,7 @@ typedef struct _ep_sock_private {
   uint32_t user_events;
   uint32_t pending_events;
   _poll_status_t poll_status;
-  unsigned deleted : 1;
+  bool delete_pending;
 } _ep_sock_private_t;
 
 static DWORD _epoll_events_to_afd_events(uint32_t epoll_events) {
@@ -155,7 +155,6 @@ static inline _ep_sock_private_t* _ep_sock_alloc(void) {
 }
 
 static inline void _ep_sock_free(_ep_sock_private_t* sock_private) {
-  assert(sock_private->poll_status == _POLL_IDLE);
   free(sock_private);
 }
 
@@ -216,30 +215,41 @@ err1:
   return NULL;
 }
 
-void ep_sock_delete(ep_port_t* port_info, ep_sock_t* sock_info) {
+static void _ep_sock_delete(ep_port_t* port_info,
+                            ep_sock_t* sock_info,
+                            bool force) {
   _ep_sock_private_t* sock_private = _ep_sock_private(sock_info);
 
-  assert(!sock_private->deleted);
+  if (!sock_private->delete_pending) {
-  sock_private->deleted = true;
+    if (sock_private->poll_status == _POLL_PENDING)
+      _ep_sock_cancel_poll(sock_private);
 
-  if (sock_private->poll_status == _POLL_PENDING)
+    ep_port_cancel_socket_update(port_info, sock_info);
-    _ep_sock_cancel_poll(sock_private);
+    ep_port_del_socket(port_info, sock_info);
 
-  ep_port_del_socket(port_info, sock_info);
+    sock_private->delete_pending = true;
-  ep_port_cancel_socket_update(port_info, sock_info);
+  }
-  ep_port_release_poll_group(port_info, sock_private->poll_group);
-  sock_private->poll_group = NULL;
 
   /* If the poll request still needs to complete, the ep_sock object can't
-   * be free()d yet. `ep_sock_feed_event` will take care of this later. */
+   * be free()d yet. `ep_sock_feed_event()` or `ep_port_close()` will take care
-  if (sock_private->poll_status == _POLL_IDLE)
+   * of this later. */
+  if (force || sock_private->poll_status == _POLL_IDLE) {
+    /* Free the sock_info now. */
+    ep_port_remove_deleted_socket(port_info, sock_info);
+    ep_port_release_poll_group(port_info, sock_private->poll_group);
     _ep_sock_free(sock_private);
+  } else {
+    /* Free the socket later. */
+    ep_port_add_deleted_socket(port_info, sock_info);
+  }
+}
+
+void ep_sock_delete(ep_port_t* port_info, ep_sock_t* sock_info) {
+  _ep_sock_delete(port_info, sock_info, false);
 }
 
 void ep_sock_force_delete(ep_port_t* port_info, ep_sock_t* sock_info) {
-  _ep_sock_private_t* sock_private = _ep_sock_private(sock_info);
+  _ep_sock_delete(port_info, sock_info, true);
-  sock_private->poll_status = _POLL_IDLE;
-  ep_sock_delete(port_info, sock_info);
 }
 
 int ep_sock_set_event(ep_port_t* port_info,
@@ -265,6 +275,7 @@ int ep_sock_update(ep_port_t* port_info, ep_sock_t* sock_info) {
   _ep_sock_private_t* sock_private = _ep_sock_private(sock_info);
   bool socket_closed = false;
 
+  assert(!sock_private->delete_pending);
   if ((sock_private->poll_status == _POLL_PENDING) &&
       (sock_private->user_events & _KNOWN_EPOLL_EVENTS &
        ~sock_private->pending_events) == 0) {
@@ -331,10 +342,10 @@ int ep_sock_feed_event(ep_port_t* port_info,
   sock_private->poll_status = _POLL_IDLE;
   sock_private->pending_events = 0;
 
-  if (sock_private->deleted) {
+  if (sock_private->delete_pending) {
-    /* Ignore completion for overlapped poll operation if the socket has been
+    /* Ignore completion for overlapped poll operation if the socket is pending
-     * deleted; instead, free the socket. */
+     * deletion; instead, delete the socket. */
-    _ep_sock_free(sock_private);
+    ep_sock_delete(port_info, sock_info);
     return 0;
   }