fix some bugs; improve memory allocators

build/ipc/ipc.pro

@@ -33,7 +33,9 @@ HEADERS += \
     ../../src/policy.h \
     ../../src/queue.h \
     ../../src/log.h \
-    ../../src/id_pool.h
+    ../../src/id_pool.h \
+    ../../src/pimpl.h \
+    ../../src/concept.h
 
 SOURCES += \
     ../../src/shm.cpp \

src/concept.h

@@ -6,7 +6,24 @@ namespace ipc {
 
 // concept helpers
 
-template <bool Cond, typename R>
+template <bool Cond, typename R = void>
-using Requires = std::enable_if_t<Cond, R>;
+using require = std::enable_if_t<Cond, R>;
+
+#ifdef IPC_CONCEPT_
+#   error "IPC_CONCEPT_ has been defined."
+#endif
+
+#define IPC_CONCEPT_(NAME, WHAT)                                       \
+template <typename T>                                                  \
+class NAME {                                                           \
+private:                                                               \
+    template <typename Type>                                           \
+    static std::true_type check(decltype(std::declval<Type>().WHAT)*); \
+    template <typename Type>                                           \
+    static std::false_type check(...);                                 \
+public:                                                                \
+    using type = decltype(check<T>(nullptr));                          \
+    constexpr static auto value = type::value;                         \
+}
 
 } // namespace ipc

src/memory/alloc.h

@@ -3,9 +3,12 @@
 #include <algorithm>
 #include <utility>
 #include <cstdlib>
+#include <map>
+#include <iterator>
 
 #include "def.h"
 #include "rw_lock.h"
+#include "concept.h"
 
 #include "platform/detail.h"
 
@@ -40,12 +43,14 @@ constexpr std::size_t aligned(std::size_t size, size_t alignment) noexcept {
     return ((size - 1) & ~(alignment - 1)) + alignment;
 }
 
+IPC_CONCEPT_(has_take, take(Type{}));
+
 class scope_alloc_base {
 protected:
     struct block_t {
         block_t   * next_;
         std::size_t size_;
-    } * list_ = nullptr;
+    } * head_ = nullptr, * tail_ = nullptr;
 
     enum : std::size_t {
         aligned_block_size = aligned(sizeof(block_t), alignof(std::max_align_t))
@@ -53,7 +58,23 @@ protected:
 
 public:
     void swap(scope_alloc_base & rhs) {
-        std::swap(this->list_, rhs.list_);
+        std::swap(head_, rhs.head_);
+        std::swap(tail_, rhs.tail_);
+    }
+
+    bool empty() const noexcept {
+        return head_ == nullptr;
+    }
+
+    void take(scope_alloc_base && rhs) {
+        if (rhs.empty()) return;
+        if (empty()) swap(rhs);
+        else {
+            std::swap(tail_->next_, rhs.head_);
+            // rhs.head_ should be nullptr here
+            tail_ = rhs.tail_;
+            rhs.tail_ = nullptr;
+        }
     }
 
     void free(void* /*p*/) {}
@@ -72,19 +93,19 @@ private:
     alloc_policy alloc_;
 
     void free_all() {
-        while (list_ != nullptr) {
+        while (!empty()) {
-            auto curr = list_;
+            auto curr = head_;
-            list_ = list_->next_;
+            head_ = head_->next_;
             alloc_.free(curr, curr->size_);
         }
-        // now list_ is nullptr
+        // now head_ is nullptr
     }
 
 public:
     scope_alloc() = default;
 
-    scope_alloc(scope_alloc&& rhs)            { this->swap(rhs); }
+    scope_alloc(scope_alloc&& rhs)            { swap(rhs); }
-    scope_alloc& operator=(scope_alloc&& rhs) { this->swap(rhs); return (*this); }
+    scope_alloc& operator=(scope_alloc&& rhs) { swap(rhs); return (*this); }
 
     ~scope_alloc() { free_all(); }
 
@@ -98,16 +119,32 @@ public:
         base_t::swap(rhs);
     }
 
+    template <typename A = AllocP>
+    auto take(scope_alloc && rhs) -> ipc::require<detail::has_take<A>::value> {
+        base_t::take(std::move(rhs));
+        alloc_.take(std::move(rhs.alloc_));
+    }
+
+    template <typename A = AllocP>
+    auto take(scope_alloc && rhs) -> ipc::require<!detail::has_take<A>::value> {
+        base_t::take(std::move(rhs));
+    }
+
     void clear() {
         free_all();
+        tail_ = nullptr;
         alloc_.~alloc_policy();
     }
 
     void* alloc(std::size_t size) {
         auto curr = static_cast<block_t*>(alloc_.alloc(size += aligned_block_size));
-        curr->next_ = list_;
+        curr->next_ = head_;
         curr->size_ = size;
-        return (reinterpret_cast<byte_t*>(list_ = curr) + aligned_block_size);
+        head_ = curr;
+        if (tail_ == nullptr) {
+            tail_ = curr;
+        }
+        return (reinterpret_cast<byte_t*>(curr) + aligned_block_size);
     }
 };
 
@@ -137,8 +174,30 @@ protected:
 
 public:
     void swap(fixed_alloc_base& rhs) {
-        std::swap(this->init_expand_, rhs.init_expand_);
+        std::swap(init_expand_, rhs.init_expand_);
-        std::swap(this->cursor_     , rhs.cursor_);
+        std::swap(cursor_     , rhs.cursor_);
+    }
+
+    bool empty() const noexcept {
+        return cursor_ == nullptr;
+    }
+
+    void take(fixed_alloc_base && rhs) {
+        init_expand_ = (ipc::detail::max)(init_expand_, rhs.init_expand_);
+        if (rhs.empty()) return;
+        auto curr = cursor_;
+        if (curr != nullptr) while (1) {
+            auto next_cur = next(curr);
+            if (next_cur == nullptr) {
+                std::swap(next(curr), rhs.cursor_);
+                return;
+            }
+            // next_cur != nullptr
+            else curr = next_cur;
+        }
+        // curr == nullptr, means cursor_ == nullptr
+        else std::swap(cursor_, rhs.cursor_);
+        // rhs.cursor_ must be nullptr
     }
 
     void free(void* p) {
@@ -191,24 +250,23 @@ private:
     alloc_policy alloc_;
 
     void* try_expand() {
-        if (this->cursor_ != nullptr) {
+        if (empty()) {
-            return this->cursor_;
+            auto size = ExpandP::template next<block_size>(init_expand_);
-        }
+            auto p = node_p(cursor_ = alloc_.alloc(size));
-        auto size = ExpandP::template next<block_size>(this->init_expand_);
-        auto p = this->node_p(this->cursor_ = alloc_.alloc(size));
             for (std::size_t i = 0; i < (size / block_size) - 1; ++i)
-            p = this->node_p((*p) = reinterpret_cast<byte_t*>(p) + block_size);
+                p = node_p((*p) = reinterpret_cast<byte_t*>(p) + block_size);
             (*p) = nullptr;
-        return this->cursor_;
+        }
+        return cursor_;
     }
 
 public:
     explicit fixed_alloc(std::size_t init_expand = 1) {
-        this->init(init_expand);
+        init(init_expand);
     }
 
-    fixed_alloc(fixed_alloc&& rhs) : fixed_alloc() { this->swap(rhs); }
+    fixed_alloc(fixed_alloc&& rhs) : fixed_alloc() { swap(rhs); }
-    fixed_alloc& operator=(fixed_alloc&& rhs)      { this->swap(rhs); return (*this); }
+    fixed_alloc& operator=(fixed_alloc&& rhs)      { swap(rhs); return (*this); }
 
     template <typename A>
     void set_allocator(A && alc) {
@@ -220,15 +278,26 @@ public:
         base_t::swap(rhs);
     }
 
+    template <typename A = AllocP>
+    auto take(fixed_alloc && rhs) -> ipc::require<detail::has_take<A>::value> {
+        base_t::take(std::move(rhs));
+        alloc_.take(std::move(rhs.alloc_));
+    }
+
+    template <typename A = AllocP>
+    auto take(fixed_alloc && rhs) -> ipc::require<!detail::has_take<A>::value> {
+        base_t::take(std::move(rhs));
+    }
+
     void clear() {
-        ExpandP::prev(this->init_expand_);
+        ExpandP::prev(init_expand_);
-        this->cursor_ = nullptr;
+        cursor_ = nullptr;
         alloc_.~alloc_policy();
     }
 
     void* alloc() {
         void* p = try_expand();
-        this->cursor_ = this->next(p);
+        cursor_ = next(p);
         return p;
     }
 
@@ -246,22 +315,43 @@ namespace detail {
 class variable_alloc_base {
 protected:
     struct head_t {
-        head_t * next_;
+        std::size_t free_;
-        size_t   size_;
-        size_t   free_;
     } * head_ = nullptr;
 
+    std::map<std::size_t, head_t*, std::greater<std::size_t>> reserves_;
+
     enum : std::size_t {
         aligned_head_size = aligned(sizeof(head_t), alignof(std::max_align_t))
     };
 
     static byte_t * buffer(head_t* p) {
-        return reinterpret_cast<byte_t*>(p) + aligned_head_size;
+        return reinterpret_cast<byte_t*>(p) + aligned_head_size + p->free_;
+    }
+
+    std::size_t remain() const noexcept {
+        return (head_ == nullptr) ? 0 : head_->free_;
     }
 
 public:
     void swap(variable_alloc_base& rhs) {
-        std::swap(this->head_, rhs.head_);
+        std::swap(head_, rhs.head_);
+    }
+
+    bool empty() const noexcept {
+        return remain() == 0;
+    }
+
+    void take(variable_alloc_base && rhs) {
+        if (rhs.remain() > remain()) {
+            if (!empty()) {
+                reserves_.emplace(head_->free_, head_);
+            }
+            head_ = rhs.head_;
+        }
+        else if (!rhs.empty()) {
+            reserves_.emplace(rhs.head_->free_, rhs.head_);
+        }
+        rhs.head_ = nullptr;
     }
 
     void free(void* /*p*/) {}
@@ -270,7 +360,7 @@ public:
 
 } // namespace detail
 
-template <std::size_t ChunkSize = (sizeof(void*) * 1024), typename AllocP = static_alloc>
+template <std::size_t ChunkSize = (sizeof(void*) * 1024), typename AllocP = scope_alloc<>>
 class variable_alloc : public detail::variable_alloc_base {
 public:
     using base_t = detail::variable_alloc_base;
@@ -280,41 +370,11 @@ public:
 private:
     alloc_policy alloc_;
 
-    head_t* alloc_head(std::size_t size) {
-        size = (ipc::detail::max)(ChunkSize, ipc::detail::max<std::size_t>(size, aligned_head_size));
-        head_t* p = static_cast<head_t*>(alloc_.alloc(size));
-        p->free_ = (p->size_ = size) - aligned_head_size;
-        return p;
-    }
-
-    void* alloc_new_chunk(std::size_t size) {
-        head_t* p = alloc_head(aligned_head_size + size);
-        if (p == nullptr) return nullptr;
-        if (size > (ChunkSize - aligned_head_size) && head_ != nullptr) {
-            p->next_ = head_->next_;
-            head_->next_ = p;
-            return base_t::buffer(p) + (p->free_ -= size);
-        }
-        p->next_ = head_;
-        return base_t::buffer(head_ = p) + (p->free_ -= size);
-    }
-
-    void free_all() {
-        while (head_ != nullptr) {
-            head_t* curr = head_;
-            head_ = head_->next_;
-            alloc_.free(curr, curr->size_);
-        }
-        // now head_ is nullptr
-    }
-
 public:
     variable_alloc() = default;
 
-    variable_alloc(variable_alloc&& rhs)            { this->swap(rhs); }
+    variable_alloc(variable_alloc&& rhs)            { swap(rhs); }
-    variable_alloc& operator=(variable_alloc&& rhs) { this->swap(rhs); return (*this); }
+    variable_alloc& operator=(variable_alloc&& rhs) { swap(rhs); return (*this); }
-
-    ~variable_alloc() { free_all(); }
 
     template <typename A>
     void set_allocator(A && alc) {
@@ -326,16 +386,44 @@ public:
         base_t::swap(rhs);
     }
 
+    template <typename A = AllocP>
+    auto take(variable_alloc && rhs) -> ipc::require<detail::has_take<A>::value> {
+        base_t::take(std::move(rhs));
+        alloc_.take(std::move(rhs.alloc_));
+    }
+
+    template <typename A = AllocP>
+    auto take(variable_alloc && rhs) -> ipc::require<!detail::has_take<A>::value> {
+        base_t::take(std::move(rhs));
+    }
+
     void clear() {
-        free_all();
         alloc_.~alloc_policy();
     }
 
-    void* alloc(size_t size) {
+    void* alloc(std::size_t size) {
-        if ((head_ == nullptr) || head_->free_ < size) {
+        if (size >= (ChunkSize - aligned_head_size)) {
-            return alloc_new_chunk(size);
+            return alloc_.alloc(size);
         }
-        return base_t::buffer(head_) + (head_->free_ -= size);
+        if (remain() < size) {
+            auto it = reserves_.begin();
+            if ((it == reserves_.end()) || (it->first < size)) {
+                head_ = static_cast<head_t*>(alloc_.alloc(ChunkSize));
+                head_->free_ = ChunkSize - aligned_head_size - size;
+            }
+            else {
+                auto temp = it->second;
+                temp->free_ -= size;
+                reserves_.erase(it);
+                if (remain() < temp->free_) {
+                    head_ = temp;
+                }
+                else return base_t::buffer(temp);
+            }
+        }
+        // size shouldn't be 0 here, otherwise behavior is undefined
+        else head_->free_ -= size;
+        return base_t::buffer(head_);
     }
 };
 

src/memory/resource.h

@@ -18,7 +18,7 @@ namespace ipc {
 namespace mem {
 
 template <std::size_t Size>
-using static_sync_fixed = static_wrapper<sync_wrapper<fixed_alloc<Size>>>;
+using static_sync_fixed = static_wrapper<async_wrapper<fixed_alloc<Size, static_alloc>>>;
 
 namespace detail {
 
@@ -34,25 +34,15 @@ struct chunk_mapping_policy {
     }
 };
 
-template <typename AllocP>
-struct chunk_alloc_recoverer : default_alloc_recoverer<AllocP> {
-    void collect(alloc_policy && alc) {
-        alc.clear(); // recycle memory to the central heap (static_chunk_alloc)
-        default_alloc_recoverer<AllocP>::collect(std::move(alc));
-    }
-};
-
 } // namespace detail
 
 using static_chunk_alloc   = variable_wrapper<static_sync_fixed, detail::chunk_mapping_policy>;
 using chunk_variable_alloc = variable_alloc<detail::chunk_mapping_policy::base_size, static_chunk_alloc>;
 
 template <std::size_t Size>
-using static_async_fixed =
+using static_async_fixed = static_wrapper<async_wrapper<fixed_alloc<Size, chunk_variable_alloc>>>;
-      static_wrapper<async_wrapper<fixed_alloc<Size/*, chunk_variable_alloc*/>/*, detail::chunk_alloc_recoverer*/>>;
 
 using async_pool_alloc = variable_wrapper<static_async_fixed>;
-//using async_pool_alloc = static_wrapper<async_wrapper<chunk_variable_alloc, detail::chunk_alloc_recoverer>>;
 
 template <typename T>
 using allocator = allocator_wrapper<T, async_pool_alloc>;

src/memory/wrapper.h

@@ -13,6 +13,7 @@
 #include "def.h"
 #include "rw_lock.h"
 #include "tls_pointer.h"
+#include "concept.h"
 
 #include "memory/alloc.h"
 #include "platform/detail.h"
@@ -157,6 +158,19 @@ public:
         }
     }
 
+    template <typename A = AllocP>
+    auto try_replenish(alloc_policy & alc) -> ipc::require<detail::has_take<A>::value> {
+        IPC_UNUSED_ auto guard = ipc::detail::unique_lock(master_lock_);
+        if (!master_allocs_.empty()) {
+            alc.take(std::move(master_allocs_.back()));
+            master_allocs_.pop_back();
+        }
+    }
+
+    template <typename A = AllocP>
+    constexpr static auto try_replenish(alloc_policy & /*alc*/) noexcept
+        -> ipc::require<!detail::has_take<A>::value> {}
+
     void collect(alloc_policy && alc) {
         IPC_UNUSED_ auto guard = ipc::detail::unique_lock(master_lock_);
         master_allocs_.emplace_back(std::move(alc));
@@ -175,6 +189,8 @@ private:
     class alloc_proxy : public AllocP {
         async_wrapper * w_ = nullptr;
 
+        IPC_CONCEPT_(has_empty, empty());
+
     public:
         alloc_proxy(alloc_proxy && rhs)
             : AllocP(std::move(rhs))
@@ -190,6 +206,20 @@ private:
             if (w_ == nullptr) return;
             w_->recoverer_.collect(std::move(*this));
         }
+
+        template <typename A = AllocP>
+        auto alloc(std::size_t size) -> ipc::require<has_empty<A>::value, void*> {
+            auto p = AllocP::alloc(size);
+            if (AllocP::empty() && (w_ != nullptr)) {
+                w_->recoverer_.try_replenish(*this);
+            }
+            return p;
+        }
+
+        template <typename A = AllocP>
+        auto alloc(std::size_t size) -> ipc::require<!has_empty<A>::value, void*> {
+            return AllocP::alloc(size);
+        }
     };
 
     friend class alloc_proxy;

src/pimpl.h

@@ -11,10 +11,10 @@ namespace ipc {
 // pimpl small object optimization helpers
 
 template <typename T, typename R = T*>
-using IsImplComfortable = Requires<(sizeof(T) <= sizeof(T*)), R>;
+using IsImplComfortable = ipc::require<(sizeof(T) <= sizeof(T*)), R>;
 
 template <typename T, typename R = T*>
-using IsImplUncomfortable = Requires<(sizeof(T) > sizeof(T*)), R>;
+using IsImplUncomfortable = ipc::require<(sizeof(T) > sizeof(T*)), R>;
 
 template <typename T, typename... P>
 constexpr auto make_impl(P&&... params) -> IsImplComfortable<T> {

src/platform/to_tchar.h

@@ -27,7 +27,7 @@ template <typename T, typename U>
 struct is_same_char<T, U, std::true_type> : std::is_same<T, typename U::value_type> {};
 
 template <typename T, typename S, typename R = S>
-using IsSameChar = ipc::Requires<is_same_char<T, S>::value, R>;
+using IsSameChar = ipc::require<is_same_char<T, S>::value, R>;
 
 ////////////////////////////////////////////////////////////////
 /// to_tchar implementation

test/test_mem.cpp

@@ -172,9 +172,9 @@ struct test_performance<AllocT, ModeT, 1> {
 };
 
 void Unit::test_static() {
-    //test_performance<ipc::mem::static_alloc, alloc_FIFO  , 8>::start();
+    test_performance<ipc::mem::static_alloc, alloc_FIFO  , 8>::start();
-    //test_performance<ipc::mem::static_alloc, alloc_LIFO  , 8>::start();
+    test_performance<ipc::mem::static_alloc, alloc_LIFO  , 8>::start();
-    //test_performance<ipc::mem::static_alloc, alloc_random, 8>::start();
+    test_performance<ipc::mem::static_alloc, alloc_random, 8>::start();
 }
 
 void Unit::test_pool() {