ChunkedList: store pointers to next/last elements instead of indices …

…to save overhead of address calculation and allow more efficient packing of ItemAccess

* this also reintroduces the chunk-size as template parameter instead of storing it as member
* initialize AtomicList with `chunk_capacity`, not allocation size
* AtomicList: construct Item/ItemControlBlock from `memory::Block`
* ChunkedList: construct Chunk from `memory::Block`
* reorder ChunkedList::Item struct for more efficient alignment
* encode `has_item` of ItemAccess into one 8-byte pointer using bitmagic

redGrapes/resource/resource.cpp

@@ -22,10 +22,7 @@ unsigned int ResourceBase::generateID()
 ResourceBase::ResourceBase()
     : id( generateID() )
     , scope_level( scope_depth() )
-    , users(
-        memory::Allocator( get_arena_id() ),
-        REDGRAPES_RUL_CHUNKSIZE
-    )
+    , users( memory::Allocator( get_arena_id() ) )
 {}
 
 unsigned ResourceBase::get_arena_id() const {

redGrapes/resource/resource.hpp

@@ -50,7 +50,7 @@ class ResourceBase
     unsigned int scope_level;
 
     SpinLock users_mutex;
-    ChunkedList< Task* > users;
+    ChunkedList< Task*, REDGRAPES_RUL_CHUNKSIZE > users;
 
     /**
      * Create a new resource with an unused ID.

redGrapes/resource/resource_user.cpp

@@ -15,8 +15,8 @@ namespace redGrapes
 
     ResourceUser::ResourceUser()
         : scope_level( SingletonContext::get().scope_depth() )
-        , access_list( 64 )
-        , unique_resources( 64 )
+        , access_list( memory::Allocator() )
+        , unique_resources( memory::Allocator() )
     {
     }
 
@@ -29,6 +29,8 @@ namespace redGrapes
 
     ResourceUser::ResourceUser( std::initializer_list< ResourceAccess > list )
         : scope_level( scope_depth() )
+        , access_list( memory::Allocator() )
+        , unique_resources( memory::Allocator() )
     {
         for( auto & ra : list )
             add_resource_access(ra);

redGrapes/resource/resource_user.hpp

@@ -18,7 +18,7 @@
 #include <redGrapes/util/chunked_list.hpp>
 #include <redGrapes/util/trace.hpp>
 
-//#include <redGrapes/resource/resource.hpp>
+#include <redGrapes/resource/resource.hpp>
 
 namespace redGrapes
 {
@@ -32,7 +32,7 @@ struct ResourceAccess;
 struct ResourceUsageEntry
 {
     std::shared_ptr< ResourceBase > resource;
-    typename ChunkedList< Task* >::MutBackwardIterator task_entry;
+    typename ChunkedList< Task*, REDGRAPES_RUL_CHUNKSIZE >::MutBackwardIterator task_entry;
 
     bool operator==( ResourceUsageEntry const & other ) const;
 };
@@ -54,8 +54,8 @@ class ResourceUser
 
     uint8_t scope_level;
 
-    ChunkedList<ResourceAccess> access_list;
-    ChunkedList<ResourceUsageEntry> unique_resources;
+    ChunkedList<ResourceAccess,     8> access_list;
+    ChunkedList<ResourceUsageEntry, 8> unique_resources;
 }; // class ResourceUser
 
 } // namespace redGrapes

redGrapes/scheduler/event.cpp

@@ -28,19 +28,21 @@ namespace scheduler
 Event::Event()
     : state(1)
     , waker_id(-1)
-    , followers( REDGRAPES_EVENT_FOLLOWER_LIST_CHUNKSIZE )
+    , followers( memory::Allocator() )
 {
 }
 
 Event::Event(Event & other)
     : state((uint16_t)other.state)
-    , waker_id(other.waker_id)
+    , waker_id( other.waker_id )
+    , followers( memory::Allocator() )
 {
 }
 
 Event::Event(Event && other)
     : state((uint16_t)other.state)
     , waker_id(other.waker_id)
+    , followers( memory::Allocator() )
 {
 }
 

redGrapes/scheduler/event.hpp

@@ -98,7 +98,7 @@ struct Event
     WakerId waker_id;
 
     //! the set of subsequent events
-    ChunkedList< EventPtr > followers;
+    ChunkedList< EventPtr, REDGRAPES_EVENT_FOLLOWER_LIST_CHUNKSIZE > followers;
 
     Event();
     Event(Event &);

redGrapes/util/atomic_list.hpp

@@ -17,6 +17,8 @@
 #include <hwloc.h>
 #include <spdlog/spdlog.h>
 
+#include <redGrapes/memory/block.hpp>
+
 namespace redGrapes
 {
 namespace memory
@@ -42,22 +44,24 @@ template <
 struct AtomicList
 {
 //private:
-    struct ItemPtr
+    struct ItemControlBlock
     {
         bool volatile deleted;
-        std::shared_ptr< ItemPtr > prev;
-        Item * item_data;
-
-        template < typename... Args >
-        ItemPtr( Item * item_data, Args&&... args )
-                : deleted(false)
-                , prev(nullptr)
-                , item_data(item_data)
+        std::shared_ptr< ItemControlBlock > prev;
+        uintptr_t item_data_ptr;
+
+        ItemControlBlock( memory::Block blk )
+            : item_data_ptr( blk.ptr )
         {
-            new ( get() ) Item ( std::forward<Args>(args)... );
+            /* put Item at front and initialize it
+             * with the remaining memory region
+             */
+            blk.ptr += sizeof(Item);
+            blk.len -= sizeof(Item);
+            new ( get() ) Item ( blk );
         }
 
-        ~ItemPtr()
+        ~ItemControlBlock()
         {
             get()->~Item();
         }
@@ -75,7 +79,7 @@ struct AtomicList
          */
         void skip_deleted_prev()
         {
-            std::shared_ptr<ItemPtr> p = std::atomic_load( &prev );
+            std::shared_ptr<ItemControlBlock> p = std::atomic_load( &prev );
             while( p && p->deleted )
                 p = std::atomic_load( &p->prev );
 
@@ -84,13 +88,13 @@ struct AtomicList
 
         Item * get() const
         {
-            return item_data;
+            return (Item*)item_data_ptr;
         }
     };
 
     Allocator alloc;
-    std::shared_ptr< ItemPtr > head;
-    size_t const chunk_size;
+    std::shared_ptr< ItemControlBlock > head;
+    size_t const chunk_capacity;
 
     /* keeps a single, predefined pointer
      * and frees it on deallocate.
@@ -126,35 +130,30 @@ struct AtomicList
         }
     };
 
-
-
 public:
-
-    AtomicList( Allocator && alloc, size_t chunk_size )
+    AtomicList( Allocator && alloc, size_t chunk_capacity )
         : alloc( alloc )
         , head( nullptr )
-        , chunk_size( chunk_size )
+        , chunk_capacity( chunk_capacity )
     {
-#ifndef NDEBUG
-        if( chunk_size <= get_controlblock_size() )
-             spdlog::error("chunksize = {}, control block ={}", chunk_size, get_controlblock_size());
-#endif
-
-        assert( chunk_size > get_controlblock_size() );
     }
 
     static constexpr size_t get_controlblock_size()
     {
         /* TODO: use sizeof( ...shared_ptr_inplace_something... )
          */
         size_t const shared_ptr_size = 512;
-
-        return sizeof(ItemPtr) + shared_ptr_size;
+        return sizeof(ItemControlBlock) + shared_ptr_size;
     }
 
     constexpr size_t get_chunk_capacity()
     {
-        return chunk_size - get_controlblock_size();
+        return chunk_capacity;
+    }
+
+    constexpr size_t get_chunk_allocsize()
+    {
+        return chunk_capacity + get_controlblock_size();
     }
 
     /* initializes a new chunk
@@ -168,30 +167,25 @@ struct AtomicList
          * - shared_ptr control block
          * - chunk control block
          * - chunk data
-         * whereby chunk data is not included by sizeof(Chunk),
+         * whereby chunk data is not included by sizeof(ItemControlBlock),
          * but reserved by StaticAlloc.
          * This works because shared_ptr control block lies at lower address.
          */
-        StaticAlloc<void> chunk_alloc( this->alloc, chunk_size );
-        uintptr_t base = (uintptr_t)chunk_alloc.ptr;
-        return append_item(
-            std::allocate_shared< ItemPtr >(
-                chunk_alloc,
-
-                /* TODO: generalize this constructor call,
-                 *  specialized for `memory chunks` now
-                 */
-                (Item*) (base + get_controlblock_size()),
-                base + get_controlblock_size() + sizeof(Item),
-                base + chunk_size
-            )
-        );
+        StaticAlloc<void> chunk_alloc( this->alloc, get_chunk_allocsize() );
+
+        // this block will contain the Item-data of ItemControlBlock
+        memory::Block blk{
+            .ptr = (uintptr_t)chunk_alloc.ptr + get_controlblock_size(),
+            .len = chunk_capacity - get_controlblock_size()
+        };
+
+        return append_item( std::allocate_shared< ItemControlBlock >( chunk_alloc, blk ) );
     }
 
     template < bool is_const = false >
     struct BackwardIterator
     {
-        std::shared_ptr< ItemPtr > c;
+        std::shared_ptr< ItemControlBlock > c;
 
         void erase()
         {
@@ -259,7 +253,7 @@ struct AtomicList
 
     MutBackwardIterator rend() const
     {
-        return MutBackwardIterator{ std::shared_ptr<ItemPtr>() };
+        return MutBackwardIterator{ std::shared_ptr<ItemControlBlock>() };
     }
 
     ConstBackwardIterator crbegin() const
@@ -269,7 +263,7 @@ struct AtomicList
 
     ConstBackwardIterator crend() const
     {
-        return ConstBackwardIterator{ std::shared_ptr<ItemPtr>() };
+        return ConstBackwardIterator{ std::shared_ptr<ItemControlBlock>() };
     }
 
     /* Flags chunk at `pos` as erased. Actual removal is delayed until
@@ -287,17 +281,17 @@ struct AtomicList
      * and returns the previous head to which the new_head
      * is now linked.
      */
-    auto append_item( std::shared_ptr< ItemPtr > new_head )
+    auto append_item( std::shared_ptr< ItemControlBlock > new_head )
     {
         TRACE_EVENT("Allocator", "AtomicList::append_item()");
-        std::shared_ptr< ItemPtr > old_head;
+        std::shared_ptr< ItemControlBlock > old_head;
 
         bool append_successful = false;
         while( ! append_successful )
         {
             old_head = std::atomic_load( &head );
             std::atomic_store( &new_head->prev, old_head );
-            append_successful = std::atomic_compare_exchange_strong<ItemPtr>( &head, &old_head, new_head );
+            append_successful = std::atomic_compare_exchange_strong<ItemControlBlock>( &head, &old_head, new_head );
         }
 
         return MutBackwardIterator{ old_head };