Zakero_MemoryPool.h

Go to the documentation of this file.

/******************************************************************************
 * Copyright 2020 Andrew Moore
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at https://mozilla.org/MPL/2.0/.
 */
 
#ifndef zakero_MemoryPool_h
#define zakero_MemoryPool_h
 
/******************************************************************************
 * Includes
 */
 
#include <cstring>
#include <functional>
#include <map>
#include <mutex>
#include <system_error>
 
// POSIX
#include <sys/mman.h>
#include <unistd.h>
 
 
/******************************************************************************
 * Macros
 */
 
// {{{ Macros
 
#define ZAKERO_MEMORYPOOL__ERROR_DATA \
        X(Error_None                  , 0 , "No Error"                                                     ) \
        X(Error_Already_Initialized   , 1 , "The Memory Pool has already been initialized."                ) \
        X(Error_Size_Too_Small        , 2 , "Invalid Size: Must be greater than 0."                        ) \
        X(Error_Size_Too_Large        , 3 , "Invalid Size: Must be less than zakero::MemoryPool::Size_Max" ) \
        X(Error_Failed_To_Create_File , 4 , "Unable to create file."                                       ) \
        X(Error_Failed_To_Resize_File , 5 , "Unable to resize file."                                       ) \
        X(Error_Failed_To_Map_File    , 6 , "Unable to memory map the file."                               ) \
        X(Error_Out_Of_Memory         , 7 , "Not enough contiguous memory."                                ) \
        X(Error_Invalid_Offset        , 8 , "The offset is not valid."                                     ) \
 
// }}}
 
namespace zakero
{
        // {{{ Declaration
 
        class MemoryPool
        {
                public:
#define X(name_, val_, mesg_) \
                        static constexpr int name_ = val_;
                        ZAKERO_MEMORYPOOL__ERROR_DATA
#undef X
                        static constexpr size_t Size_Max = (size_t)std::numeric_limits<off_t>::max();
 
                        enum class Alignment : uint8_t
                        {       Bits_8  = 0
                        ,       Bits_16 = 1
                        ,       Bits_32 = 3
                        ,       Bits_64 = 7
                        ,       Byte_1 = Bits_8
                        ,       Byte_2 = Bits_16
                        ,       Byte_4 = Bits_32
                        ,       Byte_8 = Bits_64
                        };
 
                        struct Segment
                        {
                                off_t offset;
                                off_t size;
                                bool  in_use;
                        };
 
                        using AddressMap = std::map<uint8_t*, uint8_t*>;
 
                        using LambdaSize       = std::function<void(size_t)>;
                        using LambdaAddressMap = std::function<void(const MemoryPool::AddressMap&)>;
 
                        using VectorSegment = std::vector<Segment>;
 
                        MemoryPool(const std::string&) noexcept;
                        ~MemoryPool() noexcept;
 
                                      std::error_code init(const size_t, const bool = false, const MemoryPool::Alignment = MemoryPool::Alignment::Bits_64) noexcept;
                        [[nodiscard]] int             fd() const noexcept;
                        [[nodiscard]] size_t          size() const noexcept;
                                      void            sizeOnChange(MemoryPool::LambdaSize) noexcept;
 
                        [[nodiscard]] off_t           alloc(const size_t) noexcept;
                        [[nodiscard]] off_t           alloc(const size_t, std::error_code&) noexcept;
                        [[nodiscard]] off_t           alloc(size_t, uint8_t) noexcept;
                        [[nodiscard]] off_t           alloc(size_t, uint8_t, std::error_code&) noexcept;
                        [[nodiscard]] off_t           alloc(size_t, uint32_t) noexcept;
                        [[nodiscard]] off_t           alloc(size_t, uint32_t, std::error_code&) noexcept;
                                      void            free(off_t&) noexcept;
                        [[nodiscard]] off_t           realloc(off_t, size_t) noexcept;
                        [[nodiscard]] off_t           realloc(off_t, size_t, std::error_code&) noexcept;
 
                        [[nodiscard]] uint8_t*        addressOf(off_t) const noexcept;
                                      void            onRemap(MemoryPool::LambdaAddressMap) noexcept;
                        [[nodiscard]] static uint8_t* remap(const MemoryPool::AddressMap&, uint8_t*) noexcept;
 
                        [[nodiscard]] std::string     dump(size_t, size_t) const noexcept;
                        [[nodiscard]] VectorSegment   segmentList() const noexcept;
 
                private:
                        uint8_t*                     memory;
                        std::string                  name;
                        mutable std::mutex           mutex;
                        MemoryPool::VectorSegment    segment;
                        MemoryPool::LambdaSize       size_on_change;
                        MemoryPool::LambdaAddressMap on_remap;
                        size_t                       pool_size;
                        int                          file_descriptor;
                        MemoryPool::Alignment        alignment;
                        bool                         pool_can_expand;
 
                        // -------------------------------------------------- //
 
                        [[nodiscard]] bool   expandToFit(size_t, size_t&) noexcept;
                                      
                        [[nodiscard]] bool   segmentExpand(const size_t, const size_t) noexcept;
                        [[nodiscard]] bool   segmentFindBestFit(const size_t, size_t&) noexcept;
                                      bool   segmentFindInUse(const off_t, size_t&) const noexcept;
                                      void   segmentMergeNext(const size_t) noexcept;
                                      void   segmentMergePrev(const size_t) noexcept;
                        [[nodiscard]] size_t segmentMove(const size_t, const size_t, size_t&) noexcept;
                                      void   segmentSplit(size_t, size_t) noexcept;
 
                        // -------------------------------------------------- //
 
                        MemoryPool(const MemoryPool&) = delete;
                        MemoryPool& operator=(const MemoryPool&) = delete;
        };
 
        // }}}
        // {{{ Convenience
 
        std::string to_string(const MemoryPool::VectorSegment&) noexcept;
 
        // }}}
}
 
// {{{ Implementation
 
#ifdef ZAKERO_MEMORYPOOL_IMPLEMENTATION
 
// {{{ Macros
// {{{ Macros : Doxygen
 
#ifdef ZAKERO__DOXYGEN_DEFINE_DOCS
 
// Only used for generating Doxygen documentation
 
#define ZAKERO_MEMORYPOOL_IMPLEMENTATION
 
#endif
 
// }}}
 
#define ZAKERO_MEMORYPOOL__ERROR(err_) std::error_code(err_, MemoryPoolErrorCategory);
 
// }}}
 
namespace zakero
{
// {{{ Anonymous Namespace
 
namespace
{
        class MemoryPoolErrorCategory_
                : public std::error_category
        {
                public:
                        constexpr MemoryPoolErrorCategory_() noexcept
                        {
                        }
 
                        const char* name() const noexcept override
                        {
                                return "zakero.MemoryPool";
                        }
 
                        std::string message(int condition) const override
                        {
                                switch(condition)
                                {
#define X(name_, val_, mesg_) \
                                        case val_: return mesg_;
                                        ZAKERO_MEMORYPOOL__ERROR_DATA
#undef X
                                }
 
                                return "Unknown error condition";
                        }
        } MemoryPoolErrorCategory;
 
        zakero::MemoryPool::LambdaSize       LambdaSize_DoNothing       = [](size_t) noexcept {};
        zakero::MemoryPool::LambdaAddressMap LambdaAddressMap_DoNothing = [](const zakero::MemoryPool::AddressMap&) noexcept {};
        inline off_t calculateActualSize(const off_t  size      
                , const zakero::MemoryPool::Alignment alignment 
                )
        {
                const off_t mod  = static_cast<off_t>(alignment);
                const off_t step = static_cast<off_t>(alignment) + 1;
 
                return ((size + mod) / step) * step;
        }
}
 
// }}}
// {{{ Documentation
 
// }}}
// {{{ MemoryPool
 
        MemoryPool::MemoryPool(const std::string& name 
                ) noexcept
                : memory(nullptr)
                , name(name)
                , mutex()
                , segment()
                , size_on_change(LambdaSize_DoNothing)
                , on_remap(LambdaAddressMap_DoNothing)
                , pool_size(0)
                , file_descriptor(-1)
                , alignment(zakero::MemoryPool::Alignment::Bits_64)
                , pool_can_expand(false)
        {
        }
 
 
        MemoryPool::~MemoryPool() noexcept
        {
                on_remap = LambdaAddressMap_DoNothing;
                segment.clear();
                name.clear();
 
                if(memory != nullptr)
                {
                        #ifdef ZAKERO_MEMORYPOOL_ZERO_ON_FREE
                        memset(memory, '\0', pool_size);
                        #endif
 
                        munmap(memory, pool_size);
                        memory = nullptr;
                }
 
                close(file_descriptor);
                file_descriptor = -1;
 
                pool_size = 0;
        }
 
 
        std::error_code MemoryPool::init(const size_t size       
                , const bool                          expandable 
                , const MemoryPool::Alignment         alignment  
                ) noexcept
        {
                //ZAKERO_PROFILER_DURATION("MemoryPool", "init");
 
                if(this->file_descriptor != -1)
                {
                        return ZAKERO_MEMORYPOOL__ERROR(Error_Already_Initialized);
                }
 
                if(size <= 0)
                {
                        return ZAKERO_MEMORYPOOL__ERROR(Error_Size_Too_Small);
                }
 
                if(size > MemoryPool::Size_Max)
                {
                        return ZAKERO_MEMORYPOOL__ERROR(Error_Size_Too_Large);
                }
 
                size_t pool_size = calculateActualSize(size, alignment);
 
                #ifdef __linux__
                int fd = memfd_create(name.c_str(), 0);
                #else
                #error Need more code...
                #endif
 
                if(fd == -1)
                {
                        return ZAKERO_MEMORYPOOL__ERROR(Error_Failed_To_Create_File);
                }
 
                if(ftruncate(fd, pool_size) == -1)
                {
                        return ZAKERO_MEMORYPOOL__ERROR(Error_Failed_To_Resize_File);
                }
 
                uint8_t* memory = (uint8_t*)mmap(nullptr
                        , pool_size
                        , PROT_READ | PROT_WRITE
                        , MAP_SHARED | MAP_NORESERVE
                        , fd
                        , 0
                        );
 
                if(memory == MAP_FAILED)
                {
                        close(fd);
 
                        return ZAKERO_MEMORYPOOL__ERROR(Error_Failed_To_Map_File);
                }
 
                this->segment.push_back(
                {       .offset = 0
                ,       .size   = (off_t)pool_size
                ,       .in_use = false
                });
 
                this->pool_can_expand = expandable;
                this->alignment       = alignment;
                this->pool_size       = pool_size;
                this->memory          = memory;
                this->file_descriptor = fd;
 
                return ZAKERO_MEMORYPOOL__ERROR(Error_None);
        }
 
 
        int MemoryPool::fd() const noexcept
        {
                return file_descriptor;
        }
 
 
        size_t MemoryPool::size() const noexcept
        {
                return pool_size;
        }
 
 
        void MemoryPool::sizeOnChange(MemoryPool::LambdaSize lambda 
                ) noexcept
        {
                if(lambda == nullptr)
                {
                        size_on_change = LambdaSize_DoNothing;
                }
                else
                {
                        size_on_change = lambda;
                }
        }
 
 
        off_t MemoryPool::alloc(const size_t size 
                ) noexcept
        {
                std::error_code error;
 
                return alloc(size, error);
        }
 
 
        off_t MemoryPool::alloc(const size_t size  
                , std::error_code&           error 
                ) noexcept
        {
                //ZAKERO_PROFILER_DURATION("MemoryPool", "alloc");
 
                // Validate Input
 
                if(size <= 0)
                {
                        error = ZAKERO_MEMORYPOOL__ERROR(Error_Size_Too_Small);
                        return -1;
                }
 
                if(size > MemoryPool::Size_Max)
                {
                        error = ZAKERO_MEMORYPOOL__ERROR(Error_Size_Too_Large);
                        return -1;
                }
 
                const size_t segment_size = calculateActualSize(size, alignment);
 
                // Allocate from the pool
 
                std::lock_guard<std::mutex> lock(mutex);
 
                size_t index = 0;
 
                if(segmentFindBestFit(segment_size, index) == false)
                {
                        if(pool_can_expand == false)
                        {
                                error = ZAKERO_MEMORYPOOL__ERROR(Error_Out_Of_Memory);
                                return -1;
                        }
 
                        if(expandToFit(segment_size, index) == false)
                        {
                                error = ZAKERO_MEMORYPOOL__ERROR(Error_Failed_To_Resize_File);
                                return -1;
                        }
                }
 
                error = ZAKERO_MEMORYPOOL__ERROR(Error_None);
 
                segment[index].in_use = true;
 
                if((size_t)segment[index].size == segment_size)
                {
                        return segment[index].offset;
                }
 
                segmentSplit(index, segment_size);
 
                return segment[index].offset;
        }
 
 
        off_t MemoryPool::alloc(size_t size  
                , uint8_t              value 
                ) noexcept
        {
                std::error_code error;
 
                return alloc(size, value, error);
        }
 
 
        off_t MemoryPool::alloc(size_t size  
                , uint8_t              value 
                , std::error_code&     error 
                ) noexcept
        {
                off_t offset = alloc(size, error);
 
                if(offset < 0)
                {
                        return offset;
                }
 
                uint8_t* ptr = addressOf(offset);
 
                memset(ptr, value, size);
 
                return offset;
        }
 
 
        off_t MemoryPool::alloc(size_t size  
                , uint32_t             value 
                ) noexcept
        {
                std::error_code error;
 
                return alloc(size, value, error);
        }
 
 
        off_t MemoryPool::alloc(size_t size  
                , const uint32_t       value 
                , std::error_code&     error 
                ) noexcept
        {
                off_t offset = alloc(size, error);
 
                if(offset < 0)
                {
                        return offset;
                }
 
                size_t count = size / 4;
                uint32_t* p = (uint32_t*)addressOf(offset);
 
                for(uint32_t i = 0; i < count; i++)
                {
                        p[i] = value;
                }
 
                return offset;
        }
 
 
        void MemoryPool::free(off_t& offset 
                ) noexcept
        {
                //ZAKERO_PROFILER_DURATION("MemoryPool", "free");
 
                std::lock_guard<std::mutex> lock(mutex);
 
                size_t index = 0;
 
                if(segmentFindInUse(offset, index) == false)
                {
                        return;
                }
 
                #ifdef ZAKERO_MEMORYPOOL_ZERO_ON_FREE
                memset(memory + segment[index].offset
                        , '\0'
                        , segment[index].size
                        );
                #endif
 
                segment[index].in_use = false;
 
                segmentMergeNext(index);
                segmentMergePrev(index);
 
                offset = -1;
        }
 
 
        off_t MemoryPool::realloc(off_t offset 
                , size_t                size   
                ) noexcept
        {
                std::error_code error;
 
                return realloc(offset, size, error);
        }
 
 
        off_t MemoryPool::realloc(off_t offset 
                , size_t                size   
                , std::error_code&      error  
                ) noexcept
        {
                //ZAKERO_PROFILER_DURATION("MemoryPool", "realloc");
 
                // Validate Input
 
                if(size <= 0)
                {
                        error = ZAKERO_MEMORYPOOL__ERROR(Error_Size_Too_Small);
                        return -1;
                }
 
                if(size > MemoryPool::Size_Max)
                {
                        error = ZAKERO_MEMORYPOOL__ERROR(Error_Size_Too_Large);
                        return -1;
                }
 
                const size_t segment_size = calculateActualSize(size, alignment);
 
                std::lock_guard<std::mutex> lock(mutex);
 
                size_t index_src = 0;
 
                if(segmentFindInUse(offset, index_src) == false)
                {
                        error = ZAKERO_MEMORYPOOL__ERROR(Error_Invalid_Offset);
                        return -1;
                }
 
                Segment& segment_src = segment[index_src];
 
                // Same size, nothing to do
                if(segment_size == (size_t)segment_src.size)
                {
                        error = ZAKERO_MEMORYPOOL__ERROR(Error_None);
                        return offset;
                }
 
                // New size is smaller, shrink segment
                if(segment_size < (size_t)segment_src.size)
                {
                        segmentSplit(index_src, segment_size);
 
                        error = ZAKERO_MEMORYPOOL__ERROR(Error_None);
                        return offset;
                }
 
                // Larger, try to expand into the next segment
                if(segmentExpand(index_src, segment_size) == true)
                {
                        error = ZAKERO_MEMORYPOOL__ERROR(Error_None);
                        return offset;
                }
 
                // Larger, find a new location
                size_t index_dst = 0;
 
                if(segmentFindBestFit(segment_size, index_dst) == false)
                {
                        if(pool_can_expand == false)
                        {
                                error = ZAKERO_MEMORYPOOL__ERROR(Error_Out_Of_Memory);
                                return -1;
                        }
 
                        if(expandToFit(segment_size, index_dst) == false)
                        {
                                error = ZAKERO_MEMORYPOOL__ERROR(Error_Failed_To_Resize_File);
                                return -1;
                        }
                }
 
                index_dst = segmentMove(index_src, segment_size, index_dst);
 
                offset = segment[index_dst].offset;
 
                error = ZAKERO_MEMORYPOOL__ERROR(Error_None);
                return offset;
        }
 
 
        uint8_t* MemoryPool::addressOf(off_t offset 
                ) const noexcept
        {
                //ZAKERO_PROFILER_DURATION("MemoryPool", "addressOf");
 
                size_t index = 0;
 
                if(segmentFindInUse(offset, index) == false)
                {
                        return nullptr;
                }
 
                return memory + offset;
        }
 
 
        void MemoryPool::onRemap(MemoryPool::LambdaAddressMap lambda 
                ) noexcept
        {
                if(lambda == nullptr)
                {
                        on_remap = LambdaAddressMap_DoNothing;
                }
                else
                {
                        on_remap = lambda;
                }
        }
 
 
        uint8_t* MemoryPool::remap(const MemoryPool::AddressMap& addr_map 
                , uint8_t*                                       address  
                ) noexcept
        {
                if(addr_map.contains(address))
                {
                        return addr_map.at(address);
                }
 
                return address;
        }
 
 
        std::string MemoryPool::dump(size_t bytes_per_character 
                , size_t                    characters_per_line 
                ) const noexcept
        {
                std::lock_guard<std::mutex> lock(mutex);
 
                std::string str = "{ \"name\": \"" + name + "\"\n"
                        + ", \"pool_size\": " + std::to_string(pool_size) + "\n"
                        ;
 
                size_t count = 0;
 
                str += ", \"Segment\":\n";
                std::string delim = "  [ ";
                for(const auto& seg : segment)
                {
                        str += delim;
                        str += "{ \"offset\": " + std::to_string(seg.offset)
                                + ", \"size\": "   + std::to_string(seg.size)
                                + ", \"in_use\": " + std::to_string(seg.in_use)
                                + " }\n";
 
                        count++;
                        delim = "  , ";
                }
                str += "  ]\n";
 
                count = 0;
 
                int character = 'A';
 
                str += ", \"Layout\":\n  [ \"";
                for(const auto& seg : segment)
                {
                        int ch = '.';
                        if(seg.in_use)
                        {
                                ch = character;
                        }
 
                        size_t segment_count = seg.size / bytes_per_character;
 
                        while(segment_count > 0)
                        {
                                if((count + segment_count) >= characters_per_line)
                                {
                                        size_t t = characters_per_line - count;
 
                                        str += std::string(t, ch) + "\"\n  , \"";
 
                                        count = 0;
                                        segment_count -= t;
                                }
                                else
                                {
                                        str += std::string(segment_count, ch);
                                        
                                        count += segment_count;
                                        segment_count = 0;
                                }
                        }
 
                        if(ch != '.')
                        {
                                if(character == 'Z')
                                {
                                        character = 'A';
                                }
                                else
                                {
                                        character++;
                                }
                        }
                }
 
                str += "\"\n  ]\n}";
 
                return str;
        }
 
 
        MemoryPool::VectorSegment MemoryPool::segmentList(
                ) const noexcept
        {
                std::lock_guard<std::mutex> lock(mutex);
 
                return segment;
        }
 
 
        bool MemoryPool::expandToFit(size_t size_increase 
                , size_t&                   index         
                ) noexcept
        {
                //ZAKERO_PROFILER_DURATION("MemoryPool", "expandToFit");
 
                if(pool_can_expand == false)
                {
                        return false;
                }
 
                if(segment.back().in_use == false)
                {
                        size_increase -= segment.back().size;
                }
 
                if(pool_size + size_increase > MemoryPool::Size_Max)
                {
                        return false;
                }
 
                uint8_t*     old_memory = memory;
                const size_t old_size   = pool_size;
                const size_t new_size   = pool_size + size_increase;
 
                if(ftruncate(file_descriptor, new_size) == -1)
                {
                        return false;
                }
 
                void* new_memory = mremap(old_memory
                        , old_size
                        , new_size
                        , MREMAP_MAYMOVE
                        );
 
                if(new_memory == MAP_FAILED)
                {
                        return false;
                }
 
                pool_size = new_size;
 
                index = segment.size();
                if(index > 0
                        && segment[index - 1].in_use == false
                        )
                {
                        index--;
                        segment[index].size += (off_t)size_increase;
                }
                else
                {
                        segment.push_back(
                        {       .offset = (off_t)old_size
                        ,       .size   = (off_t)size_increase
                        ,       .in_use = false
                        });
                }
 
                size_on_change(pool_size);
 
                if(new_memory != old_memory)
                {
                        memory = (uint8_t*)new_memory;
 
                        MemoryPool::AddressMap addr_map;
 
                        for(const auto& seg : segment)
                        {
                                if(seg.in_use)
                                {
                                        uint8_t* old_addr = old_memory + seg.offset;
                                        uint8_t* new_addr = memory + seg.offset;
 
                                        addr_map[old_addr] = new_addr;
                                }
                        }
 
                        on_remap(addr_map);
                }
 
                return true;
        }
 
 
        bool MemoryPool::segmentExpand(const size_t index 
                , const size_t                      size  
                ) noexcept
        {
                //ZAKERO_PROFILER_DURATION("MemoryPool::Segment", "Expand");
 
                const size_t index_next = index + 1;
                if(index_next >= segment.size())
                {
                        return false;
                }
 
                Segment& segment_next = segment[index_next];
 
                if(segment_next.in_use == true)
                {
                        return false;
                }
 
                Segment& segment_this = segment[index];
 
                if(size > size_t(segment_this.size + segment_next.size))
                {
                        return false;
                }
 
                const size_t size_next = size - segment_this.size;
 
                segment_this.size = size;
 
                if(size_next == 0)
                {
                        segment.erase(std::begin(segment) + index_next);
                }
                else
                {
                        segment_next.offset += size_next;
                        segment_next.size   -= size_next;
                }
 
                return true;
        }
 
 
        bool MemoryPool::segmentFindBestFit(const size_t size 
                , size_t& index                               
                ) noexcept
        {
                //ZAKERO_PROFILER_DURATION("MemoryPool::Segment", "FindBestFit");
 
                for(size_t i = 0; i < segment.size(); i++)
                {
                        const Segment& seg = segment[i];
 
                        if(seg.in_use == false && (size_t)seg.size >= size)
                        {
                                index = i;
 
                                return true;
                        }
                }
 
                return false;
        }
 
 
        bool MemoryPool::segmentFindInUse(const off_t offset 
                , size_t&                             index  
                ) const noexcept
        {
                //ZAKERO_PROFILER_DURATION("MemoryPool::Segment", "FindInUse");
 
                for(size_t i = 0; i < segment.size(); i++)
                {
                        const Segment& seg = segment[i];
 
                        if(seg.offset > offset)
                        {
                                return false;
                        }
 
                        if(seg.offset == offset && seg.in_use == true)
                        {
                                index = i;
 
                                return true;
                        }
                }
 
                return false;
        }
 
 
        void MemoryPool::segmentMergeNext(const size_t index 
                ) noexcept
        {
                //ZAKERO_PROFILER_DURATION("MemoryPool::Segment", "MergeNext");
 
                size_t index_next = index + 1;
 
                if(index_next >= segment.size())
                {
                        return;
                }
 
                Segment& segment_next = segment[index_next];
 
                if(segment_next.in_use == true)
                {
                        return;
                }
 
                segment[index].size += segment[index_next].size;
 
                segment.erase(std::begin(segment) + index_next);
        }
 
 
        void MemoryPool::segmentMergePrev(const size_t index 
                ) noexcept
        {
                //ZAKERO_PROFILER_DURATION("MemoryPool::Segment", "MergePrev");
 
                if(index == 0)
                {
                        return;
                }
 
                const size_t index_prev = index - 1;
 
                Segment& segment_prev = segment[index_prev];
 
                if(segment_prev.in_use == true)
                {
                        return;
                }
 
                segment_prev.size += segment[index].size;
 
                segment.erase(std::begin(segment) + index);
        }
 
 
        size_t MemoryPool::segmentMove(const size_t src_index 
                , const size_t                      dst_size  
                , size_t&                           dst_index 
                ) noexcept
        {
                //ZAKERO_PROFILER_DURATION("MemoryPool::Segment", "Move");
 
                Segment&       src_segment = segment[src_index];
                const uint8_t* src_addr    = memory + src_segment.offset;
                const size_t   src_size    = src_segment.size;
 
                Segment&     dst_segment = segment[dst_index];
                uint8_t*     dst_addr    = memory + dst_segment.offset;
                const size_t dst_offset  = dst_segment.offset;
 
                dst_segment.in_use = true;
 
                memcpy(dst_addr, src_addr, src_size);
 
                #ifdef ZAKERO_MEMORYPOOL_ZERO_ON_FREE
                memset(src_addr, '\0', src_size);
                #endif
                src_segment.in_use = false;
 
                if(src_index > dst_index)
                {
                        segmentMergeNext(src_index);
                        segmentMergePrev(src_index);
 
                        segmentSplit(dst_index, dst_size);
                }
                else
                {
                        segmentSplit(dst_index, dst_size);
 
                        segmentMergeNext(src_index);
                        segmentMergePrev(src_index);
 
                        segmentFindInUse(dst_offset, dst_index);
                }
 
                return dst_index;
        }
 
 
        void MemoryPool::segmentSplit(size_t index 
                , size_t                     size  
                ) noexcept
        {
                //ZAKERO_PROFILER_DURATION("MemoryPool::Segment", "Split");
 
                Segment& this_segment = segment[index];
 
                const size_t index_next  = index + 1;
                const off_t  offset_next = this_segment.offset + size;
                const off_t  size_next   = this_segment.size - size;
 
                this_segment.size = size;
 
                if(this_segment.in_use)
                {
                        uint8_t* addr = memory + offset_next;
 
                        memset(addr, '\0', size_next);
                }
 
                if(index_next >= segment.size())
                {
                        segment.push_back(
                        {       .offset = offset_next
                        ,       .size   = size_next
                        ,       .in_use = false
                        });
                }
                else
                {
                        Segment& segment_next = segment[index_next];
 
                        if(segment_next.in_use)
                        {
                                segment.insert(std::begin(segment) + index_next,
                                {       .offset = offset_next
                                ,       .size   = size_next
                                ,       .in_use = false
                                });
                        }
                        else // Not in use
                        {
                                segment_next.offset  = offset_next;
                                segment_next.size   += size_next;
                        }
                }
        }
 
// }}}
// {{{ Convenience
 
        std::string to_string(const MemoryPool::VectorSegment& segment 
                ) noexcept
        {
                std::string str = "";
 
                std::string delim = "[ ";
                for(const auto& seg : segment)
                {
                        str += delim;
                        str += "{ \"offset\": " + std::to_string(seg.offset)
                                + ", \"size\": "   + std::to_string(seg.size)
                                + ", \"in_use\": " + std::to_string(seg.in_use)
                                + " }\n";
 
                        delim = ", ";
                }
                str += "]\n";
 
                return str;
        }
 
// }}}
 
}
 
#endif // ZAKERO_MEMORYPOOL_IMPLEMENTATION
 
// }}}
 
#endif // zakero_MemoryPool_h