Thrill  0.1
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
thrill::mem Namespace Reference

Classes

class  AlignedAllocator
 
struct  AlignedAllocatorSettings
 
class  Allocator
 
class  Allocator< void >
 
class  Deleter
 std::default_deleter with Manager tracking More...
 
class  FixedAllocator
 
class  FixedAllocator< void, manager_ >
 
class  FixedPoolAllocator
 
class  GPoolDeleter
 deleter for unique_ptr with memory from mem::g_pool. More...
 
class  Manager
 Object shared by allocators and other classes to track memory allocations. More...
 
class  Pool
 A simple memory allocation manager. More...
 
class  PoolAllocator
 

Typedefs

template<typename T >
using by_deque = std::deque< T, BypassAllocator< T > >
 deque without malloc tracking More...
 
using by_string = std::basic_string< char, std::char_traits< char >, BypassAllocator< char > >
 string without malloc tracking More...
 
using by_stringbuf = std::basic_stringbuf< char, std::char_traits< char >, BypassAllocator< char > >
 stringbuf without malloc tracking More...
 
template<typename T >
using by_vector = std::vector< T, BypassAllocator< T > >
 vector without malloc tracking More...
 
template<typename Type >
using BypassAllocator = FixedAllocator< Type, g_bypass_manager >
 instantiate FixedAllocator as BypassAllocator More...
 
using CounterType = ssize_t
 
template<typename T >
using deque = std::deque< T, Allocator< T > >
 deque with Manager tracking More...
 
using free_type = void(*)(void *)
 
template<typename Type >
using GPoolAllocator = FixedPoolAllocator< Type, GPool >
 template alias for allocating from mem::g_pool. More...
 
using malloc_type = void *(*)(size_t)
 function pointer to the real procedures, loaded using dlsym() More...
 
using realloc_type = void *(*)(void *, size_t)
 
using safe_ostringstream = std::basic_ostringstream< char, std::char_traits< char >, mem::GPoolAllocator< char > >
 
using safe_string = std::basic_string< char, std::char_traits< char >, mem::GPoolAllocator< char > >
 
template<typename T >
using safe_unique_ptr = std::unique_ptr< T, GPoolDeleter< T > >
 unique_ptr with memory from mem::g_pool. More...
 
using string = std::basic_string< char, std::char_traits< char >, Allocator< char > >
 string with Manager tracking More...
 
using stringbuf = std::basic_stringbuf< char, std::char_traits< char >, Allocator< char > >
 stringbuf with Manager tracking More...
 
template<typename T >
using unique_ptr = std::unique_ptr< T, Deleter< T > >
 unique_ptr with Manager tracking More...
 
template<typename T >
using vector = std::vector< T, Allocator< T > >
 vector with Manager tracking More...
 

Functions

static void * aligned_alloc (size_t size, size_t meta_info_size=0)
 
static void aligned_dealloc (void *ptr, size_t size, size_t meta_info_size=0)
 
template<typename T >
void by_delete (T *value)
 operator delete with our Allocator More...
 
template<typename T , typename... Args>
Tby_new (Args &&...args)
 operator new with our Allocator More...
 
void bypass_free (void *ptr, size_t size) noexcept
 bypass malloc tracker and access free() directly More...
 
void * bypass_malloc (size_t size) noexcept
 bypass malloc tracker and access malloc() directly More...
 
static void dec_count (size_t dec)
 decrement allocation to statistics More...
 
void flush_memory_statistics ()
 method to flush thread-local memory statistics when memory_exceeded More...
 
Manager g_bypass_manager (nullptr,"Bypass")
 
static ssize_t get (const CounterType &a)
 
PoolGPool ()
 singleton instance of global pool for I/O data structures More...
 
static void inc_count (size_t inc)
 add allocation to statistics More...
 
template<typename T , typename... Args>
unique_ptr< Tmake_unique (Manager &manager, Args &&...args)
 make_unique with Manager tracking More...
 
ssize_t malloc_tracker_current ()
 user function to return the currently allocated amount of memory More...
 
ssize_t malloc_tracker_peak ()
 user function to return the peak allocation More...
 
void malloc_tracker_print_status ()
 user function which prints current and peak allocation to stderr More...
 
void malloc_tracker_reset_peak ()
 user function to reset the peak allocation to current More...
 
ssize_t malloc_tracker_total_allocs ()
 user function to return total number of allocations More...
 
template<typename T >
void mm_delete (Manager &manager, T *value)
 operator delete with our Allocator More...
 
template<typename T , typename... Args>
Tmm_new (Manager &manager, Args &&...args)
 operator new with our Allocator More...
 
template<typename T , typename... Args>
safe_unique_ptr< Tsafe_make_unique (Args &&...args)
 make_unique with Manager tracking More...
 
void set_memory_limit_indication (ssize_t size)
 
void StartMemProfiler (common::ProfileThread &sched, common::JsonLogger &logger)
 launch profiler task More...
 
static ssize_t sync_add_and_fetch (CounterType &curr, ssize_t inc)
 
static ssize_t sync_sub_and_fetch (CounterType &curr, ssize_t dec)
 
static by_string to_string (int val)
 convert to string More...
 
static by_string to_string (unsigned val)
 convert to string More...
 
static by_string to_string (long val)
 convert to string More...
 
static by_string to_string (unsigned long val)
 convert to string More...
 
static void update_memprofile (ssize_t float_current, ssize_t base_current)
 
void update_peak (ssize_t float_curr, ssize_t base_curr)
 

Variables

static CounterType base_curr { 0 }
 
static CounterType current_allocs { 0 }
 
static CounterType float_curr { 0 }
 
Manager g_bypass_manager
 global bypass memory manager More...
 
static constexpr size_t init_alignment = sizeof(size_t)
 align allocations to init_heap to this number by rounding up allocations More...
 
static char init_heap [1024 *1024]
 
static CounterType init_heap_use { 0 }
 
static constexpr int log_operations = 0
 
static constexpr int log_operations_init_heap = 0
 
static constexpr size_t log_operations_threshold = 100000
 <– set this to 1 for log output More...
 
bool memory_exceeded = false
 memory limit exceeded indicator More...
 
ssize_t memory_limit_indication = std::numeric_limits<ssize_t>::max()
 
static OhlcBar mp_base
 
static constexpr bool mp_enable = true
 
static OhlcBar mp_float
 
static CounterType mp_next_bar { 0 }
 
static constexpr size_t padding = 16
 
static CounterType peak_bytes { 0 }
 
static free_type real_free = nullptr
 
static malloc_type real_malloc = nullptr
 
static realloc_type real_realloc = nullptr
 
static constexpr size_t sentinel = 0xDEADC0DE
 a sentinel value prefixed to each allocation More...
 
static const ssize_t tl_delay_threshold = 1024 * 1024
 
static thread_local LocalStats tl_stats = { 0, 0, 0 }
 
static CounterType total_allocs { 0 }
 
static CounterType total_bytes { 0 }
 

Typedef Documentation

using by_deque = std::deque<T, BypassAllocator<T> >

deque without malloc tracking

Definition at line 187 of file allocator_base.hpp.

using by_string = std::basic_string< char, std::char_traits<char>, BypassAllocator<char> >

string without malloc tracking

Definition at line 175 of file allocator_base.hpp.

using by_stringbuf = std::basic_stringbuf< char, std::char_traits<char>, BypassAllocator<char> >

stringbuf without malloc tracking

Definition at line 179 of file allocator_base.hpp.

using by_vector = std::vector<T, BypassAllocator<T> >

vector without malloc tracking

Definition at line 183 of file allocator_base.hpp.

instantiate FixedAllocator as BypassAllocator

Definition at line 151 of file allocator_base.hpp.

using CounterType = ssize_t

CounterType is used for atomic counters, and get to retrieve their contents. Due to the thread-local cached statistics, the overall memory usage counter can actually go negative!

Definition at line 99 of file malloc_tracker.cpp.

using deque = std::deque<T, Allocator<T> >

deque with Manager tracking

Definition at line 232 of file allocator.hpp.

using free_type = void (*)(void*)

Definition at line 79 of file malloc_tracker.cpp.

template alias for allocating from mem::g_pool.

Definition at line 308 of file pool.hpp.

using malloc_type = void* (*)(size_t)

function pointer to the real procedures, loaded using dlsym()

Definition at line 78 of file malloc_tracker.cpp.

using realloc_type = void* (*)(void*, size_t)

Definition at line 80 of file malloc_tracker.cpp.

using safe_ostringstream = std::basic_ostringstream< char, std::char_traits<char>, mem::GPoolAllocator<char> >

alias for std::ostringstream except that it uses the safer g_pool as allocator for the internal string buffer

Definition at line 340 of file pool.hpp.

using safe_string = std::basic_string< char, std::char_traits<char>, mem::GPoolAllocator<char> >

alias for std::string except that its memory is allocated in the safer g_pool.

Definition at line 335 of file pool.hpp.

using safe_unique_ptr = std::unique_ptr<T, GPoolDeleter<T> >

unique_ptr with memory from mem::g_pool.

Definition at line 323 of file pool.hpp.

using string = std::basic_string< char, std::char_traits<char>, Allocator<char> >

string with Manager tracking

Examples:
examples/tutorial/k-means_step5.cpp, and examples/tutorial/k-means_step6.cpp.

Definition at line 220 of file allocator.hpp.

using stringbuf = std::basic_stringbuf< char, std::char_traits<char>, Allocator<char> >

stringbuf with Manager tracking

Definition at line 224 of file allocator.hpp.

using unique_ptr = std::unique_ptr<T, Deleter<T> >

unique_ptr with Manager tracking

Definition at line 208 of file allocator.hpp.

using vector = std::vector<T, Allocator<T> >

vector with Manager tracking

Definition at line 228 of file allocator.hpp.

Function Documentation

static void* thrill::mem::aligned_alloc ( size_t  size,
size_t  meta_info_size = 0 
)
inlinestatic

Definition at line 207 of file aligned_allocator.hpp.

static void thrill::mem::aligned_dealloc ( void *  ptr,
size_t  size,
size_t  meta_info_size = 0 
)
inlinestatic

Definition at line 212 of file aligned_allocator.hpp.

void thrill::mem::by_delete ( T value)

operator delete with our Allocator

Definition at line 164 of file allocator_base.hpp.

References FixedAllocator< Type, manager_ >::deallocate(), and AllocatorBase< Type >::destroy().

T* thrill::mem::by_new ( Args &&...  args)
void bypass_free ( void *  ptr,
size_t  size 
)
noexcept
void * bypass_malloc ( size_t  size)
noexcept
static void thrill::mem::dec_count ( size_t  dec)
static
void flush_memory_statistics ( )

method to flush thread-local memory statistics when memory_exceeded

Definition at line 191 of file malloc_tracker.cpp.

References base_curr, current_allocs, float_curr, memory_exceeded, memory_limit_indication, sync_add_and_fetch(), tl_stats, total_allocs, total_bytes, update_memprofile(), and update_peak().

Referenced by dec_count(), and inc_count().

Manager thrill::mem::g_bypass_manager ( nullptr  ,
"Bypass"   
)
static ssize_t thrill::mem::get ( const CounterType a)
inlinestatic

Definition at line 106 of file malloc_tracker.cpp.

static void thrill::mem::inc_count ( size_t  inc)
static
unique_ptr<T> thrill::mem::make_unique ( Manager &  manager,
Args &&...  args 
)

make_unique with Manager tracking

Definition at line 212 of file allocator.hpp.

ssize_t malloc_tracker_current ( )

user function to return the currently allocated amount of memory

returns the currently allocated amount of memory

Definition at line 255 of file malloc_tracker.cpp.

References float_curr.

ssize_t malloc_tracker_peak ( )

user function to return the peak allocation

returns the current peak memory allocation

Definition at line 260 of file malloc_tracker.cpp.

References peak_bytes.

void malloc_tracker_print_status ( )

user function which prints current and peak allocation to stderr

Definition at line 275 of file malloc_tracker.cpp.

References base_curr, float_curr, peak_bytes, and PPREFIX.

Referenced by thrill::data::OurNewHandler(), and DIABase::RunScope().

void malloc_tracker_reset_peak ( )

user function to reset the peak allocation to current

resets the peak memory allocation to current

Definition at line 265 of file malloc_tracker.cpp.

References float_curr, and peak_bytes.

ssize_t malloc_tracker_total_allocs ( )

user function to return total number of allocations

returns the total number of allocations

Definition at line 270 of file malloc_tracker.cpp.

References total_allocs.

void thrill::mem::mm_delete ( Manager &  manager,
T value 
)

operator delete with our Allocator

Definition at line 180 of file allocator.hpp.

References Allocator< Type >::deallocate(), and AllocatorBase< Type >::destroy().

T* thrill::mem::mm_new ( Manager &  manager,
Args &&...  args 
)

operator new with our Allocator

Definition at line 171 of file allocator.hpp.

References Allocator< Type >::allocate(), AllocatorBase< Type >::construct(), and gen_data::value.

safe_unique_ptr<T> thrill::mem::safe_make_unique ( Args &&...  args)

make_unique with Manager tracking

Definition at line 327 of file pool.hpp.

References GPool(), and Pool::make().

void set_memory_limit_indication ( ssize_t  size)

set the malloc tracking system to set memory_exceeded when this limit is exceed. it does not actually limit allocation!

Definition at line 280 of file malloc_tracker.cpp.

References memory_limit_indication.

Referenced by MemoryConfig::apply().

void StartMemProfiler ( common::ProfileThread &  sched,
common::JsonLogger &  logger 
)

launch profiler task

Definition at line 371 of file malloc_tracker.cpp.

References ProfileThread::Add().

Referenced by HostContext::HostContext().

static ssize_t thrill::mem::sync_add_and_fetch ( CounterType curr,
ssize_t  inc 
)
inlinestatic

Definition at line 115 of file malloc_tracker.cpp.

Referenced by bypass_malloc(), flush_memory_statistics(), and inc_count().

static ssize_t thrill::mem::sync_sub_and_fetch ( CounterType curr,
ssize_t  dec 
)
inlinestatic

Definition at line 124 of file malloc_tracker.cpp.

Referenced by bypass_free(), and dec_count().

static by_string thrill::mem::to_string ( unsigned  val)
inlinestatic

convert to string

Definition at line 195 of file allocator_base.hpp.

static by_string thrill::mem::to_string ( long  val)
inlinestatic

convert to string

Definition at line 200 of file allocator_base.hpp.

static by_string thrill::mem::to_string ( unsigned long  val)
inlinestatic

convert to string

Definition at line 205 of file allocator_base.hpp.

static void update_memprofile ( ssize_t  float_current,
ssize_t  base_current 
)
static
void thrill::mem::update_peak ( ssize_t  float_curr,
ssize_t  base_curr 
)

Definition at line 185 of file malloc_tracker.cpp.

References base_curr, and peak_bytes.

Referenced by bypass_malloc(), flush_memory_statistics(), and inc_count().

Variable Documentation

CounterType current_allocs { 0 }
static
Manager g_bypass_manager(nullptr,"Bypass")

global bypass memory manager

constexpr size_t init_alignment = sizeof(size_t)
static

align allocations to init_heap to this number by rounding up allocations

Definition at line 139 of file malloc_tracker.cpp.

Referenced by preinit_malloc().

char init_heap[1024 *1024]
static

Definition at line 134 of file malloc_tracker.cpp.

Referenced by free(), preinit_malloc(), and realloc().

CounterType init_heap_use { 0 }
static

Definition at line 135 of file malloc_tracker.cpp.

Referenced by free(), preinit_malloc(), and realloc().

constexpr int log_operations = 0
static

Definition at line 62 of file malloc_tracker.cpp.

Referenced by free(), malloc(), and realloc().

constexpr int log_operations_init_heap = 0
static

Definition at line 136 of file malloc_tracker.cpp.

Referenced by preinit_free(), preinit_malloc(), and preinit_realloc().

constexpr size_t log_operations_threshold = 100000
static

<– set this to 1 for log output

Definition at line 63 of file malloc_tracker.cpp.

Referenced by free(), malloc(), and realloc().

ssize_t memory_limit_indication = std::numeric_limits<ssize_t>::max()
OhlcBar mp_base
static

Definition at line 309 of file malloc_tracker.cpp.

Referenced by update_memprofile().

constexpr bool mp_enable = true
static

Definition at line 288 of file malloc_tracker.cpp.

Referenced by update_memprofile().

OhlcBar mp_float
static

Definition at line 309 of file malloc_tracker.cpp.

Referenced by update_memprofile().

CounterType mp_next_bar { 0 }
static

Definition at line 290 of file malloc_tracker.cpp.

Referenced by update_memprofile().

constexpr size_t padding = 16
static

In the generic hook implementation, we add to each allocation additional data for bookkeeping.

Definition at line 75 of file malloc_tracker.cpp.

Referenced by free(), malloc(), preinit_free(), preinit_malloc(), preinit_realloc(), realloc(), thrill::api::Zip(), and thrill::api::ZipWindow().

CounterType peak_bytes { 0 }
static
free_type real_free = nullptr
static

Definition at line 83 of file malloc_tracker.cpp.

Referenced by bypass_free(), and free().

malloc_type real_malloc = nullptr
static

Definition at line 82 of file malloc_tracker.cpp.

Referenced by bypass_malloc(), and malloc().

realloc_type real_realloc = nullptr
static

Definition at line 84 of file malloc_tracker.cpp.

const ssize_t tl_delay_threshold = 1024 * 1024
static

Definition at line 181 of file malloc_tracker.cpp.

Referenced by dec_count(), and inc_count().

thread_local LocalStats tl_stats = { 0, 0, 0 }
static

Definition at line 180 of file malloc_tracker.cpp.

Referenced by dec_count(), flush_memory_statistics(), and inc_count().

CounterType total_allocs { 0 }
static
CounterType total_bytes { 0 }
static