Contains many common tools and helpers shared by higher layers. See common. More...

Namespaces
	qsort_local

Classes
struct	alias_cast_helper

class	AtomicMovable
	This is a derivative of std::atomic which enables easier and less error-prone writing of move-only classes by implementing a move constructor. More...

class	BinaryHeap

class	ComponentSum
	template for computing the component-wise sum of std::array or std::vector. More...

class	ComponentSum< std::array< Type, N >, Operation >
	Compute the component-wise sum of two std::array<T,N> of same sizes. More...

class	ComponentSum< std::vector< Type >, Operation >
	Compute the component-wise sum of two std::vector<T> of same sizes. More...

class	ConcurrentBoundedQueue
	This is a queue, similar to std::queue and tbb::concurrent_bounded_queue, except that it uses mutexes for synchronization. More...

class	ConcurrentQueue
	This is a queue, similar to std::queue and tbb::concurrent_queue, except that it uses mutexes for synchronization. More...

class	ErrnoException
	An Exception which is thrown on system errors and contains errno information. More...

struct	FunctionTraits< ReturnType(*)(Args...)>
	specialize for function pointers More...

struct	FunctionTraits< ReturnType(ClassType::*)(Args...) const >
	specialize for pointers to const member function More...

struct	FunctionTraits< ReturnType(ClassType::*)(Args...)>
	specialize for pointers to mutable member function More...

struct	HashCrc32Fallback

struct	HashDataSwitch
	Hashing helper that decides what is hashed. More...

struct	HashDataSwitch< std::string >

class	hypergeometric_distribution

struct	Identity
	Identity functor, very useful for default parameters. More...

class	ItemReaderToolsBase
	CRTP class to enhance item/memory reader classes with Varint decoding and String decoding. More...

class	ItemWriterToolsBase
	CRTP class to enhance item/memory writer classes with Varint encoding and String encoding. More...

class	JsonBeginObj
	A special class to output verbatim text. More...

class	JsonEndObj
	A special class to output verbatim text. More...

class	JsonLine
	JsonLine is an object used to aggregate a set of key:value pairs for output into a JSON log. More...

struct	JsonLinePutSwitch

class	JsonLogger
	JsonLogger is a receiver of JSON output objects for logging. More...

class	JsonVerbatim
	A special class to output verbatim text. More...

class	Matrix
	A simple m x n dense matrix for generating statistics. More...

class	maximum

class	minimum

class	ProfileTask

class	ProfileTaskRegistration

class	ProfileThread

class	RadixSort
	SortAlgorithm class for use with api::Sort() which calls radix_sort_CI() if K is small enough. More...

class	Range
	represents a 1 dimensional range (interval) [begin,end) More...

class	ReservoirSampling
	Implementation of reservoir sampling using Vitter's Algorithm R. More...

class	ReservoirSamplingFast
	Fast exact implementation of reservoir sampling using skip values. More...

class	ReservoirSamplingGrow
	Implementation of a fast approximation of adaptive reservoir sampling using http://erikerlandson.github.io/blog/2015/11/20/very-fast-reservoir-sampling/ The reservoir size grows logarithmically with the number given to the sampler, new items replace old ones such that all items in the stream are sampled with the same approximately uniform probability. More...

class	RunTimer
	RIAA class for running the timer until destruction. More...

class	Sampling

class	SharedState

class	Singleton

class	StatsCounter
	This class provides a statistical counter that can easily be deactivated using a boolean template switch. More...

class	StatsCounter< ValueType_, false >

class	StatsCounter< ValueType_, true >

class	StatsTimerBase
	This class provides a statistical stop watch timer that can easily be deactivated using a boolean template switch. More...

class	StatsTimerBase< false >

class	StatsTimerBase< true >

class	StatsTimerBaseStarted

class	StatsTimerBaseStopped

class	SystemException
	An Exception which is thrown on system errors. More...

class	TabulationHashing
	Tabulation Hashing, see https://en.wikipedia.org/wiki/Tabulation_hashing. More...

class	TimedCounter

class	UIntPair
	Construct an 40-bit or 48-bit unsigned integer stored in five or six bytes. More...

class	Vector
	A compile-time fixed-length D-dimensional point with double precision. More...

class	VectorConcat
	Compute the concatenation of two std::vector<T>s. More...

class	VVector
	A variable-length D-dimensional point with double precision. More...

class	ZipfDistribution
	A class for producing random integers distributed according to the Zipf-Mandelbrot probability mass function: More...

Typedefs
template<typename Type >
using	Array = Type[]
	A template to make writing temporary arrays easy: Array<int>{ 1, 2, 3 }. More...

using	FakeStatsTimer = StatsTimerBase< false >

using	FakeStatsTimerStart = StatsTimerBaseStarted< false >

using	FakeStatsTimerStopped = StatsTimerBaseStopped< false >

template<typename T >
using	hash = HashCrc32< T >
	Select a hashing method. More...

template<typename T >
using	HashCrc32 = HashCrc32Fallback< T >

template<typename T >
using	HashTabulated = TabulationHashing< sizeof(T)>
	Tabulation hashing. More...

using	hypergeometric = hypergeometric_distribution<>

template<typename T >
using	is_trivially_copyable = std::is_trivially_copyable< T >

template<typename Type , typename Allocator = std::allocator<Type>>
using	RingBuffer = tlx::RingBuffer< Type, Allocator >

using	StatsTimer = StatsTimerBase< true >

using	StatsTimerStart = StatsTimerBaseStarted< true >

using	StatsTimerStopped = StatsTimerBaseStopped< true >

using	ThreadBarrier = tlx::ThreadBarrierSpin

using	uint40 = UIntPair< uint8_t >
	Construct a 40-bit unsigned integer stored in five bytes. More...

using	uint48 = UIntPair< uint16_t >
	Construct a 48-bit unsigned integer stored in six bytes. More...

Functions
template<typename IntegerType , unsigned bits = (8 * sizeof(IntegerType))>
static IntegerType	AddTruncToType (const IntegerType &a, const IntegerType &b)

template<typename To , typename From >
To &	alias_cast (From &raw_data)

template<typename To , typename From >
const To &	alias_cast (const From &raw_data)

static size_t	CalcOneFactorPeer (size_t r, size_t p, size_t n)
	Calculate a Perfect Matching (1-Factor) on a Complete Graph. More...

static size_t	CalcOneFactorSize (size_t n)
	Number of rounds in Perfect Matching (1-Factor). More...

static Range	CalculateLocalRange (size_t global_size, size_t p, size_t i)

static size_t	CalculatePartition (size_t global_size, size_t p, size_t k)

uint32_t	crc32_slicing_by_8 (uint32_t crc, const void *data, size_t length)

template<typename... Args>
std::thread	CreateThread (Args &&... args)
	create a std::thread and repeat creation if it fails More...

template<typename T >
T	from_cstr (const char nptr, char *endptr=nullptr, int base=10)
	Number parsing helpers, wraps strto{f,d,ld,l,ul,ll,ull}() via type switch. More...

template<>
double	from_cstr< double > (const char nptr, char *endptr, int)

template<>
float	from_cstr< float > (const char nptr, char *endptr, int)

template<>
long	from_cstr< long > (const char nptr, char *endptr, int base)

template<>
long double	from_cstr< long double > (const char nptr, char *endptr, int)

template<>
long long	from_cstr< long long > (const char nptr, char *endptr, int base)

template<>
unsigned long	from_cstr< unsigned long > (const char nptr, char *endptr, int base)

template<>
unsigned long long	from_cstr< unsigned long long > (const char nptr, char *endptr, int base)

template<typename Type >
static bool	from_str (const std::string &str, Type &outval)
	Template transformation function which uses std::istringstream to parse any istreamable type from a std::string. More...

std::string	GetHostname ()
	get hostname More...

std::string	GetNameForThisThread ()
	Returns the name of the current thread or 'unknown [id]'. More...

static uint64_t	Hash128to64 (const uint64_t upper, const uint64_t lower)

static uint32_t	Hash64to32 (uint64_t key)
	Returns a uint32_t hash of a uint64_t. More...

void	LogCmdlineParams (JsonLogger &logger)

void	MakePipe (int out_pipefds[2])
	create a pair of pipe file descriptors More...

template<typename Type , typename Functor >
auto	MapVector (const std::vector< Type > &input, const Functor &f) -> std::vector< typename std::result_of< Functor(Type)>::type >

template<typename T >
static constexpr const T &	max (const T &a, const T &b)
	template for constexpr max, because std::max is not good enough. More...

template<typename T >
static constexpr const T &	min (const T &a, const T &b)
	template for constexpr min, because std::min is not good enough. More...

void	NameThisThread (const std::string &name)
	Defines a name for the current thread, only if no name was set previously. More...

template<size_t D>
Vector< D >	operator* (const double a, const Vector< D > &b)

VVector< double >	operator* (const double a, const VVector< double > &b)

void	PortSetCloseOnExec (int fd)
	set FD_CLOEXEC on file descriptor (if possible) More...

static JsonLine &	Put (JsonLine &line, bool const &value)

static JsonLine &	Put (JsonLine &line, int const &value)

static JsonLine &	Put (JsonLine &line, unsigned int const &value)

static JsonLine &	Put (JsonLine &line, long const &value)

static JsonLine &	Put (JsonLine &line, unsigned long const &value)

static JsonLine &	Put (JsonLine &line, long long const &value)

static JsonLine &	Put (JsonLine &line, unsigned long long const &value)

static JsonLine &	Put (JsonLine &line, double const &value)

static JsonLine &	Put (JsonLine &line, const char *const &str)

static JsonLine &	Put (JsonLine &line, std::string const &str)

template<typename Type , std::size_t N>
static JsonLine &	Put (JsonLine &line, const Type(&arr)[N])

template<typename Type >
static JsonLine &	Put (JsonLine &line, std::initializer_list< Type > const &list)

template<typename Type >
static JsonLine &	Put (JsonLine &line, std::vector< Type > const &vec)

template<typename Type , std::size_t N>
static JsonLine &	Put (JsonLine &line, std::array< Type, N > const &arr)

static JsonLine &	Put (JsonLine &line, JsonVerbatim const &verbatim)

template<typename Compare , typename Iterator >
void	qsort_three_pivots (Iterator left, Iterator right, Compare cmp)

template<typename Compare , typename Iterator >
void	qsort_two_pivots_yaroslavskiy (Iterator lo, Iterator hi, Compare cmp)

template<size_t MaxDepth, typename Iterator , typename Char , typename Comparator = std::less<typename std::iterator_traits<Iterator>::value_type>, typename SubSorter = tlx::NoOperation<void>>
static void	radix_sort_CI (Iterator begin, Iterator end, size_t K, const Comparator &cmp, const SubSorter &sub_sort, size_t depth, Char *char_cache)
	Internal helper method, use radix_sort_CI below. More...

template<size_t MaxDepth, typename Iterator , typename Comparator = std::less<typename std::iterator_traits<Iterator>::value_type>, typename SubSorter = tlx::NoOperation<void>>
static void	radix_sort_CI (Iterator begin, Iterator end, size_t K, const Comparator &cmp=Comparator(), const SubSorter &sub_sort=SubSorter())
	Radix sort the iterator range [begin,end). More...

template<typename RandomEngine = std::default_random_engine>
static std::string	RandomString (std::string::size_type size, RandomEngine rng, const std::string &letters="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz")
	Generate a random string of given length. More...

void	SetCpuAffinity (std::thread &thread, size_t cpu_id)
	set cpu/core affinity of a thread More...

void	SetCpuAffinity (size_t cpu_id)
	set cpu/core affinity of current thread More...

void	StartLinuxProcStatsProfiler (ProfileThread &sched, JsonLogger &logger)
	launch profiler task More...

template<typename Type >
static std::string	to_str (const Type &t)

struct dirent *	ts_readdir (DIR *dirp)
	mutex-locked readdir() call More...

template<typename T , size_t N>
static std::string	VecToStr (const std::array< T, N > &data)
	Logging helper to print arrays as [a1,a2,a3,...]. More...

template<typename T >
static std::string	VecToStr (const std::vector< T > &data)
	Logging helper to print vectors as [a1,a2,a3,...]. More...

Variables
const uint32_t	crc_tableil8_o32 [256]

const uint32_t	crc_tableil8_o40 [256]

const uint32_t	crc_tableil8_o48 [256]

const uint32_t	crc_tableil8_o56 [256]

const uint32_t	crc_tableil8_o64 [256]

const uint32_t	crc_tableil8_o72 [256]

const uint32_t	crc_tableil8_o80 [256]

const uint32_t	crc_tableil8_o88 [256]

static constexpr unsigned	g_cache_line_size = 64
	Finding cache line size is hard - we assume 64 byte. More...

static constexpr bool	g_debug_mode = !g_ndebug
	debug mode is active, if NDEBUG is false. More...

static constexpr bool	g_debug_push_file = false

mem::Manager	g_logger_mem_manager (nullptr, "Logger")
	memory manager singleton for Logger More...

static constexpr bool	g_ndebug = false
	global ndebug flag as a boolean, NDEBUG means no debug in Release mode. More...

static constexpr bool	g_profile_thread = true
	global flag to enable background profiler thread More...

static constexpr bool	g_self_verify = g_debug_mode

static ThreadLoggerPrefixHook	s_default_logger
	default logger singleton More...

static thread_local size_t	s_message_counter = 0
	thread message counter More...

static thread_local char	s_thread_name [64] = { 0 }
	thread name More...

class thrill::common::UIntPair	TLX_ATTRIBUTE_PACKED

Detailed Description

Contains many common tools and helpers shared by higher layers. See common.

Typedef Documentation

◆ Array

using Array = Type[]

A template to make writing temporary arrays easy: Array<int>{ 1, 2, 3 }.

Definition at line 64 of file json_logger.hpp.

◆ FakeStatsTimer

using FakeStatsTimer = StatsTimerBase<false>

Definition at line 292 of file stats_timer.hpp.

◆ FakeStatsTimerStart

using FakeStatsTimerStart = StatsTimerBaseStarted<false>

Definition at line 293 of file stats_timer.hpp.

◆ FakeStatsTimerStopped

using FakeStatsTimerStopped = StatsTimerBaseStopped<false>

Definition at line 294 of file stats_timer.hpp.

◆ hash

using hash = HashCrc32<T>

Select a hashing method.

Definition at line 262 of file hash.hpp.

◆ HashCrc32

using HashCrc32 = HashCrc32Fallback<T>

Definition at line 205 of file hash.hpp.

◆ HashTabulated

using HashTabulated = TabulationHashing<sizeof(T)>

Tabulation hashing.

Definition at line 258 of file hash.hpp.

◆ hypergeometric

using hypergeometric = hypergeometric_distribution<>

Definition at line 241 of file hypergeometric_distribution.hpp.

◆ is_trivially_copyable

using is_trivially_copyable = std::is_trivially_copyable<T>

Definition at line 54 of file defines.hpp.

◆ RingBuffer

using RingBuffer = tlx::RingBuffer<Type, Allocator>

Definition at line 21 of file ring_buffer.hpp.

◆ StatsTimer

using StatsTimer = StatsTimerBase<true>

Definition at line 288 of file stats_timer.hpp.

◆ StatsTimerStart

using StatsTimerStart = StatsTimerBaseStarted<true>

Definition at line 289 of file stats_timer.hpp.

◆ StatsTimerStopped

using StatsTimerStopped = StatsTimerBaseStopped<true>

Definition at line 290 of file stats_timer.hpp.

◆ ThreadBarrier

using ThreadBarrier = tlx::ThreadBarrierSpin

Definition at line 26 of file thread_barrier.hpp.

◆ uint40

using uint40 = UIntPair<uint8_t>

Construct a 40-bit unsigned integer stored in five bytes.

Definition at line 260 of file uint_types.hpp.

◆ uint48

using uint48 = UIntPair<uint16_t>

Construct a 48-bit unsigned integer stored in six bytes.

Definition at line 263 of file uint_types.hpp.

Function Documentation

◆ AddTruncToType()

static IntegerType thrill::common::AddTruncToType	(	const IntegerType &	a,
		const IntegerType &	b
	)

inlinestatic

Add x + y but truncate result upwards such that it fits into original datatype

Definition at line 31 of file math.hpp.

Referenced by GroupByNode< ValueType, KeyExtractor, GroupFunction, HashFunction, UseLocationDetection >::HashCount::operator+(), JoinNode< ValueType, FirstDIA, SecondDIA, KeyExtractor1, KeyExtractor2, JoinFunction, HashFunction, UseLocationDetection >::HashCount::operator+(), GroupByNode< ValueType, KeyExtractor, GroupFunction, HashFunction, UseLocationDetection >::HashCount::operator+=(), and JoinNode< ValueType, FirstDIA, SecondDIA, KeyExtractor1, KeyExtractor2, JoinFunction, HashFunction, UseLocationDetection >::HashCount::operator+=().

◆ alias_cast() [1/2]

To& thrill::common::alias_cast ( From & raw_data )

Definition at line 49 of file hash.hpp.

References alias_cast_helper< To, From >::in, and alias_cast_helper< To, From >::out.

◆ alias_cast() [2/2]

const To& thrill::common::alias_cast ( const From & raw_data )

Definition at line 56 of file hash.hpp.

References alias_cast_helper< To, From >::in, and alias_cast_helper< To, From >::out.

◆ CalcOneFactorPeer()

static size_t thrill::common::CalcOneFactorPeer	(	size_t	r,
		size_t	p,
		size_t	n
	)

inlinestatic

Calculate a Perfect Matching (1-Factor) on a Complete Graph.

Used by collective network algorithms.

Parameters

r	round [0..n-1) of matching
p	rank of this processor 0..n-1
n	number of processors (graph size)

Returns: peer processor in this round

Definition at line 140 of file math.hpp.

References CalcOneFactorSize().

Referenced by Group::OneFactorPeer().

◆ CalcOneFactorSize()

static size_t thrill::common::CalcOneFactorSize ( size_t n )

inlinestatic

Number of rounds in Perfect Matching (1-Factor).

Definition at line 127 of file math.hpp.

Referenced by CalcOneFactorPeer(), and Group::OneFactorSize().

◆ CalculateLocalRange()

static Range thrill::common::CalculateLocalRange	(	size_t	global_size,
		size_t	p,
		size_t	i
	)

inlinestatic

given a global range [0,global_size) and p PEs to split the range, calculate the [local_begin,local_end) index range assigned to the PE i.

Definition at line 110 of file math.hpp.

References Range::Range().

Referenced by Context::CalculateLocalRange(), Context::CalculateLocalRangeOnHost(), CalculatePartition(), DistributeNode< ValueType >::Execute(), GroupToIndexNode< ValueType, KeyExtractor, GroupFunction >::GroupToIndexNode(), DuplicateDetection::WriteEncodedHashes(), and LocationDetection< thrill::api::JoinNode::HashCount >::WriteOccurenceCounts().

◆ CalculatePartition()

static size_t thrill::common::CalculatePartition	(	size_t	global_size,
		size_t	p,
		size_t	k
	)

inlinestatic

Definition at line 115 of file math.hpp.

References Range::begin, CalculateLocalRange(), and Range::end.

Referenced by GroupToIndexNode< ValueType, KeyExtractor, GroupFunction >::PreOp().

◆ crc32_slicing_by_8()

uint32_t crc32_slicing_by_8	(	uint32_t	crc,
		const void *	data,
		size_t	length
	)

Definition at line 489 of file hash.cpp.

Referenced by HashCrc32Fallback< ValueType >::operator()(), and HashDataSwitch< std::string >::size().

◆ CreateThread()

std::thread thrill::common::CreateThread ( Args &&... args )

create a std::thread and repeat creation if it fails

Definition at line 44 of file porting.hpp.

References GetHostname(), LOG1, SetCpuAffinity(), and ts_readdir().

Referenced by SortNode< ValueType, CompareFunction, SortAlgorithm, Stable >::MainOp(), thrill::api::RunBackendLoopback(), thrill::api::RunLoopbackThreads(), ReduceNode< ValueType, KeyExtractor, ReduceFunction, ReduceConfig, KeyHashFunction, KeyEqualFunction, VolatileKey, UseDuplicateDetection >::StartPreOp(), and ReduceToIndexNode< ValueType, KeyExtractor, ReduceFunction, ReduceConfig, VolatileKey, SkipPreReducePhase >::StartPreOp().

◆ from_cstr()

T thrill::common::from_cstr	(	const char *	nptr,
		char **	endptr = `nullptr`,
		int	base = `10`
	)

Number parsing helpers, wraps strto{f,d,ld,l,ul,ll,ull}() via type switch.

Referenced by from_str().

◆ from_cstr< double >()

double thrill::common::from_cstr< double >	(	const char *	nptr,
		char **	endptr,
		int
	)

inline

Definition at line 74 of file string.hpp.

◆ from_cstr< float >()

float thrill::common::from_cstr< float >	(	const char *	nptr,
		char **	endptr,
		int
	)

inline

Definition at line 67 of file string.hpp.

◆ from_cstr< long >()

long thrill::common::from_cstr< long >	(	const char *	nptr,
		char **	endptr,
		int	base
	)

inline

Definition at line 91 of file string.hpp.

◆ from_cstr< long double >()

long double thrill::common::from_cstr< long double >	(	const char *	nptr,
		char **	endptr,
		int
	)

inline

Definition at line 81 of file string.hpp.

◆ from_cstr< long long >()

long long thrill::common::from_cstr< long long >	(	const char *	nptr,
		char **	endptr,
		int	base
	)

inline

Definition at line 105 of file string.hpp.

◆ from_cstr< unsigned long >()

unsigned long thrill::common::from_cstr< unsigned long >	(	const char *	nptr,
		char **	endptr,
		int	base
	)

inline

Definition at line 97 of file string.hpp.

◆ from_cstr< unsigned long long >()

unsigned long long thrill::common::from_cstr< unsigned long long >	(	const char *	nptr,
		char **	endptr,
		int	base
	)

inline

Definition at line 111 of file string.hpp.

◆ from_str()

static bool thrill::common::from_str	(	const std::string &	str,
		Type &	outval
	)

inlinestatic

Template transformation function which uses std::istringstream to parse any istreamable type from a std::string.

Returns true only if the whole string was parsed.

Definition at line 49 of file string.hpp.

References from_cstr().

◆ GetHostname()

std::string GetHostname ( )

get hostname

Definition at line 143 of file porting.cpp.

Referenced by CreateThread(), and thrill::api::RunBackendLoopback().

◆ GetNameForThisThread()

std::string thrill::common::GetNameForThisThread ( )

Returns the name of the current thread or 'unknown [id]'.

◆ Hash128to64()

static uint64_t thrill::common::Hash128to64	(	const uint64_t	upper,
		const uint64_t	lower
	)

inlinestatic

This is the Hash128to64 function from Google's cityhash (available under the MIT License).

Definition at line 64 of file hash.hpp.

References ull().

Referenced by ReduceByHash< Key, HashFunction >::operator()().

◆ Hash64to32()

static uint32_t thrill::common::Hash64to32 ( uint64_t key )

inlinestatic

Returns a uint32_t hash of a uint64_t.

This comes from http://www.concentric.net/~ttwang/tech/inthash.htm

This hash gives no guarantees on the cryptographic suitability nor the quality of randomness produced, and the mapping may change in the future.

Definition at line 83 of file hash.hpp.

◆ LogCmdlineParams()

void LogCmdlineParams ( JsonLogger & logger )

try to figure out the command line options of the current process and log it to json logger

Definition at line 68 of file porting.cpp.

References tlx::replace_all(), thrill::mem::to_string(), and tlx::unused().

Referenced by HostContext::HostContext().

◆ MakePipe()

void MakePipe ( int out_pipefds[2] )

create a pair of pipe file descriptors

Definition at line 52 of file porting.cpp.

References O_BINARY, and PortSetCloseOnExec().

Referenced by SelectDispatcher::SelectDispatcher(), thrill::vfs::SysOpenReadStream(), and thrill::vfs::SysOpenWriteStream().

◆ MapVector()

auto thrill::common::MapVector	(	const std::vector< Type > &	input,
		const Functor &	f
	)		-> std::vector<typename std::result_of<Functor(Type)>::type>

inline

apply the Functor to each item in a std::vector<T> and return a new std::vector<U> with different type.

Definition at line 78 of file functional.hpp.

Referenced by ConcatNode< ValueType >::ConcatNode(), and UnionNode< ValueType >::UnionNode().

◆ max()

static constexpr const T& thrill::common::max	(	const T &	a,
		const T &	b
	)

inlinestatic

template for constexpr max, because std::max is not good enough.

Definition at line 45 of file functional.hpp.

◆ min()

static constexpr const T& thrill::common::min	(	const T &	a,
		const T &	b
	)

inlinestatic

template for constexpr min, because std::min is not good enough.

Definition at line 39 of file functional.hpp.

Referenced by ComponentSum< std::vector< Type >, Operation >::operator()().

◆ NameThisThread()

void NameThisThread ( const std::string & name )

Defines a name for the current thread, only if no name was set previously.

Defines a name for the current thread.

Definition at line 40 of file logger.cpp.

Referenced by thrill::api::Run(), thrill::api::RunBackendLoopback(), thrill::api::RunLocalSameThread(), thrill::api::RunLoopbackThreads(), and DispatcherThread::Work().

◆ operator*() [1/2]

Vector<D> thrill::common::operator*	(	const double	a,
		const Vector< D > &	b
	)

Definition at line 115 of file vector.hpp.

References gen_data::D, and Vector< D, Type >::x.

Referenced by thrill::core::hyperloglog::mergeSameIndices().

◆ operator*() [2/2]

VVector<double> thrill::common::operator*	(	const double	a,
		const VVector< double > &	b
	)

Definition at line 211 of file vector.hpp.

References VVector< Type >::x.

◆ PortSetCloseOnExec()

void PortSetCloseOnExec ( int fd )

set FD_CLOEXEC on file descriptor (if possible)

Definition at line 42 of file porting.cpp.

References tlx::unused().

Referenced by MakePipe(), thrill::vfs::SysOpenReadStream(), and thrill::vfs::SysOpenWriteStream().

◆ Put() [1/15]

static JsonLine& thrill::common::Put	(	JsonLine &	line,
		bool const &	value
	)

inlinestatic

Definition at line 279 of file json_logger.hpp.

References JsonLine::os_.

Referenced by ReducePostPhaseEmitter< TableItem, ValueType, thrill::api::ReduceNode::Emitter, VolatileKey >::Emit(), JsonLine::operator<<(), and Put().

◆ Put() [2/15]

static JsonLine& thrill::common::Put	(	JsonLine &	line,
		int const &	value
	)

inlinestatic

Definition at line 285 of file json_logger.hpp.

References JsonLine::os_, and gen_data::value.

◆ Put() [3/15]

static JsonLine& thrill::common::Put	(	JsonLine &	line,
		unsigned int const &	value
	)

inlinestatic

Definition at line 291 of file json_logger.hpp.

References JsonLine::os_, and gen_data::value.

◆ Put() [4/15]

static JsonLine& thrill::common::Put	(	JsonLine &	line,
		long const &	value
	)

inlinestatic

Definition at line 297 of file json_logger.hpp.

References JsonLine::os_, and gen_data::value.

◆ Put() [5/15]

static JsonLine& thrill::common::Put	(	JsonLine &	line,
		unsigned long const &	value
	)

inlinestatic

Definition at line 303 of file json_logger.hpp.

References JsonLine::os_, and gen_data::value.

◆ Put() [6/15]

static JsonLine& thrill::common::Put	(	JsonLine &	line,
		long long const &	value
	)

inlinestatic

Definition at line 309 of file json_logger.hpp.

References JsonLine::os_, and gen_data::value.

◆ Put() [7/15]

static JsonLine& thrill::common::Put	(	JsonLine &	line,
		unsigned long long const &	value
	)

inlinestatic

Definition at line 315 of file json_logger.hpp.

References JsonLine::os_, and gen_data::value.

◆ Put() [8/15]

static JsonLine& thrill::common::Put	(	JsonLine &	line,
		double const &	value
	)

inlinestatic

Definition at line 321 of file json_logger.hpp.

References JsonLine::os_, and gen_data::value.

◆ Put() [9/15]

static JsonLine& thrill::common::Put	(	JsonLine &	line,
		const char *const &	str
	)

inlinestatic

Definition at line 327 of file json_logger.hpp.

References JsonLine::os_, and JsonLine::PutEscapedChar().

◆ Put() [10/15]

static JsonLine& thrill::common::Put	(	JsonLine &	line,
		std::string const &	str
	)

inlinestatic

Definition at line 335 of file json_logger.hpp.

References JsonLine::os_, and JsonLine::PutEscapedChar().

◆ Put() [11/15]

static JsonLine& thrill::common::Put	(	JsonLine &	line,
		const Type(&)	arr[N]
	)

inlinestatic

Definition at line 345 of file json_logger.hpp.

References gen_data::N, JsonLine::os_, and Put().

◆ Put() [12/15]

static JsonLine& thrill::common::Put	(	JsonLine &	line,
		std::initializer_list< Type > const &	list
	)

inlinestatic

Definition at line 357 of file json_logger.hpp.

References JsonLine::os_, and Put().

◆ Put() [13/15]

static JsonLine& thrill::common::Put	(	JsonLine &	line,
		std::vector< Type > const &	vec
	)

inlinestatic

Definition at line 371 of file json_logger.hpp.

References JsonLine::os_, and Put().

◆ Put() [14/15]

static JsonLine& thrill::common::Put	(	JsonLine &	line,
		std::array< Type, N > const &	arr
	)

inlinestatic

Definition at line 385 of file json_logger.hpp.

References JsonLine::os_, and Put().

◆ Put() [15/15]

static JsonLine& thrill::common::Put	(	JsonLine &	line,
		JsonVerbatim const &	verbatim
	)

inlinestatic

Definition at line 398 of file json_logger.hpp.

References JsonLine::items_, JsonLine::os_, and JsonVerbatim::str_.

◆ qsort_three_pivots()

void thrill::common::qsort_three_pivots	(	Iterator	left,
		Iterator	right,
		Compare	cmp
	)

Definition at line 244 of file qsort.hpp.

References thrill::common::qsort_local::InsertionSort(), thrill::common::qsort_local::rotate3(), thrill::common::qsort_local::rotate4(), sLOG0, thrill::common::qsort_local::sort_samples(), and tlx::swap().

Referenced by main().

◆ qsort_two_pivots_yaroslavskiy()

void thrill::common::qsort_two_pivots_yaroslavskiy	(	Iterator	lo,
		Iterator	hi,
		Compare	cmp
	)

Definition at line 186 of file qsort.hpp.

References thrill::common::qsort_local::rotate3(), thrill::common::qsort_local::sort_samples(), and tlx::swap().

Referenced by main().

◆ radix_sort_CI() [1/2]

static void thrill::common::radix_sort_CI	(	Iterator	begin,
		Iterator	end,
		size_t	K,
		const Comparator &	cmp,
		const SubSorter &	sub_sort,
		size_t	depth,
		Char *	char_cache
	)

inlinestatic

Internal helper method, use radix_sort_CI below.

Definition at line 38 of file radix_sort.hpp.

References tlx::digest_detail::K, and tlx::swap().

◆ radix_sort_CI() [2/2]

static void thrill::common::radix_sort_CI	(	Iterator	begin,
		Iterator	end,
		size_t	K,
		const Comparator &	cmp = `Comparator()`,
		const SubSorter &	sub_sort = `SubSorter()`
	)

inlinestatic

Radix sort the iterator range [begin,end).

Sort unconditionally up to depth MaxDepth, then call the sub_sort method for further sorting. Small buckets are sorted using std::sort() with given comparator. Characters are extracted from items in the range using the at_radix(depth) method. All character values must be less than K (the counting array size).

Definition at line 119 of file radix_sort.hpp.

References tlx::digest_detail::K.

◆ RandomString()

static std::string thrill::common::RandomString	(	std::string::size_type	size,
		RandomEngine	rng,
		const std::string &	letters = `"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"`
	)

inlinestatic

Generate a random string of given length.

The set of available bytes/characters is given as the second argument. Each byte is equally probable. Uses the pseudo-random number generator from stdlib; take care to seed it using srand() before calling this function.

Parameters

size	length of result
rng	Random number generator to use, e.g. std::default_random_engine.
letters	character set to choose from

Returns: random string of given length

Definition at line 162 of file string.hpp.

◆ SetCpuAffinity() [1/2]

void SetCpuAffinity	(	std::thread &	thread,
		size_t	cpu_id
	)

set cpu/core affinity of a thread

Definition at line 111 of file porting.cpp.

References LOG1, and tlx::unused().

Referenced by CreateThread(), SortNode< ValueType, CompareFunction, SortAlgorithm, Stable >::MainOp(), thrill::api::RunBackendLoopback(), thrill::api::RunLoopbackThreads(), and DispatcherThread::Work().

◆ SetCpuAffinity() [2/2]

void SetCpuAffinity ( size_t cpu_id )

set cpu/core affinity of current thread

Definition at line 128 of file porting.cpp.

References LOG1, and tlx::unused().

◆ StartLinuxProcStatsProfiler()

void StartLinuxProcStatsProfiler	(	ProfileThread &	,
		JsonLogger &
	)

launch profiler task

Definition at line 860 of file linux_proc_stats.cpp.

Referenced by HostContext::HostContext().

◆ to_str()

static std::string thrill::common::to_str ( const Type & t )

inlinestatic

Use ostream to output any type as string. You generally DO NOT want to use this, instead create a larger ostringstream.

Definition at line 36 of file string.hpp.

Referenced by thrill::api::RunCheckUnlinkBinary().

◆ ts_readdir()

struct dirent * ts_readdir ( DIR * dirp )

mutex-locked readdir() call

Definition at line 153 of file porting.cpp.

Referenced by CreateThread(), thrill::vfs::SysGlobWalkRecursive(), and TemporaryDirectory::wipe_directory().

◆ VecToStr() [1/2]

static std::string thrill::common::VecToStr ( const std::array< T, N > & data )

static

Logging helper to print arrays as [a1,a2,a3,...].

Definition at line 121 of file string.hpp.

Referenced by ZipNode< ValueType, ZipFunction, Pad, UnequalCheck, NoRebalance, kNumInputs >::MainOp(), and ZipWindowNode< ValueType, ZipFunction_, Pad_, UnequalCheck, UseStdArray, kNumInputs_ >::MainOp().

◆ VecToStr() [2/2]

static std::string thrill::common::VecToStr ( const std::vector< T > & data )

static

Logging helper to print vectors as [a1,a2,a3,...].

Definition at line 136 of file string.hpp.

Variable Documentation

◆ crc_tableil8_o32

const uint32_t crc_tableil8_o32[256]

Definition at line 71 of file hash.cpp.

◆ crc_tableil8_o40

const uint32_t crc_tableil8_o40[256]

Definition at line 125 of file hash.cpp.

◆ crc_tableil8_o48

const uint32_t crc_tableil8_o48[256]

Definition at line 179 of file hash.cpp.

◆ crc_tableil8_o56

const uint32_t crc_tableil8_o56[256]

Definition at line 233 of file hash.cpp.

◆ crc_tableil8_o64

const uint32_t crc_tableil8_o64[256]

Definition at line 287 of file hash.cpp.

◆ crc_tableil8_o72

const uint32_t crc_tableil8_o72[256]

Definition at line 341 of file hash.cpp.

◆ crc_tableil8_o80

const uint32_t crc_tableil8_o80[256]

Definition at line 395 of file hash.cpp.

◆ crc_tableil8_o88

const uint32_t crc_tableil8_o88[256]

Definition at line 449 of file hash.cpp.

◆ g_cache_line_size

constexpr unsigned g_cache_line_size = 64

static

Finding cache line size is hard - we assume 64 byte.

Definition at line 38 of file config.hpp.

Referenced by FlowControlChannel::LocalData::IncCounter().

◆ g_debug_mode

constexpr bool g_debug_mode = !g_ndebug

static

debug mode is active, if NDEBUG is false.

Definition at line 28 of file config.hpp.

◆ g_debug_push_file

constexpr bool g_debug_push_file = false

static

global flag to warn user when two DIANodes could push data directly via a File (skipping one data round trip) if the function stack were empty. This can be used to find lambda which could be be fused to reduce the number of data round trips.

Definition at line 44 of file config.hpp.

Referenced by SizeNode< ValueType >::OnPreOpFile(), RebalanceNode< ValueType >::OnPreOpFile(), CacheNode< ValueType >::OnPreOpFile(), PrefixSumNode< ValueType, SumFunction, Inclusive >::OnPreOpFile(), AllGatherNode< ValueType >::OnPreOpFile(), ZipWithIndexNode< ValueType, ZipFunction >::OnPreOpFile(), BaseWindowNode< ValueType, Input, WindowFunction, PartialWindowFunction >::OnPreOpFile(), ZipNode< ValueType, ZipFunction, Pad, UnequalCheck, NoRebalance, kNumInputs >::OnPreOpFile(), SortNode< ValueType, CompareFunction, SortAlgorithm, Stable >::OnPreOpFile(), and ConcatNode< ValueType >::OnPreOpFile().

◆ g_logger_mem_manager

mem::Manager g_logger_mem_manager

memory manager singleton for Logger

Definition at line 28 of file logger.hpp.

◆ g_ndebug

constexpr bool g_ndebug = false

static

global ndebug flag as a boolean, NDEBUG means no debug in Release mode.

Definition at line 24 of file config.hpp.

◆ g_profile_thread

constexpr bool g_profile_thread = true

static

global flag to enable background profiler thread

Definition at line 35 of file config.hpp.

Referenced by ProfileThread::ProfileThread(), and ProfileThread::~ProfileThread().

◆ g_self_verify

constexpr bool g_self_verify = g_debug_mode

static

global flag to enable code parts doing self-verification. Later this may be set false if NDEBUG is set in production mode.

Definition at line 32 of file config.hpp.

◆ s_default_logger

ThreadLoggerPrefixHook s_default_logger

static

default logger singleton

Definition at line 92 of file logger.cpp.

◆ s_message_counter

thread_local size_t s_message_counter = 0

static

thread message counter

Definition at line 37 of file logger.cpp.

◆ s_thread_name

thread_local char s_thread_name[64] = { 0 }

static

thread name

Definition at line 34 of file logger.cpp.

◆ TLX_ATTRIBUTE_PACKED

class thrill::common::UIntPair TLX_ATTRIBUTE_PACKED

Referenced by UIntPair< High_ >::max().

Namespaces

Classes

Typedefs

Functions

Variables

Detailed Description

Typedef Documentation

◆ Array

◆ FakeStatsTimer

◆ FakeStatsTimerStart

◆ FakeStatsTimerStopped

◆ hash

◆ HashCrc32

◆ HashTabulated

◆ hypergeometric

◆ is_trivially_copyable

◆ RingBuffer

◆ StatsTimer

◆ StatsTimerStart

◆ StatsTimerStopped

◆ ThreadBarrier

◆ uint40

◆ uint48

Function Documentation

◆ AddTruncToType()

◆ alias_cast() [1/2]

◆ alias_cast() [2/2]

◆ CalcOneFactorPeer()

◆ CalcOneFactorSize()

◆ CalculateLocalRange()

◆ CalculatePartition()

◆ crc32_slicing_by_8()

◆ CreateThread()

◆ from_cstr()

◆ from_cstr< double >()

◆ from_cstr< float >()

◆ from_cstr< long >()

◆ from_cstr< long double >()

◆ from_cstr< long long >()

◆ from_cstr< unsigned long >()

◆ from_cstr< unsigned long long >()

◆ from_str()

◆ GetHostname()

◆ GetNameForThisThread()

◆ Hash128to64()

◆ Hash64to32()

◆ LogCmdlineParams()

◆ MakePipe()

◆ MapVector()

◆ max()

◆ min()

◆ NameThisThread()

◆ operator*() [1/2]

◆ operator*() [2/2]

◆ PortSetCloseOnExec()

◆ Put() [1/15]

◆ Put() [2/15]

◆ Put() [3/15]

◆ Put() [4/15]

◆ Put() [5/15]

◆ Put() [6/15]

◆ Put() [7/15]

◆ Put() [8/15]

◆ Put() [9/15]

◆ Put() [10/15]

◆ Put() [11/15]

◆ Put() [12/15]

◆ Put() [13/15]

◆ Put() [14/15]

◆ Put() [15/15]

◆ qsort_three_pivots()

◆ qsort_two_pivots_yaroslavskiy()

◆ radix_sort_CI() [1/2]

◆ radix_sort_CI() [2/2]

◆ RandomString()

◆ SetCpuAffinity() [1/2]

◆ SetCpuAffinity() [2/2]

◆ StartLinuxProcStatsProfiler()

◆ to_str()

◆ ts_readdir()