Thrill  0.1
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
thrill::api Namespace Reference

Classes

class  ActionNode
 
class  ActionResultNode
 
exception  AllGatherNode
 
exception  AllReduceNode
 
struct  ArrayTag
 tag structure for ZipWindow() More...
 
class  BaseWindowNode
 
class  BernoulliSampleNode
 
exception  CacheNode
 A DOpNode which caches all items in an external file. More...
 
exception  CollapseNode
 
struct  CollapseSwitch
 Template switch to generate a CollapseNode if there is a non-empty Stack. More...
 
struct  CollapseSwitch< ValueType, FunctionStack< ValueType > >
 Template switch to NOT generate a CollapseNode if there is an empty Stack. More...
 
exception  ConcatNode
 
exception  ConcatToDIANode
 
class  Context
 The Context of a job is a unique instance per worker which holds references to all underlying parts of Thrill. More...
 
struct  CutTag
 tag structure for Zip() More...
 
class  DefaultReduceConfig
 
class  DefaultReduceToIndexConfig
 
class  DefaultSortAlgorithm
 
class  DIA
 DIA is the interface between the user and the Thrill framework. More...
 
class  DIABase
 The DIABase is the untyped super class of DIANode. More...
 
class  DIAMemUse
 Description of the amount of RAM the internal data structures of a DIANode require. More...
 
class  DIANode
 A DIANode is a typed node representing and operation in Thrill. More...
 
struct  DisjointTag
 tag structure for Window() and FlatWindow() More...
 
exception  DisjointWindowNode
 
exception  DistributeNode
 
class  DOpNode
 A DOpNode is a typed node representing and distributed operations in Thrill. More...
 
struct  DuplicateDetectionFlag
 tag structure for ReduceByKey() More...
 
exception  EqualToDIANode
 
class  FunctionStack
 A FunctionStack is a chain of lambda functions that can be folded to a single lambda functions. More...
 
class  Future
 The return type class for all ActionFutures. More...
 
class  Future< void >
 Specialized template class for ActionFuture which return void. More...
 
exception  GatherNode
 
exception  GenerateNode
 A DIANode which performs a Generate operation. More...
 
class  GroupByIterator
 
class  GroupByMultiwayMergeIterator
 
exception  GroupByNode
 
exception  GroupToIndexNode
 
class  HostContext
 The HostContext contains all data structures shared among workers on the same host. More...
 
exception  JoinNode
 Performs an inner join between two DIAs. More...
 
struct  LocalStorageTag
 tag structure for Read() More...
 
struct  LocationDetectionFlag
 tag structure for GroupByKey(), and InnerJoin() More...
 
class  MemoryConfig
 
class  MergeNode
 Implementation of Thrill's merge. More...
 
struct  NoRebalanceTag
 tag structure for Zip() More...
 
exception  OverlapWindowNode
 
struct  PadTag
 tag structure for Zip() More...
 
exception  PrefixSumNode
 
exception  ReadBinaryNode
 A DIANode which performs a line-based Read operation. More...
 
exception  ReadLinesNode
 A DIANode which performs a line-based Read operation. More...
 
exception  RebalanceNode
 
exception  ReduceNode
 A DIANode which performs a Reduce operation. More...
 
exception  ReduceToIndexNode
 A DIANode which performs a ReduceToIndex operation. More...
 
exception  SampleNode
 
exception  SizeNode
 
exception  SortNode
 A DIANode which performs a Sort operation. More...
 
class  SourceNode
 
exception  UnionNode
 Implements a Union operation by hooking any number of input DIANodes and forwarding the output immediately to all children. More...
 
struct  VolatileKeyFlag
 tag structure for ReduceByKey(), and ReduceToIndex() More...
 
exception  WriteBinaryNode
 
exception  WriteLinesNode
 
exception  WriteLinesOneNode
 
exception  ZipNode
 A DIANode which performs a Zip operation. More...
 
exception  ZipWindowNode
 
class  ZipWindowReader
 
class  ZipWindowReader< ZipWindowNode, false >
 template specialization Reader which delivers std::vector<>s to ZipFunction More...
 
class  ZipWindowReader< ZipWindowNode, true >
 template specialization Reader which delivers std::array<>s to ZipFunction More...
 
struct  ZipWindowTraits< ReturnType(*)(Args...)>
 specialize for function pointers More...
 
struct  ZipWindowTraits< ReturnType(ClassType::*)(Args...) const >
 specialize for pointers to const member function More...
 
struct  ZipWindowTraits< ReturnType(ClassType::*)(Args...)>
 specialize for pointers to mutable member function More...
 
exception  ZipWithIndexNode
 A DIANode which calculates the array index for each items and performs a Zip-like operation without extra rebalancing of the DIA data. More...
 

Typedefs

template<typename T >
using mm_set = std::set< T, std::less< T >, mem::Allocator< T > >
 
using TestGroup = net::tcp::Group
 

Functions

template<typename FirstDIA , typename... DIAs>
auto Concat (const FirstDIA &first_dia, const DIAs &...dias)
 Concat is a DOp, which concatenates any number of DIAs to a single DIA. More...
 
template<typename ValueType >
auto Concat (const std::initializer_list< DIA< ValueType > > &dias)
 Concat is a DOp, which concatenates any number of DIAs to a single DIA. More...
 
template<typename ValueType >
auto Concat (const std::vector< DIA< ValueType > > &dias)
 Concat is a DOp, which concatenates any number of DIAs to a single DIA. More...
 
template<typename ValueType >
auto ConcatToDIA (Context &ctx, const std::vector< ValueType > &in_vector)
 ConcatToDIA is a Source-DOp, which takes a vector of data on all workers, and CONCATENATES them into a DIA. More...
 
template<typename ValueType >
auto ConcatToDIA (Context &ctx, std::vector< ValueType > &&in_vector)
 ConcatToDIA is a Source-DOp, which takes a vector of data on all workers, and CONCATENATES them into a DIA. More...
 
template<typename NetGroup >
static std::vector
< std::unique_ptr< HostContext > > 
ConstructLoopbackHostContexts (const MemoryConfig &mem_config, size_t num_hosts, size_t workers_per_host)
 Generic network constructor for net backends supporting loopback tests. More...
 
template<>
common::CountingPtr
< data::CatStream
Context::GetNewStream< data::CatStream > (size_t dia_id)
 
template<>
common::CountingPtr
< data::MixStream
Context::GetNewStream< data::MixStream > (size_t dia_id)
 
static bool Deinitialize ()
 
static const char * DetectNetBackend ()
 
template<typename ValueType >
auto EqualToDIA (Context &ctx, const std::vector< ValueType > &in_vector)
 EqualToDIA is a Source-DOp, which takes a vector of data EQUAL on all workers, and returns the data in a DIA. More...
 
template<typename ValueType >
auto EqualToDIA (Context &ctx, std::vector< ValueType > &&in_vector)
 EqualToDIA is an Source-DOp, which takes a vector of data EQUAL on all workers, and returns the data in a DIA. More...
 
static void FindStages (Context &ctx, const DIABasePtr &action, mm_set< Stage > &stages)
 
template<typename GenerateFunction >
auto Generate (Context &ctx, size_t size, const GenerateFunction &generate_function)
 Generate is a Source-DOp, which creates a DIA of given size using a generator function. More...
 
static auto Generate (Context &ctx, size_t size)
 Generate is a Source-DOp, which creates a DIA of given size containing the size_t indexes [0,size). More...
 
static bool Initialize ()
 
template<bool LocationDetectionValue, typename FirstDIA , typename SecondDIA , typename KeyExtractor1 , typename KeyExtractor2 , typename JoinFunction , typename HashFunction = std::hash<typename common::FunctionTraits<KeyExtractor1>::result_type>>
auto InnerJoin (const LocationDetectionFlag< LocationDetectionValue > &, const FirstDIA &first_dia, const SecondDIA &second_dia, const KeyExtractor1 &key_extractor1, const KeyExtractor2 &key_extractor2, const JoinFunction &join_function, const HashFunction &hash_function=HashFunction())
 Performs an inner join between this DIA and the DIA given in the first parameter. More...
 
template<typename FirstDIA , typename SecondDIA , typename KeyExtractor1 , typename KeyExtractor2 , typename JoinFunction , typename HashFunction = std::hash<typename common::FunctionTraits<KeyExtractor1>::result_type>>
auto InnerJoin (const FirstDIA &first_dia, const SecondDIA &second_dia, const KeyExtractor1 &key_extractor1, const KeyExtractor2 &key_extractor2, const JoinFunction &join_function, const HashFunction &hash_function=HashFunction())
 Performs an inner join between this DIA and the DIA given in the first parameter. More...
 
template<typename Comparator , typename FirstDIA , typename... DIAs>
auto Merge (const Comparator &comparator, const FirstDIA &first_dia, const DIAs &...dias)
 Merge is a DOp, which merges any number of sorted DIAs to a single sorted DIA. More...
 
std::ostream & operator<< (std::ostream &os, const DIABase &d)
 make ostream-able. More...
 
template<typename ValueType >
DIA< ValueType > ReadBinary (Context &ctx, const std::vector< std::string > &filepath, uint64_t size_limit=ReadBinaryNode< ValueType >::no_size_limit_)
 ReadBinary is a DOp, which reads a file written by WriteBinary from the file system and creates a DIA. More...
 
template<typename ValueType >
DIA< ValueType > ReadBinary (struct LocalStorageTag, Context &ctx, const std::vector< std::string > &filepath, uint64_t size_limit=ReadBinaryNode< ValueType >::no_size_limit_)
 
template<typename ValueType >
DIA< ValueType > ReadBinary (Context &ctx, const std::string &filepath, uint64_t size_limit=ReadBinaryNode< ValueType >::no_size_limit_)
 ReadBinary is a DOp, which reads a file written by WriteBinary from the file system and creates a DIA. More...
 
template<typename ValueType >
DIA< ValueType > ReadBinary (struct LocalStorageTag, Context &ctx, const std::string &filepath, uint64_t size_limit=ReadBinaryNode< ValueType >::no_size_limit_)
 
DIA< std::string > ReadLines (Context &ctx, const std::string &filepath)
 ReadLines is a DOp, which reads a file from the file system and creates an ordered DIA according to a given read function. More...
 
DIA< std::string > ReadLines (Context &ctx, const std::vector< std::string > &filepaths)
 ReadLines is a DOp, which reads a file from the file system and creates an ordered DIA according to a given read function. More...
 
DIA< std::string > ReadLines (struct LocalStorageTag, Context &ctx, const std::string &filepath)
 
DIA< std::string > ReadLines (struct LocalStorageTag, Context &ctx, const std::vector< std::string > &filepaths)
 
int Run (const std::function< void(Context &)> &job_startpoint)
 Runs the given job startpoint with a Context instance. More...
 
template<typename NetGroup >
static int RunBackendLoopback (const char *backend, const std::function< void(Context &)> &job_startpoint)
 Run() implementation which uses a loopback net backend ("mock" or "tcp"). More...
 
template<typename NetGroup >
static void RunLoopbackThreads (const MemoryConfig &mem_config, size_t num_hosts, size_t workers_per_host, size_t core_offset, const std::function< void(Context &)> &job_startpoint)
 Generic runner for backends supporting loopback tests. More...
 
int RunNotSupported (const char *env_net)
 
static bool SetupBlockSize ()
 
static void TopoSortStages (mm_set< Stage > &stages, mem::vector< Stage > &result)
 
static void TopoSortVisit (const Stage &s, mm_set< Stage > &stages, mem::vector< Stage > &result)
 
template<typename FirstDIA , typename... DIAs>
auto Union (const FirstDIA &first_dia, const DIAs &...dias)
 Union is a LOp, which creates the union of all items from any number of DIAs as a single DIA, where the items are in an arbitrary order. More...
 
template<typename ValueType >
auto Union (const std::initializer_list< DIA< ValueType > > &dias)
 Union is a LOp, which creates the union of all items from any number of DIAs as a single DIA, where the items are in an arbitrary order. More...
 
template<typename ValueType >
auto Union (const std::vector< DIA< ValueType > > &dias)
 Union is a LOp, which creates the union of all items from any number of DIAs as a single DIA, where the items are in an arbitrary order. More...
 
int wrap_setenv (const char *name, const char *value, int overwrite)
 
template<typename ZipFunction , typename FirstDIAType , typename FirstDIAStack , typename... DIAs>
auto Zip (const ZipFunction &zip_function, const DIA< FirstDIAType, FirstDIAStack > &first_dia, const DIAs &...dias)
 Zips two DIAs of equal size in style of functional programming by applying zip_function to the i-th elements of both input DIAs to form the i-th element of the output DIA. More...
 
template<typename ZipFunction , typename FirstDIAType , typename FirstDIAStack , typename... DIAs>
auto Zip (struct CutTag, const ZipFunction &zip_function, const DIA< FirstDIAType, FirstDIAStack > &first_dia, const DIAs &...dias)
 Zips any number of DIAs of equal size in style of functional programming by applying zip_function to the i-th elements of both input DIAs to form the i-th element of the output DIA. More...
 
template<typename ZipFunction , typename FirstDIAType , typename FirstDIAStack , typename... DIAs>
auto Zip (struct PadTag, const ZipFunction &zip_function, const DIA< FirstDIAType, FirstDIAStack > &first_dia, const DIAs &...dias)
 Zips any number of DIAs in style of functional programming by applying zip_function to the i-th elements of both input DIAs to form the i-th element of the output DIA. More...
 
template<typename ZipFunction , typename FirstDIAType , typename FirstDIAStack , typename... DIAs>
auto Zip (struct PadTag, const ZipFunction &zip_function, const typename common::FunctionTraits< ZipFunction >::args_tuple_plain &padding, const DIA< FirstDIAType, FirstDIAStack > &first_dia, const DIAs &...dias)
 Zips any number of DIAs in style of functional programming by applying zip_function to the i-th elements of both input DIAs to form the i-th element of the output DIA. More...
 
template<typename ZipFunction , typename FirstDIAType , typename FirstDIAStack , typename... DIAs>
auto Zip (struct NoRebalanceTag, const ZipFunction &zip_function, const DIA< FirstDIAType, FirstDIAStack > &first_dia, const DIAs &...dias)
 Zips any number of DIAs in style of functional programming by applying zip_function to the i-th elements of both input DIAs to form the i-th element of the output DIA. More...
 
template<typename ZipFunction , typename FirstDIAType , typename FirstDIAStack , typename... DIAs>
auto ZipWindow (const std::array< size_t, 1+sizeof...(DIAs)> &window_size, const ZipFunction &zip_function, const DIA< FirstDIAType, FirstDIAStack > &first_dia, const DIAs &...dias)
 Zips two DIAs of equal size in style of functional programming by applying zip_function to the i-th fixed-sized windows of both input DIAs to form the i-th element of the output DIA. More...
 
template<typename ZipFunction , typename FirstDIAType , typename FirstDIAStack , typename... DIAs>
auto ZipWindow (struct CutTag, const std::array< size_t, 1+sizeof...(DIAs)> &window_size, const ZipFunction &zip_function, const DIA< FirstDIAType, FirstDIAStack > &first_dia, const DIAs &...dias)
 Zips two DIAs of equal size in style of functional programming by applying zip_function to the i-th fixed-sized windows of both input DIAs to form the i-th element of the output DIA. More...
 
template<typename ZipFunction , typename FirstDIAType , typename FirstDIAStack , typename... DIAs>
auto ZipWindow (struct PadTag, const std::array< size_t, 1+sizeof...(DIAs)> &window_size, const ZipFunction &zip_function, const typename ZipWindowTraits< ZipFunction >::value_type_tuple_plain &padding, const DIA< FirstDIAType, FirstDIAStack > &first_dia, const DIAs &...dias)
 Zips two DIAs of equal size in style of functional programming by applying zip_function to the i-th fixed-sized windows of both input DIAs to form the i-th element of the output DIA. More...
 
template<typename ZipFunction , typename FirstDIAType , typename FirstDIAStack , typename... DIAs>
auto ZipWindow (struct PadTag, const std::array< size_t, 1+sizeof...(DIAs)> &window_size, const ZipFunction &zip_function, const DIA< FirstDIAType, FirstDIAStack > &first_dia, const DIAs &...dias)
 Zips two DIAs of equal size in style of functional programming by applying zip_function to the i-th fixed-sized windows of both input DIAs to form the i-th element of the output DIA. More...
 
template<typename ZipFunction , typename FirstDIAType , typename FirstDIAStack , typename... DIAs>
auto ZipWindow (struct ArrayTag, struct PadTag, const std::array< size_t, 1+sizeof...(DIAs)> &window_size, const ZipFunction &zip_function, const typename ZipWindowTraits< ZipFunction >::value_type_tuple_plain &padding, const DIA< FirstDIAType, FirstDIAStack > &first_dia, const DIAs &...dias)
 Zips two DIAs of equal size in style of functional programming by applying zip_function to the i-th fixed-sized windows of both input DIAs to form the i-th element of the output DIA. More...
 
template<typename ZipFunction , typename FirstDIAType , typename FirstDIAStack , typename... DIAs>
auto ZipWindow (struct ArrayTag, struct PadTag, const std::array< size_t, 1+sizeof...(DIAs)> &window_size, const ZipFunction &zip_function, const DIA< FirstDIAType, FirstDIAStack > &first_dia, const DIAs &...dias)
 Zips two DIAs of equal size in style of functional programming by applying zip_function to the i-th fixed-sized windows of both input DIAs to form the i-th element of the output DIA. More...
 
Run Methods with Internal Networks for Testing
void RunLocalMock (const MemoryConfig &mem_config, size_t num_hosts, size_t workers_per_host, const std::function< void(Context &)> &job_startpoint)
 Function to run a number of mock hosts as locally independent threads, which communicate via internal stream sockets. More...
 
void RunLocalTests (const std::function< void(Context &)> &job_startpoint)
 Helper Function to execute RunLocalMock() tests using mock networks in test suite for many different numbers of workers and hosts as independent threads in one program. More...
 
void RunLocalTests (size_t ram, const std::function< void(Context &)> &job_startpoint)
 Helper Function to execute RunLocalMock() tests using mock networks in test suite for many different numbers of workers and hosts as independent threads in one program. More...
 
void RunLocalSameThread (const std::function< void(Context &)> &job_startpoint)
 Runs the given job_startpoint within the same thread with a test network –> run test with one host and one thread. More...
 
int RunCheckDieWithParent ()
 Check environment variable THRILL_DIE_WITH_PARENT and enable process flag: this is useful for ssh/invoke.sh: it kills spawned processes when the ssh connection breaks. More...
 
int RunCheckUnlinkBinary ()
 Check environment variable THRILL_UNLINK_BINARY and unlink given program path: this is useful for ssh/invoke.sh: it removes the copied program files while it is running, hence it is gone even if the program crashes. More...
 
template<typename ValueType >
auto Distribute (Context &ctx, const std::vector< ValueType > &in_vector, size_t source_id=0)
 Distribute is a Source DOp, which scatters the vector data from the source_id to all workers, partitioning equally, and returning the data in a DIA. More...
 
template<typename ValueType >
auto Distribute (Context &ctx, std::vector< ValueType > &&in_vector, size_t source_id=0)
 Distribute is a Source DOp, which scatters the vector data from the source_id to all workers, partitioning equally, and returning the data in a DIA. More...
 
template<typename Lambda >
auto RunEmitter (const Lambda &lambda)
 Base case for the chaining of lambda functions. More...
 
template<typename Lambda , typename... MoreLambdas>
auto RunEmitter (const Lambda &lambda, const MoreLambdas &...rest)
 Recursive case for the chaining of lambda functions. More...
 
template<typename Input , typename Lambda >
static auto MakeFunctionStack (const Lambda &lambda)
 

Variables

struct ArrayTag ArrayTag
 global const ArrayTag instance More...
 
struct VolatileKeyFlag< true > VolatileKeyTag
 global const VolatileKeyFlag instance More...
 
struct VolatileKeyFlag< false > NoVolatileKeyTag
 global const VolatileKeyFlag instance More...
 
struct DisjointTag DisjointTag
 global const DisjointTag instance More...
 
struct CutTag CutTag
 global const CutTag instance More...
 
struct PadTag PadTag
 global const PadTag instance More...
 
struct NoRebalanceTag NoRebalanceTag
 global const NoRebalanceTag instance More...
 
struct LocalStorageTag LocalStorageTag
 global const LocalStorageTag instance More...
 
struct DuplicateDetectionFlag
< true > 
DuplicateDetectionTag
 global const DuplicateDetectionFlag instance More...
 
struct DuplicateDetectionFlag
< false > 
NoDuplicateDetectionTag
 global const DuplicateDetectionFlag instance More...
 
struct LocationDetectionFlag
< true > 
LocationDetectionTag
 global const LocationDetectionFlag instance More...
 
struct LocationDetectionFlag
< false > 
NoLocationDetectionTag
 global const LocationDetectionFlag instance More...
 
enum  DIAState { NEW, EXECUTED, DISPOSED }
 Possible states a DIABase can be in. More...
 
using DIABasePtr = common::CountingPtr< DIABase >
 

Typedef Documentation

using mm_set = std::set<T, std::less<T>, mem::Allocator<T> >

Definition at line 297 of file dia_base.cpp.

Definition at line 205 of file context.cpp.

Function Documentation

auto thrill::api::ConcatToDIA ( Context &  ctx,
std::vector< ValueType > &&  in_vector 
)

ConcatToDIA is a Source-DOp, which takes a vector of data on all workers, and CONCATENATES them into a DIA.

Use Distribute to actually distribute data from a single worker, ConcatToDIA is a wrapper if the data is already distributed.

Parameters
ctxReference to the Context object
in_vectorVector to concatenate into a DIA, the contents is MOVED into the DIANode.

Definition at line 93 of file concat_to_dia.hpp.

static std::vector<std::unique_ptr<HostContext> > thrill::api::ConstructLoopbackHostContexts ( const MemoryConfig &  mem_config,
size_t  num_hosts,
size_t  workers_per_host 
)
inlinestatic

Generic network constructor for net backends supporting loopback tests.

Definition at line 86 of file context.cpp.

References Manager::kGroupCount.

static bool thrill::api::Deinitialize ( )
inlinestatic

Definition at line 192 of file context.cpp.

References thrill::vfs::Deinitialize().

Referenced by RunBackendLoopback().

static const char* thrill::api::DetectNetBackend ( )
inlinestatic

Definition at line 725 of file context.cpp.

Referenced by Run().

static void thrill::api::FindStages ( Context &  ctx,
const DIABasePtr &  action,
mm_set< Stage > &  stages 
)
static

Do a BFS on parents to find all DIANodes (Stages) needed to Execute or PushData to calculate this action node.

Definition at line 301 of file dia_base.cpp.

References debug, DIABase::ForwardDataOnly(), LOG, DIABase::mem_manager(), Context::my_rank(), NEW, DIABase::parents(), DIABase::RequireParentPushData(), and DIABase::state().

Referenced by DIABase::RunScope().

static bool thrill::api::Initialize ( )
inlinestatic
static auto thrill::api::MakeFunctionStack ( const Lambda &  lambda)
inlinestatic

Definition at line 146 of file function_stack.hpp.

std::ostream& thrill::api::operator<< ( std::ostream &  os,
const DIABase &  d 
)
DIA<ValueType> thrill::api::ReadBinary ( struct LocalStorageTag  ,
Context &  ctx,
const std::vector< std::string > &  filepath,
uint64_t  size_limit = ReadBinaryNode<ValueType>::no_size_limit_ 
)

Definition at line 358 of file read_binary.hpp.

DIA<ValueType> thrill::api::ReadBinary ( struct LocalStorageTag  ,
Context &  ctx,
const std::string &  filepath,
uint64_t  size_limit = ReadBinaryNode<ValueType>::no_size_limit_ 
)

Definition at line 392 of file read_binary.hpp.

DIA<std::string> thrill::api::ReadLines ( struct LocalStorageTag  ,
Context &  ctx,
const std::string &  filepath 
)

Definition at line 472 of file read_lines.hpp.

DIA<std::string> thrill::api::ReadLines ( struct LocalStorageTag  ,
Context &  ctx,
const std::vector< std::string > &  filepaths 
)

Definition at line 479 of file read_lines.hpp.

int Run ( const std::function< void(Context &)> &  job_startpoint)

Runs the given job startpoint with a Context instance.

Startpoints may be called multiple times with concurrent threads and different context instances across different workers. The Thrill configuration is taken from environment variables starting the THRILL_.

THRILL_NET is the network backend to use, e.g.: mock, local, tcp, or mpi.

THRILL_RANK contains the rank of this worker

THRILL_HOSTLIST contains a space- or comma-separated list of host:ports to connect to.

THRILL_WORKERS_PER_HOST is the number of workers (threads) per host.

Additional variables:

THRILL_DIE_WITH_PARENT sets a flag which terminates the program if the caller terminates (this is automatically set by ssh/invoke.sh). No more zombies.

THRILL_UNLINK_BINARY deletes a file. Used by ssh/invoke.sh to unlink a copied program binary while it is running. Hence, it can keep /tmp clean.

Returns
0 if execution was fine on all threads.
Examples:
examples/tutorial/k-means_step1.cpp, examples/tutorial/k-means_step2.cpp, examples/tutorial/k-means_step3.cpp, examples/tutorial/k-means_step4.cpp, examples/tutorial/k-means_step5.cpp, and examples/tutorial/k-means_step6.cpp.

Definition at line 796 of file context.cpp.

References DetectNetBackend(), RunCheckDieWithParent(), RunCheckUnlinkBinary(), and RunNotSupported().

Referenced by main().

static int thrill::api::RunBackendLoopback ( const char *  backend,
const std::function< void(Context &)> &  job_startpoint 
)
inlinestatic

Run() implementation which uses a loopback net backend ("mock" or "tcp").

Definition at line 310 of file context.cpp.

References Deinitialize(), Initialize(), MemoryConfig::print(), and MemoryConfig::setup_detect().

int RunCheckDieWithParent ( )

Check environment variable THRILL_DIE_WITH_PARENT and enable process flag: this is useful for ssh/invoke.sh: it kills spawned processes when the ssh connection breaks.

Hence: no more zombies.

Definition at line 751 of file context.cpp.

Referenced by Run().

int RunCheckUnlinkBinary ( )

Check environment variable THRILL_UNLINK_BINARY and unlink given program path: this is useful for ssh/invoke.sh: it removes the copied program files while it is running, hence it is gone even if the program crashes.

Definition at line 782 of file context.cpp.

Referenced by Run().

auto thrill::api::RunEmitter ( const Lambda &  lambda)

Base case for the chaining of lambda functions.

The last lambda function receives an input element, no emitter and should have no return type. It should therefore store the input parameter externally.

Parameters
lambdaLambda function that represents the chain end.

Definition at line 39 of file function_stack.hpp.

Referenced by FunctionStack< Input_, Lambdas >::EmitSequence(), and RunEmitter().

auto thrill::api::RunEmitter ( const Lambda &  lambda,
const MoreLambdas &...  rest 
)

Recursive case for the chaining of lambda functions.

The given lambda function receives an input element, an emitter and should have no return type. The emitter will be built using the chain of remaining lambda functions.

Parameters
lambdaCurrent lambda function to be chained.
restRemaining lambda functions.

Definition at line 59 of file function_stack.hpp.

References RunEmitter().

void RunLocalMock ( const MemoryConfig &  mem_config,
size_t  num_hosts,
size_t  workers_per_host,
const std::function< void(Context &)> &  job_startpoint 
)

Function to run a number of mock hosts as locally independent threads, which communicate via internal stream sockets.

Definition at line 208 of file context.cpp.

Referenced by RunLocalTests().

void RunLocalSameThread ( const std::function< void(Context &)> &  job_startpoint)

Runs the given job_startpoint within the same thread with a test network –> run test with one host and one thread.

Definition at line 271 of file context.cpp.

References Group::ConstructLoopbackMesh(), Manager::kGroupCount, thrill::common::NameThisThread(), MemoryConfig::print(), MemoryConfig::setup(), thrill::mem::to_string(), and MemoryConfig::verbose_.

void RunLocalTests ( const std::function< void(Context &)> &  job_startpoint)

Helper Function to execute RunLocalMock() tests using mock networks in test suite for many different numbers of workers and hosts as independent threads in one program.

Use this function in most test cases.

Definition at line 244 of file context.cpp.

void RunLocalTests ( size_t  ram,
const std::function< void(Context &)> &  job_startpoint 
)

Helper Function to execute RunLocalMock() tests using mock networks in test suite for many different numbers of workers and hosts as independent threads in one program.

Use this function in most test cases.

Definition at line 249 of file context.cpp.

References RunLocalMock(), MemoryConfig::setup(), MemoryConfig::verbose_, and wrap_setenv().

static void thrill::api::RunLoopbackThreads ( const MemoryConfig &  mem_config,
size_t  num_hosts,
size_t  workers_per_host,
size_t  core_offset,
const std::function< void(Context &)> &  job_startpoint 
)
inlinestatic
int thrill::api::RunNotSupported ( const char *  env_net)

Definition at line 718 of file context.cpp.

Referenced by Run().

static bool thrill::api::SetupBlockSize ( )
inlinestatic

Definition at line 158 of file context.cpp.

References thrill::data::default_block_size, and thrill::data::start_block_size.

Referenced by Initialize().

static void thrill::api::TopoSortStages ( mm_set< Stage > &  stages,
mem::vector< Stage > &  result 
)
static

Definition at line 372 of file dia_base.cpp.

References TopoSortVisit().

Referenced by DIABase::RunScope().

static void thrill::api::TopoSortVisit ( const Stage &  s,
mm_set< Stage > &  stages,
mem::vector< Stage > &  result 
)
static

Definition at line 349 of file dia_base.cpp.

References die_unless.

Referenced by TopoSortStages().

int thrill::api::wrap_setenv ( const char *  name,
const char *  value,
int  overwrite 
)

Definition at line 239 of file context.cpp.

Referenced by RunLocalTests().

Variable Documentation

struct CutTag CutTag

global const CutTag instance

Definition at line 67 of file dia.hpp.

struct DuplicateDetectionFlag< true > DuplicateDetectionTag

global const DuplicateDetectionFlag instance

Definition at line 101 of file dia.hpp.

global const LocalStorageTag instance

Definition at line 91 of file dia.hpp.

struct LocationDetectionFlag< true > LocationDetectionTag

global const LocationDetectionFlag instance

Definition at line 114 of file dia.hpp.

Referenced by InnerJoin().

struct DuplicateDetectionFlag< false > NoDuplicateDetectionTag

global const DuplicateDetectionFlag instance

Definition at line 104 of file dia.hpp.

Referenced by DIA< ValueType_, Stack_ >::ReduceByKey(), and DIA< ValueType_, Stack_ >::ReducePair().

struct LocationDetectionFlag< false > NoLocationDetectionTag

global const LocationDetectionFlag instance

Definition at line 117 of file dia.hpp.

Referenced by DIA< ValueType_, Stack_ >::GroupByKey().

struct VolatileKeyFlag< false > NoVolatileKeyTag

global const VolatileKeyFlag instance

Definition at line 51 of file dia.hpp.

Referenced by DIA< ValueType_, Stack_ >::ReduceByKey(), and DIA< ValueType_, Stack_ >::ReduceToIndex().

struct PadTag PadTag

global const PadTag instance

Definition at line 75 of file dia.hpp.

Referenced by examples::suffix_sorting::DC3Recursive(), and examples::suffix_sorting::DC7Recursive().

struct VolatileKeyFlag< true > VolatileKeyTag

global const VolatileKeyFlag instance

Definition at line 48 of file dia.hpp.