Thrill  0.1
thrill::api Namespace Reference

Classes

class  ActionNode
 
class  ActionResultNode
 
class  AllGatherNode
 
class  AllReduceNode
 
struct  ArrayTag
 tag structure for ZipWindow() More...
 
class  BaseWindowNode
 
class  BernoulliSampleNode
 
class  CacheNode
 A DOpNode which caches all items in an external file. More...
 
class  CollapseNode
 
struct  CollapseSwitch
 Template switch to generate a CollapseNode if there is a non-empty Stack. More...
 
struct  CollapseSwitch< ValueType, tlx::FunctionStack< ValueType > >
 Template switch to NOT generate a CollapseNode if there is an empty Stack. More...
 
class  ConcatNode
 
class  ConcatToDIANode
 
class  Context
 The Context of a job is a unique instance per worker which holds references to all underlying parts of Thrill. More...
 
struct  CutTag
 tag structure for Zip() More...
 
class  DefaultReduceConfig
 
class  DefaultReduceToIndexConfig
 
class  DefaultSortAlgorithm
 
class  DefaultStableSortAlgorithm
 
class  DIA
 DIA is the interface between the user and the Thrill framework. More...
 
class  DIABase
 The DIABase is the untyped super class of DIANode. More...
 
class  DIAMemUse
 Description of the amount of RAM the internal data structures of a DIANode require. More...
 
class  DIANode
 A DIANode is a typed node representing and operation in Thrill. More...
 
struct  DisjointTag
 tag structure for Window() and FlatWindow() More...
 
class  DisjointWindowNode
 
class  DistributeNode
 
class  DOpNode
 A DOpNode is a typed node representing and distributed operations in Thrill. More...
 
struct  DuplicateDetectionFlag
 tag structure for ReduceByKey() More...
 
class  EqualToDIANode
 
class  Future
 The return type class for all ActionFutures. More...
 
class  Future< void >
 Specialized template class for ActionFuture which return void. More...
 
class  GatherNode
 
class  GenerateNode
 A DIANode which performs a Generate operation. More...
 
class  GroupByIterator
 
class  GroupByMultiwayMergeIterator
 
class  GroupByNode
 
class  GroupToIndexNode
 
class  HostContext
 The HostContext contains all data structures shared among workers on the same host. More...
 
class  HyperLogLogNode
 
class  JoinNode
 Performs an inner join between two DIAs. More...
 
struct  LocalStorageTag
 tag structure for Read() More...
 
struct  LocationDetectionFlag
 tag structure for GroupByKey(), and InnerJoin() More...
 
class  MemoryConfig
 
class  MergeNode
 Implementation of Thrill's merge. More...
 
struct  NoRebalanceTag
 tag structure for Zip() More...
 
class  OverlapWindowNode
 
struct  PadTag
 tag structure for Zip() More...
 
class  PrefixSumNode
 
class  ReadBinaryNode
 A DIANode which performs a line-based Read operation. More...
 
class  ReadLinesNode
 A DIANode which performs a line-based Read operation. More...
 
class  RebalanceNode
 
class  ReduceNode
 A DIANode which performs a Reduce operation. More...
 
class  ReduceToIndexNode
 A DIANode which performs a ReduceToIndex operation. More...
 
class  SampleNode
 A DIANode which performs sampling without replacement. More...
 
class  SizeNode
 
struct  SkipPreReducePhaseTag
 tag structure for ReduceToIndex() More...
 
class  SortNode
 A DIANode which performs a Sort operation. More...
 
class  SourceNode
 
class  UnionNode
 Implements a Union operation by hooking any number of input DIANodes and forwarding the output immediately to all children. More...
 
struct  VolatileKeyFlag
 tag structure for ReduceByKey(), and ReduceToIndex() More...
 
class  WriteBinaryNode
 
class  WriteLinesNode
 
class  WriteLinesOneNode
 
class  ZipNode
 A DIANode which performs a Zip operation. More...
 
class  ZipWindowNode
 
class  ZipWindowReader
 
class  ZipWindowReader< ZipWindowNode, false >
 template specialization Reader which delivers std::vector<>s to ZipFunction More...
 
class  ZipWindowReader< ZipWindowNode, true >
 template specialization Reader which delivers std::array<>s to ZipFunction More...
 
struct  ZipWindowTraits< ReturnType(*)(Args...)>
 specialize for function pointers More...
 
struct  ZipWindowTraits< ReturnType(ClassType::*)(Args...) const >
 specialize for pointers to const member function More...
 
struct  ZipWindowTraits< ReturnType(ClassType::*)(Args...)>
 specialize for pointers to mutable member function More...
 
class  ZipWithIndexNode
 A DIANode which calculates the array index for each items and performs a Zip-like operation without extra rebalancing of the DIA data. More...
 

Typedefs

template<typename T >
using mm_set = std::set< T, std::less< T >, mem::Allocator< T > >
 
using TestGroup = net::tcp::Group
 

Functions

template<typename FirstDIA , typename... DIAs>
auto Concat (const FirstDIA &first_dia, const DIAs &... dias)
 Concat is a DOp, which concatenates any number of DIAs to a single DIA. More...
 
template<typename ValueType >
auto Concat (const std::initializer_list< DIA< ValueType > > &dias)
 Concat is a DOp, which concatenates any number of DIAs to a single DIA. More...
 
template<typename ValueType >
auto Concat (const std::vector< DIA< ValueType > > &dias)
 Concat is a DOp, which concatenates any number of DIAs to a single DIA. More...
 
template<typename ValueType >
auto ConcatToDIA (Context &ctx, const std::vector< ValueType > &in_vector)
 ConcatToDIA is a Source-DOp, which takes a vector of data on all workers, and CONCATENATES them into a DIA. More...
 
template<typename ValueType >
auto ConcatToDIA (Context &ctx, std::vector< ValueType > &&in_vector)
 ConcatToDIA is a Source-DOp, which takes a vector of data on all workers, and CONCATENATES them into a DIA. More...
 
template<typename NetGroup >
static std::vector< std::unique_ptr< HostContext > > ConstructLoopbackHostContexts (const MemoryConfig &mem_config, size_t num_hosts, size_t workers_per_host)
 Generic network constructor for net backends supporting loopback tests. More...
 
template<>
data::CatStreamPtr Context::GetNewStream< data::CatStream > (size_t dia_id)
 
template<>
data::MixStreamPtr Context::GetNewStream< data::MixStream > (size_t dia_id)
 
static bool Deinitialize ()
 
static const char * DetectNetBackend ()
 
template<typename ValueType >
auto EqualToDIA (Context &ctx, const std::vector< ValueType > &in_vector)
 EqualToDIA is a Source-DOp, which takes a vector of data EQUAL on all workers, and returns the data in a DIA. More...
 
template<typename ValueType >
auto EqualToDIA (Context &ctx, std::vector< ValueType > &&in_vector)
 EqualToDIA is an Source-DOp, which takes a vector of data EQUAL on all workers, and returns the data in a DIA. More...
 
static void FindStages (Context &ctx, const DIABasePtr &action, mm_set< Stage > *stages)
 
static size_t FindWorkersPerHost (const char *&str_workers_per_host, const char *&env_workers_per_host)
 
template<typename GenerateFunction >
auto Generate (Context &ctx, size_t size, const GenerateFunction &generate_function)
 Generate is a Source-DOp, which creates a DIA of given size using a generator function. More...
 
static auto Generate (Context &ctx, size_t size)
 Generate is a Source-DOp, which creates a DIA of given size containing the size_t indexes [0,size). More...
 
static bool Initialize ()
 
template<bool LocationDetectionValue, typename FirstDIA , typename SecondDIA , typename KeyExtractor1 , typename KeyExtractor2 , typename JoinFunction , typename HashFunction = std::hash<typename common::FunctionTraits<KeyExtractor1>::result_type>>
auto InnerJoin (const LocationDetectionFlag< LocationDetectionValue > &, const FirstDIA &first_dia, const SecondDIA &second_dia, const KeyExtractor1 &key_extractor1, const KeyExtractor2 &key_extractor2, const JoinFunction &join_function, const HashFunction &hash_function=HashFunction())
 Performs an inner join between this DIA and the DIA given in the first parameter. More...
 
template<typename FirstDIA , typename SecondDIA , typename KeyExtractor1 , typename KeyExtractor2 , typename JoinFunction , typename HashFunction = std::hash<typename common::FunctionTraits<KeyExtractor1>::result_type>>
auto InnerJoin (const FirstDIA &first_dia, const SecondDIA &second_dia, const KeyExtractor1 &key_extractor1, const KeyExtractor2 &key_extractor2, const JoinFunction &join_function, const HashFunction &hash_function=HashFunction())
 Performs an inner join between this DIA and the DIA given in the first parameter. More...
 
template<typename Comparator , typename FirstDIA , typename... DIAs>
auto Merge (const Comparator &comparator, const FirstDIA &first_dia, const DIAs &... dias)
 Merge is a DOp, which merges any number of sorted DIAs to a single sorted DIA. More...
 
std::ostream & operator<< (std::ostream &os, const DIABase &d)
 make ostream-able. More...
 
template<typename ValueType >
DIA< ValueType > ReadBinary (Context &ctx, const std::vector< std::string > &filepath, uint64_t size_limit=ReadBinaryNode< ValueType >::no_size_limit_)
 ReadBinary is a DOp, which reads a file written by WriteBinary from the file system and creates a DIA. More...
 
template<typename ValueType >
DIA< ValueType > ReadBinary (struct LocalStorageTag, Context &ctx, const std::vector< std::string > &filepath, uint64_t size_limit=ReadBinaryNode< ValueType >::no_size_limit_)
 ReadBinary is a DOp, which reads a file written by WriteBinary from the file system and creates a DIA. More...
 
template<typename ValueType >
DIA< ValueType > ReadBinary (Context &ctx, const std::string &filepath, uint64_t size_limit=ReadBinaryNode< ValueType >::no_size_limit_)
 ReadBinary is a DOp, which reads a file written by WriteBinary from the file system and creates a DIA. More...
 
template<typename ValueType >
DIA< ValueType > ReadBinary (struct LocalStorageTag, Context &ctx, const std::string &filepath, uint64_t size_limit=ReadBinaryNode< ValueType >::no_size_limit_)
 ReadBinary is a DOp, which reads a file written by WriteBinary from the file system and creates a DIA. More...
 
DIA< std::string > ReadLines (Context &ctx, const std::string &filepath)
 ReadLines is a DOp, which reads a file from the file system and creates an ordered DIA according to a given read function. More...
 
DIA< std::string > ReadLines (struct LocalStorageTag, Context &ctx, const std::string &filepath)
 ReadLines is a DOp, which reads a file from the file system and creates an ordered DIA according to a given read function. More...
 
DIA< std::string > ReadLines (Context &ctx, const std::vector< std::string > &filepaths)
 ReadLines is a DOp, which reads a file from the file system and creates an ordered DIA according to a given read function. More...
 
DIA< std::string > ReadLines (struct LocalStorageTag, Context &ctx, const std::vector< std::string > &filepaths)
 ReadLines is a DOp, which reads a file from the file system and creates an ordered DIA according to a given read function. More...
 
int Run (const std::function< void(Context &)> &job_startpoint)
 Runs the given job startpoint with a Context instance. More...
 
template<typename NetGroup >
static int RunBackendLoopback (const char *backend, const std::function< void(Context &)> &job_startpoint)
 Run() implementation which uses a loopback net backend ("mock" or "tcp"). More...
 
template<typename NetGroup >
static void RunLoopbackThreads (const MemoryConfig &mem_config, size_t num_hosts, size_t workers_per_host, size_t core_offset, const std::function< void(Context &)> &job_startpoint)
 Generic runner for backends supporting loopback tests. More...
 
int RunNotSupported (const char *env_net)
 
void RunSetupFoxxll ()
 
static bool SetupBlockSize ()
 
static void TopoSortStages (mm_set< Stage > *stages, mem::vector< Stage > *result)
 
static void TopoSortVisit (const Stage &s, mm_set< Stage > *stages, mem::vector< Stage > *result)
 
template<typename FirstDIA , typename... DIAs>
auto Union (const FirstDIA &first_dia, const DIAs &... dias)
 Union is a LOp, which creates the union of all items from any number of DIAs as a single DIA, where the items are in an arbitrary order. More...
 
template<typename ValueType >
auto Union (const std::initializer_list< DIA< ValueType > > &dias)
 Union is a LOp, which creates the union of all items from any number of DIAs as a single DIA, where the items are in an arbitrary order. More...
 
template<typename ValueType >
auto Union (const std::vector< DIA< ValueType > > &dias)
 Union is a LOp, which creates the union of all items from any number of DIAs as a single DIA, where the items are in an arbitrary order. More...
 
template<typename ZipFunction , typename FirstDIAType , typename FirstDIAStack , typename... DIAs>
auto Zip (const ZipFunction &zip_function, const DIA< FirstDIAType, FirstDIAStack > &first_dia, const DIAs &... dias)
 Zips two DIAs of equal size in style of functional programming by applying zip_function to the i-th elements of both input DIAs to form the i-th element of the output DIA. More...
 
template<typename ZipFunction , typename FirstDIAType , typename FirstDIAStack , typename... DIAs>
auto Zip (struct CutTag, const ZipFunction &zip_function, const DIA< FirstDIAType, FirstDIAStack > &first_dia, const DIAs &... dias)
 Zips any number of DIAs of equal size in style of functional programming by applying zip_function to the i-th elements of both input DIAs to form the i-th element of the output DIA. More...
 
template<typename ZipFunction , typename FirstDIAType , typename FirstDIAStack , typename... DIAs>
auto Zip (struct PadTag, const ZipFunction &zip_function, const DIA< FirstDIAType, FirstDIAStack > &first_dia, const DIAs &... dias)
 Zips any number of DIAs in style of functional programming by applying zip_function to the i-th elements of both input DIAs to form the i-th element of the output DIA. More...
 
template<typename ZipFunction , typename FirstDIAType , typename FirstDIAStack , typename... DIAs>
auto Zip (struct PadTag, const ZipFunction &zip_function, const typename common::FunctionTraits< ZipFunction >::args_tuple_plain &padding, const DIA< FirstDIAType, FirstDIAStack > &first_dia, const DIAs &... dias)
 Zips any number of DIAs in style of functional programming by applying zip_function to the i-th elements of both input DIAs to form the i-th element of the output DIA. More...
 
template<typename ZipFunction , typename FirstDIAType , typename FirstDIAStack , typename... DIAs>
auto Zip (struct NoRebalanceTag, const ZipFunction &zip_function, const DIA< FirstDIAType, FirstDIAStack > &first_dia, const DIAs &... dias)
 Zips any number of DIAs in style of functional programming by applying zip_function to the i-th elements of both input DIAs to form the i-th element of the output DIA. More...
 
template<typename ZipFunction , typename FirstDIAType , typename FirstDIAStack , typename... DIAs>
auto ZipWindow (const std::array< size_t, 1+sizeof ...(DIAs)> &window_size, const ZipFunction &zip_function, const DIA< FirstDIAType, FirstDIAStack > &first_dia, const DIAs &... dias)
 Zips two DIAs of equal size in style of functional programming by applying zip_function to the i-th fixed-sized windows of both input DIAs to form the i-th element of the output DIA. More...
 
template<typename ZipFunction , typename FirstDIAType , typename FirstDIAStack , typename... DIAs>
auto ZipWindow (struct CutTag, const std::array< size_t, 1+sizeof ...(DIAs)> &window_size, const ZipFunction &zip_function, const DIA< FirstDIAType, FirstDIAStack > &first_dia, const DIAs &... dias)
 Zips two DIAs of equal size in style of functional programming by applying zip_function to the i-th fixed-sized windows of both input DIAs to form the i-th element of the output DIA. More...
 
template<typename ZipFunction , typename FirstDIAType , typename FirstDIAStack , typename... DIAs>
auto ZipWindow (struct PadTag, const std::array< size_t, 1+sizeof ...(DIAs)> &window_size, const ZipFunction &zip_function, const typename ZipWindowTraits< ZipFunction >::value_type_tuple_plain &padding, const DIA< FirstDIAType, FirstDIAStack > &first_dia, const DIAs &... dias)
 Zips two DIAs of equal size in style of functional programming by applying zip_function to the i-th fixed-sized windows of both input DIAs to form the i-th element of the output DIA. More...
 
template<typename ZipFunction , typename FirstDIAType , typename FirstDIAStack , typename... DIAs>
auto ZipWindow (struct PadTag, const std::array< size_t, 1+sizeof ...(DIAs)> &window_size, const ZipFunction &zip_function, const DIA< FirstDIAType, FirstDIAStack > &first_dia, const DIAs &... dias)
 Zips two DIAs of equal size in style of functional programming by applying zip_function to the i-th fixed-sized windows of both input DIAs to form the i-th element of the output DIA. More...
 
template<typename ZipFunction , typename FirstDIAType , typename FirstDIAStack , typename... DIAs>
auto ZipWindow (struct ArrayTag, struct PadTag, const std::array< size_t, 1+sizeof ...(DIAs)> &window_size, const ZipFunction &zip_function, const typename ZipWindowTraits< ZipFunction >::value_type_tuple_plain &padding, const DIA< FirstDIAType, FirstDIAStack > &first_dia, const DIAs &... dias)
 Zips two DIAs of equal size in style of functional programming by applying zip_function to the i-th fixed-sized windows of both input DIAs to form the i-th element of the output DIA. More...
 
template<typename ZipFunction , typename FirstDIAType , typename FirstDIAStack , typename... DIAs>
auto ZipWindow (struct ArrayTag, struct PadTag, const std::array< size_t, 1+sizeof ...(DIAs)> &window_size, const ZipFunction &zip_function, const DIA< FirstDIAType, FirstDIAStack > &first_dia, const DIAs &... dias)
 Zips two DIAs of equal size in style of functional programming by applying zip_function to the i-th fixed-sized windows of both input DIAs to form the i-th element of the output DIA. More...
 
Run Methods with Internal Networks for Testing
void RunLocalMock (const MemoryConfig &mem_config, size_t num_hosts, size_t workers_per_host, const std::function< void(Context &)> &job_startpoint)
 Function to run a number of mock hosts as locally independent threads, which communicate via internal stream sockets. More...
 
void RunLocalTests (const std::function< void(Context &)> &job_startpoint)
 Helper Function to execute RunLocalMock() tests using mock networks in test suite for many different numbers of workers and hosts as independent threads in one program. More...
 
void RunLocalTests (size_t ram, const std::function< void(Context &)> &job_startpoint)
 Helper Function to execute RunLocalMock() tests using mock networks in test suite for many different numbers of workers and hosts as independent threads in one program. More...
 
void RunLocalSameThread (const std::function< void(Context &)> &job_startpoint)
 Runs the given job_startpoint within the same thread with a test network –> run test with one host and one thread. More...
 
int RunCheckDieWithParent ()
 Check environment variable THRILL_DIE_WITH_PARENT and enable process flag: this is useful for ssh/invoke.sh: it kills spawned processes when the ssh connection breaks. More...
 
int RunCheckUnlinkBinary ()
 Check environment variable THRILL_UNLINK_BINARY and unlink given program path: this is useful for ssh/invoke.sh: it removes the copied program files while it is running, hence it is gone even if the program crashes. More...
 
template<typename ValueType >
auto Distribute (Context &ctx, const std::vector< ValueType > &in_vector, size_t source_id=0)
 Distribute is a Source DOp, which scatters the vector data from the source_id to all workers, partitioning equally, and returning the data in a DIA. More...
 
template<typename ValueType >
auto Distribute (Context &ctx, std::vector< ValueType > &&in_vector, size_t source_id=0)
 Distribute is a Source DOp, which scatters the vector data from the source_id to all workers, partitioning equally, and returning the data in a DIA. More...
 

Variables

const struct ArrayTag ArrayTag
 global const ArrayTag instance More...
 
const struct VolatileKeyFlag< true > VolatileKeyTag
 global const VolatileKeyFlag instance More...
 
const struct VolatileKeyFlag< false > NoVolatileKeyTag
 global const VolatileKeyFlag instance More...
 
const struct SkipPreReducePhaseTag SkipPreReducePhaseTag
 global const SkipPreReducePhaseTag instance More...
 
const struct DisjointTag DisjointTag
 global const DisjointTag instance More...
 
const struct CutTag CutTag
 global const CutTag instance More...
 
const struct PadTag PadTag
 global const PadTag instance More...
 
const struct NoRebalanceTag NoRebalanceTag
 global const NoRebalanceTag instance More...
 
const struct LocalStorageTag LocalStorageTag
 global const LocalStorageTag instance More...
 
const struct DuplicateDetectionFlag< true > DuplicateDetectionTag
 global const DuplicateDetectionFlag instance More...
 
const struct DuplicateDetectionFlag< false > NoDuplicateDetectionTag
 global const DuplicateDetectionFlag instance More...
 
const struct LocationDetectionFlag< true > LocationDetectionTag
 global const LocationDetectionFlag instance More...
 
const struct LocationDetectionFlag< false > NoLocationDetectionTag
 global const LocationDetectionFlag instance More...
 
enum  DIAState { NEW, EXECUTED, DISPOSED }
 Possible states a DIABase can be in. More...
 
using DIABasePtr = tlx::CountingPtr< DIABase >
 

Typedef Documentation

◆ mm_set

using mm_set = std::set<T, std::less<T>, mem::Allocator<T> >

Definition at line 298 of file dia_base.cpp.

◆ TestGroup

Definition at line 296 of file context.cpp.

Function Documentation

◆ ConcatToDIA()

auto thrill::api::ConcatToDIA ( Context ctx,
std::vector< ValueType > &&  in_vector 
)

ConcatToDIA is a Source-DOp, which takes a vector of data on all workers, and CONCATENATES them into a DIA.

Use Distribute to actually distribute data from a single worker, ConcatToDIA is a wrapper if the data is already distributed.

ConcatToDIA.svg
Parameters
ctxReference to the Context object
in_vectorVector to concatenate into a DIA, the contents is MOVED into the DIANode.

Definition at line 99 of file concat_to_dia.hpp.

References ConcatToDIA().

◆ ConstructLoopbackHostContexts()

static std::vector<std::unique_ptr<HostContext> > thrill::api::ConstructLoopbackHostContexts ( const MemoryConfig mem_config,
size_t  num_hosts,
size_t  workers_per_host 
)
inlinestatic

Generic network constructor for net backends supporting loopback tests.

Definition at line 95 of file context.cpp.

References Manager::kGroupCount.

◆ Context::GetNewStream< data::CatStream >()

◆ Context::GetNewStream< data::MixStream >()

◆ Deinitialize()

static bool thrill::api::Deinitialize ( )
inlinestatic

Definition at line 283 of file context.cpp.

References thrill::vfs::Deinitialize().

Referenced by RunBackendLoopback().

◆ DetectNetBackend()

static const char* thrill::api::DetectNetBackend ( )
inlinestatic

Definition at line 822 of file context.cpp.

Referenced by Run().

◆ FindStages()

static void thrill::api::FindStages ( Context ctx,
const DIABasePtr action,
mm_set< Stage > *  stages 
)
static

Do a BFS on parents to find all DIANodes (Stages) needed to Execute or PushData to calculate this action node.

Definition at line 302 of file dia_base.cpp.

References debug, LOG, Context::my_rank(), and NEW.

Referenced by DIABase::RunScope().

◆ FindWorkersPerHost()

static size_t thrill::api::FindWorkersPerHost ( const char *&  str_workers_per_host,
const char *&  env_workers_per_host 
)
inlinestatic

Definition at line 204 of file context.cpp.

Referenced by RunBackendLoopback().

◆ Initialize()

static bool thrill::api::Initialize ( )
inlinestatic

Definition at line 274 of file context.cpp.

References thrill::vfs::Initialize(), and SetupBlockSize().

Referenced by thrill::net::tcp::Construct(), and RunBackendLoopback().

◆ operator<<()

◆ Run()

int Run ( const std::function< void(Context &)> &  job_startpoint)

Runs the given job startpoint with a Context instance.

Startpoints may be called multiple times with concurrent threads and different context instances across different workers. The Thrill configuration is taken from environment variables starting the THRILL_.

THRILL_NET is the network backend to use, e.g.: mock, local, tcp, or mpi.

THRILL_RANK contains the rank of this worker

THRILL_HOSTLIST contains a space- or comma-separated list of host:ports to connect to.

THRILL_WORKERS_PER_HOST is the number of workers (threads) per host.

Additional variables:

THRILL_DIE_WITH_PARENT sets a flag which terminates the program if the caller terminates (this is automatically set by ssh/invoke.sh). No more zombies.

THRILL_UNLINK_BINARY deletes a file. Used by ssh/invoke.sh to unlink a copied program binary while it is running. Hence, it can keep /tmp clean.

Returns
0 if execution was fine on all threads.
Examples:
examples/tutorial/k-means_step1.cpp, examples/tutorial/k-means_step2.cpp, examples/tutorial/k-means_step3.cpp, examples/tutorial/k-means_step4.cpp, examples/tutorial/k-means_step5.cpp, and examples/tutorial/k-means_step6.cpp.

Definition at line 947 of file context.cpp.

References DetectNetBackend(), thrill::common::NameThisThread(), RunCheckDieWithParent(), RunCheckUnlinkBinary(), RunNotSupported(), and RunSetupFoxxll().

Referenced by main().

◆ RunBackendLoopback()

◆ RunCheckDieWithParent()

int RunCheckDieWithParent ( )

Check environment variable THRILL_DIE_WITH_PARENT and enable process flag: this is useful for ssh/invoke.sh: it kills spawned processes when the ssh connection breaks.

Hence: no more zombies.

Definition at line 849 of file context.cpp.

Referenced by Run().

◆ RunCheckUnlinkBinary()

int RunCheckUnlinkBinary ( )

Check environment variable THRILL_UNLINK_BINARY and unlink given program path: this is useful for ssh/invoke.sh: it removes the copied program files while it is running, hence it is gone even if the program crashes.

Definition at line 880 of file context.cpp.

References disk_config::autogrow, TLX_LOG1, thrill::common::to_str(), and disk_config::unlink_on_open.

Referenced by Run().

◆ RunLocalMock()

void RunLocalMock ( const MemoryConfig mem_config,
size_t  num_hosts,
size_t  workers_per_host,
const std::function< void(Context &)> &  job_startpoint 
)

Function to run a number of mock hosts as locally independent threads, which communicate via internal stream sockets.

Definition at line 299 of file context.cpp.

Referenced by RunLocalTests().

◆ RunLocalSameThread()

void RunLocalSameThread ( const std::function< void(Context &)> &  job_startpoint)

Runs the given job_startpoint within the same thread with a test network –> run test with one host and one thread.

Definition at line 376 of file context.cpp.

References Group::ConstructLoopbackMesh(), Manager::kGroupCount, HostContext::mem_config(), thrill::common::NameThisThread(), MemoryConfig::print(), MemoryConfig::setup(), thrill::mem::to_string(), and MemoryConfig::verbose_.

◆ RunLocalTests() [1/2]

void RunLocalTests ( const std::function< void(Context &)> &  job_startpoint)

Helper Function to execute RunLocalMock() tests using mock networks in test suite for many different numbers of workers and hosts as independent threads in one program.

Use this function in most test cases.

Definition at line 319 of file context.cpp.

◆ RunLocalTests() [2/2]

void RunLocalTests ( size_t  ram,
const std::function< void(Context &)> &  job_startpoint 
)

Helper Function to execute RunLocalMock() tests using mock networks in test suite for many different numbers of workers and hosts as independent threads in one program.

Use this function in most test cases.

Definition at line 324 of file context.cpp.

References MemoryConfig::enable_proc_profiler_, LOG0, HostContext::mem_config(), RunLocalMock(), tlx::setenv(), MemoryConfig::setup(), and MemoryConfig::verbose_.

◆ RunLoopbackThreads()

static void thrill::api::RunLoopbackThreads ( const MemoryConfig mem_config,
size_t  num_hosts,
size_t  workers_per_host,
size_t  core_offset,
const std::function< void(Context &)> &  job_startpoint 
)
inlinestatic

◆ RunNotSupported()

int thrill::api::RunNotSupported ( const char *  env_net)

Definition at line 815 of file context.cpp.

Referenced by Run().

◆ RunSetupFoxxll()

void thrill::api::RunSetupFoxxll ( )

Definition at line 940 of file context.cpp.

Referenced by Run().

◆ SetupBlockSize()

static bool thrill::api::SetupBlockSize ( )
inlinestatic

Definition at line 179 of file context.cpp.

References thrill::data::default_block_size, and thrill::data::start_block_size.

Referenced by Initialize().

◆ TopoSortStages()

static void thrill::api::TopoSortStages ( mm_set< Stage > *  stages,
mem::vector< Stage > *  result 
)
static

Definition at line 373 of file dia_base.cpp.

References TopoSortVisit().

Referenced by DIABase::RunScope().

◆ TopoSortVisit()

static void thrill::api::TopoSortVisit ( const Stage &  s,
mm_set< Stage > *  stages,
mem::vector< Stage > *  result 
)
static

Definition at line 350 of file dia_base.cpp.

References die_unless.

Referenced by TopoSortStages().

Variable Documentation

◆ ArrayTag

const struct ArrayTag ArrayTag

global const ArrayTag instance

Definition at line 48 of file zip_window.hpp.

Referenced by JsonLine::arr(), and ZipWindow().

◆ CutTag

const struct CutTag CutTag

global const CutTag instance

Definition at line 75 of file dia.hpp.

Referenced by DIA< BfsNode >::next_dia_id().

◆ DisjointTag

const struct DisjointTag DisjointTag

global const DisjointTag instance

Definition at line 67 of file dia.hpp.

Referenced by ConstructRLBWT(), and DIA< BfsNode >::next_dia_id().

◆ DuplicateDetectionTag

const struct DuplicateDetectionFlag< true > DuplicateDetectionTag

global const DuplicateDetectionFlag instance

Definition at line 109 of file dia.hpp.

Referenced by DIA< BfsNode >::next_dia_id().

◆ LocalStorageTag

global const LocalStorageTag instance

Definition at line 99 of file dia.hpp.

◆ LocationDetectionTag

const struct LocationDetectionFlag< true > LocationDetectionTag

global const LocationDetectionFlag instance

Definition at line 122 of file dia.hpp.

Referenced by InnerJoin(), and DIA< BfsNode >::next_dia_id().

◆ NoDuplicateDetectionTag

const struct DuplicateDetectionFlag< false > NoDuplicateDetectionTag

◆ NoLocationDetectionTag

const struct LocationDetectionFlag< false > NoLocationDetectionTag

global const LocationDetectionFlag instance

Definition at line 125 of file dia.hpp.

Referenced by DIA< BfsNode >::GroupByKey(), and DIA< BfsNode >::next_dia_id().

◆ NoRebalanceTag

global const NoRebalanceTag instance

Definition at line 91 of file dia.hpp.

Referenced by DIA< BfsNode >::next_dia_id().

◆ NoVolatileKeyTag

const struct VolatileKeyFlag< false > NoVolatileKeyTag

global const VolatileKeyFlag instance

Definition at line 51 of file dia.hpp.

Referenced by DIA< BfsNode >::next_dia_id(), DIA< BfsNode >::ReduceByKey(), and DIA< BfsNode >::ReduceToIndex().

◆ PadTag

const struct PadTag PadTag

global const PadTag instance

Definition at line 83 of file dia.hpp.

Referenced by DIA< BfsNode >::next_dia_id().

◆ SkipPreReducePhaseTag

global const SkipPreReducePhaseTag instance

Definition at line 59 of file dia.hpp.

Referenced by DIA< BfsNode >::next_dia_id().

◆ VolatileKeyTag

const struct VolatileKeyFlag< true > VolatileKeyTag

global const VolatileKeyFlag instance

Definition at line 48 of file dia.hpp.

Referenced by DIA< BfsNode >::next_dia_id().