Thrill  0.1
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
context.hpp
Go to the documentation of this file.
1 /*******************************************************************************
2  * thrill/api/context.hpp
3  *
4  * Part of Project Thrill - http://project-thrill.org
5  *
6  * Copyright (C) 2015 Alexander Noe <[email protected]>
7  * Copyright (C) 2015 Tobias Sturm <[email protected]>
8  * Copyright (C) 2015 Timo Bingmann <[email protected]>
9  *
10  * All rights reserved. Published under the BSD-2 license in the LICENSE file.
11  ******************************************************************************/
12 
13 #pragma once
14 #ifndef THRILL_API_CONTEXT_HEADER
15 #define THRILL_API_CONTEXT_HEADER
16 
17 #include <thrill/common/config.hpp>
23 #include <thrill/data/file.hpp>
28 #include <thrill/net/manager.hpp>
29 
30 #include <algorithm>
31 #include <cassert>
32 #include <functional>
33 #include <numeric>
34 #include <string>
35 #include <tuple>
36 #include <vector>
37 
38 namespace thrill {
39 namespace api {
40 
41 //! \ingroup api_layer
42 //! \{
43 
44 // forward declarations
45 class DIABase;
46 
48 {
49 public:
50  //! detect memory configuration from environment
51  int setup_detect();
52 
53  //! setup memory size
54  void setup(size_t ram);
55 
56  MemoryConfig divide(size_t hosts) const;
57  void apply();
58 
59  void print(size_t workers_per_host) const;
60 
61  //! total amount of physical ram detected or THRILL_RAM
62  size_t ram_;
63 
64  //! amount of RAM dedicated to data::BlockPool -- hard limit
66 
67  //! amount of RAM dedicated to data::BlockPool -- soft limit
69 
70  //! total amount of RAM for DIANode data structures such as the reduce
71  //! tables. divide by the number of worker threads before use.
72  size_t ram_workers_;
73 
74  //! remaining free-floating RAM used for user and Thrill data structures.
75  size_t ram_floating_;
76 
77  //! StageBuilder verbosity flag
78  bool verbose_ = true;
79 };
80 
81 /*!
82  * The HostContext contains all data structures shared among workers on the same
83  * host. It is used to construct and destroy them. For testing multiple
84  * instances are run in the same process.
85  */
87 {
88 public:
89 #ifndef SWIG
90  //! constructor from existing net Groups. Used by the construction methods.
92  std::array<net::GroupPtr, net::Manager::kGroupCount>&& groups,
93  size_t workers_per_host);
94 
95  //! Construct a number of mock hosts running in this process.
96  static std::vector<std::unique_ptr<HostContext> >
97  ConstructLoopback(size_t num_hosts, size_t workers_per_host);
98 #endif
99 
100  //! create host log
102 
103  //! Returns local_host_id_
104  size_t local_host_id() const { return local_host_id_; }
105 
106  //! number of workers per host (all have the same).
107  size_t workers_per_host() const { return workers_per_host_; }
108 
109  //! memory limit of each worker Context for local data structures
110  size_t worker_mem_limit() const {
112  }
113 
114  //! host-global memory config
116 
117  //! host-global memory manager
119 
120  //! net manager constructs communication groups to other hosts.
122 
123  //! Returns id of this host in the cluser. A host is a machine in the
124  //! cluster that hosts multiple workers
125  size_t host_rank() const { return net_manager_.my_host_rank(); }
126 
127  //! the flow control group is used for collective communication.
129 
130  //! the block manager keeps all data blocks moving through the system.
132 
133  //! data multiplexer transmits large amounts of data asynchronously.
135 
136 private:
137  //! memory configuration
139 
140 public:
141  //! \name Logging System
142  //! \{
143 
144  //! base logger exclusive for this host context
146 
147  //! public member which delivers key:value pairs as JSON log lines. this
148  //! logger is local to this Context which is exclusive for one worker
149  //! thread.
151 
152  //! thread for scheduling profiling methods for statistical output
153  std::unique_ptr<common::ProfileThread> profiler_;
154 
155  //! \}
156 
157 private:
158  //! id among all _local_ hosts (in test program runs)
160 
161  //! number of workers per host (all have the same).
163 
164  //! host-global memory manager for internal memory only
165  mem::Manager mem_manager_ { nullptr, "HostContext" };
166 
167  //! net manager constructs communication groups to other hosts.
169 
170 #if !THRILL_HAVE_THREAD_SANITIZER
171  //! register net_manager_'s profiling method
173  std::chrono::milliseconds(500), *profiler_, &net_manager_
174  };
175 #endif
176 
177  //! the flow control group is used for collective communication.
180  };
181 
182  //! data block pool
186  };
187 
188 #if !THRILL_HAVE_THREAD_SANITIZER
189  //! register BlockPool's profiling method
191  std::chrono::milliseconds(500), *profiler_, &block_pool_
192  };
193 #endif
194 
195  //! data multiplexer transmits large amounts of data asynchronously.
199  };
200 };
201 
202 /*!
203  * The Context of a job is a unique instance per worker which holds references
204  * to all underlying parts of Thrill. The context is able to give references to
205  * the \ref data::Multiplexer "stream multiplexer", the \ref net::Group "net
206  * group" \ref common::Stats "stats" and \ref common::StatsGraph "stats graph".
207  * Threads share the stream multiplexer and the net group via the context
208  * object.
209  */
210 class Context
211 {
212 public:
213  Context(HostContext& host_context, size_t local_worker_id)
214  : local_host_id_(host_context.local_host_id()),
215  local_worker_id_(local_worker_id),
216  workers_per_host_(host_context.workers_per_host()),
217  mem_limit_(host_context.worker_mem_limit()),
218  mem_config_(host_context.mem_config()),
219  mem_manager_(host_context.mem_manager()),
220  net_manager_(host_context.net_manager()),
221  flow_manager_(host_context.flow_manager()),
222  block_pool_(host_context.block_pool()),
223  multiplexer_(host_context.data_multiplexer()),
224  base_logger_(&host_context.base_logger_) {
225  assert(local_worker_id < workers_per_host());
226  }
227 
228  //! method used to launch a job's main procedure. it wraps it in log output.
229  void Launch(const std::function<void(Context&)>& job_startpoint);
230 
231  //! \name System Information
232  //! \{
233 
234  //! Returns the total number of hosts.
235  size_t num_hosts() const {
236  return net_manager_.num_hosts();
237  }
238 
239  //! Returns the number of workers that is hosted on each host
240  size_t workers_per_host() const {
241  return workers_per_host_;
242  }
243 
244  //! Global rank of this worker among all other workers in the system.
245  size_t my_rank() const {
246  return workers_per_host() * host_rank() + local_worker_id();
247  }
248 
249  //! memory limit of this worker Context for local data structures
250  size_t mem_limit() const { return mem_limit_; }
251 
252  //! Global number of workers in the system.
253  size_t num_workers() const {
254  return num_hosts() * workers_per_host();
255  }
256 
257  //! Returns id of this host in the cluser
258  //! A host is a machine in the cluster that hosts multiple workers
259  size_t host_rank() const {
260  return net_manager_.my_host_rank();
261  }
262 
263  //! Returns the local id ot this worker on the host
264  //! A worker is _locally_ identified by this id
265  size_t local_worker_id() const {
266  return local_worker_id_;
267  }
268 
269  //! id among all _local_ hosts (in test program runs)
270  size_t local_host_id() const { return local_host_id_; }
271 
272 #ifndef SWIG
273  //! Outputs the context as [host id]:[local worker id] to an std::ostream
274  friend std::ostream& operator << (std::ostream& os, const Context& ctx) {
275  return os << ctx.host_rank() << ":" << ctx.local_worker_id();
276  }
277 #endif
278 
279  //! \}
280 
281  //! \name Data Subsystem
282  //! \{
283 
284  //! Returns a new File object containing a sequence of local Blocks.
285  data::File GetFile(size_t dia_id) {
286  return data::File(block_pool_, local_worker_id_, dia_id);
287  }
288 
289  //! Returns a new File object containing a sequence of local Blocks.
290  data::File GetFile(DIABase* dia);
291 
292  //! Returns a new File, wrapped in a CountingPtr, containing a sequence of
293  //! local Blocks.
294  data::FilePtr GetFilePtr(size_t dia_id);
295 
296  //! Returns a new File, wrapped in a CountingPtr, containing a sequence of
297  //! local Blocks.
299 
300  //! Returns a reference to a new CatStream. This method alters the state of
301  //! the context and must be called on all Workers to ensure correct
302  //! communication coordination.
303  data::CatStreamPtr GetNewCatStream(size_t dia_id);
304 
305  //! Returns a reference to a new CatStream. This method alters the state of
306  //! the context and must be called on all Workers to ensure correct
307  //! communication coordination.
309 
310  //! Returns a reference to a new MixStream. This method alters the state
311  //! of the context and must be called on all Workers to ensure correct
312  //! communication coordination.
313  data::MixStreamPtr GetNewMixStream(size_t dia_id);
314 
315  //! Returns a reference to a new MixStream. This method alters the state
316  //! of the context and must be called on all Workers to ensure correct
317  //! communication coordination.
319 
320  //! Returns a reference to a new CatStream or MixStream, selectable via
321  //! template parameter.
322  template <typename Stream>
323  tlx::CountingPtr<Stream> GetNewStream(size_t dia_id);
324 
325  //! the block manager keeps all data blocks moving through the system.
327 
328  //! \}
329 
330  //! host-global memory config
331  const MemoryConfig& mem_config() const { return mem_config_; }
332 
333  //! returns the host-global memory manager
335 
337 
338  //! given a global range [0,global_size) and p PEs to split the range, calculate
339  //! the [local_begin,local_end) index range assigned to the PE i. Takes the
340  //! information from the Context.
341  common::Range CalculateLocalRange(size_t global_size) const {
343  global_size, num_workers(), my_rank());
344  }
345 
346  common::Range CalculateLocalRangeOnHost(size_t global_size) const {
348  global_size, workers_per_host(), local_worker_id());
349  }
350 
351  //! Perform collectives and print min, max, mean, stdev, and all local
352  //! values.
353  template <typename Type>
354  void PrintCollectiveMeanStdev(const char* text, const Type& local) {
355  std::vector<Type> svec = { local };
356  svec = net.Reduce(svec, 0, common::VectorConcat<Type>());
357  if (my_rank() == 0) {
358  double sum = std::accumulate(svec.begin(), svec.end(), 0.0);
359  double mean = sum / svec.size();
360 
361  double sq_sum = std::inner_product(
362  svec.begin(), svec.end(), svec.begin(), 0.0);
363  double stdev = std::sqrt(sq_sum / svec.size() - mean * mean);
364 
365  double min = *std::min_element(svec.begin(), svec.end());
366  double max = *std::max_element(svec.begin(), svec.end());
367 
368  LOG1 << text << " mean " << mean
369  << " max " << max << " stdev " << stdev
370  << " = " << (stdev / mean * 100.0) << "%"
371  << " max-min " << max - min
372  << " = " << ((max - min) / min * 100.0) << "%"
373  << " max-mean " << max - mean
374  << " = " << ((max - mean) / mean * 100.0) << "%"
375  << " svec " << common::VecToStr(svec);
376  }
377  }
378 
379  //! return value of consume flag.
380  bool consume() const { return consume_; }
381 
382  /*!
383  * Sets consume-mode flag such that DIA contents may be consumed during
384  * PushData(). When in consume mode the DIA contents is destroyed online
385  * when it is transmitted to the next operation. This enables reusing the
386  * space of the consume operations. This enabled processing more data with
387  * less space. However, by default this mode is DISABLED, because it
388  * requires deliberate insertion of .Keep() calls.
389  */
390  void enable_consume(bool consume = true) { consume_ = consume; }
391 
392  //! Returns next_dia_id_ to generate DIA::id_ serial.
393  size_t next_dia_id() { return ++last_dia_id_; }
394 
395 private:
396  //! id among all _local_ hosts (in test program runs)
398 
399  //! number of this host context, 0..p-1, within this host
401 
402  //! number of workers hosted per host
404 
405  //! memory limit of this worker Context for local data structures
406  size_t mem_limit_;
407 
408  //! memory configuration in HostContext
410 
411  //! host-global memory manager
413 
414  //! net::Manager instance that is shared among workers
416 
417  //! net::FlowControlChannelManager instance that is shared among workers
419 
420  //! data block pool
422 
423  //! data::Multiplexer instance that is shared among workers
425 
426  //! flag to set which enables selective consumption of DIA contents!
427  bool consume_ = false;
428 
429  //! the number of valid DIA ids. 0 is reserved for invalid.
430  size_t last_dia_id_ = 0;
431 
432 public:
433  //! \name Network Subsystem
434  //! \{
435 
436  //! public member which exposes all network primitives from
437  //! FlowControlChannel for DOp implementations. Use it as
438  //! `context_.net.Method()`.
441  };
442 
443  //! \}
444 
445 public:
446  //! \name Logging System
447  //! \{
448 
449  //! base logger exclusive for this worker
451 
452  //! public member which delivers key:value pairs as JSON log lines. this
453  //! logger is local to this Context which is exclusive for one worker
454  //! thread.
456  &base_logger_, "host_rank", host_rank(), "worker_rank", my_rank()
457  };
458 
459  //! \}
460 };
461 
462 //! \name Run Methods with Internal Networks for Testing
463 //! \{
464 
465 /*!
466  * Function to run a number of mock hosts as locally independent threads, which
467  * communicate via internal stream sockets.
468  */
469 void RunLocalMock(const MemoryConfig& mem_config,
470  size_t num_hosts, size_t workers_per_host,
471  const std::function<void(Context&)>& job_startpoint);
472 
473 /*!
474  * Helper Function to execute RunLocalMock() tests using mock networks in test
475  * suite for many different numbers of workers and hosts as independent threads
476  * in one program. Use this function in most test cases.
477  */
478 void RunLocalTests(const std::function<void(Context&)>& job_startpoint);
479 
480 /*!
481  * Helper Function to execute RunLocalMock() tests using mock networks in test
482  * suite for many different numbers of workers and hosts as independent threads
483  * in one program. Use this function in most test cases.
484  */
485 void RunLocalTests(
486  size_t ram, const std::function<void(Context&)>& job_startpoint);
487 
488 /*!
489  * Runs the given job_startpoint within the same thread with a test network -->
490  * run test with one host and one thread.
491  */
492 void RunLocalSameThread(const std::function<void(Context&)>& job_startpoint);
493 
494 /*!
495  * Check environment variable THRILL_DIE_WITH_PARENT and enable process flag:
496  * this is useful for ssh/invoke.sh: it kills spawned processes when the ssh
497  * connection breaks. Hence: no more zombies.
498  */
500 
501 /*!
502  * Check environment variable THRILL_UNLINK_BINARY and unlink given program
503  * path: this is useful for ssh/invoke.sh: it removes the copied program files
504  * _while_ it is running, hence it is gone even if the program crashes.
505  */
507 
508 //! \}
509 
510 /*!
511  * Runs the given job startpoint with a Context instance. Startpoints may be
512  * called multiple times with concurrent threads and different context instances
513  * across different workers. The Thrill configuration is taken from environment
514  * variables starting the THRILL_.
515  *
516  * THRILL_NET is the network backend to use, e.g.: mock, local, tcp, or mpi.
517  *
518  * THRILL_RANK contains the rank of this worker
519  *
520  * THRILL_HOSTLIST contains a space- or comma-separated list of host:ports to
521  * connect to.
522  *
523  * THRILL_WORKERS_PER_HOST is the number of workers (threads) per host.
524  *
525  * Additional variables:
526  *
527  * THRILL_DIE_WITH_PARENT sets a flag which terminates the program if the caller
528  * terminates (this is automatically set by ssh/invoke.sh). No more zombies.
529  *
530  * THRILL_UNLINK_BINARY deletes a file. Used by ssh/invoke.sh to unlink a copied
531  * program binary while it is running. Hence, it can keep /tmp clean.
532  *
533  * \returns 0 if execution was fine on all threads.
534  */
535 int Run(const std::function<void(Context&)>& job_startpoint);
536 
537 //! \}
538 
539 } // namespace api
540 
541 //! imported from api namespace
542 using api::HostContext;
543 
544 //! imported from api namespace
545 using api::Context;
546 
547 //! imported from api namespace
548 using api::Run;
549 
550 } // namespace thrill
551 
552 #endif // !THRILL_API_CONTEXT_HEADER
553 
554 /******************************************************************************/
mem::Manager & mem_manager()
returns the host-global memory manager
Definition: context.hpp:334
HostContext(size_t local_host_id, const MemoryConfig &mem_config, std::array< net::GroupPtr, net::Manager::kGroupCount > &&groups, size_t workers_per_host)
constructor from existing net Groups. Used by the construction methods.
Definition: context.cpp:973
size_t mem_limit_
memory limit of this worker Context for local data structures
Definition: context.hpp:406
size_t ram_
total amount of physical ram detected or THRILL_RAM
Definition: context.hpp:62
size_t local_host_id_
id among all local hosts (in test program runs)
Definition: context.hpp:397
net::FlowControlChannel & net
Definition: context.hpp:439
size_t ram_floating_
remaining free-floating RAM used for user and Thrill data structures.
Definition: context.hpp:75
MemoryConfig divide(size_t hosts) const
Definition: context.cpp:945
size_t num_hosts() const
Returns the total number of hosts.
Definition: context.hpp:235
size_t worker_mem_limit() const
memory limit of each worker Context for local data structures
Definition: context.hpp:110
Group & GetDataGroup()
Returns the net::Group for the data manager.
Definition: manager.hpp:77
size_t host_rank() const
Definition: context.hpp:259
void PrintCollectiveMeanStdev(const char *text, const Type &local)
Definition: context.hpp:354
size_t my_rank() const
Global rank of this worker among all other workers in the system.
Definition: context.hpp:245
bool consume() const
return value of consume flag.
Definition: context.hpp:380
net::Manager & net_manager_
net::Manager instance that is shared among workers
Definition: context.hpp:415
#define LOG1
Definition: logger.hpp:171
std::string MakeHostLogPath(size_t host_rank)
create host log
Definition: context.cpp:998
A File is an ordered sequence of Block objects for storing items.
Definition: file.hpp:55
int Run(const std::function< void(Context &)> &job_startpoint)
Runs the given job startpoint with a Context instance.
Definition: context.cpp:801
size_t workers_per_host_
number of workers hosted per host
Definition: context.hpp:403
Type
VFS object type.
Definition: file_io.hpp:52
net::Manager & net_manager()
net manager constructs communication groups to other hosts.
Definition: context.hpp:121
data::BlockPool & block_pool()
the block manager keeps all data blocks moving through the system.
Definition: context.hpp:131
common::JsonLogger logger_
Definition: context.hpp:150
represents a 1 dimensional range (interval) [begin,end)
Definition: math.hpp:40
const MemoryConfig & mem_config() const
host-global memory config
Definition: context.hpp:331
data::Multiplexer & data_multiplexer()
data multiplexer transmits large amounts of data asynchronously.
Definition: context.hpp:134
size_t num_hosts() const
Definition: manager.hpp:54
int RunCheckDieWithParent()
Check environment variable THRILL_DIE_WITH_PARENT and enable process flag: this is useful for ssh/inv...
Definition: context.cpp:756
friend std::ostream & operator<<(std::ostream &os, const Context &ctx)
Outputs the context as [host id]:[local worker id] to an std::ostream.
Definition: context.hpp:274
Compute the concatenation of two std::vector<T>s.
Definition: functional.hpp:159
int setup_detect()
detect memory configuration from environment
Definition: context.cpp:873
The Context of a job is a unique instance per worker which holds references to all underlying parts o...
Definition: context.hpp:210
size_t workers_per_host() const
number of workers per host (all have the same).
Definition: context.hpp:107
mem::Manager & mem_manager_
host-global memory manager
Definition: context.hpp:412
Multiplexes virtual Connections on Dispatcher.
Definition: multiplexer.hpp:64
void enable_consume(bool consume=true)
Sets consume-mode flag such that DIA contents may be consumed during PushData().
Definition: context.hpp:390
FlowControlChannel & GetFlowControlChannel(size_t thread_id)
Gets the flow control channel for a certain thread.
size_t host_rank() const
Definition: context.hpp:125
The DIABase is the untyped super class of DIANode.
Definition: dia_base.hpp:87
int RunCheckUnlinkBinary()
Check environment variable THRILL_UNLINK_BINARY and unlink given program path: this is useful for ssh...
Definition: context.cpp:787
data::BlockPool & block_pool_
data block pool
Definition: context.hpp:421
size_t workers_per_host() const
Returns the number of workers that is hosted on each host.
Definition: context.hpp:240
mem::Manager & mem_manager()
host-global memory manager
Definition: context.hpp:118
data::CatStreamPtr GetNewCatStream(size_t dia_id)
Definition: context.cpp:1036
const MemoryConfig & mem_config_
memory configuration in HostContext
Definition: context.hpp:409
Group & GetFlowGroup()
Returns the net::Group for the flow control channel.
Definition: manager.hpp:72
static Range CalculateLocalRange(size_t global_size, size_t p, size_t i)
Definition: math.hpp:81
data::Multiplexer data_multiplexer_
data multiplexer transmits large amounts of data asynchronously.
Definition: context.hpp:196
common::JsonLogger base_logger_
base logger exclusive for this host context
Definition: context.hpp:145
void print(size_t workers_per_host) const
Definition: context.cpp:957
Pool to allocate, keep, swap out/in, and free all ByteBlocks on the host.
Definition: block_pool.hpp:40
void RunLocalSameThread(const std::function< void(Context &)> &job_startpoint)
Runs the given job_startpoint within the same thread with a test network –> run test with one host an...
Definition: context.cpp:276
size_t next_dia_id()
Returns next_dia_id_ to generate DIA::id_ serial.
Definition: context.hpp:393
size_t local_worker_id_
number of this host context, 0..p-1, within this host
Definition: context.hpp:400
size_t local_host_id() const
id among all local hosts (in test program runs)
Definition: context.hpp:270
static std::string VecToStr(const std::array< T, N > &data)
Logging helper to print arrays as [a1,a2,a3,...].
Definition: string.hpp:179
Provides a blocking collection for communication.
size_t ram_block_pool_hard_
amount of RAM dedicated to data::BlockPool – hard limit
Definition: context.hpp:65
void RunLocalMock(const MemoryConfig &mem_config, size_t num_hosts, size_t workers_per_host, const std::function< void(Context &)> &job_startpoint)
Function to run a number of mock hosts as locally independent threads, which communicate via internal...
Definition: context.cpp:213
common::JsonLogger logger_
Definition: context.hpp:455
T TLX_ATTRIBUTE_WARN_UNUSED_RESULT Reduce(const T &value, size_t root=0, const BinarySumOp &sum_op=BinarySumOp())
Reduces a value of a serializable type T over all workers to the given worker, provided a certain red...
common::Range CalculateLocalRange(size_t global_size) const
Definition: context.hpp:341
std::basic_string< char, std::char_traits< char >, Allocator< char > > string
string with Manager tracking
Definition: allocator.hpp:220
data::FilePtr GetFilePtr(size_t dia_id)
Definition: context.cpp:1027
void RunLocalTests(const std::function< void(Context &)> &job_startpoint)
Helper Function to execute RunLocalMock() tests using mock networks in test suite for many different ...
Definition: context.cpp:249
High-performance smart pointer used as a wrapping reference counting pointer.
common::Range CalculateLocalRangeOnHost(size_t global_size) const
Definition: context.hpp:346
Context(HostContext &host_context, size_t local_worker_id)
Definition: context.hpp:213
data::File GetFile(size_t dia_id)
Returns a new File object containing a sequence of local Blocks.
Definition: context.hpp:285
data::MixStreamPtr GetNewMixStream(size_t dia_id)
Definition: context.cpp:1044
common::ProfileTaskRegistration net_manager_profiler_
register net_manager_'s profiling method
Definition: context.hpp:172
net::FlowControlChannelManager & flow_manager()
the flow control group is used for collective communication.
Definition: context.hpp:128
bool consume_
flag to set which enables selective consumption of DIA contents!
Definition: context.hpp:427
void Launch(const std::function< void(Context &)> &job_startpoint)
method used to launch a job's main procedure. it wraps it in log output.
Definition: context.cpp:1104
data::Multiplexer & multiplexer_
data::Multiplexer instance that is shared among workers
Definition: context.hpp:424
size_t ram_block_pool_soft_
amount of RAM dedicated to data::BlockPool – soft limit
Definition: context.hpp:68
Object shared by allocators and other classes to track memory allocations.
Definition: manager.hpp:28
net::FlowControlChannelManager flow_manager_
the flow control group is used for collective communication.
Definition: context.hpp:178
mem::Manager mem_manager_
host-global memory manager for internal memory only
Definition: context.hpp:165
The HostContext contains all data structures shared among workers on the same host.
Definition: context.hpp:86
size_t local_worker_id() const
Definition: context.hpp:265
net::Manager & net_manager()
Definition: context.hpp:336
void setup(size_t ram)
setup memory size
Definition: context.cpp:868
common::ProfileTaskRegistration block_pool_profiler_
register BlockPool's profiling method
Definition: context.hpp:190
size_t num_workers() const
Global number of workers in the system.
Definition: context.hpp:253
JsonLogger is a receiver of JSON output objects for logging.
Definition: json_logger.hpp:69
static std::vector< std::unique_ptr< HostContext > > ConstructLoopback(size_t num_hosts, size_t workers_per_host)
Construct a number of mock hosts running in this process.
Definition: context.cpp:222
MemoryConfig & mem_config()
host-global memory config
Definition: context.hpp:115
net::FlowControlChannelManager & flow_manager_
net::FlowControlChannelManager instance that is shared among workers
Definition: context.hpp:418
MemoryConfig mem_config_
memory configuration
Definition: context.hpp:138
data::BlockPool block_pool_
data block pool
Definition: context.hpp:183
static constexpr const T & min(const T &a, const T &b)
template for constexpr min, because std::min is not good enough.
Definition: functional.hpp:59
size_t last_dia_id_
the number of valid DIA ids. 0 is reserved for invalid.
Definition: context.hpp:430
std::unique_ptr< common::ProfileThread > profiler_
thread for scheduling profiling methods for statistical output
Definition: context.hpp:153
size_t local_host_id() const
Returns local_host_id_.
Definition: context.hpp:104
Initializes communication channels, manages communication channels and handles errors.
Definition: manager.hpp:40
common::JsonLogger base_logger_
base logger exclusive for this worker
Definition: context.hpp:450
tlx::CountingPtr< Stream > GetNewStream(size_t dia_id)
net::Manager net_manager_
net manager constructs communication groups to other hosts.
Definition: context.hpp:168
size_t my_host_rank() const
Definition: manager.hpp:50
size_t local_host_id_
id among all local hosts (in test program runs)
Definition: context.hpp:159
size_t workers_per_host_
number of workers per host (all have the same).
Definition: context.hpp:162
size_t mem_limit() const
memory limit of this worker Context for local data structures
Definition: context.hpp:250
bool verbose_
StageBuilder verbosity flag.
Definition: context.hpp:78
data::BlockPool & block_pool()
the block manager keeps all data blocks moving through the system.
Definition: context.hpp:326
static constexpr const T & max(const T &a, const T &b)
template for constexpr max, because std::max is not good enough.
Definition: functional.hpp:65