Detailed Description

template<typename ValueType>
class thrill::api::ConcatNode< ValueType >

Definition at line 35 of file concat.hpp.

Inheritance diagram for ConcatNode< ValueType >:

Collaboration diagram for ConcatNode< ValueType >:

#include <concat.hpp>

Classes
class	RegisterParent

Public Types
using	Super = DOpNode< ValueType >

Public Types inherited from DOpNode< ValueType >
using	Super = DIANode< ValueType >

Public Types inherited from DIANode< ValueType >
using	Callback = tlx::delegate< void(const ValueType &)>

Public Types inherited from DIABase
using	DIABasePtr = tlx::CountingPtr< DIABase >

Public Member Functions
template<typename ParentDIA0 , typename... ParentDIAs>
	ConcatNode (const ParentDIA0 &parent0, const ParentDIAs &... parents)

template<typename ParentDIA >
	ConcatNode (const std::vector< ParentDIA > &parents)

template<typename ParentDIA >
	ConcatNode (const std::initializer_list< ParentDIA > &parents)

void	Dispose () final
	Virtual clear method. Triggers actual disposing in sub-classes. More...

void	Execute () final
	Executes the concat operation. More...

bool	OnPreOpFile (const data::File &file, size_t parent_index) final
	Receive a whole data::File of ValueType, but only if our stack is empty. More...

void	PrintWarning ()

void	PushData (bool consume) final
	Virtual method for pushing data. Triggers actual pushing in sub-classes. More...

void	StopPreOp (size_t parent_index) final
	Virtual method for preparing end of PushData. More...

Public Member Functions inherited from DOpNode< ValueType >
	DOpNode (Context &ctx, const char *label, const std::initializer_list< size_t > &parent_ids, const std::initializer_list< DIABasePtr > &parents)
	Constructor for a DOpNode, which sets references to the parent nodes. More...

	DOpNode (Context &ctx, const char *label, std::vector< size_t > &&parent_ids, std::vector< DIABasePtr > &&parents)
	Constructor for a DOpNode, which sets references to the parent nodes. More...

Public Member Functions inherited from DIANode< ValueType >
	DIANode (Context &ctx, const char *label, const std::initializer_list< size_t > &parent_ids, const std::initializer_list< DIABasePtr > &parents)
	Constructor for a DIANode, which sets references to the parent nodes. More...

	DIANode (Context &ctx, const char *label, std::vector< size_t > &&parent_ids, std::vector< DIABasePtr > &&parents)
	Constructor for a DIANode, which sets references to the parent nodes. More...

virtual void	AddChild (DIABase *node, const Callback &callback=Callback(), size_t parent_index=0)
	Enables children to push their "folded" function chains to their parent. More...

std::vector< DIABase * >	children () const override
	Returns the children of this DIABase. More...

void	PushFile (data::File &file, bool consume) const

void	PushItem (const ValueType &item) const
	Method for derived classes to Push a single item to all children. More...

void	RemoveAllChildren () override

void	RemoveChild (DIABase *node) override

void	RunPushData () override

Public Member Functions inherited from DIABase
	DIABase (Context &ctx, const char *label, const std::initializer_list< size_t > &parent_ids, const std::initializer_list< DIABasePtr > &parents)
	The constructor for a DIABase. More...

	DIABase (Context &ctx, const char *label, std::vector< size_t > &&parent_ids, std::vector< DIABasePtr > &&parents)
	The constructor for a DIABase. More...

	DIABase (const DIABase &)=delete
	non-copyable: delete copy-constructor More...

	DIABase (DIABase &&)=default
	move-constructor: default More...

virtual	~DIABase ()
	Virtual destructor for a DIABase. More...

virtual size_t	consume_counter () const
	Returns consume_counter_. More...

Context &	context ()
	Returns the api::Context of this DIABase. More...

virtual void	DecConsumeCounter (size_t counter)

const size_t &	dia_id () const
	return unique id of DIANode subclass as stored by StatsNode More...

virtual bool	ForwardDataOnly () const

virtual void	IncConsumeCounter (size_t counter)

const char *	label () const
	return label() of DIANode subclass as stored by StatsNode More...

mem::Manager &	mem_manager ()
	Return the Context's memory manager. More...

DIABase &	operator= (const DIABase &)=delete
	non-copyable: delete assignment operator More...

DIABase &	operator= (DIABase &&)=default
	move-assignment operator: default More...

std::vector< size_t >	parent_ids () const
	Returns the parents of this DIABase. More...

const std::vector< DIABasePtr > &	parents () const
	Returns the parents of this DIABase. More...

void	RemoveParent (DIABase *p)
	Remove a parent. More...

virtual bool	RequireParentPushData (size_t) const

void	RunScope ()

void	set_mem_limit (const DIAMemUse &mem_limit)

void	set_state (const DIAState &state)

virtual void	SetConsumeCounter (size_t counter)

DIAState	state () const

virtual DIAMemUse	PreOpMemUse ()
	Amount of RAM used by PreOp after StartPreOp() More...

virtual void	StartPreOp (size_t)
	Virtual method for preparing start of PushData. More...

virtual DIAMemUse	ExecuteMemUse ()
	Amount of RAM used by Execute() More...

virtual DIAMemUse	PushDataMemUse ()
	Amount of RAM used by PushData() More...

Public Member Functions inherited from ReferenceCounter
	ReferenceCounter () noexcept
	new objects have zero reference count More...

	ReferenceCounter (const ReferenceCounter &) noexcept
	coping still creates a new object with zero reference count More...

	~ReferenceCounter ()

bool	dec_reference () const noexcept
	Call whenever resetting (i.e. More...

void	inc_reference () const noexcept
	Call whenever setting a pointer to the object. More...

ReferenceCounter &	operator= (const ReferenceCounter &) noexcept
	assignment operator, leaves pointers unchanged More...

size_t	reference_count () const noexcept
	Return the number of references to this object (for debugging) More...

bool	unique () const noexcept
	Test if the ReferenceCounter is referenced by only one CountingPtr. More...

Static Public Attributes
static constexpr bool	debug = false

Static Public Attributes inherited from DIABase
static constexpr size_t	kNeverConsume = static_cast<size_t>(-1)
	Never full consume. More...

Private Attributes
std::vector< data::File >	files_
	Files for intermediate storage. More...

const size_t	num_inputs_
	number of input DIAs More...

const std::vector< bool >	parent_stack_empty_
	Whether the parent stack is empty. More...

std::vector< data::CatStreamPtr >	streams_
	Array of CatStreams for exchange. More...

std::vector< data::File::Writer >	writers_
	Writers to intermediate files. More...

Additional Inherited Members
Public Attributes inherited from DIABase
common::JsonLogger	logger_

Protected Attributes inherited from DIANode< ValueType >
std::vector< Child >	children_
	Callback functions from the child nodes. More...

Protected Attributes inherited from DIABase
Context &	context_
	associated Context More...

const size_t	dia_id_
	DIA serial id. More...

const char *const	label_
	DOp node static label. More...

DIAState	state_ = DIAState::NEW
	State of the DIANode. State is NEW on creation. More...

std::vector< DIABasePtr >	parents_
	Parents of this DIABase. More...

DIAMemUse	mem_limit_ = 0

size_t	consume_counter_ = 1

Member Typedef Documentation

◆ Super

using Super = DOpNode<ValueType>

Definition at line 40 of file concat.hpp.

Constructor & Destructor Documentation

◆ ConcatNode() [1/3]

ConcatNode	(	const ParentDIA0 &	parent0,
		const ParentDIAs &...	parents
	)

inlineexplicit

Constructor for variant with variadic parent parameter pack, which each parent may have a different FunctionStack.

Definition at line 46 of file concat.hpp.

References tlx::call_foreach_with_index(), DIABase::context_, ConcatNode< ValueType >::files_, Context::GetFile(), ConcatNode< ValueType >::num_inputs_, ConcatNode< ValueType >::parent_stack_empty_, DIABase::parents(), ConcatNode< ValueType >::PrintWarning(), and ConcatNode< ValueType >::writers_.

◆ ConcatNode() [2/3]

ConcatNode ( const std::vector< ParentDIA > & parents )

inlineexplicit

Constructor for variant with a std::vector of parents all with the same (usually empty) FunctionStack.

Definition at line 73 of file concat.hpp.

References DIABase::context_, ConcatNode< ValueType >::files_, Context::GetFile(), thrill::common::MapVector(), ConcatNode< ValueType >::num_inputs_, ConcatNode< ValueType >::parent_stack_empty_, DIABase::parents(), ConcatNode< ValueType >::PrintWarning(), BlockWriter< BlockSink >::Put(), and ConcatNode< ValueType >::writers_.

◆ ConcatNode() [3/3]

ConcatNode ( const std::initializer_list< ParentDIA > & parents )

inlineexplicit

Constructor for variant with a std::vector of parents all with the same (usually empty) FunctionStack.

Definition at line 114 of file concat.hpp.

Member Function Documentation

◆ Dispose()

void Dispose ( )

inlinefinalvirtual

Virtual clear method. Triggers actual disposing in sub-classes.

Reimplemented from DIABase.

Definition at line 296 of file concat.hpp.

References ConcatNode< ValueType >::files_, ConcatNode< ValueType >::streams_, and ConcatNode< ValueType >::writers_.

◆ Execute()

void Execute ( )

inlinefinalvirtual

Executes the concat operation.

Implements DIABase.

Definition at line 188 of file concat.hpp.

References FlowControlChannel::AllReduce(), DIABase::context_, ConcatNode< ValueType >::files_, Context::GetNewCatStream(), LOG, min(), Context::my_rank(), Context::net, ConcatNode< ValueType >::num_inputs_, Context::num_workers(), FlowControlChannel::PrefixSum(), sLOG, and ConcatNode< ValueType >::streams_.

◆ OnPreOpFile()

bool OnPreOpFile	(	const data::File &	file,
		size_t	parent_index
	)

inlinefinalvirtual

Receive a whole data::File of ValueType, but only if our stack is empty.

Reimplemented from DIABase.

Definition at line 155 of file concat.hpp.

References ConcatNode< ValueType >::files_, thrill::common::g_debug_push_file, LOGC, ConcatNode< ValueType >::num_inputs_, and ConcatNode< ValueType >::parent_stack_empty_.

◆ PrintWarning()

void PrintWarning ( )

inline

Definition at line 117 of file concat.hpp.

References LOG1.

Referenced by ConcatNode< ValueType >::ConcatNode().

◆ PushData()

void PushData ( bool consume )

inlinefinalvirtual

Virtual method for pushing data. Triggers actual pushing in sub-classes.

Implements DIABase.

Definition at line 280 of file concat.hpp.

References BlockReader< BlockSource >::HasNext(), LOG, BlockReader< BlockSource >::Next(), ConcatNode< ValueType >::num_inputs_, DIANode< ValueType >::PushItem(), and ConcatNode< ValueType >::streams_.

◆ StopPreOp()

void StopPreOp ( size_t )

inlinefinalvirtual

Virtual method for preparing end of PushData.

Reimplemented from DIABase.

Definition at line 183 of file concat.hpp.

References ConcatNode< ValueType >::writers_.

Member Data Documentation

◆ debug

constexpr bool debug = false

static

Definition at line 38 of file concat.hpp.

◆ files_

std::vector<data::File> files_

private

Files for intermediate storage.

Definition at line 310 of file concat.hpp.

Referenced by ConcatNode< ValueType >::ConcatNode(), ConcatNode< ValueType >::Dispose(), ConcatNode< ValueType >::Execute(), and ConcatNode< ValueType >::OnPreOpFile().

◆ num_inputs_

const size_t num_inputs_

private

number of input DIAs

Definition at line 304 of file concat.hpp.

Referenced by ConcatNode< ValueType >::ConcatNode(), ConcatNode< ValueType >::Execute(), ConcatNode< ValueType >::OnPreOpFile(), and ConcatNode< ValueType >::PushData().

◆ parent_stack_empty_

const std::vector<bool> parent_stack_empty_

private

Whether the parent stack is empty.

Definition at line 307 of file concat.hpp.

Referenced by ConcatNode< ValueType >::ConcatNode(), and ConcatNode< ValueType >::OnPreOpFile().

◆ streams_

std::vector<data::CatStreamPtr> streams_

private

Array of CatStreams for exchange.

Definition at line 315 of file concat.hpp.

Referenced by ConcatNode< ValueType >::Dispose(), ConcatNode< ValueType >::Execute(), and ConcatNode< ValueType >::PushData().

◆ writers_

std::vector<data::File::Writer> writers_

private

Writers to intermediate files.

Definition at line 312 of file concat.hpp.

Referenced by ConcatNode< ValueType >::ConcatNode(), ConcatNode< ValueType >::Dispose(), ConcatNode< ValueType >::RegisterParent::operator()(), and ConcatNode< ValueType >::StopPreOp().

The documentation for this class was generated from the following file:

thrill/api/concat.hpp

Detailed Description

template<typename ValueType> class thrill::api::ConcatNode< ValueType >

Classes

Public Types

Public Member Functions

Static Public Attributes

Private Attributes

Additional Inherited Members

Member Typedef Documentation

◆ Super

Constructor & Destructor Documentation

◆ ConcatNode() [1/3]

◆ ConcatNode() [2/3]

◆ ConcatNode() [3/3]

Member Function Documentation

◆ Dispose()

◆ Execute()

◆ OnPreOpFile()

◆ PrintWarning()

◆ PushData()

◆ StopPreOp()

Member Data Documentation

◆ debug

◆ files_

◆ num_inputs_

◆ parent_stack_empty_

◆ streams_

◆ writers_

template<typename ValueType>
class thrill::api::ConcatNode< ValueType >