Thrill  0.1
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
block.hpp
Go to the documentation of this file.
1 /*******************************************************************************
2  * thrill/data/block.hpp
3  *
4  * Part of Project Thrill - http://project-thrill.org
5  *
6  * Copyright (C) 2015 Timo Bingmann <[email protected]>
7  *
8  * All rights reserved. Published under the BSD-2 license in the LICENSE file.
9  ******************************************************************************/
10 
11 #pragma once
12 #ifndef THRILL_DATA_BLOCK_HEADER
13 #define THRILL_DATA_BLOCK_HEADER
14 
15 #include <thrill/common/logger.hpp>
17 #include <thrill/mem/manager.hpp>
18 #include <thrill/mem/pool.hpp>
19 #include <tlx/counting_ptr.hpp>
20 
21 #include <cassert>
22 #include <ostream>
23 #include <string>
24 
25 namespace thrill {
26 namespace data {
27 
28 //! \addtogroup data_layer
29 //! \{
30 
31 class PinnedBlock;
32 class PinRequest;
33 using PinRequestPtr =
35 
36 /*!
37  * Block combines a reference to a read-only \ref ByteBlock and book-keeping
38  * information. The book-keeping meta-information currently is the start of the
39  * first item, the ends of the item range, and the number of items in the range.
40  *
41  * Multiple Block instances can share the same ByteBlock but have different
42  * book-keeping / meta- information!
43  *
44  * <pre>
45  * +--+---------+---------+-------------+---------+-----+
46  * | |Item1 |Item2 |Item3 |Item4 |Item5|(partial)
47  * +--+---------+---------+-------------+---------+-----+
48  * ^ ^ ^
49  * begin first_item num_items=5 end
50  * </pre>
51  */
52 class Block
53 {
54 public:
55  //! default-ctor: create invalid Block.
56  Block() = default;
57 
58  Block(const Block& other) = default;
59  Block& operator = (const Block& other) = default;
60  Block(Block&& other) = default;
61  Block& operator = (Block&& other) = default;
62 
63  //! Creates a block that points to the given data::ByteBlock with the given
64  //! offsets The block can be initialized as pinned or not.
66  size_t begin, size_t end, size_t first_item, size_t num_items,
67  bool typecode_verify)
68  : byte_block_(std::move(byte_block)),
69  begin_(begin), end_(end),
70  first_item_(first_item), num_items_(num_items),
71  typecode_verify_(typecode_verify) { }
72 
73  //! Return whether the enclosed ByteBlock is valid.
74  bool IsValid() const {
75  return byte_block_;
76  }
77 
78  //! access to byte_block_
79  const ByteBlockPtr& byte_block() const { return byte_block_; }
80 
81  //! mutable access to byte_block_
83 
84  //! return number of items beginning in this block
85  size_t num_items() const { return num_items_; }
86 
87  //! return number of pins in underlying ByteBlock
88  size_t pin_count(size_t local_worker_id) const {
89  assert(byte_block_);
90  return byte_block_->pin_count(local_worker_id);
91  }
92 
93  //! accessor to begin_
94  void set_begin(size_t i) { begin_ = i; }
95 
96  //! accessor to end_
97  void set_end(size_t i) { end_ = i; }
98 
99  //! return length of valid data in bytes.
100  size_t size() const { return end_ - begin_; }
101 
102  //! accessor to first_item_ (absolute in ByteBlock)
103  size_t first_item_absolute() const { return first_item_; }
104 
105  //! return the first_item_offset relative to data_begin().
106  size_t first_item_relative() const { return first_item_ - begin_; }
107 
108  //! Returns typecode_verify_
109  bool typecode_verify() const { return typecode_verify_; }
110 
111  friend std ::ostream& operator << (std::ostream& os, const Block& b);
112 
113  //! Creates a pinned copy of this Block. If the underlying data::ByteBlock
114  //! is already pinned, the Future is directly filled with a copy if this
115  //! block. Otherwise an async pin call will be issued.
116  PinRequestPtr Pin(size_t local_worker_id) const;
117 
118  //! Convenience function to call Pin() and wait for the future.
119  PinnedBlock PinWait(size_t local_worker_id) const;
120 
121 protected:
122  static constexpr bool debug = false;
123 
124  //! referenced ByteBlock
126 
127  //! beginning offset of valid bytes to read
128  size_t begin_ = 0;
129 
130  //! one byte beyond the end of the valid bytes in the ByteBlock (can be used
131  //! to virtually shorten a ByteBlock)
132  size_t end_ = 0;
133 
134  //! offset of first valid element in the ByteBlock in absolute bytes from
135  //! byte_block_->begin().
136  size_t first_item_ = 0;
137 
138  //! number of valid items that _start_ in this block (includes cut-off
139  //! element at the end)
140  size_t num_items_ = 0;
141 
142  //! flag whether the underlying data contains self verify type codes from
143  //! BlockReader, this is false to needed to read external files.
144  bool typecode_verify_ = false;
145 };
146 
147 /*!
148  * A pinned / pin-counted derivative of a Block. By holding a pin, it is a
149  * guaranteed that the contained ByteBlock's data is loaded in RAM. Since pins
150  * are counted per thread, the PinnedBlock is a counting pointer plus a thread
151  * id. An ordinary Block can be pinned by calling Pin(), which delivers a future
152  * PinnedBlock, which is available once the data is actually loaded.
153  *
154  * Be careful to move PinnedBlock as must as possible, since copying costs a
155  * pinning and an unpinning operation, whereas moving is free.
156  */
157 class PinnedBlock : private Block
158 {
159 public:
160  //! Create invalid PinnedBlock.
161  PinnedBlock() = default;
162 
163  //! Creates a block that points to the given data::PinnedByteBlock with the
164  //! given offsets. The returned block is also pinned, the pin is transfered!
166  size_t begin, size_t end, size_t first_item, size_t num_items,
167  bool typecode_verify)
168  : Block(std::move(byte_block),
169  begin, end, first_item, num_items, typecode_verify),
170  local_worker_id_(byte_block.local_worker_id()) {
171  LOG << "PinnedBlock::Acquire() from new PinnedByteBlock"
172  << " for local_worker_id=" << local_worker_id_;
173  }
174 
175  //! copy-ctor: increment underlying's pin count
176  PinnedBlock(const PinnedBlock& pb) noexcept
177  : Block(pb), local_worker_id_(pb.local_worker_id_) {
178  if (byte_block_) byte_block_->IncPinCount(local_worker_id_);
179  }
180 
181  //! move-ctor: move underlying's pin
182  PinnedBlock(PinnedBlock&& pb) noexcept
183  : Block(std::move(pb)), local_worker_id_(pb.local_worker_id_) {
184  assert(!pb.byte_block_);
185  }
186 
187  //! copy-assignment: copy underlying and increase pin count
189  if (this == &pb) return *this;
190  // first acquire other's pin count
191  if (pb.byte_block_) pb.byte_block_->IncPinCount(pb.local_worker_id_);
192  // then release the current one
193  if (byte_block_) byte_block_->DecPinCount(local_worker_id_);
194  // copy over Block information
195  Block::operator = (pb);
196  local_worker_id_ = pb.local_worker_id_;
197  return *this;
198  }
199 
200  //! move-assignment: move underlying, release current's pin
202  if (this == &pb) return *this;
203  // release the current one
204  if (byte_block_) byte_block_->DecPinCount(local_worker_id_);
205  // move over Block information, keep other's pin count
206  Block::operator = (std::move(pb));
207  local_worker_id_ = pb.local_worker_id_;
208  // invalidate other block
209  assert(!pb.byte_block_);
210  return *this;
211  }
212 
214  LOG << "~PinnedBlock() byte_block_=" << byte_block_.get();
215  if (byte_block_)
216  byte_block_->DecPinCount(local_worker_id_);
217  }
218 
219  //! \name Accessors to Super-Class
220  //! \{
221 
222  //! Return whether the enclosed ByteBlock is valid.
223  bool IsValid() const { return Block::IsValid(); }
224 
225  //! access to byte_block_
226  const ByteBlockPtr& byte_block() const { return Block::byte_block(); }
227 
228  //! mutable access to byte_block_
230 
231  //! return number of items beginning in this block
232  size_t num_items() const { return Block::num_items(); }
233 
234  //! return number of pins in underlying ByteBlock
235  size_t pin_count(size_t local_worker_id) const
236  { return Block::pin_count(local_worker_id); }
237 
238  //! accessor to begin_
239  void set_begin(size_t i) { return Block::set_begin(i); }
240 
241  //! accessor to end_
242  void set_end(size_t i) { return Block::set_end(i); }
243 
244  //! return length of valid data in bytes.
245  size_t size() const { return Block::size(); }
246 
247  //! accessor to first_item_ (absolute in ByteBlock)
248  size_t first_item_absolute() const { return Block::first_item_absolute(); }
249 
250  //! return the first_item_offset relative to data_begin().
251  size_t first_item_relative() const { return Block::first_item_relative(); }
252 
253  //! return typecode_verify from Block
254  bool typecode_verify() const { return Block::typecode_verify(); }
255 
256  //! \}
257 
258  //! return pointer to beginning of valid data
259  const Byte * data_begin() const {
260  assert(byte_block_);
261  return byte_block_->begin() + begin_;
262  }
263 
264  //! return pointer to end of valid data
265  const Byte * data_end() const {
266  assert(byte_block_);
267  return byte_block_->begin() + end_;
268  }
269 
270  //! release pin on block and reset Block pointer to nullptr
271  void Reset() {
272  if (byte_block_) {
273  byte_block_->DecPinCount(local_worker_id_);
274  byte_block_.reset();
275  }
276  }
277 
278  //! extract Block has an unpinned copy
279  Block ToBlock() const {
280  return Block(*this);
281  }
282 
283  //! extract Block has an unpinned move
285  byte_block_->DecPinCount(local_worker_id_);
286  return Block(std::move(*this));
287  }
288 
289  //! extract ByteBlock including it's pin. afterwards, this PinnedBlock is
290  //! invalid.
292  return PinnedByteBlockPtr(std::move(byte_block_), local_worker_id_);
293  }
294 
295  //! copy the underlying byte_block_ into a new PinnedByteBlockPtr, which
296  //! increases the pin count. use StealPinnedByteBlock to move the underlying
297  //! pin out (cheaper).
300  if (pbb.valid()) pbb->IncPinCount(local_worker_id_);
301  return pbb;
302  }
303 
304  //! Return block as std::string (for debugging), includes eventually cut off
305  //! elements form the beginning included
306  std::string ToString() const;
307 
308  //! not available in PinnedBlock
309  PinRequestPtr Pin() const = delete;
310 
311  //! not available in PinnedBlock
312  PinnedBlock PinWait() const = delete;
313 
314  //! make ostreamable for debugging
315  friend std ::ostream& operator << (std::ostream& os, const PinnedBlock& b);
316 
317 private:
318  //! protected construction from an unpinned block AFTER the pin was taken,
319  //! this method does NOT pin it.
320  PinnedBlock(const Block& b, size_t local_worker_id)
321  : Block(b), local_worker_id_(local_worker_id) { }
322 
323  //! thread id of holder of pin
325 
326  //! friend for creating PinnedBlock from unpinned Block in PinBlock() using
327  //! protected constructor.
328  friend class BlockPool;
329 };
330 
332 {
333 public:
334  //! wait and get the PinnedBlock. this may block until the read is complete.
335  PinnedBlock Wait();
336 
337  //! whether the read is completed, cannot block.
338  bool ready() const { return ready_; }
339 
341 
342 private:
343  //! calls BlockPool::OnReadComplete used to tlx::delegate
344  void OnComplete(foxxll::request* req, bool success);
345 
346  PinRequest(BlockPool* block_pool, PinnedBlock&& block, bool ready = true)
347  : block_pool_(block_pool), block_(std::move(block)), ready_(ready) { }
348 
349  //! reference back to BlockPool
351  //! pinned block which will be returned, this PinnedBlock may already be
352  //! partially initialized for the read!
354  //! running read request
356 
357  //! indication that the PinnedBlocks ready
358  std::atomic<bool> ready_;
359 
360  //! for access to protected data
361  friend class BlockPool;
362  friend class mem::Pool;
363 };
364 
365 //! \}
366 
367 } // namespace data
368 } // namespace thrill
369 
370 #endif // !THRILL_DATA_BLOCK_HEADER
371 
372 /******************************************************************************/
Block combines a reference to a read-only ByteBlock and book-keeping information. ...
Definition: block.hpp:52
const ByteBlockPtr & byte_block() const
access to byte_block_
Definition: block.hpp:79
PinnedBlock PinWait() const =delete
not available in PinnedBlock
const ByteBlockPtr & byte_block() const
access to byte_block_
Definition: block.hpp:226
Block ToBlock() const
extract Block has an unpinned copy
Definition: block.hpp:279
bool typecode_verify() const
Returns typecode_verify_.
Definition: block.hpp:109
friend std::ostream & operator<<(std::ostream &os, const Block &b)
Definition: block.cpp:22
PinnedBlock Wait()
wait and get the PinnedBlock. this may block until the read is complete.
Definition: block.cpp:64
ByteBlockPtr & byte_block()
mutable access to byte_block_
Definition: block.hpp:82
PinnedBlock(const Block &b, size_t local_worker_id)
Definition: block.hpp:320
PinRequestPtr Pin(size_t local_worker_id) const
Definition: block.cpp:39
PinnedByteBlockPtr CopyPinnedByteBlock() const
Definition: block.hpp:298
bool ready() const
whether the read is completed, cannot block.
Definition: block.hpp:338
PinRequestPtr Pin() const =delete
not available in PinnedBlock
Block(ByteBlockPtr &&byte_block, size_t begin, size_t end, size_t first_item, size_t num_items, bool typecode_verify)
Definition: block.hpp:65
PinnedBlock()=default
Create invalid PinnedBlock.
PinnedBlock & operator=(PinnedBlock &pb) noexcept
copy-assignment: copy underlying and increase pin count
Definition: block.hpp:188
A pinned / pin-counted pointer to a ByteBlock.
Definition: byte_block.hpp:182
void reset()
release contained pointer, frees object if this is the last reference.
PinnedBlock(const PinnedBlock &pb) noexcept
copy-ctor: increment underlying's pin count
Definition: block.hpp:176
void set_end(size_t i)
accessor to end_
Definition: block.hpp:97
size_t num_items() const
return number of items beginning in this block
Definition: block.hpp:85
size_t first_item_absolute() const
accessor to first_item_ (absolute in ByteBlock)
Definition: block.hpp:103
Block MoveToBlock()&&
extract Block has an unpinned move
Definition: block.hpp:284
size_t first_item_
Definition: block.hpp:136
size_t local_worker_id_
thread id of holder of pin
Definition: block.hpp:324
size_t size() const
return length of valid data in bytes.
Definition: block.hpp:245
std::string ToString() const
Definition: block.cpp:47
bool IsValid() const
Return whether the enclosed ByteBlock is valid.
Definition: block.hpp:74
void set_begin(size_t i)
accessor to begin_
Definition: block.hpp:94
bool typecode_verify() const
return typecode_verify from Block
Definition: block.hpp:254
void OnComplete(foxxll::request *req, bool success)
calls BlockPool::OnReadComplete used to tlx::delegate
Definition: block_pool.cpp:719
Type * get() const noexcept
return the enclosed pointer.
void Reset()
release pin on block and reset Block pointer to nullptr
Definition: block.hpp:271
Block & operator=(const Block &other)=default
A simple memory allocation manager.
Definition: pool.hpp:74
BlockPool * block_pool_
reference back to BlockPool
Definition: block.hpp:350
ByteBlockPtr & byte_block()
Definition: block.hpp:340
uint8_t Byte
type of underlying memory area
Definition: byte_block.hpp:37
Pool to allocate, keep, swap out/in, and free all ByteBlocks on the host.
Definition: block_pool.hpp:42
PinnedBlock PinWait(size_t local_worker_id) const
Convenience function to call Pin() and wait for the future.
Definition: block.cpp:35
ByteBlockPtr & byte_block()
mutable access to byte_block_
Definition: block.hpp:229
const Byte * data_end() const
return pointer to end of valid data
Definition: block.hpp:265
void set_end(size_t i)
accessor to end_
Definition: block.hpp:242
std::basic_string< char, std::char_traits< char >, Allocator< char > > string
string with Manager tracking
Definition: allocator.hpp:220
ByteBlockPtr byte_block_
referenced ByteBlock
Definition: block.hpp:125
size_t first_item_relative() const
return the first_item_offset relative to data_begin().
Definition: block.hpp:251
size_t size() const
return length of valid data in bytes.
Definition: block.hpp:100
foxxll::request_ptr req_
running read request
Definition: block.hpp:355
const Byte * data_begin() const
return pointer to beginning of valid data
Definition: block.hpp:259
PinRequest(BlockPool *block_pool, PinnedBlock &&block, bool ready=true)
Definition: block.hpp:346
High-performance smart pointer used as a wrapping reference counting pointer.
size_t pin_count(size_t local_worker_id) const
return number of pins in underlying ByteBlock
Definition: block.hpp:88
size_t first_item_relative() const
return the first_item_offset relative to data_begin().
Definition: block.hpp:106
size_t num_items() const
return number of items beginning in this block
Definition: block.hpp:232
A pinned / pin-counted derivative of a Block.
Definition: block.hpp:157
PinnedBlock(PinnedByteBlockPtr &&byte_block, size_t begin, size_t end, size_t first_item, size_t num_items, bool typecode_verify)
Definition: block.hpp:165
std::atomic< bool > ready_
indication that the PinnedBlocks ready
Definition: block.hpp:358
Request object encapsulating basic properties like file and offset.
Definition: request.hpp:49
PinnedBlock(PinnedBlock &&pb) noexcept
move-ctor: move underlying's pin
Definition: block.hpp:182
tlx::CountingPtr< PinRequest, mem::GPoolDeleter< PinRequest > > PinRequestPtr
Definition: block.hpp:34
friend std::ostream & operator<<(std::ostream &os, const PinnedBlock &b)
make ostreamable for debugging
Definition: block.cpp:53
PinnedBlock block_
Definition: block.hpp:353
bool IsValid() const
Return whether the enclosed ByteBlock is valid.
Definition: block.hpp:223
size_t begin_
beginning offset of valid bytes to read
Definition: block.hpp:128
PinnedByteBlockPtr StealPinnedByteBlock()&&
Definition: block.hpp:291
#define LOG
Default logging method: output if the local debug variable is true.
Definition: logger.hpp:24
size_t first_item_absolute() const
accessor to first_item_ (absolute in ByteBlock)
Definition: block.hpp:248
size_t pin_count(size_t local_worker_id) const
return number of pins in underlying ByteBlock
Definition: block.hpp:235
static constexpr bool debug
Definition: block.hpp:122
void set_begin(size_t i)
accessor to begin_
Definition: block.hpp:239
Provides reference counting abilities for use with CountingPtr.
Block()=default
default-ctor: create invalid Block.