Thrill  0.1
byte_block.hpp
Go to the documentation of this file.
1 /*******************************************************************************
2  * thrill/data/byte_block.hpp
3  *
4  * Part of Project Thrill - http://project-thrill.org
5  *
6  * Copyright (C) 2015 Timo Bingmann <[email protected]>
7  *
8  * All rights reserved. Published under the BSD-2 license in the LICENSE file.
9  ******************************************************************************/
10 
11 #pragma once
12 #ifndef THRILL_DATA_BYTE_BLOCK_HEADER
13 #define THRILL_DATA_BYTE_BLOCK_HEADER
14 
15 #include <thrill/mem/pool.hpp>
16 
17 #include <foxxll/io/file.hpp>
18 #include <foxxll/mng/bid.hpp>
19 #include <tlx/counting_ptr.hpp>
20 
21 #include <string>
22 #include <vector>
23 
24 namespace thrill {
25 namespace data {
26 
27 //! \addtogroup data_layer
28 //! \{
29 
30 //! starting size of blocks in BlockWriter.
31 extern size_t start_block_size;
32 
33 //! default size of blocks in File, Channel, BlockQueue, etc.
34 extern size_t default_block_size;
35 
36 //! type of underlying memory area
37 using Byte = uint8_t;
38 
39 // forward declarations.
40 class BlockPool;
41 
42 /*!
43  * A ByteBlock is the basic storage units of containers like File, BlockQueue,
44  * etc. It consists of a fixed number of bytes without any type and meta
45  * information. Conceptually a ByteBlock is written _once_ and can then be
46  * shared read-only between containers using CountingPtr<const ByteBlock>
47  * reference counting inside a Block, which adds meta information.
48  *
49  * ByteBlocks can be swapped to disk, which decreases their size to 0.
50  */
52 {
53  static constexpr bool debug = false;
54 
55 public:
56  //! deleter for CountingPtr<ByteBlock>
57  class Deleter
58  {
59  public:
60  void operator () (ByteBlock* bb) const;
61  void operator () (const ByteBlock* bb) const;
62  };
63 
66 
67 public:
68  //! mutable data accessor to memory block
69  Byte * data() { return data_; }
70  //! const data accessor to memory block
71  const Byte * data() const { return data_; }
72 
73  //! mutable data accessor to beginning of memory block
74  Byte * begin() { return data_; }
75  //! const data accessor to beginning of memory block
76  const Byte * begin() const { return data_; }
77 
78  //! mutable data accessor beyond end of memory block
79  Byte * end() { return data_ + size_; }
80  //! const data accessor beyond end of memory block
81  const Byte * end() const { return data_ + size_; }
82 
83  //! the block size
84  size_t size() const { return size_; }
85 
86  //! Returns whether the ByteBlock is in an external file.
87  bool has_ext_file() const { return ext_file_.get() != nullptr; }
88 
89  //! return current pin count
90  size_t pin_count(size_t local_worker_id) const {
91  return pin_count_[local_worker_id];
92  }
93 
94  //! return string list of pin_counts
95  std::string pin_count_str() const;
96 
97  //! true if block resides in memory
98  bool in_memory() const {
99  return data_ != nullptr;
100  }
101 
102  //! true if being deleted
103  bool is_deleted() const {
104  return pin_count_.empty();
105  }
106 
107  //! increment pin count, must be >= 1 before.
108  void IncPinCount(size_t local_worker_id);
109 
110  //! decrement pin count, possibly signal block pool that if it reaches zero.
111  void DecPinCount(size_t local_worker_id);
112 
113 private:
114  //! the memory block itself is referenced as it is in a a separate memory
115  //! region that can be swapped out
117 
118  //! the allocated size of the buffer in bytes
119  const size_t size_;
120 
121  //! reference to BlockPool for deletion.
123 
124  //! counts the number of pins in this block per thread_id.
125  std::vector<size_t, mem::GPoolAllocator<size_t> > pin_count_;
126 
127  //! counts the total number of pins, the data_ may be swapped out when this
128  //! reaches zero.
129  size_t total_pins_ = 0;
130 
131  //! external memory block, which contains a pointer to foxxll::file, an
132  //! offset into the file, and (unfortunately) also the size.
134 
135  //! shared pointer to external file, if this is != nullptr then the Block
136  //! was created for directly reading binary files.
138 
139  // BlockPool is a friend to call ctor and to manipulate data_.
140  friend class BlockPool;
141  // Block is a friend to call {Increase,Reduce}PinCount()
142  friend class Block;
143  friend class PinnedBlock;
144  // for calling protected constructor
145  friend class mem::Pool;
146 
147  //! No default construction of Byteblock
148  ByteBlock() = delete;
149 
150  /*!
151  * Constructor to initialize ByteBlock in a buffer of memory. Protected,
152  * used BlockPool::AllocateByteBlock() for construction.
153  *
154  * \param data the memory address of the byte-blocks data. nullptr if swapped out
155  * \param size the size of the block in bytes
156  * \param block_pool the block pool that manages this ByteBlock
157  */
158  ByteBlock(BlockPool* block_pool, Byte* data, size_t size);
159 
160  //! Constructor to initialize ByteBlock as a mapping to an external
161  //! foxxll::file area.
162  ByteBlock(BlockPool* block_pool, const foxxll::file_ptr& ext_file,
163  int64_t offset, size_t size);
164 
165  friend std::ostream& operator << (std::ostream& os, const ByteBlock& b);
166 
167  //! forwarded to block_pool_
168  void OnWriteComplete(foxxll::request* req, bool success);
169 };
170 
171 class PinnedByteBlockPtr;
172 
173 /*!
174  * A non-pinned counting pointer to a ByteBlock
175  */
177 {
179 
180 public:
181  //! default ctor: returns an empty ByteBlockPtr
182  ByteBlockPtr() = default;
183 
184 protected:
185  //! (protected) ctor from PinnedByteBlockPtr
187  //! (protected) ctor from PinnedByteBlockPtr
189  //! (protected) ctor from plain pointer
190  ByteBlockPtr(ByteBlock* ptr) : Super(ptr) { }
191 
192  friend class PinnedByteBlockPtr;
193  friend class BlockPool;
194 };
195 
196 /*!
197  * A pinned / pin-counted pointer to a ByteBlock. By holding a pin, it is a
198  * guaranteed that the ByteBlock's underlying memory is loaded in RAM. Since
199  * pins are counted per thread, the PinnedByteBlockPtr is a counting pointer
200  * plus a thread id.
201  *
202  * Be careful to move PinnedByteBlockPtr as must as possible, since copying
203  * costs a pinning and an unpinning operation, whereas moving is free.
204  */
206 {
207 public:
208  //! default ctor: contains a nullptr pointer.
209  PinnedByteBlockPtr() noexcept = default;
210 
211  //! copy-ctor: increment underlying's pin count
213  : ByteBlockPtr(pbb), local_worker_id_(pbb.local_worker_id_) {
214  if (valid()) get()->IncPinCount(local_worker_id_);
215  }
216 
217  //! move-ctor: move underlying's pin
219  : ByteBlockPtr(std::move(pbb)), local_worker_id_(pbb.local_worker_id_) {
220  assert(!pbb.valid());
221  }
222 
223  //! copy-assignment: transfer underlying's pin count
225  if (this == &pbb) return *this;
226  // first acquire other's pin count
227  if (pbb.valid()) pbb->IncPinCount(pbb.local_worker_id_);
228  // then release the current one
229  if (valid()) get()->DecPinCount(local_worker_id_);
230  // copy over information, keep pin
232  local_worker_id_ = pbb.local_worker_id_;
233  return *this;
234  }
235 
236  //! move-assignment: move underlying's pin
238  if (this == &pbb) return *this;
239  // release the current one
240  if (valid()) get()->DecPinCount(local_worker_id_);
241  // move over information, keep other's pin
242  ByteBlockPtr::operator = (std::move(pbb));
243  local_worker_id_ = pbb.local_worker_id_;
244  // invalidated other block
245  assert(!pbb.valid());
246  return *this;
247  }
248 
249  //! destructor: remove pin
251  if (valid()) get()->DecPinCount(local_worker_id_);
252  }
253 
254  //! local worker id of holder of pin
255  size_t local_worker_id() const { return local_worker_id_; }
256 
257  //! relinquish pin
259  if (valid()) get()->DecPinCount(local_worker_id_);
260  return std::move(*this);
261  }
262 
263 private:
264  //! protected ctor for calling from Acquire().
265  PinnedByteBlockPtr(ByteBlock* ptr, size_t local_worker_id) noexcept
266  : ByteBlockPtr(ptr), local_worker_id_(local_worker_id) { }
267 
268  //! protected ctor for calling from Acquire().
269  PinnedByteBlockPtr(ByteBlockPtr&& ptr, size_t local_worker_id) noexcept
270  : ByteBlockPtr(std::move(ptr)), local_worker_id_(local_worker_id) { }
271 
272  //! local worker id of holder of pin
274 
275  //! for access to protected constructor to transfer pin
276  friend class PinnedBlock;
277  //! for access to protected constructor to AllocateByteBlock().
278  friend class BlockPool;
279 };
280 
281 inline
283 
284 inline
286 
287 //! \}
288 
289 } // namespace data
290 } // namespace thrill
291 
292 #endif // !THRILL_DATA_BYTE_BLOCK_HEADER
293 
294 /******************************************************************************/
Block combines a reference to a read-only ByteBlock and book-keeping information. ...
Definition: block.hpp:52
std::vector< size_t, mem::GPoolAllocator< size_t > > pin_count_
counts the number of pins in this block per thread_id.
Definition: byte_block.hpp:125
size_t size() const
the block size
Definition: byte_block.hpp:84
bool is_deleted() const
true if being deleted
Definition: byte_block.hpp:103
void OnWriteComplete(foxxll::request *req, bool success)
forwarded to block_pool_
Definition: byte_block.cpp:75
friend std::ostream & operator<<(std::ostream &os, const ByteBlock &b)
Definition: byte_block.cpp:79
size_t default_block_size
default size of blocks in File, Channel, BlockQueue, etc.
Definition: byte_block.cpp:25
BlockPool * block_pool_
reference to BlockPool for deletion.
Definition: byte_block.hpp:122
Specialization of block identifier class (BID) for variable size block size.
Definition: bid.hpp:112
ReferenceCounter & operator=(const ReferenceCounter &) noexcept
assignment operator, leaves pointers unchanged
A pinned / pin-counted pointer to a ByteBlock.
Definition: byte_block.hpp:205
STL namespace.
tlx::CountingPtr< ByteBlock, Deleter > ByteBlockPtr
Definition: byte_block.hpp:64
size_t local_worker_id() const
local worker id of holder of pin
Definition: byte_block.hpp:255
PinnedByteBlockPtr(PinnedByteBlockPtr &&pbb) noexcept
move-ctor: move underlying&#39;s pin
Definition: byte_block.hpp:218
A non-pinned counting pointer to a ByteBlock.
Definition: byte_block.hpp:176
void IncPinCount(size_t local_worker_id)
increment pin count, must be >= 1 before.
Definition: byte_block.cpp:67
ByteBlockPtr()=default
default ctor: returns an empty ByteBlockPtr
Byte * data()
mutable data accessor to memory block
Definition: byte_block.hpp:69
std::string pin_count_str() const
return string list of pin_counts
Definition: byte_block.cpp:63
void operator()(ByteBlock *bb) const
Definition: byte_block.cpp:45
const Byte * data() const
const data accessor to memory block
Definition: byte_block.hpp:71
foxxll::BID< 0 > em_bid_
Definition: byte_block.hpp:133
PinnedByteBlockPtr(const PinnedByteBlockPtr &pbb) noexcept
copy-ctor: increment underlying&#39;s pin count
Definition: byte_block.hpp:212
CountingPtr & operator=(const CountingPtr &other) noexcept
ByteBlock()=delete
No default construction of Byteblock.
Type * get() const noexcept
return the enclosed pointer.
A simple memory allocation manager.
Definition: pool.hpp:74
foxxll::file_ptr ext_file_
Definition: byte_block.hpp:137
ByteBlockPtr(ByteBlock *ptr)
(protected) ctor from plain pointer
Definition: byte_block.hpp:190
size_t start_block_size
starting size of blocks in BlockWriter.
Definition: byte_block.cpp:24
const size_t size_
the allocated size of the buffer in bytes
Definition: byte_block.hpp:119
deleter for CountingPtr<ByteBlock>
Definition: byte_block.hpp:57
uint8_t Byte
type of underlying memory area
Definition: byte_block.hpp:37
Pool to allocate, keep, swap out/in, and free all ByteBlocks on the host.
Definition: block_pool.hpp:42
A ByteBlock is the basic storage units of containers like File, BlockQueue, etc.
Definition: byte_block.hpp:51
std::basic_string< char, std::char_traits< char >, Allocator< char > > string
string with Manager tracking
Definition: allocator.hpp:220
bool in_memory() const
true if block resides in memory
Definition: byte_block.hpp:98
PinnedByteBlockPtr(ByteBlock *ptr, size_t local_worker_id) noexcept
protected ctor for calling from Acquire().
Definition: byte_block.hpp:265
size_t pin_count(size_t local_worker_id) const
return current pin count
Definition: byte_block.hpp:90
High-performance smart pointer used as a wrapping reference counting pointer.
~PinnedByteBlockPtr()
destructor: remove pin
Definition: byte_block.hpp:250
const Byte * begin() const
const data accessor to beginning of memory block
Definition: byte_block.hpp:76
bool has_ext_file() const
Returns whether the ByteBlock is in an external file.
Definition: byte_block.hpp:87
A pinned / pin-counted derivative of a Block.
Definition: block.hpp:157
size_t local_worker_id_
local worker id of holder of pin
Definition: byte_block.hpp:273
Byte * end()
mutable data accessor beyond end of memory block
Definition: byte_block.hpp:79
PinnedByteBlockPtr(ByteBlockPtr &&ptr, size_t local_worker_id) noexcept
protected ctor for calling from Acquire().
Definition: byte_block.hpp:269
Request object encapsulating basic properties like file and offset.
Definition: request.hpp:49
static constexpr bool debug
Definition: byte_block.hpp:53
Byte * begin()
mutable data accessor to beginning of memory block
Definition: byte_block.hpp:74
const Byte * end() const
const data accessor beyond end of memory block
Definition: byte_block.hpp:81
ByteBlockPtr ReleasePin() &&
relinquish pin
Definition: byte_block.hpp:258
void DecPinCount(size_t local_worker_id)
decrement pin count, possibly signal block pool that if it reaches zero.
Definition: byte_block.cpp:71
Provides reference counting abilities for use with CountingPtr.