Thrill  0.1
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
byte_block.hpp
Go to the documentation of this file.
1 /*******************************************************************************
2  * thrill/data/byte_block.hpp
3  *
4  * Part of Project Thrill - http://project-thrill.org
5  *
6  * Copyright (C) 2015 Timo Bingmann <[email protected]>
7  *
8  * All rights reserved. Published under the BSD-2 license in the LICENSE file.
9  ******************************************************************************/
10 
11 #pragma once
12 #ifndef THRILL_DATA_BYTE_BLOCK_HEADER
13 #define THRILL_DATA_BYTE_BLOCK_HEADER
14 
15 #include <thrill/mem/pool.hpp>
16 
17 #include <foxxll/io/file.hpp>
18 #include <foxxll/mng/bid.hpp>
19 #include <tlx/counting_ptr.hpp>
20 
21 #include <string>
22 #include <vector>
23 
24 namespace thrill {
25 namespace data {
26 
27 //! \addtogroup data_layer
28 //! \{
29 
30 //! starting size of blocks in BlockWriter.
31 extern size_t start_block_size;
32 
33 //! default size of blocks in File, Channel, BlockQueue, etc.
34 extern size_t default_block_size;
35 
36 //! type of underlying memory area
37 using Byte = uint8_t;
38 
39 // forward declarations.
40 class BlockPool;
41 
42 /*!
43  * A ByteBlock is the basic storage units of containers like File, BlockQueue,
44  * etc. It consists of a fixed number of bytes without any type and meta
45  * information. Conceptually a ByteBlock is written _once_ and can then be
46  * shared read-only between containers using CountingPtr<const ByteBlock>
47  * reference counting inside a Block, which adds meta information.
48  *
49  * ByteBlocks can be swapped to disk, which decreases their size to 0.
50  */
52 {
53  static constexpr bool debug = false;
54 
55 public:
56  //! deleter for CountingPtr<ByteBlock>
57  class Deleter
58  {
59  public:
60  void operator () (ByteBlock* bb) const;
61  void operator () (const ByteBlock* bb) const;
62  };
63 
66 
67 public:
68  //! mutable data accessor to memory block
69  Byte * data() { return data_; }
70  //! const data accessor to memory block
71  const Byte * data() const { return data_; }
72 
73  //! mutable data accessor to beginning of memory block
74  Byte * begin() { return data_; }
75  //! const data accessor to beginning of memory block
76  const Byte * begin() const { return data_; }
77 
78  //! mutable data accessor beyond end of memory block
79  Byte * end() { return data_ + size_; }
80  //! const data accessor beyond end of memory block
81  const Byte * end() const { return data_ + size_; }
82 
83  //! the block size
84  size_t size() const { return size_; }
85 
86  //! Returns whether the ByteBlock is in an external file.
87  bool has_ext_file() const { return ext_file_.get() != nullptr; }
88 
89  //! return current pin count
90  size_t pin_count(size_t local_worker_id) const {
91  return pin_count_[local_worker_id];
92  }
93 
94  //! return string list of pin_counts
95  std::string pin_count_str() const;
96 
97  //! true if block resides in memory
98  bool in_memory() const {
99  return data_ != nullptr;
100  }
101 
102  //! true if being deleted
103  bool is_deleted() const {
104  return pin_count_.empty();
105  }
106 
107  //! increment pin count, must be >= 1 before.
108  void IncPinCount(size_t local_worker_id);
109 
110  //! decrement pin count, possibly signal block pool that if it reaches zero.
111  void DecPinCount(size_t local_worker_id);
112 
113 private:
114  //! the memory block itself is referenced as it is in a a separate memory
115  //! region that can be swapped out
117 
118  //! the allocated size of the buffer in bytes
119  const size_t size_;
120 
121  //! reference to BlockPool for deletion.
123 
124  //! counts the number of pins in this block per thread_id.
125  std::vector<size_t, mem::GPoolAllocator<size_t> > pin_count_;
126 
127  //! counts the total number of pins, the data_ may be swapped out when this
128  //! reaches zero.
129  size_t total_pins_ = 0;
130 
131  //! external memory block, which contains a pointer to foxxll::file, an
132  //! offset into the file, and (unfortunately) also the size.
134 
135  //! shared pointer to external file, if this is != nullptr then the Block
136  //! was created for directly reading binary files.
138 
139  // BlockPool is a friend to call ctor and to manipulate data_.
140  friend class BlockPool;
141  // Block is a friend to call {Increase,Reduce}PinCount()
142  friend class Block;
143  friend class PinnedBlock;
144  // for calling protected constructor
145  friend class mem::Pool;
146 
147  //! No default construction of Byteblock
148  ByteBlock() = delete;
149 
150  /*!
151  * Constructor to initialize ByteBlock in a buffer of memory. Protected,
152  * used BlockPool::AllocateByteBlock() for construction.
153  *
154  * \param data the memory address of the byte-blocks data. nullptr if swapped out
155  * \param size the size of the block in bytes
156  * \param block_pool the block pool that manages this ByteBlock
157  */
158  ByteBlock(BlockPool* block_pool, Byte* data, size_t size);
159 
160  //! Constructor to initialize ByteBlock as a mapping to an external
161  //! foxxll::file area.
162  ByteBlock(BlockPool* block_pool, const foxxll::file_ptr& ext_file,
163  int64_t offset, size_t size);
164 
165  friend std ::ostream& operator << (std::ostream& os, const ByteBlock& b);
166 
167  //! forwarded to block_pool_
168  void OnWriteComplete(foxxll::request* req, bool success);
169 };
170 
172 
173 /*!
174  * A pinned / pin-counted pointer to a ByteBlock. By holding a pin, it is a
175  * guaranteed that the ByteBlock's underlying memory is loaded in RAM. Since
176  * pins are counted per thread, the PinnedByteBlockPtr is a counting pointer
177  * plus a thread id.
178  *
179  * Be careful to move PinnedByteBlockPtr as must as possible, since copying
180  * costs a pinning and an unpinning operation, whereas moving is free.
181  */
183 {
184 public:
185  //! default ctor: contains a nullptr pointer.
186  PinnedByteBlockPtr() noexcept = default;
187 
188  //! copy-ctor: increment underlying's pin count
190  : ByteBlockPtr(pbb), local_worker_id_(pbb.local_worker_id_) {
191  if (valid()) get()->IncPinCount(local_worker_id_);
192  }
193 
194  //! move-ctor: move underlying's pin
196  : ByteBlockPtr(std::move(pbb)), local_worker_id_(pbb.local_worker_id_) {
197  assert(!pbb.valid());
198  }
199 
200  //! copy-assignment: transfer underlying's pin count
202  if (this == &pbb) return *this;
203  // first acquire other's pin count
204  if (pbb.valid()) pbb->IncPinCount(pbb.local_worker_id_);
205  // then release the current one
206  if (valid()) get()->DecPinCount(local_worker_id_);
207  // copy over information, keep pin
209  local_worker_id_ = pbb.local_worker_id_;
210  return *this;
211  }
212 
213  //! move-assignment: move underlying's pin
215  if (this == &pbb) return *this;
216  // release the current one
217  if (valid()) get()->DecPinCount(local_worker_id_);
218  // move over information, keep other's pin
219  ByteBlockPtr::operator = (std::move(pbb));
220  local_worker_id_ = pbb.local_worker_id_;
221  // invalidated other block
222  assert(!pbb.valid());
223  return *this;
224  }
225 
226  //! destructor: remove pin
228  if (valid()) get()->DecPinCount(local_worker_id_);
229  }
230 
231  //! local worker id of holder of pin
232  size_t local_worker_id() const { return local_worker_id_; }
233 
234 private:
235  //! protected ctor for calling from Acquire().
238 
239  //! protected ctor for calling from Acquire().
241  : ByteBlockPtr(std::move(ptr)), local_worker_id_(local_worker_id) { }
242 
243  //! local worker id of holder of pin
245 
246  //! for access to protected constructor to transfer pin
247  friend class PinnedBlock;
248  //! for access to protected constructor to AllocateByteBlock().
249  friend class BlockPool;
250 };
251 
252 //! \}
253 
254 } // namespace data
255 } // namespace thrill
256 
257 #endif // !THRILL_DATA_BYTE_BLOCK_HEADER
258 
259 /******************************************************************************/
Block combines a reference to a read-only ByteBlock and book-keeping information. ...
Definition: block.hpp:52
bool is_deleted() const
true if being deleted
Definition: byte_block.hpp:103
std::vector< size_t, mem::GPoolAllocator< size_t > > pin_count_
counts the number of pins in this block per thread_id.
Definition: byte_block.hpp:125
void OnWriteComplete(foxxll::request *req, bool success)
forwarded to block_pool_
Definition: byte_block.cpp:75
size_t default_block_size
default size of blocks in File, Channel, BlockQueue, etc.
Definition: byte_block.cpp:25
std::string pin_count_str() const
return string list of pin_counts
Definition: byte_block.cpp:63
size_t pin_count(size_t local_worker_id) const
return current pin count
Definition: byte_block.hpp:90
bool has_ext_file() const
Returns whether the ByteBlock is in an external file.
Definition: byte_block.hpp:87
bool in_memory() const
true if block resides in memory
Definition: byte_block.hpp:98
BlockPool * block_pool_
reference to BlockPool for deletion.
Definition: byte_block.hpp:122
Specialization of block identifier class (BID) for variable size block size.
Definition: bid.hpp:112
A pinned / pin-counted pointer to a ByteBlock.
Definition: byte_block.hpp:182
tlx::CountingPtr< ByteBlock, Deleter > ByteBlockPtr
Definition: byte_block.hpp:64
bool valid() const noexcept
test for a non-nullptr pointer
PinnedByteBlockPtr(PinnedByteBlockPtr &&pbb) noexcept
move-ctor: move underlying's pin
Definition: byte_block.hpp:195
void IncPinCount(size_t local_worker_id)
increment pin count, must be >= 1 before.
Definition: byte_block.cpp:67
Byte * data()
mutable data accessor to memory block
Definition: byte_block.hpp:69
friend std::ostream & operator<<(std::ostream &os, const ByteBlock &b)
Definition: byte_block.cpp:79
foxxll::BID< 0 > em_bid_
Definition: byte_block.hpp:133
PinnedByteBlockPtr(const PinnedByteBlockPtr &pbb) noexcept
copy-ctor: increment underlying's pin count
Definition: byte_block.hpp:189
const Byte * begin() const
const data accessor to beginning of memory block
Definition: byte_block.hpp:76
CountingPtr & operator=(const CountingPtr &other) noexcept
ByteBlock()=delete
No default construction of Byteblock.
Type * get() const noexcept
return the enclosed pointer.
A simple memory allocation manager.
Definition: pool.hpp:74
foxxll::file_ptr ext_file_
Definition: byte_block.hpp:137
size_t start_block_size
starting size of blocks in BlockWriter.
Definition: byte_block.cpp:24
const size_t size_
the allocated size of the buffer in bytes
Definition: byte_block.hpp:119
PinnedByteBlockPtr & operator=(PinnedByteBlockPtr &pbb) noexcept
copy-assignment: transfer underlying's pin count
Definition: byte_block.hpp:201
deleter for CountingPtr<ByteBlock>
Definition: byte_block.hpp:57
uint8_t Byte
type of underlying memory area
Definition: byte_block.hpp:37
Pool to allocate, keep, swap out/in, and free all ByteBlocks on the host.
Definition: block_pool.hpp:42
A ByteBlock is the basic storage units of containers like File, BlockQueue, etc.
Definition: byte_block.hpp:51
std::basic_string< char, std::char_traits< char >, Allocator< char > > string
string with Manager tracking
Definition: allocator.hpp:220
PinnedByteBlockPtr(ByteBlock *ptr, size_t local_worker_id) noexcept
protected ctor for calling from Acquire().
Definition: byte_block.hpp:236
High-performance smart pointer used as a wrapping reference counting pointer.
~PinnedByteBlockPtr()
destructor: remove pin
Definition: byte_block.hpp:227
void operator()(ByteBlock *bb) const
Definition: byte_block.cpp:45
A pinned / pin-counted derivative of a Block.
Definition: block.hpp:157
const Byte * end() const
const data accessor beyond end of memory block
Definition: byte_block.hpp:81
size_t local_worker_id_
local worker id of holder of pin
Definition: byte_block.hpp:244
Byte * end()
mutable data accessor beyond end of memory block
Definition: byte_block.hpp:79
PinnedByteBlockPtr(ByteBlockPtr &&ptr, size_t local_worker_id) noexcept
protected ctor for calling from Acquire().
Definition: byte_block.hpp:240
Request object encapsulating basic properties like file and offset.
Definition: request.hpp:49
static constexpr bool debug
Definition: byte_block.hpp:53
Byte * begin()
mutable data accessor to beginning of memory block
Definition: byte_block.hpp:74
PinnedByteBlockPtr() noexcept=default
default ctor: contains a nullptr pointer.
const Byte * data() const
const data accessor to memory block
Definition: byte_block.hpp:71
size_t local_worker_id() const
local worker id of holder of pin
Definition: byte_block.hpp:232
void DecPinCount(size_t local_worker_id)
decrement pin count, possibly signal block pool that if it reaches zero.
Definition: byte_block.cpp:71
ByteBlock::ByteBlockPtr ByteBlockPtr
Definition: byte_block.hpp:171
Provides reference counting abilities for use with CountingPtr.
size_t size() const
the block size
Definition: byte_block.hpp:84