Thrill  0.1
buffer_builder.hpp
Go to the documentation of this file.
1 /*******************************************************************************
2  * thrill/net/buffer_builder.hpp
3  *
4  * Classes BufferBuilder and BinaryBufferReader to construct data blocks with variable
5  * length content. Programs construct blocks using BufferBuilder::Put<type>()
6  * and read them using BufferReader::Get<type>(). The operation sequences must
7  * match. See test-binary-builder.cpp for an example.
8  *
9  * Part of Project Thrill - http://project-thrill.org
10  *
11  * Copyright (C) 2013-2015 Timo Bingmann <[email protected]>
12  *
13  * All rights reserved. Published under the BSD-2 license in the LICENSE file.
14  ******************************************************************************/
15 
16 #pragma once
17 #ifndef THRILL_NET_BUFFER_BUILDER_HEADER
18 #define THRILL_NET_BUFFER_BUILDER_HEADER
19 
22 #include <thrill/net/buffer.hpp>
23 
24 #include <algorithm>
25 #include <cassert>
26 #include <cstdlib>
27 #include <stdexcept>
28 #include <string>
29 #include <type_traits>
30 
31 namespace thrill {
32 namespace net {
33 
34 //! \addtogroup net_layer
35 //! \{
36 
37 /*!
38  * BufferBuilder represents a dynamically growable area of memory, which
39  * can be modified by appending integral data types via Put() and other basic
40  * operations.
41  */
43  : public common::ItemWriterToolsBase<BufferBuilder>
44 {
45 private:
46  //! type used to store the bytes
47  using Byte = unsigned char;
48 
49  //! simple pointer iterators
50  using iterator = Byte *;
51  //! simple pointer iterators
52  using const_iterator = const Byte *;
53  //! simple pointer references
54  using reference = Byte&;
55  //! simple pointer references
56  using const_reference = const Byte&;
57 
58  //! Allocated buffer pointer.
59  Byte* data_ = nullptr;
60 
61  //! Size of valid data.
62  size_t size_ = 0;
63 
64  //! Total capacity of buffer.
65  size_t capacity_ = 0;
66 
67 public:
68  //! \name Construction, Movement, Destruction
69  //! \{
70 
71  //! Create a new empty object
72  BufferBuilder() = default;
73 
74  //! Copy-Constructor, duplicates memory content.
75  BufferBuilder(const BufferBuilder& other) {
76  Assign(other);
77  }
78 
79  //! Move-Constructor, moves memory area.
80  BufferBuilder(BufferBuilder&& other) noexcept
81  : data_(other.data_), size_(other.size_), capacity_(other.capacity_) {
82  other.data_ = nullptr;
83  other.size_ = 0;
84  other.capacity_ = 0;
85  }
86 
87  //! Constructor, copy memory area.
88  BufferBuilder(const void* data, size_t size) {
89  Assign(data, size);
90  }
91 
92  //! Constructor, create object with n bytes pre-allocated.
93  explicit BufferBuilder(size_t size) {
94  Reserve(size);
95  }
96 
97  //! Constructor from std::string, COPIES string content.
98  explicit BufferBuilder(const std::string& str) {
99  Assign(str.data(), str.size());
100  }
101 
102  //! Assignment operator: copy other's memory range into buffer.
104  if (&other != this)
105  Assign(other.data(), other.size());
106 
107  return *this;
108  }
109 
110  //! Move-Assignment operator: move other's memory area into buffer.
112  if (this != &other)
113  {
114  if (data_) free(data_);
115  data_ = other.data_;
116  size_ = other.size_;
117  capacity_ = other.capacity_;
118  other.data_ = nullptr;
119  other.size_ = 0;
120  other.capacity_ = 0;
121  }
122  return *this;
123  }
124 
125  //! Destroys the memory space.
127  Deallocate();
128  }
129 
130  //! Deallocates the kept memory space (we use dealloc() instead of free()
131  //! as a name, because sometimes "free" is replaced by the preprocessor)
133  if (data_) free(data_);
134  data_ = nullptr;
135  size_ = capacity_ = 0;
136 
137  return *this;
138  }
139 
140  //! \}
141 
142  //! \name Data, Size, and Capacity Accessors
143  //! \{
144 
145  //! Return a pointer to the currently kept memory area.
146  const Byte * data() const {
147  return data_;
148  }
149 
150  //! Return a writeable pointer to the currently kept memory area.
151  Byte * data() {
152  return data_;
153  }
154 
155  //! Return the currently used length in bytes.
156  size_t size() const {
157  return size_;
158  }
159 
160  //! Return the currently allocated buffer capacity.
161  size_t capacity() const {
162  return capacity_;
163  }
164 
165  //! \} //do not append empty end-of-stream buffer
166 
167  //! \name Buffer Growing, Clearing, and other Management
168  //! \{
169 
170  //! Clears the memory contents, does not deallocate the memory.
172  size_ = 0;
173  return *this;
174  }
175 
176  //! Set the valid bytes in the buffer, use if the buffer is filled
177  //! directly.
178  BufferBuilder& set_size(size_t n) {
179  assert(n <= capacity_);
180  size_ = n;
181 
182  return *this;
183  }
184 
185  //! Make sure that at least n bytes are allocated.
186  BufferBuilder& Reserve(size_t n) {
187  if (capacity_ < n)
188  {
189  capacity_ = n;
190  data_ = static_cast<Byte*>(realloc(data_, capacity_));
191  }
192 
193  return *this;
194  }
195 
196  //! Dynamically allocate more memory. At least n bytes will be available,
197  //! probably more to compensate future growth.
199  if (capacity_ < n)
200  {
201  // place to adapt the buffer growing algorithm as need.
202  size_t newsize = capacity_;
203 
204  while (newsize < n) {
205  if (newsize < 256) newsize = 512;
206  else if (newsize < 1024 * 1024) newsize = 2 * newsize;
207  else newsize += 1024 * 1024;
208  }
209 
210  Reserve(newsize);
211  }
212 
213  return *this;
214  }
215 
216  //! Detach the memory from the object, returns the memory pointer.
217  const Byte * Detach() {
218  const Byte* data = data_;
219  data_ = nullptr;
220  size_ = capacity_ = 0;
221  return data;
222  }
223 
224  //! Explicit conversion to std::string (copies memory of course).
226  return std::string(reinterpret_cast<const char*>(data_), size_);
227  }
228 
229  //! Explicit conversion to Buffer MOVING the memory ownership.
231  Buffer b = Buffer::Acquire(data_, size_);
232  Detach();
233  return b;
234  }
235 
236  //! \}
237 
238  //! \name Assignment or Alignment
239  //! \{
240 
241  //! Copy a memory range into the buffer, overwrites all current
242  //! data. Roughly equivalent to clear() followed by append().
243  BufferBuilder& Assign(const void* data, size_t len) {
244  if (len > capacity_) Reserve(len);
245 
246  const Byte* cdata = reinterpret_cast<const Byte*>(data);
247  std::copy(cdata, cdata + len, data_);
248  size_ = len;
249 
250  return *this;
251  }
252 
253  //! Copy the contents of another buffer object into this buffer, overwrites
254  //! all current data. Roughly equivalent to clear() followed by append().
256  if (&other != this)
257  Assign(other.data(), other.size());
258 
259  return *this;
260  }
261 
262  //! Align the size of the buffer to a multiple of n. Fills up with 0s.
263  BufferBuilder& Align(size_t n) {
264  assert(n > 0);
265  size_t rem = size_ % n;
266  if (rem != 0)
267  {
268  size_t add = n - rem;
269  if (size_ + add > capacity_) DynReserve(size_ + add);
270  std::fill(data_ + size_, data_ + size_ + add, 0);
271  size_ += add;
272  }
273  assert((size_ % n) == 0);
274 
275  return *this;
276  }
277 
278  //! \}
279 
280  //! \name Appending Write Functions
281  //! \{
282 
283  //! Append a memory range to the buffer
284  BufferBuilder& Append(const void* data, size_t len) {
285  if (size_ + len > capacity_) DynReserve(size_ + len);
286 
287  const Byte* cdata = reinterpret_cast<const Byte*>(data);
288  std::copy(cdata, cdata + len, data_ + size_);
289  size_ += len;
290 
291  return *this;
292  }
293 
294  //! Append the contents of a different buffer object to this one.
295  BufferBuilder& Append(const class BufferBuilder& bb) {
296  return Append(bb.data(), bb.size());
297  }
298 
299  //! Append to contents of a std::string, excluding the null (which isn't
300  //! contained in the string size anyway).
302  return Append(s.data(), s.size());
303  }
304 
305  //! Put (append) a single item of the template type T to the buffer. Be
306  //! careful with implicit type conversions!
307  template <typename Type>
308  BufferBuilder& Put(const Type& item) {
309  static_assert(
311  "You only want to Put() trivially copyable types as raw values.");
312 
313  if (size_ + sizeof(Type) > capacity_) DynReserve(size_ + sizeof(Type));
314 
315  *reinterpret_cast<Type*>(data_ + size_) = item;
316  size_ += sizeof(Type);
317 
318  return *this;
319  }
320 
321  //! Put a single byte to the buffer (used via CRTP from ItemWriterToolsBase)
323  return Put<uint8_t>(data);
324  }
325 
326  //! Put (append) a single item of the template type T to the buffer. Be
327  //! careful with implicit type conversions!
328  template <typename Type>
329  BufferBuilder& PutRaw(const Type& item) {
330  return Put<Type>(item);
331  }
332 
333  //! \}
334 
335  //! \name Access
336  //! \{
337 
338  //! return mutable iterator to first element
340  { return data_; }
341  //! return constant iterator to first element
343  { return data_; }
344  //! return constant iterator to first element
346  { return begin(); }
347 
348  //! return mutable iterator beyond last element
350  { return data_ + size_; }
351  //! return constant iterator beyond last element
353  { return data_ + size_; }
354  //! return constant iterator beyond last element
356  { return end(); }
357 
358  //! return the i-th position of the vector
360  assert(i < size_);
361  return *(begin() + i);
362  }
363 
364  //! \}
365 };
366 
367 //! \}
368 
369 } // namespace net
370 } // namespace thrill
371 
372 #endif // !THRILL_NET_BUFFER_BUILDER_HEADER
373 
374 /******************************************************************************/
size_t size() const
Return the currently used length in bytes.
iterator begin()
return mutable iterator to first element
const_iterator cend() const
return constant iterator beyond last element
const Byte * const_iterator
simple pointer iterators
BufferBuilder & AppendString(const std::string &s)
const Byte & const_reference
simple pointer references
BufferBuilder & DynReserve(size_t n)
Type
VFS object type.
Definition: file_io.hpp:52
const_iterator begin() const
return constant iterator to first element
size_t capacity() const
Return the currently allocated buffer capacity.
~BufferBuilder()
Destroys the memory space.
Byte * iterator
simple pointer iterators
Buffer ToBuffer()
Explicit conversion to Buffer MOVING the memory ownership.
BufferBuilder & PutByte(Byte data)
Put a single byte to the buffer (used via CRTP from ItemWriterToolsBase)
BufferBuilder & Append(const void *data, size_t len)
Append a memory range to the buffer.
BufferBuilder(const std::string &str)
Constructor from std::string, COPIES string content.
BufferBuilder()=default
Create a new empty object.
BufferBuilder(size_t size)
Constructor, create object with n bytes pre-allocated.
Byte * data()
Return a writeable pointer to the currently kept memory area.
Byte & reference
simple pointer references
BufferBuilder & Deallocate()
const Byte * data() const
Return a pointer to the currently kept memory area.
BufferBuilder & Assign(const BufferBuilder &other)
size_t size_
Size of valid data.
BufferBuilder & operator=(const BufferBuilder &other)
Assignment operator: copy other&#39;s memory range into buffer.
static Buffer Acquire(void *data, size_type size)
Definition: buffer.hpp:110
BufferBuilder(const BufferBuilder &other)
Copy-Constructor, duplicates memory content.
BufferBuilder & Align(size_t n)
Align the size of the buffer to a multiple of n. Fills up with 0s.
reference operator[](size_t i)
return the i-th position of the vector
unsigned char Byte
type used to store the bytes
BufferBuilder & Append(const class BufferBuilder &bb)
Append the contents of a different buffer object to this one.
BufferBuilder & Clear()
Clears the memory contents, does not deallocate the memory.
const_iterator cbegin() const
return constant iterator to first element
std::is_trivially_copyable< T > is_trivially_copyable
Definition: defines.hpp:54
std::basic_string< char, std::char_traits< char >, Allocator< char > > string
string with Manager tracking
Definition: allocator.hpp:220
CRTP class to enhance item/memory writer classes with Varint encoding and String encoding.
std::string ToString() const
Explicit conversion to std::string (copies memory of course).
BufferBuilder & Assign(const void *data, size_t len)
BufferBuilder(const void *data, size_t size)
Constructor, copy memory area.
BufferBuilder & Reserve(size_t n)
Make sure that at least n bytes are allocated.
BufferBuilder(BufferBuilder &&other) noexcept
Move-Constructor, moves memory area.
BufferBuilder represents a dynamically growable area of memory, which can be modified by appending in...
BufferBuilder & PutRaw(const Type &item)
Byte * data_
Allocated buffer pointer.
Simple buffer of characters without initialization or growing functionality.
Definition: buffer.hpp:40
void free(void *ptr) NOEXCEPT
exported free symbol that overrides loading from libc
BufferBuilder & Put(const Type &item)
const Byte * Detach()
Detach the memory from the object, returns the memory pointer.
iterator end()
return mutable iterator beyond last element
void * realloc(void *ptr, size_t size) NOEXCEPT
exported realloc() symbol that overrides loading from libc
BufferBuilder & set_size(size_t n)
size_t capacity_
Total capacity of buffer.
const_iterator end() const
return constant iterator beyond last element