Thrill  0.1
json_logger.hpp
Go to the documentation of this file.
1 /*******************************************************************************
2  * thrill/common/json_logger.hpp
3  *
4  * Logger for statistical output in JSON format for post-processing.
5  *
6  * Part of Project Thrill - http://project-thrill.org
7  *
8  * Copyright (C) 2016 Timo Bingmann <[email protected]>
9  *
10  * All rights reserved. Published under the BSD-2 license in the LICENSE file.
11  ******************************************************************************/
12 
13 #pragma once
14 #ifndef THRILL_COMMON_JSON_LOGGER_HEADER
15 #define THRILL_COMMON_JSON_LOGGER_HEADER
16 
18 
19 #include <array>
20 #include <cassert>
21 #include <initializer_list>
22 #include <memory>
23 #include <mutex>
24 #include <sstream>
25 #include <string>
26 #include <vector>
27 
28 namespace thrill {
29 namespace common {
30 
31 // forward declarations
32 class JsonLine;
33 class ScheduleThread;
34 template <typename Type>
36 
37 //! A special class to output verbatim text
39 {
40 public:
41  explicit JsonVerbatim(const std::string& str = std::string())
42  : str_(str) { }
44 };
45 
46 //! A special class to output verbatim text
48 {
49 public:
50  explicit JsonBeginObj(const std::string& str = std::string())
51  : str_(str) { }
53 };
54 
55 //! A special class to output verbatim text
57 {
58 public:
59  JsonEndObj() { }
60 };
61 
62 //! A template to make writing temporary arrays easy: Array<int>{ 1, 2, 3 }.
63 template <typename Type>
64 using Array = Type[];
65 
66 /*!
67  * JsonLogger is a receiver of JSON output objects for logging.
68  */
70 {
71 public:
72  //! open JsonLogger with ofstream uninitialized to discard log output.
73  JsonLogger() = default;
74 
75  //! open JsonLogger with ofstream. if path is empty, output goes to stdout
76  explicit JsonLogger(const std::string& path);
77 
78  //! open JsonLogger with a super logger
79  explicit JsonLogger(JsonLogger* super);
80 
81  //! open JsonLogger with a super logger and some additional common key:value
82  //! pairs
83  template <typename... Args>
84  explicit JsonLogger(JsonLogger* super, const Args& ... args);
85 
86  //! create new JsonLine instance which will be written to this logger.
87  JsonLine line();
88 
89  template <typename Type>
90  JsonLine operator << (const Type& t);
91 
92  //! launch background profiler
93  void StartProfiler();
94 
95 public:
96  //! output to superior JsonLogger
97  JsonLogger* super_ = nullptr;
98 
99  //! direct output stream for top loggers
100  std::unique_ptr<std::ostream> os_;
101 
102  //! mutex to lock logger output
103  std::mutex mutex_;
104 
105  //! common items outputted to each line
107 
108  //! friends for sending to os_
109  friend class JsonLine;
110 
111  template <typename Type>
112  friend struct JsonLinePutSwitch;
113 };
114 
115 /*!
116  * JsonLine is an object used to aggregate a set of key:value pairs for output
117  * into a JSON log.
118  */
119 class JsonLine
120 {
121 public:
122  //! ctor: bind output
123  JsonLine(JsonLogger* logger, std::ostream& os)
124  : logger_(logger), os_(os) {
125  if (logger)
126  lock_ = std::unique_lock<std::mutex>(logger_->mutex_);
127  }
128 
129  //! non-copyable: delete copy-constructor
130  JsonLine(const JsonLine&) = delete;
131  //! non-copyable: delete assignment operator
132  JsonLine& operator = (const JsonLine&) = delete;
133  //! move-constructor: unlink pointer
135  : logger_(o.logger_), lock_(std::move(o.lock_)),
136  os_(o.os_), items_(o.items_), sub_dict_(o.sub_dict_)
137  { o.logger_ = nullptr; }
138 
139  struct ArrayTag { };
140  struct DictionaryTag { };
141 
142  //! output any type
143  template <typename Type>
144  JsonLine& operator << (Type const& t);
145 
147  // write key
148  operator << (t.str_);
149  PutSeparator();
150  os_ << '{';
151  items_ = 0;
152  return *this;
153  }
154 
156  os_ << '}';
157  return *this;
158  }
159 
160  //! destructor: deliver to output
162  Close();
163  }
164 
165  //! close the line
166  void Close() {
167  if (logger_ && items_ != 0) {
168  assert(items_ % 2 == 0);
169  os_ << '}' << std::endl;
170  items_ = 0;
171  }
172  else if (!logger_ && sub_dict_) {
173  os_ << '}';
174  sub_dict_ = false;
175  }
176  else if (!logger_ && sub_array_) {
177  os_ << ']';
178  sub_array_ = false;
179  }
180  }
181 
182  //! number of items already put
183  size_t items() const { return items_; }
184 
185  //! return JsonLine has sub-dictionary of this one
186  template <typename Key>
187  JsonLine sub(const Key& key) {
188  // write key
189  operator << (key);
190  PutSeparator();
191  os_ << '{';
192  return JsonLine(DictionaryTag(), *this);
193  }
194 
195  //! return JsonLine has sub-dictionary of this one
196  template <typename Key>
197  JsonLine arr(const Key& key) {
198  // write key
199  operator << (key);
200  PutSeparator();
201  os_ << '[';
202  return JsonLine(ArrayTag(), *this);
203  }
204 
205  //! return JsonLine has sub-dictionary of this one
207  if (items_ > 0)
208  os_ << ',';
209  os_ << '{';
210  items_++;
211  return JsonLine(DictionaryTag(), *this);
212  }
213 
214  //! put an items separator (either ',' or ':') and increment counter.
215  void PutSeparator() {
216  if (items_ > 0) {
217  os_ << (items_ % 2 == 0 ? ',' : ':');
218  }
219  items_++;
220  }
221 
222  void PutEscapedChar(char ch) {
223  // from: http://stackoverflow.com/a/7725289
224  switch (ch) {
225  case '\\': os_ << '\\' << '\\';
226  break;
227  case '"': os_ << '\\' << '"';
228  break;
229  case '/': os_ << '\\' << '/';
230  break;
231  case '\b': os_ << '\\' << 'b';
232  break;
233  case '\f': os_ << '\\' << 'f';
234  break;
235  case '\n': os_ << '\\' << 'n';
236  break;
237  case '\r': os_ << '\\' << 'r';
238  break;
239  case '\t': os_ << '\\' << 't';
240  break;
241  default: os_ << ch;
242  break;
243  }
244  }
245 
246 private:
247  //! when destructed this object is delivered to the output.
248  JsonLogger* logger_ = nullptr;
249 
250  //! lock on the logger output stream
251  std::unique_lock<std::mutex> lock_;
252 
253  //! construct sub-dictionary
254  JsonLine(struct DictionaryTag, JsonLine& parent)
255  : os_(parent.os_), sub_dict_(true) { }
256 
257  //! construct sub-dictionary
258  JsonLine(struct ArrayTag, JsonLine& parent)
259  : os_(parent.os_), sub_array_(true) { }
260 
261 public:
262  //! reference to output stream
263  std::ostream& os_;
264 
265  //! items counter for output stream
266  size_t items_ = 0;
267 
268  //! indicator for sub-dictionaries.
269  bool sub_dict_ = false;
270 
271  //! indicator for sub-array.
272  bool sub_array_ = false;
273 };
274 
275 /******************************************************************************/
276 // Template specializations for JsonLine
277 
278 static inline
279 JsonLine& Put(JsonLine& line, bool const& value) {
280  line.os_ << (value ? "true" : "false");
281  return line;
282 }
283 
284 static inline
285 JsonLine& Put(JsonLine& line, int const& value) {
286  line.os_ << value;
287  return line;
288 }
289 
290 static inline
291 JsonLine& Put(JsonLine& line, unsigned int const& value) {
292  line.os_ << value;
293  return line;
294 }
295 
296 static inline
297 JsonLine& Put(JsonLine& line, long const& value) {
298  line.os_ << value;
299  return line;
300 }
301 
302 static inline
303 JsonLine& Put(JsonLine& line, unsigned long const& value) {
304  line.os_ << value;
305  return line;
306 }
307 
308 static inline
309 JsonLine& Put(JsonLine& line, long long const& value) {
310  line.os_ << value;
311  return line;
312 }
313 
314 static inline
315 JsonLine& Put(JsonLine& line, unsigned long long const& value) {
316  line.os_ << value;
317  return line;
318 }
319 
320 static inline
321 JsonLine& Put(JsonLine& line, double const& value) {
322  line.os_ << value;
323  return line;
324 }
325 
326 static inline
327 JsonLine& Put(JsonLine& line, const char* const& str) {
328  line.os_ << '"';
329  for (const char* s = str; *s; ++s) line.PutEscapedChar(*s);
330  line.os_ << '"';
331  return line;
332 }
333 
334 static inline
335 JsonLine& Put(JsonLine& line, std::string const& str) {
336  line.os_ << '"';
337  for (std::string::const_iterator i = str.begin(); i != str.end(); ++i)
338  line.PutEscapedChar(*i);
339  line.os_ << '"';
340  return line;
341 }
342 
343 template <typename Type, std::size_t N>
344 static inline
345 JsonLine& Put(JsonLine& line, const Type (& arr)[N]) {
346  line.os_ << '[';
347  for (size_t i = 0; i < N; ++i) {
348  if (i != 0) line.os_ << ',';
349  Put(line, arr[i]);
350  }
351  line.os_ << ']';
352  return line;
353 }
354 
355 template <typename Type>
356 static inline
357 JsonLine& Put(JsonLine& line, std::initializer_list<Type> const& list) {
358  line.os_ << '[';
359  for (typename std::initializer_list<Type>::const_iterator it = list.begin();
360  it != list.end(); ++it) {
361  if (it != list.begin())
362  line.os_ << ',';
363  Put(line, *it);
364  }
365  line.os_ << ']';
366  return line;
367 }
368 
369 template <typename Type>
370 static inline
371 JsonLine& Put(JsonLine& line, std::vector<Type> const& vec) {
372  line.os_ << '[';
373  for (typename std::vector<Type>::const_iterator it = vec.begin();
374  it != vec.end(); ++it) {
375  if (it != vec.begin())
376  line.os_ << ',';
377  Put(line, *it);
378  }
379  line.os_ << ']';
380  return line;
381 }
382 
383 template <typename Type, std::size_t N>
384 static inline
385 JsonLine& Put(JsonLine& line, std::array<Type, N> const& arr) {
386  line.os_ << '[';
387  for (typename std::array<Type, N>::const_iterator it = arr.begin();
388  it != arr.end(); ++it) {
389  if (it != arr.begin())
390  line.os_ << ',';
391  Put(line, *it);
392  }
393  line.os_ << ']';
394  return line;
395 }
396 
397 static inline
398 JsonLine& Put(JsonLine& line, JsonVerbatim const& verbatim) {
399  // undo increment of item counter
400  --line.items_;
401  line.os_ << verbatim.str_;
402  return line;
403 }
404 
405 //! template << forwards to ::Put for ADL type switching
406 template <typename Type>
408  PutSeparator();
409  return Put(*this, t);
410 }
411 
412 /******************************************************************************/
413 // JsonLogger
414 
415 template <typename... Args>
416 JsonLogger::JsonLogger(JsonLogger* super, const Args& ... args)
417  : JsonLogger(super) {
418 
419  std::ostringstream oss;
420  {
421  // use JsonLine writer without a Logger to generate a valid string.
422  JsonLine json(nullptr, oss);
423  // -tb: do not use tlx::vexpand() because the order of argument
424  // evaluation is undefined.
425  tlx::call_foreach([&json](const auto& a) { json << a; }, args...);
426  }
427  common_ = JsonVerbatim(oss.str());
428 }
429 
430 template <typename Type>
432  JsonLine out = line();
433  out << t;
434  return out;
435 }
436 
437 } // namespace common
438 } // namespace thrill
439 
440 #endif // !THRILL_COMMON_JSON_LOGGER_HEADER
441 
442 /******************************************************************************/
JsonLine operator<<(const Type &t)
JsonVerbatim(const std::string &str=std::string())
Definition: json_logger.hpp:41
A special class to output verbatim text.
Definition: json_logger.hpp:47
Type[] Array
A template to make writing temporary arrays easy: Array<int>{ 1, 2, 3 }.
Definition: json_logger.hpp:64
JsonLine(JsonLogger *logger, std::ostream &os)
ctor: bind output
Type
VFS object type.
Definition: file_io.hpp:52
JsonLine obj()
return JsonLine has sub-dictionary of this one
JsonLogger()=default
open JsonLogger with ofstream uninitialized to discard log output.
void Close()
close the line
STL namespace.
size_t items() const
number of items already put
A special class to output verbatim text.
Definition: json_logger.hpp:56
std::mutex mutex_
mutex to lock logger output
JsonLine line()
create new JsonLine instance which will be written to this logger.
Definition: json_logger.cpp:57
~JsonLine()
destructor: deliver to output
int N
Definition: gen_data.py:15
JsonLine(JsonLine &&o)
move-constructor: unlink pointer
int value
Definition: gen_data.py:41
std::ostream & os_
reference to output stream
void PutSeparator()
put an items separator (either &#39;,&#39; or &#39;:&#39;) and increment counter.
JsonLine(struct ArrayTag, JsonLine &parent)
construct sub-dictionary
std::unique_lock< std::mutex > lock_
lock on the logger output stream
std::basic_string< char, std::char_traits< char >, Allocator< char > > string
string with Manager tracking
Definition: allocator.hpp:220
JsonVerbatim common_
common items outputted to each line
JsonLine & operator<<(Type const &t)
output any type
void PutEscapedChar(char ch)
JsonLine sub(const Key &key)
return JsonLine has sub-dictionary of this one
std::unique_ptr< std::ostream > os_
direct output stream for top loggers
static JsonLine & Put(JsonLine &line, bool const &value)
size_t items_
items counter for output stream
JsonLogger is a receiver of JSON output objects for logging.
Definition: json_logger.hpp:69
A special class to output verbatim text.
Definition: json_logger.hpp:38
void call_foreach(Functor &&f, Args &&... args)
const struct ArrayTag ArrayTag
global const ArrayTag instance
Definition: zip_window.hpp:48
JsonLine arr(const Key &key)
return JsonLine has sub-dictionary of this one
JsonBeginObj(const std::string &str=std::string())
Definition: json_logger.hpp:50
std::ostream & operator<<(std::ostream &os, const DIABase &d)
make ostream-able.
Definition: dia_base.cpp:449
JsonLine is an object used to aggregate a set of key:value pairs for output into a JSON log...
JsonLine(struct DictionaryTag, JsonLine &parent)
construct sub-dictionary