Thrill  0.1
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
config.cpp
Go to the documentation of this file.
1 /***************************************************************************
2  * foxxll/mng/config.cpp
3  *
4  * Part of FOXXLL. See http://foxxll.org
5  *
6  * Copyright (C) 2002 Roman Dementiev <[email protected]>
7  * Copyright (C) 2007, 2009 Johannes Singler <[email protected]>
8  * Copyright (C) 2008, 2009 Andreas Beckmann <[email protected]>
9  * Copyright (C) 2013 Timo Bingmann <[email protected]>
10  *
11  * Distributed under the Boost Software License, Version 1.0.
12  * (See accompanying file LICENSE_1_0.txt or copy at
13  * http://www.boost.org/LICENSE_1_0.txt)
14  **************************************************************************/
15 
16 #include <cassert>
17 #include <fstream>
18 
19 #include <tlx/logger/core.hpp>
20 
22 #include <foxxll/common/utils.hpp>
23 #include <foxxll/config.hpp>
24 #include <foxxll/io/file.hpp>
25 #include <foxxll/mng/config.hpp>
26 #include <foxxll/version.hpp>
29 #include <tlx/string/split.hpp>
30 
31 #if FOXXLL_WINDOWS
32  #ifndef NOMINMAX
33  #define NOMINMAX
34  #endif
35  #include <windows.h>
36 #else
37  #include <unistd.h>
38 #endif
39 
40 namespace foxxll {
41 
42 static inline bool exist_file(const std::string& path)
43 {
44  TLX_LOG0 << "Checking " << path << " for disk configuration.";
45  std::ifstream in(path.c_str());
46  return in.good();
47 }
48 
50  : is_initialized(false)
51 {
54 }
55 
57 {
58  for (disk_list_type::const_iterator it = disks_list.begin();
59  it != disks_list.end(); it++)
60  {
61  if (it->delete_on_exit)
62  {
63  TLX_LOG1 << "Removing disk file: " << it->path;
64  unlink(it->path.c_str());
65  }
66  }
67 }
68 
70 {
71  // if disks_list is empty, then try to load disk configuration files
72  if (disks_list.size() == 0)
73  {
74  find_config();
75  }
76 
77  max_device_id_ = 0;
78 
79  is_initialized = true;
80 }
81 
83 {
84  // check several locations for disk configuration files
85 
86  // check STXXLCFG environment path
87  const char* cfg = getenv("STXXLCFG");
88  if (cfg && exist_file(cfg))
89  return load_config_file(cfg);
90 
91 #if !FOXXLL_WINDOWS
92  // read environment, unix style
93  const char* hostname = getenv("HOSTNAME");
94  const char* home = getenv("HOME");
95  const char* suffix = "";
96 #else
97  // read environment, windows style
98  const char* hostname = getenv("COMPUTERNAME");
99  const char* home = getenv("APPDATA");
100  const char* suffix = ".txt";
101 #endif
102 
103  // check current directory
104  {
105  std::string basepath = "./.stxxl";
106 
107  if (hostname && exist_file(basepath + "." + hostname + suffix))
108  return load_config_file(basepath + "." + hostname + suffix);
109 
110  if (exist_file(basepath + suffix))
111  return load_config_file(basepath + suffix);
112  }
113 
114  // check home directory
115  if (home)
116  {
117  std::string basepath = std::string(home) + "/.stxxl";
118 
119  if (hostname && exist_file(basepath + "." + hostname + suffix))
120  return load_config_file(basepath + "." + hostname + suffix);
121 
122  if (exist_file(basepath + suffix))
123  return load_config_file(basepath + suffix);
124  }
125 
126  // load default configuration
128 }
129 
131 {
132  TLX_LOG1 << "Warning: no config file found.";
133  TLX_LOG1 << "Using default disk configuration.";
134  disk_config entry1(default_disk_path(), 1000 * 1024 * 1024, default_disk_io_impl());
135  entry1.delete_on_exit = true;
136  entry1.autogrow = true;
137  disks_list.push_back(entry1);
138 
139  // no flash disks
140  first_flash = static_cast<unsigned int>(disks_list.size());
141 }
142 
143 void config::load_config_file(const std::string& config_path)
144 {
145  std::vector<disk_config> flash_list;
146  std::ifstream cfg_file(config_path.c_str());
147 
148  if (!cfg_file)
149  return load_default_config();
150 
151  std::string line;
152 
153  while (std::getline(cfg_file, line))
154  {
155  // skip comments
156  if (line.size() == 0 || line[0] == '#') continue;
157 
158  disk_config entry;
159  entry.parse_line(line); // throws on errors
160 
161  if (!entry.flash)
162  disks_list.push_back(entry);
163  else
164  flash_list.push_back(entry);
165  }
166  cfg_file.close();
167 
168  // put flash devices after regular disks
169  first_flash = static_cast<unsigned int>(disks_list.size());
170  disks_list.insert(disks_list.end(), flash_list.begin(), flash_list.end());
171 
172  if (disks_list.empty()) {
173  FOXXLL_THROW(
174  std::runtime_error,
175  "No disks found in '" << config_path << "'."
176  );
177  }
178 }
179 
181 {
182  disks_list.push_back(cfg);
183  return *this;
184 }
185 
186 unsigned int config::max_device_id()
187 {
188  return max_device_id_;
189 }
190 
192 {
193  return max_device_id_++;
194 }
195 
196 void config::update_max_device_id(unsigned int devid)
197 {
198  if (max_device_id_ < devid + 1)
199  max_device_id_ = devid + 1;
200 }
201 
202 std::pair<unsigned, unsigned> config::regular_disk_range() const
203 {
204  assert(is_initialized);
205  return std::pair<unsigned, unsigned>(0, first_flash);
206 }
207 
208 std::pair<unsigned, unsigned> config::flash_range() const
209 {
210  assert(is_initialized);
211  return std::pair<unsigned, unsigned>(first_flash, static_cast<unsigned>(disks_list.size()));
212 }
213 
215 {
217  assert(disk < disks_list.size());
218  return disks_list[disk];
219 }
220 
221 const std::string& config::disk_path(size_t disk) const
222 {
223  assert(is_initialized);
224  return disks_list[disk].path;
225 }
226 
228 {
229 #if !FOXXLL_WINDOWS
230  return "/var/tmp/foxxll";
231 #else
232  char* tmpstr = new char[255];
233  if (GetTempPathA(255, tmpstr) == 0)
234  FOXXLL_THROW_WIN_LASTERROR(resource_error, "GetTempPathA()");
235  std::string result = tmpstr;
236  result += "foxxll.tmp";
237  delete[] tmpstr;
238  return result;
239 #endif
240 }
241 
243 {
244  assert(is_initialized);
245  return disks_list[disk].size;
246 }
247 
248 const std::string& config::disk_io_impl(size_t disk) const
249 {
250  assert(is_initialized);
251  return disks_list[disk].io_impl;
252 }
253 
255 {
256 #if !FOXXLL_WINDOWS
257  return "syscall";
258 #else
259  return "wincall";
260 #endif
261 }
262 
264 {
265  assert(is_initialized);
266 
268 
269  for (disk_list_type::const_iterator it = disks_list.begin();
270  it != disks_list.end(); it++)
271  {
272  total_size += it->size;
273  }
274 
275  return total_size;
276 }
277 
278 ////////////////////////////////////////////////////////////////////////////////
279 
281  : size(0),
282  autogrow(true),
283  delete_on_exit(false),
284  direct(DIRECT_TRY),
285  flash(false),
286  queue(file::DEFAULT_QUEUE),
287  device_id(file::DEFAULT_DEVICE_ID),
288  raw_device(false),
289  unlink_on_open(false),
290  queue_length(0)
291 { }
292 
294  const std::string& _io_impl)
295  : path(_path),
296  size(_size),
297  io_impl(_io_impl),
298  autogrow(true),
299  delete_on_exit(false),
300  direct(DIRECT_TRY),
301  flash(false),
302  queue(file::DEFAULT_QUEUE),
303  device_id(file::DEFAULT_DEVICE_ID),
304  raw_device(false),
305  unlink_on_open(false),
306  queue_length(0)
307 {
308  parse_fileio();
309 }
310 
312  : size(0),
313  autogrow(true),
314  delete_on_exit(false),
315  direct(DIRECT_TRY),
316  flash(false),
317  queue(file::DEFAULT_QUEUE),
318  device_id(file::DEFAULT_DEVICE_ID),
319  raw_device(false),
320  unlink_on_open(false),
321  queue_length(0)
322 {
323  parse_line(line);
324 }
325 
327 {
328  // split off disk= or flash=
329  std::vector<std::string> eqfield = tlx::split('=', line, 2, 2);
330 
331  if (eqfield[0] == "disk") {
332  flash = false;
333  }
334  else if (eqfield[0] == "flash") {
335  flash = true;
336  }
337  else {
338  FOXXLL_THROW(
339  std::runtime_error,
340  "Unknown configuration token " << eqfield[0]
341  );
342  }
343 
344  // *** Set Default Extra Options ***
345 
346  autogrow = true; // was default for a long time, have to keep it this way
347  delete_on_exit = false;
348  direct = DIRECT_TRY;
349  // flash is already set
352  unlink_on_open = false;
353 
354  // *** Save Basic Options ***
355 
356  // split at commands, at least 3 fields
357  std::vector<std::string> cmfield = tlx::split(',', eqfield[1], 3, 3);
358 
359  // path:
361  // replace $$ -> pid in path
362  {
363  std::string::size_type pos;
364  if ((pos = path.find("$$")) != std::string::npos)
365  {
366 #if !FOXXLL_WINDOWS
367  int pid = getpid();
368 #else
369  DWORD pid = GetCurrentProcessId();
370 #endif
371  path.replace(pos, 3, to_str(pid));
372  }
373  }
374 
375  // size: (default unit MiB)
376  if (!tlx::parse_si_iec_units(cmfield[1], &size, 'M')) {
377  FOXXLL_THROW(
378  std::runtime_error,
379  "Invalid disk size '" << cmfield[1] << "' in disk configuration file."
380  );
381  }
382 
383  if (size == 0) {
384  autogrow = true;
385  delete_on_exit = true;
386  }
387 
388  // io_impl:
389  io_impl = cmfield[2];
390  parse_fileio();
391 }
392 
394 {
395  // skip over leading spaces
396  size_t leadspace = io_impl.find_first_not_of(' ');
397  if (leadspace > 0)
398  io_impl = io_impl.substr(leadspace);
399 
400  // split off extra fileio parameters
401  size_t spacepos = io_impl.find(' ');
402  if (spacepos == std::string::npos)
403  return; // no space in fileio
404 
405  // *** Parse Extra Fileio Parameters ***
406 
407  std::string paramstr = io_impl.substr(spacepos + 1);
408  io_impl = io_impl.substr(0, spacepos);
409 
410  std::vector<std::string> param = tlx::split(' ', paramstr);
411 
412  for (std::vector<std::string>::const_iterator p = param.begin();
413  p != param.end(); ++p)
414  {
415  // split at equal sign
416  std::vector<std::string> eq = tlx::split('=', *p, 2, 2);
417 
418  // *** PLEASE try to keep the elseifs sorted by parameter name!
419  if (*p == "") {
420  // skip blank options
421  }
422  else if (*p == "autogrow" || *p == "noautogrow" || eq[0] == "autogrow")
423  {
424  // TODO: which fileio implementation support autogrow?
425 
426  if (*p == "autogrow") autogrow = true;
427  else if (*p == "noautogrow") autogrow = false;
428  else if (eq[1] == "off") autogrow = false;
429  else if (eq[1] == "on") autogrow = true;
430  else if (eq[1] == "no") autogrow = false;
431  else if (eq[1] == "yes") autogrow = true;
432  else
433  {
434  FOXXLL_THROW(
435  std::runtime_error,
436  "Invalid parameter '" << *p << "' in disk configuration file."
437  );
438  }
439  }
440  else if (*p == "delete" || *p == "delete_on_exit")
441  {
442  delete_on_exit = true;
443  }
444  else if (*p == "direct" || *p == "nodirect" || eq[0] == "direct")
445  {
446  // io_impl is not checked here, but I guess that is okay for DIRECT
447  // since it depends highly platform _and_ build-time configuration.
448 
449  if (*p == "direct") direct = DIRECT_ON; // force ON
450  else if (*p == "nodirect") direct = DIRECT_OFF; // force OFF
451  else if (eq[1] == "off") direct = DIRECT_OFF;
452  else if (eq[1] == "try") direct = DIRECT_TRY;
453  else if (eq[1] == "on") direct = DIRECT_ON;
454  else if (eq[1] == "no") direct = DIRECT_OFF;
455  else if (eq[1] == "yes") direct = DIRECT_ON;
456  else
457  {
458  FOXXLL_THROW(
459  std::runtime_error,
460  "Invalid parameter '" << *p << "' in disk configuration file."
461  );
462  }
463  }
464  else if (eq[0] == "queue")
465  {
466  if (io_impl == "linuxaio") {
467  FOXXLL_THROW(std::runtime_error, "Parameter '" << *p << "' invalid for fileio '" << io_impl << "' in disk configuration file.");
468  }
469 
470  char* endp;
471  queue = static_cast<int>(strtoul(eq[1].c_str(), &endp, 10));
472  if (endp && *endp != 0) {
473  FOXXLL_THROW(
474  std::runtime_error,
475  "Invalid parameter '" << *p << "' in disk configuration file."
476  );
477  }
478  }
479  else if (eq[0] == "queue_length")
480  {
481  if (io_impl != "linuxaio") {
482  FOXXLL_THROW(
483  std::runtime_error, "Parameter '" << *p << "' "
484  "is only valid for fileio linuxaio "
485  "in disk configuration file."
486  );
487  }
488 
489  char* endp;
490  queue_length = static_cast<int>(strtoul(eq[1].c_str(), &endp, 10));
491  if (endp && *endp != 0) {
492  FOXXLL_THROW(
493  std::runtime_error,
494  "Invalid parameter '" << *p << "' in disk configuration file."
495  );
496  }
497  }
498  else if (eq[0] == "device_id" || eq[0] == "devid")
499  {
500  char* endp;
501  device_id = static_cast<int>(strtoul(eq[1].c_str(), &endp, 10));
502  if (endp && *endp != 0) {
503  FOXXLL_THROW(
504  std::runtime_error,
505  "Invalid parameter '" << *p << "' in disk configuration file."
506  );
507  }
508  }
509  else if (*p == "raw_device")
510  {
511  if (!(io_impl == "syscall")) {
512  FOXXLL_THROW(std::runtime_error, "Parameter '" << *p << "' invalid for fileio '" << io_impl << "' in disk configuration file.");
513  }
514 
515  raw_device = true;
516  }
517  else if (*p == "unlink" || *p == "unlink_on_open")
518  {
519  if (!(io_impl == "syscall" || io_impl == "linuxaio" ||
520  io_impl == "mmap"))
521  {
522  FOXXLL_THROW(std::runtime_error, "Parameter '" << *p << "' invalid for fileio '" << io_impl << "' in disk configuration file.");
523  }
524 
525  unlink_on_open = true;
526  }
527  else
528  {
529  FOXXLL_THROW(
530  std::runtime_error,
531  "Invalid optional parameter '" << *p << "' in disk configuration file."
532  );
533  }
534  }
535 }
536 
538 {
539  std::ostringstream oss;
540 
541  oss << io_impl;
542 
543  if (!autogrow)
544  oss << " autogrow=no";
545 
546  if (delete_on_exit)
547  oss << " delete_on_exit";
548 
549  // tristate direct variable: OFF, TRY, ON
550  if (direct == DIRECT_OFF)
551  {
552  oss << " direct=off";
553  }
554  else if (direct == DIRECT_TRY)
555  { } // silenced: oss << " direct=try";
556  else if (direct == DIRECT_ON)
557  {
558  oss << " direct=on";
559  }
560  else
561  FOXXLL_THROW(std::runtime_error, "Invalid setting for 'direct' option.");
562 
563  if (flash) {
564  oss << " flash";
565  }
566 
568  oss << " queue=" << queue;
569  }
570 
572  oss << " devid=" << device_id;
573  }
574 
575  if (raw_device) {
576  oss << " raw_device";
577  }
578 
579  if (unlink_on_open) {
580  oss << " unlink_on_open";
581  }
582 
583  if (queue_length != 0) {
584  oss << " queue_length=" << queue_length;
585  }
586 
587  return oss.str();
588 }
589 
590 } // namespace foxxll
591 
592 /**************************************************************************/
external_size_type size
file size to initially allocate
Definition: config.hpp:46
std::pair< unsigned, unsigned > regular_disk_range() const
Returns contiguous range of regular disks w/o flash devices in the array of all disks.
Definition: config.cpp:202
void update_max_device_id(unsigned int devid)
Update the automatic physical device id counter.
Definition: config.cpp:196
unsigned int next_device_id()
Returns next automatic physical device id counter.
Definition: config.cpp:191
std::string & expand_environment_variables(std::string *sp)
Expand substrings $ABC_123 and ${ABC_123} into the corresponding environment variables.
std::string path
the file path used by the io implementation
Definition: config.hpp:43
void initialize()
Definition: config.cpp:69
void check_initialized()
Definition: config.hpp:151
void parse_fileio()
parse the "io_impl" parameter into the optional parameter fields.
Definition: config.cpp:393
static const unsigned int DEFAULT_DEVICE_ID
Definition: file.hpp:92
unsigned int max_device_id()
Returns automatic physical device id counter.
Definition: config.cpp:186
#define FOXXLL_THROW(exception_type, error_message)
Throws exception_type with "Error in [function] : [error_message]".
void load_config_file(const std::string &config_path)
Load disk configuration file.
Definition: config.cpp:143
bool unlink_on_open
unlink file immediately after opening (available on most Unix)
Definition: config.hpp:103
static std::string default_disk_path()
Definition: config.cpp:227
std::string io_impl
io implementation to access file
Definition: config.hpp:49
disk_config()
default constructor
Definition: config.cpp:280
static std::string default_disk_io_impl()
Returns name of the default I/O implementation.
Definition: config.cpp:254
unsigned int max_device_id_
static counter for automatic physical device enumeration
Definition: config.hpp:175
std::string to_str(const Type &t)
Format any ostream-able type into a string.
Definition: utils.hpp:36
void parse_line(const std::string &line)
Definition: config.cpp:326
std::vector< std::string > split(char sep, const std::string &str, std::string::size_type limit)
Split the given string at each separator character into distinct substrings.
Definition: split.cpp:20
void find_config()
Search several places for a config file.
Definition: config.cpp:82
~config()
deletes autogrow files
Definition: config.cpp:56
disk_config & disk(size_t disk)
Returns mutable disk_config structure for additional disk parameters.
Definition: config.cpp:214
static bool exist_file(const std::string &path)
Definition: config.cpp:42
disk_list_type disks_list
list of configured disks
Definition: config.hpp:122
bool parse_si_iec_units(const char *str, uint64_t *out_size, char default_unit)
Parse a string like "343KB" or "44 GiB" into the corresponding size in bytes.
external_size_type total_size() const
Returns the total size over all disks.
Definition: config.cpp:263
bool flash
marks flash drives (configuration entries with flash= instead of disk=)
Definition: config.hpp:89
static const int DEFAULT_QUEUE
Definition: file.hpp:89
std::basic_string< char, std::char_traits< char >, Allocator< char > > string
string with Manager tracking
Definition: allocator.hpp:220
#define TLX_LOG0
Override default output: never or always output log.
Definition: core.hpp:144
bool is_initialized
Finished initializing config.
Definition: config.hpp:128
std::string fileio_string() const
return formatted fileio name and optional configuration parameters
Definition: config.cpp:537
const std::string & disk_path(size_t disk) const
Definition: config.cpp:221
void print_library_version_mismatch()
Check and print mismatch between header and library versions.
Definition: version.hpp:84
const std::string & disk_io_impl(size_t disk) const
Definition: config.cpp:248
bool raw_device
turned on by syscall fileio when the path points to a raw block device
Definition: config.hpp:100
int queue_length
desired queue length for linuxaio_file and linuxaio_queue
Definition: config.hpp:106
static const int DEFAULT_LINUXAIO_QUEUE
Definition: file.hpp:90
external_size_type disk_size(size_t disk) const
Definition: config.cpp:242
config & add_disk(const disk_config &cfg)
Definition: config.cpp:180
bool delete_on_exit
delete file on program exit (default for autoconfigurated files)
Definition: config.hpp:81
#define TLX_LOG1
Definition: core.hpp:145
void load_default_config()
Load default configuration.
Definition: config.cpp:130
bool autogrow
autogrow file if more disk space is needed, automatically set if size == 0.
Definition: config.hpp:78
unsigned int device_id
Definition: config.hpp:97
std::pair< unsigned, unsigned > flash_range() const
Definition: config.cpp:208
unsigned first_flash
In disks_list, flash devices come after all regular disks.
Definition: config.hpp:125
uint64_t external_size_type
Definition: types.hpp:27
enum foxxll::disk_config::direct_type direct
std::string get_version_string_long()
Return longer "X.Y.Z (feature) (version)" version string (of headers)
Definition: version.hpp:40