1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 """Classes to handle advanced configuration in simple to complex applications.
19
20 Allows to load the configuration from a file or from command line
21 options, to generate a sample configuration file or to display
22 program's usage. Fills the gap between optik/optparse and ConfigParser
23 by adding data types (which are also available as a standalone optik
24 extension in the `optik_ext` module).
25
26
27 Quick start: simplest usage
28 ---------------------------
29
30 .. python ::
31
32 >>> import sys
33 >>> from logilab.common.configuration import Configuration
34 >>> options = [('dothis', {'type':'yn', 'default': True, 'metavar': '<y or n>'}),
35 ... ('value', {'type': 'string', 'metavar': '<string>'}),
36 ... ('multiple', {'type': 'csv', 'default': ('yop',),
37 ... 'metavar': '<comma separated values>',
38 ... 'help': 'you can also document the option'}),
39 ... ('number', {'type': 'int', 'default':2, 'metavar':'<int>'}),
40 ... ]
41 >>> config = Configuration(options=options, name='My config')
42 >>> print config['dothis']
43 True
44 >>> print config['value']
45 None
46 >>> print config['multiple']
47 ('yop',)
48 >>> print config['number']
49 2
50 >>> print config.help()
51 Usage: [options]
52
53 Options:
54 -h, --help show this help message and exit
55 --dothis=<y or n>
56 --value=<string>
57 --multiple=<comma separated values>
58 you can also document the option [current: none]
59 --number=<int>
60
61 >>> f = open('myconfig.ini', 'w')
62 >>> f.write('''[MY CONFIG]
63 ... number = 3
64 ... dothis = no
65 ... multiple = 1,2,3
66 ... ''')
67 >>> f.close()
68 >>> config.load_file_configuration('myconfig.ini')
69 >>> print config['dothis']
70 False
71 >>> print config['value']
72 None
73 >>> print config['multiple']
74 ['1', '2', '3']
75 >>> print config['number']
76 3
77 >>> sys.argv = ['mon prog', '--value', 'bacon', '--multiple', '4,5,6',
78 ... 'nonoptionargument']
79 >>> print config.load_command_line_configuration()
80 ['nonoptionargument']
81 >>> print config['value']
82 bacon
83 >>> config.generate_config()
84 # class for simple configurations which don't need the
85 # manager / providers model and prefer delegation to inheritance
86 #
87 # configuration values are accessible through a dict like interface
88 #
89 [MY CONFIG]
90
91 dothis=no
92
93 value=bacon
94
95 # you can also document the option
96 multiple=4,5,6
97
98 number=3
99
100 Note : starting with Python 2.7 ConfigParser is able to take into
101 account the order of occurrences of the options into a file (by
102 using an OrderedDict). If you have two options changing some common
103 state, like a 'disable-all-stuff' and a 'enable-some-stuff-a', their
104 order of appearance will be significant : the last specified in the
105 file wins. For earlier version of python and logilab.common newer
106 than 0.61 the behaviour is unspecified.
107
108 """
109
110 from __future__ import print_function
111
112 __docformat__ = "restructuredtext en"
113
114 __all__ = ('OptionsManagerMixIn', 'OptionsProviderMixIn',
115 'ConfigurationMixIn', 'Configuration',
116 'OptionsManager2ConfigurationAdapter')
117
118 import os
119 import sys
120 import re
121 from os.path import exists, expanduser
122 from copy import copy
123 from warnings import warn
124
125 from six import integer_types, string_types
126 from six.moves import range, configparser as cp, input
127
128 from logilab.common.compat import str_encode as _encode
129 from logilab.common.deprecation import deprecated
130 from logilab.common.textutils import normalize_text, unquote
131 from logilab.common import optik_ext
132
133 OptionError = optik_ext.OptionError
134
135 REQUIRED = []
136
138 """raised by set_option when it doesn't know what to do for an action"""
139
140
142 encoding = encoding or getattr(stream, 'encoding', None)
143 if not encoding:
144 import locale
145 encoding = locale.getpreferredencoding()
146 return encoding
147
148
149
150
151
152
153
155 """validate and return a converted value for option of type 'choice'
156 """
157 if not value in optdict['choices']:
158 msg = "option %s: invalid value: %r, should be in %s"
159 raise optik_ext.OptionValueError(msg % (name, value, optdict['choices']))
160 return value
161
163 """validate and return a converted value for option of type 'choice'
164 """
165 choices = optdict['choices']
166 values = optik_ext.check_csv(None, name, value)
167 for value in values:
168 if not value in choices:
169 msg = "option %s: invalid value: %r, should be in %s"
170 raise optik_ext.OptionValueError(msg % (name, value, choices))
171 return values
172
174 """validate and return a converted value for option of type 'csv'
175 """
176 return optik_ext.check_csv(None, name, value)
177
179 """validate and return a converted value for option of type 'yn'
180 """
181 return optik_ext.check_yn(None, name, value)
182
184 """validate and return a converted value for option of type 'named'
185 """
186 return optik_ext.check_named(None, name, value)
187
189 """validate and return a filepath for option of type 'file'"""
190 return optik_ext.check_file(None, name, value)
191
193 """validate and return a valid color for option of type 'color'"""
194 return optik_ext.check_color(None, name, value)
195
197 """validate and return a string for option of type 'password'"""
198 return optik_ext.check_password(None, name, value)
199
201 """validate and return a mx DateTime object for option of type 'date'"""
202 return optik_ext.check_date(None, name, value)
203
205 """validate and return a time object for option of type 'time'"""
206 return optik_ext.check_time(None, name, value)
207
209 """validate and return an integer for option of type 'bytes'"""
210 return optik_ext.check_bytes(None, name, value)
211
212
213 VALIDATORS = {'string': unquote,
214 'int': int,
215 'float': float,
216 'file': file_validator,
217 'font': unquote,
218 'color': color_validator,
219 'regexp': re.compile,
220 'csv': csv_validator,
221 'yn': yn_validator,
222 'bool': yn_validator,
223 'named': named_validator,
224 'password': password_validator,
225 'date': date_validator,
226 'time': time_validator,
227 'bytes': bytes_validator,
228 'choice': choice_validator,
229 'multiple_choice': multiple_choice_validator,
230 }
231
233 if opttype not in VALIDATORS:
234 raise Exception('Unsupported type "%s"' % opttype)
235 try:
236 return VALIDATORS[opttype](optdict, option, value)
237 except TypeError:
238 try:
239 return VALIDATORS[opttype](value)
240 except optik_ext.OptionValueError:
241 raise
242 except:
243 raise optik_ext.OptionValueError('%s value (%r) should be of type %s' %
244 (option, value, opttype))
245
246
247
248
249
250
251
260
264
276 return input_validator
277
278 INPUT_FUNCTIONS = {
279 'string': input_string,
280 'password': input_password,
281 }
282
283 for opttype in VALIDATORS.keys():
284 INPUT_FUNCTIONS.setdefault(opttype, _make_input_function(opttype))
285
286
287
289 """monkey patch OptionParser.expand_default since we have a particular
290 way to handle defaults to avoid overriding values in the configuration
291 file
292 """
293 if self.parser is None or not self.default_tag:
294 return option.help
295 optname = option._long_opts[0][2:]
296 try:
297 provider = self.parser.options_manager._all_options[optname]
298 except KeyError:
299 value = None
300 else:
301 optdict = provider.get_option_def(optname)
302 optname = provider.option_attrname(optname, optdict)
303 value = getattr(provider.config, optname, optdict)
304 value = format_option_value(optdict, value)
305 if value is optik_ext.NO_DEFAULT or not value:
306 value = self.NO_DEFAULT_VALUE
307 return option.help.replace(self.default_tag, str(value))
308
309
311 """return a validated value for an option according to its type
312
313 optional argument name is only used for error message formatting
314 """
315 try:
316 _type = optdict['type']
317 except KeyError:
318
319 return value
320 return _call_validator(_type, optdict, name, value)
321 convert = deprecated('[0.60] convert() was renamed _validate()')(_validate)
322
323
324
329
346
361
380
388
405
406 format_section = ini_format_section
407
426
427
428
430 """MixIn to handle a configuration from both a configuration file and
431 command line options
432 """
433
434 - def __init__(self, usage, config_file=None, version=None, quiet=0):
435 self.config_file = config_file
436 self.reset_parsers(usage, version=version)
437
438 self.options_providers = []
439
440 self._all_options = {}
441 self._short_options = {}
442 self._nocallback_options = {}
443 self._mygroups = dict()
444
445 self.quiet = quiet
446 self._maxlevel = 0
447
449
450 self.cfgfile_parser = cp.ConfigParser()
451
452 self.cmdline_parser = optik_ext.OptionParser(usage=usage, version=version)
453 self.cmdline_parser.options_manager = self
454 self._optik_option_attrs = set(self.cmdline_parser.option_class.ATTRS)
455
457 """register an options provider"""
458 assert provider.priority <= 0, "provider's priority can't be >= 0"
459 for i in range(len(self.options_providers)):
460 if provider.priority > self.options_providers[i].priority:
461 self.options_providers.insert(i, provider)
462 break
463 else:
464 self.options_providers.append(provider)
465 non_group_spec_options = [option for option in provider.options
466 if 'group' not in option[1]]
467 groups = getattr(provider, 'option_groups', ())
468 if own_group and non_group_spec_options:
469 self.add_option_group(provider.name.upper(), provider.__doc__,
470 non_group_spec_options, provider)
471 else:
472 for opt, optdict in non_group_spec_options:
473 self.add_optik_option(provider, self.cmdline_parser, opt, optdict)
474 for gname, gdoc in groups:
475 gname = gname.upper()
476 goptions = [option for option in provider.options
477 if option[1].get('group', '').upper() == gname]
478 self.add_option_group(gname, gdoc, goptions, provider)
479
481 """add an option group including the listed options
482 """
483 assert options
484
485 if group_name in self._mygroups:
486 group = self._mygroups[group_name]
487 else:
488 group = optik_ext.OptionGroup(self.cmdline_parser,
489 title=group_name.capitalize())
490 self.cmdline_parser.add_option_group(group)
491 group.level = provider.level
492 self._mygroups[group_name] = group
493
494 if group_name != "DEFAULT":
495 self.cfgfile_parser.add_section(group_name)
496
497 for opt, optdict in options:
498 self.add_optik_option(provider, group, opt, optdict)
499
501 if 'inputlevel' in optdict:
502 warn('[0.50] "inputlevel" in option dictionary for %s is deprecated,'
503 ' use "level"' % opt, DeprecationWarning)
504 optdict['level'] = optdict.pop('inputlevel')
505 args, optdict = self.optik_option(provider, opt, optdict)
506 option = optikcontainer.add_option(*args, **optdict)
507 self._all_options[opt] = provider
508 self._maxlevel = max(self._maxlevel, option.level or 0)
509
511 """get our personal option definition and return a suitable form for
512 use with optik/optparse
513 """
514 optdict = copy(optdict)
515 others = {}
516 if 'action' in optdict:
517 self._nocallback_options[provider] = opt
518 else:
519 optdict['action'] = 'callback'
520 optdict['callback'] = self.cb_set_provider_option
521
522
523 if 'default' in optdict:
524 if ('help' in optdict
525 and optdict.get('default') is not None
526 and not optdict['action'] in ('store_true', 'store_false')):
527 optdict['help'] += ' [current: %default]'
528 del optdict['default']
529 args = ['--' + str(opt)]
530 if 'short' in optdict:
531 self._short_options[optdict['short']] = opt
532 args.append('-' + optdict['short'])
533 del optdict['short']
534
535 for key in list(optdict.keys()):
536 if not key in self._optik_option_attrs:
537 optdict.pop(key)
538 return args, optdict
539
541 """optik callback for option setting"""
542 if opt.startswith('--'):
543
544 opt = opt[2:]
545 else:
546
547 opt = self._short_options[opt[1:]]
548
549 if value is None:
550 value = 1
551 self.global_set_option(opt, value)
552
554 """set option on the correct option provider"""
555 self._all_options[opt].set_option(opt, value)
556
558 """write a configuration file according to the current configuration
559 into the given stream or stdout
560 """
561 options_by_section = {}
562 sections = []
563 for provider in self.options_providers:
564 for section, options in provider.options_by_section():
565 if section is None:
566 section = provider.name
567 if section in skipsections:
568 continue
569 options = [(n, d, v) for (n, d, v) in options
570 if d.get('type') is not None]
571 if not options:
572 continue
573 if not section in sections:
574 sections.append(section)
575 alloptions = options_by_section.setdefault(section, [])
576 alloptions += options
577 stream = stream or sys.stdout
578 encoding = _get_encoding(encoding, stream)
579 printed = False
580 for section in sections:
581 if printed:
582 print('\n', file=stream)
583 format_section(stream, section.upper(), options_by_section[section],
584 encoding)
585 printed = True
586
587 - def generate_manpage(self, pkginfo, section=1, stream=None):
588 """write a man page for the current configuration into the given
589 stream or stdout
590 """
591 self._monkeypatch_expand_default()
592 try:
593 optik_ext.generate_manpage(self.cmdline_parser, pkginfo,
594 section, stream=stream or sys.stdout,
595 level=self._maxlevel)
596 finally:
597 self._unmonkeypatch_expand_default()
598
599
600
602 """initialize configuration using default values"""
603 for provider in self.options_providers:
604 provider.load_defaults()
605
610
612 """read the configuration file but do not load it (i.e. dispatching
613 values to each options provider)
614 """
615 helplevel = 1
616 while helplevel <= self._maxlevel:
617 opt = '-'.join(['long'] * helplevel) + '-help'
618 if opt in self._all_options:
619 break
620 def helpfunc(option, opt, val, p, level=helplevel):
621 print(self.help(level))
622 sys.exit(0)
623 helpmsg = '%s verbose help.' % ' '.join(['more'] * helplevel)
624 optdict = {'action' : 'callback', 'callback' : helpfunc,
625 'help' : helpmsg}
626 provider = self.options_providers[0]
627 self.add_optik_option(provider, self.cmdline_parser, opt, optdict)
628 provider.options += ( (opt, optdict), )
629 helplevel += 1
630 if config_file is None:
631 config_file = self.config_file
632 if config_file is not None:
633 config_file = expanduser(config_file)
634 if config_file and exists(config_file):
635 parser = self.cfgfile_parser
636 parser.read([config_file])
637
638 for sect, values in list(parser._sections.items()):
639 if not sect.isupper() and values:
640 parser._sections[sect.upper()] = values
641 elif not self.quiet:
642 msg = 'No config file found, using default configuration'
643 print(msg, file=sys.stderr)
644 return
645
663
665 """dispatch values previously read from a configuration file to each
666 options provider)
667 """
668 parser = self.cfgfile_parser
669 for section in parser.sections():
670 for option, value in parser.items(section):
671 try:
672 self.global_set_option(option, value)
673 except (KeyError, OptionError):
674
675 continue
676
678 """override configuration according to given parameters
679 """
680 for opt, opt_value in kwargs.items():
681 opt = opt.replace('_', '-')
682 provider = self._all_options[opt]
683 provider.set_option(opt, opt_value)
684
686 """override configuration according to command line parameters
687
688 return additional arguments
689 """
690 self._monkeypatch_expand_default()
691 try:
692 if args is None:
693 args = sys.argv[1:]
694 else:
695 args = list(args)
696 (options, args) = self.cmdline_parser.parse_args(args=args)
697 for provider in self._nocallback_options.keys():
698 config = provider.config
699 for attr in config.__dict__.keys():
700 value = getattr(options, attr, None)
701 if value is None:
702 continue
703 setattr(config, attr, value)
704 return args
705 finally:
706 self._unmonkeypatch_expand_default()
707
708
709
710
719
721
722 try:
723 self.__expand_default_backup = optik_ext.HelpFormatter.expand_default
724 optik_ext.HelpFormatter.expand_default = expand_default
725 except AttributeError:
726
727 pass
729
730 if hasattr(optik_ext.HelpFormatter, 'expand_default'):
731
732 optik_ext.HelpFormatter.expand_default = self.__expand_default_backup
733
734 - def help(self, level=0):
735 """return the usage string for available options """
736 self.cmdline_parser.formatter.output_level = level
737 self._monkeypatch_expand_default()
738 try:
739 return self.cmdline_parser.format_help()
740 finally:
741 self._unmonkeypatch_expand_default()
742
743
745 """used to ease late binding of default method (so you can define options
746 on the class using default methods on the configuration instance)
747 """
749 self.method = methname
750 self._inst = None
751
752 - def bind(self, instance):
753 """bind the method to its instance"""
754 if self._inst is None:
755 self._inst = instance
756
758 assert self._inst, 'unbound method'
759 return getattr(self._inst, self.method)(*args, **kwargs)
760
761
762
764 """Mixin to provide options to an OptionsManager"""
765
766
767 priority = -1
768 name = 'default'
769 options = ()
770 level = 0
771
773 self.config = optik_ext.Values()
774 for option in self.options:
775 try:
776 option, optdict = option
777 except ValueError:
778 raise Exception('Bad option: %r' % option)
779 if isinstance(optdict.get('default'), Method):
780 optdict['default'].bind(self)
781 elif isinstance(optdict.get('callback'), Method):
782 optdict['callback'].bind(self)
783 self.load_defaults()
784
786 """initialize the provider using default values"""
787 for opt, optdict in self.options:
788 action = optdict.get('action')
789 if action != 'callback':
790
791 default = self.option_default(opt, optdict)
792 if default is REQUIRED:
793 continue
794 self.set_option(opt, default, action, optdict)
795
797 """return the default value for an option"""
798 if optdict is None:
799 optdict = self.get_option_def(opt)
800 default = optdict.get('default')
801 if callable(default):
802 default = default()
803 return default
804
806 """get the config attribute corresponding to opt
807 """
808 if optdict is None:
809 optdict = self.get_option_def(opt)
810 return optdict.get('dest', opt.replace('-', '_'))
811 option_name = deprecated('[0.60] OptionsProviderMixIn.option_name() was renamed to option_attrname()')(option_attrname)
812
814 """get the current value for the given option"""
815 return getattr(self.config, self.option_attrname(opt), None)
816
817 - def set_option(self, opt, value, action=None, optdict=None):
818 """method called to set an option (registered in the options list)
819 """
820 if optdict is None:
821 optdict = self.get_option_def(opt)
822 if value is not None:
823 value = _validate(value, optdict, opt)
824 if action is None:
825 action = optdict.get('action', 'store')
826 if optdict.get('type') == 'named':
827 optname = self.option_attrname(opt, optdict)
828 currentvalue = getattr(self.config, optname, None)
829 if currentvalue:
830 currentvalue.update(value)
831 value = currentvalue
832 if action == 'store':
833 setattr(self.config, self.option_attrname(opt, optdict), value)
834 elif action in ('store_true', 'count'):
835 setattr(self.config, self.option_attrname(opt, optdict), 0)
836 elif action == 'store_false':
837 setattr(self.config, self.option_attrname(opt, optdict), 1)
838 elif action == 'append':
839 opt = self.option_attrname(opt, optdict)
840 _list = getattr(self.config, opt, None)
841 if _list is None:
842 if isinstance(value, (list, tuple)):
843 _list = value
844 elif value is not None:
845 _list = []
846 _list.append(value)
847 setattr(self.config, opt, _list)
848 elif isinstance(_list, tuple):
849 setattr(self.config, opt, _list + (value,))
850 else:
851 _list.append(value)
852 elif action == 'callback':
853 optdict['callback'](None, opt, value, None)
854 else:
855 raise UnsupportedAction(action)
856
877
879 """return the dictionary defining an option given it's name"""
880 assert self.options
881 for option in self.options:
882 if option[0] == opt:
883 return option[1]
884 raise OptionError('no such option %s in section %r'
885 % (opt, self.name), opt)
886
887
889 """return an iterator on available options for this provider
890 option are actually described by a 3-uple:
891 (section, option name, option dictionary)
892 """
893 for section, options in self.options_by_section():
894 if section is None:
895 if self.name is None:
896 continue
897 section = self.name.upper()
898 for option, optiondict, value in options:
899 yield section, option, optiondict
900
902 """return an iterator on options grouped by section
903
904 (section, [list of (optname, optdict, optvalue)])
905 """
906 sections = {}
907 for optname, optdict in self.options:
908 sections.setdefault(optdict.get('group'), []).append(
909 (optname, optdict, self.option_value(optname)))
910 if None in sections:
911 yield None, sections.pop(None)
912 for section, options in sorted(sections.items()):
913 yield section.upper(), options
914
920
921
922
924 """basic mixin for simple configurations which don't need the
925 manager / providers model
926 """
943
952
955
957 return iter(self.config.__dict__.items())
958
960 try:
961 return getattr(self.config, self.option_attrname(key))
962 except (optik_ext.OptionValueError, AttributeError):
963 raise KeyError(key)
964
967
968 - def get(self, key, default=None):
973
974
976 """class for simple configurations which don't need the
977 manager / providers model and prefer delegation to inheritance
978
979 configuration values are accessible through a dict like interface
980 """
981
982 - def __init__(self, config_file=None, options=None, name=None,
983 usage=None, doc=None, version=None):
991
992
994 """Adapt an option manager to behave like a
995 `logilab.common.configuration.Configuration` instance
996 """
998 self.config = provider
999
1001 return getattr(self.config, key)
1002
1004 provider = self.config._all_options[key]
1005 try:
1006 return getattr(provider.config, provider.option_attrname(key))
1007 except AttributeError:
1008 raise KeyError(key)
1009
1012
1013 - def get(self, key, default=None):
1014 provider = self.config._all_options[key]
1015 try:
1016 return getattr(provider.config, provider.option_attrname(key))
1017 except AttributeError:
1018 return default
1019
1020
1021
1023 """initialize newconfig from a deprecated configuration file
1024
1025 possible changes:
1026 * ('renamed', oldname, newname)
1027 * ('moved', option, oldgroup, newgroup)
1028 * ('typechanged', option, oldtype, newvalue)
1029 """
1030
1031 changesindex = {}
1032 for action in changes:
1033 if action[0] == 'moved':
1034 option, oldgroup, newgroup = action[1:]
1035 changesindex.setdefault(option, []).append((action[0], oldgroup, newgroup))
1036 continue
1037 if action[0] == 'renamed':
1038 oldname, newname = action[1:]
1039 changesindex.setdefault(newname, []).append((action[0], oldname))
1040 continue
1041 if action[0] == 'typechanged':
1042 option, oldtype, newvalue = action[1:]
1043 changesindex.setdefault(option, []).append((action[0], oldtype, newvalue))
1044 continue
1045 if action[1] in ('added', 'removed'):
1046 continue
1047 raise Exception('unknown change %s' % action[0])
1048
1049 options = []
1050 for optname, optdef in newconfig.options:
1051 for action in changesindex.pop(optname, ()):
1052 if action[0] == 'moved':
1053 oldgroup, newgroup = action[1:]
1054 optdef = optdef.copy()
1055 optdef['group'] = oldgroup
1056 elif action[0] == 'renamed':
1057 optname = action[1]
1058 elif action[0] == 'typechanged':
1059 oldtype = action[1]
1060 optdef = optdef.copy()
1061 optdef['type'] = oldtype
1062 options.append((optname, optdef))
1063 if changesindex:
1064 raise Exception('unapplied changes: %s' % changesindex)
1065 oldconfig = Configuration(options=options, name=newconfig.name)
1066
1067 oldconfig.load_file_configuration(configfile)
1068
1069 changes.reverse()
1070 done = set()
1071 for action in changes:
1072 if action[0] == 'renamed':
1073 oldname, newname = action[1:]
1074 newconfig[newname] = oldconfig[oldname]
1075 done.add(newname)
1076 elif action[0] == 'typechanged':
1077 optname, oldtype, newvalue = action[1:]
1078 newconfig[optname] = newvalue
1079 done.add(optname)
1080 for optname, optdef in newconfig.options:
1081 if optdef.get('type') and not optname in done:
1082 newconfig.set_option(optname, oldconfig[optname], optdict=optdef)
1083
1084
1086 """preprocess a list of options and remove duplicates, returning a new list
1087 (tuple actually) of options.
1088
1089 Options dictionaries are copied to avoid later side-effect. Also, if
1090 `otpgroup` argument is specified, ensure all options are in the given group.
1091 """
1092 alloptions = {}
1093 options = list(options)
1094 for i in range(len(options)-1, -1, -1):
1095 optname, optdict = options[i]
1096 if optname in alloptions:
1097 options.pop(i)
1098 alloptions[optname].update(optdict)
1099 else:
1100 optdict = optdict.copy()
1101 options[i] = (optname, optdict)
1102 alloptions[optname] = optdict
1103 if optgroup is not None:
1104 alloptions[optname]['group'] = optgroup
1105 return tuple(options)
1106