Browse Source

Heart beat, config generator support

Change-Id: I705583361caa747aa290883d8f33b7f499279f43
Kanagaraj Manickam 3 years ago
parent
commit
26c8878d12

+ 9
- 0
config-generator.conf View File

@@ -0,0 +1,9 @@
1
+[DEFAULT]
2
+output_file = etc/namos/namos.conf.sample
3
+wrap_width = 79
4
+namespace = namos.common.config
5
+namespace = oslo.messaging
6
+namespace = oslo.middleware
7
+namespace = oslo.db
8
+namespace = oslo.log
9
+namespace = oslo.service.service

+ 1
- 1
etc/namos.conf View File

@@ -10,4 +10,4 @@ rabbit_hosts = 172.241.0.101
10 10
 connection = mysql+pymysql://root:password@172.241.0.101/namos?charset=utf8
11 11
 
12 12
 [conductor]
13
-workers=1
13
+workers=3

+ 26
- 0
etc/oslo-config-schema.sync View File

@@ -0,0 +1,26 @@
1
+# List of config generator conf files for syncing the conf with namos
2
+heat=/opt/stack/heat/config-generator.conf
3
+namos=/home/manickan/workspace/namos/openstack/namos/config-generator.conf
4
+keystone=/opt/stack/keystone/config-generator/keystone.conf
5
+neutron-bgp-dragent=/opt/stack/neutron/etc/oslo-config-generator/bgp_dragent.ini
6
+neutron-dhcp-agent=/opt/stack/neutron/etc/oslo-config-generator/dhcp_agent.ini
7
+neutron-l3-agent=/opt/stack/neutron/etc/oslo-config-generator/l3_agent.ini
8
+neutron-linuxbridge-agent=/opt/stack/neutron/etc/oslo-config-generator/linuxbridge_agent.ini
9
+neutron-metadata-agent=/opt/stack/neutron/etc/oslo-config-generator/metadata_agent.ini
10
+neutron-metering-agent=/opt/stack/neutron/etc/oslo-config-generator/metering_agent.ini
11
+neutron-ml2=/opt/stack/neutron/etc/oslo-config-generator/ml2_conf.ini
12
+neutron-ml2-sriov=/opt/stack/neutron/etc/oslo-config-generator/ml2_conf_sriov.ini
13
+neutron=/opt/stack/neutron/etc/oslo-config-generator/neutron.conf
14
+neutron-openvswitch-agent=/opt/stack/neutron/etc/oslo-config-generator/openvswitch_agent.ini
15
+neutron-sriov-agent=/opt/stack/neutron/etc/oslo-config-generator/sriov_agent.ini
16
+lbaas-agent=/opt/stack/neutron-lbaas/etc/oslo-config-generator/lbaas_agent.ini
17
+neutron-lbaas=/opt/stack/neutron-lbaas/etc/oslo-config-generator/neutron_lbaas.conf
18
+services-lbaas=/opt/stack/neutron-lbaas/etc/oslo-config-generator/services_lbaas.conf
19
+glance-api=/opt/stack/glance/etc/oslo-config-generator/glance-api.conf
20
+glance-cache=/opt/stack/glance/etc/oslo-config-generator/glance-cache.conf
21
+glance-glare=/opt/stack/glance/etc/oslo-config-generator/glance-glare.conf
22
+glance-registry=/opt/stack/glance/etc/oslo-config-generator/glance-registry.conf
23
+glance-scrubber=/opt/stack/glance/etc/oslo-config-generator/glance-scrubber.conf
24
+glance-manage=/opt/stack/glance/etc/oslo-config-generator/glance-manage.conf
25
+nova=/opt/stack/nova/etc/nova/nova-config-generator.conf
26
+cinder=/opt/stack/cinder/cinder/config/cinder-config-generator.conf

+ 97
- 0
namos/cmd/manage.py View File

@@ -15,8 +15,11 @@
15 15
 import sys
16 16
 
17 17
 from oslo_config import cfg
18
+from oslo_utils import timeutils
18 19
 
19 20
 from namos.common import config
21
+from namos.common import exception
22
+from namos.db import api
20 23
 from namos.db import sample
21 24
 from namos.db.sqlalchemy import migration
22 25
 
@@ -25,6 +28,90 @@ CONF = cfg.CONF
25 28
 MANAGE_COMMAND_NAME = 'namos-manage'
26 29
 
27 30
 
31
+class HeartBeat(object):
32
+    def find_status(self, sw, report_interval=60):
33
+        status = False
34
+        if sw.updated_at is not None:
35
+            if ((timeutils.utcnow() - sw.updated_at).total_seconds()
36
+                    <= report_interval):
37
+                status = True
38
+        else:
39
+            if ((timeutils.utcnow() - sw.created_at).total_seconds()
40
+                    <= report_interval):
41
+                status = True
42
+
43
+        return status
44
+
45
+    def report_status(self):
46
+        # TODO(mrkanag) Make like Node: Service: worker: status
47
+        for sw in api.service_worker_get_all(None):
48
+            print ('[', 'T' if self.find_status(sw) else 'F', ']', sw.name)
49
+
50
+
51
+class OsloConfigSchemaManager(object):
52
+    def gen_schema(self):
53
+        import json
54
+        cfg_ns = dict()
55
+        for cfg_ in api.config_schema_get_all(None):
56
+            if cfg_.namespace not in cfg_ns:
57
+                cfg_ns[cfg_.namespace] = dict()
58
+            if cfg_.group_name not in cfg_ns[cfg_.namespace]:
59
+                cfg_ns[cfg_.namespace][cfg_.group_name] = dict()
60
+            cfg_ns[cfg_.namespace][cfg_.group_name][cfg_.name] = cfg_.to_dict()
61
+
62
+        open(CONF.command.outputfile, 'w').write(json.dumps(cfg_ns))
63
+
64
+    def sync(self):
65
+        if CONF.command.gen:
66
+            self.gen_schema()
67
+            return
68
+
69
+        sync_map = {}
70
+        with open(CONF.command.syncfile) as f:
71
+            for line in f:
72
+                if line.startswith("#"):
73
+                    continue
74
+                kv = line.split("=")
75
+                sync_map[kv[0]] = kv[1].replace("\n", "")
76
+
77
+        for k, v in sync_map.items():
78
+            out_file = '%s/%s.json' % (CONF.command.outputdir or '/tmp', k)
79
+            cmd = ('oslo-config-generator --config-file %s '
80
+                   '--output-file %s --output-format json' %
81
+                   (v, out_file))
82
+            print ("\nSyncing %s " % cmd)
83
+            import os
84
+            os.system(cmd)
85
+
86
+            if CONF.command.dbsync:
87
+                import json
88
+                conf_dict = json.loads(open(out_file).read())
89
+                for grp, namespaces in conf_dict.items():
90
+                    for namespace, opts in namespaces.items():
91
+                        for name, opt in opts.items():
92
+                            conf_ = dict(
93
+                                namespace=namespace,
94
+                                group_name=grp,
95
+                                name=name,
96
+                                default_value=opt['default'],
97
+                                type=opt['type']['name'],
98
+                                help=opt['help'],
99
+                                required=opt['required'],
100
+                                secret=opt['secret'],
101
+                                mutable=opt['mutable']
102
+                            )
103
+
104
+                            try:
105
+                                api.config_schema_create(None,
106
+                                                         conf_)
107
+                                _a = 'T'
108
+                            except exception.AlreadyExist:
109
+                                _a = 'F'
110
+
111
+                            print ('[', _a, ']', namespace, ':', grp, ':',
112
+                                   name)
113
+
114
+
28 115
 class DBCommand(object):
29 116
 
30 117
     def upgrade(self):
@@ -88,6 +175,16 @@ def add_command_parsers(subparsers):
88 175
     parser.add_argument('-p', '--purge', action='store_true')
89 176
     parser.set_defaults(func=command_object.demo_data)
90 177
 
178
+    parser = subparsers.add_parser('oslo_config_schema')
179
+    parser.add_argument('-f', '--syncfile')
180
+    parser.add_argument('-o', '--outputdir')
181
+    parser.add_argument('-j', '--outputfile')
182
+    parser.add_argument('-s', '--dbsync', action='store_true')
183
+    parser.add_argument('-g', '--gen', action='store_true')
184
+    parser.set_defaults(func=OsloConfigSchemaManager().sync)
185
+
186
+    parser = subparsers.add_parser('status')
187
+    parser.set_defaults(func=HeartBeat().report_status)
91 188
 
92 189
 command_opt = cfg.SubCommandOpt('command',
93 190
                                 title='Command',

+ 4
- 0
namos/common/config.py View File

@@ -49,3 +49,7 @@ def init_log(project=PROJECT_NAME):
49 49
     logging.setup(cfg.CONF,
50 50
                   project,
51 51
                   version=VERSION)
52
+
53
+
54
+def list_opts():
55
+    yield 'conductor', conductor_opts

+ 5
- 0
namos/common/exception.py View File

@@ -130,3 +130,8 @@ class ConfigNotFound(NotFound):
130 130
 class ConfigFileNotFound(NotFound):
131 131
     msg_fmt = ("Config file %(config_file_id)s does not found")
132 132
     error_code = 0x0b001
133
+
134
+
135
+class ConfigSchemaNotFound(NotFound):
136
+    msg_fmt = ("Config schema %(config_schema_id)s does not found")
137
+    error_code = 0x0c001

+ 503
- 0
namos/common/generator.py.bkup View File

@@ -0,0 +1,503 @@
1
+# Copyright 2012 SINA Corporation
2
+# Copyright 2014 Cisco Systems, Inc.
3
+# All Rights Reserved.
4
+# Copyright 2014 Red Hat, Inc.
5
+#
6
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
7
+#    not use this file except in compliance with the License. You may obtain
8
+#    a copy of the License at
9
+#
10
+#         http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+#    Unless required by applicable law or agreed to in writing, software
13
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15
+#    License for the specific language governing permissions and limitations
16
+#    under the License.
17
+
18
+"""Sample configuration generator
19
+
20
+Tool for generating a sample configuration file. See
21
+../doc/source/generator.rst for details.
22
+
23
+.. versionadded:: 1.4
24
+"""
25
+
26
+# TODO(mrkanag) copied from oslo_config.generator having the changes for
27
+# generating the conf in json format
28
+
29
+import collections
30
+import logging
31
+import operator
32
+import sys
33
+import textwrap
34
+
35
+import pkg_resources
36
+import six
37
+
38
+from oslo_config import cfg
39
+import stevedore.named  # noqa
40
+
41
+LOG = logging.getLogger(__name__)
42
+
43
+_generator_opts = [
44
+    cfg.StrOpt('output-file',
45
+               help='Path of the file to write to. Defaults to stdout.'),
46
+    cfg.StrOpt('output-format',
47
+               default='txt',
48
+               help='Output format either txt or json. Defaults to txt.'),
49
+    cfg.IntOpt('wrap-width',
50
+               default=70,
51
+               help='The maximum length of help lines.'),
52
+    cfg.MultiStrOpt('namespace',
53
+                    required=True,
54
+                    help='Option namespace under "oslo.config.opts" in which '
55
+                         'to query for options.'),
56
+]
57
+
58
+
59
+def register_cli_opts(conf):
60
+    """Register the formatter's CLI options with a ConfigOpts instance.
61
+
62
+    Note, this must be done before the ConfigOpts instance is called to parse
63
+    the configuration.
64
+
65
+    :param conf: a ConfigOpts instance
66
+    :raises: DuplicateOptError, ArgsAlreadyParsedError
67
+    """
68
+    conf.register_cli_opts(_generator_opts)
69
+
70
+
71
+def _format_defaults(opt):
72
+    "Return a list of formatted default values."
73
+    if isinstance(opt, cfg.MultiStrOpt):
74
+        if opt.sample_default is not None:
75
+            defaults = opt.sample_default
76
+        elif not opt.default:
77
+            defaults = ['']
78
+        else:
79
+            defaults = opt.default
80
+    else:
81
+        if opt.sample_default is not None:
82
+            default_str = str(opt.sample_default)
83
+        elif opt.default is None:
84
+            default_str = '<None>'
85
+        elif (isinstance(opt, cfg.StrOpt) or
86
+              isinstance(opt, cfg.IPOpt) or
87
+              isinstance(opt, cfg.HostnameOpt)):
88
+            default_str = opt.default
89
+        elif isinstance(opt, cfg.BoolOpt):
90
+            default_str = str(opt.default).lower()
91
+        elif isinstance(opt, (cfg.IntOpt, cfg.FloatOpt,
92
+                              cfg.PortOpt)):
93
+            default_str = str(opt.default)
94
+        elif isinstance(opt, (cfg.ListOpt, cfg._ConfigFileOpt,
95
+                              cfg._ConfigDirOpt)):
96
+            default_str = ','.join(opt.default)
97
+        elif isinstance(opt, cfg.DictOpt):
98
+            sorted_items = sorted(opt.default.items(),
99
+                                  key=operator.itemgetter(0))
100
+            default_str = ','.join(['%s:%s' % i for i in sorted_items])
101
+        else:
102
+            LOG.warning('Unknown option type: %s', repr(opt))
103
+            default_str = str(opt.default)
104
+        defaults = [default_str]
105
+
106
+    results = []
107
+    for default_str in defaults:
108
+        if default_str.strip() != default_str:
109
+            default_str = '"%s"' % default_str
110
+        results.append(default_str)
111
+    return results
112
+
113
+
114
+class _OptFormatter(object):
115
+
116
+    """Format configuration option descriptions to a file."""
117
+
118
+    def __init__(self, output_file=None, wrap_width=70):
119
+        """Construct an OptFormatter object.
120
+
121
+        :param output_file: a writeable file object
122
+        :param wrap_width: The maximum length of help lines, 0 to not wrap
123
+        """
124
+        self.output_file = output_file or sys.stdout
125
+        self.wrap_width = wrap_width
126
+
127
+    def _format_help(self, help_text):
128
+        """Format the help for a group or option to the output file.
129
+
130
+        :param help_text: The text of the help string
131
+        """
132
+        if self.wrap_width is not None and self.wrap_width > 0:
133
+            wrapped = ""
134
+            for line in help_text.splitlines():
135
+                text = "\n".join(textwrap.wrap(line, self.wrap_width,
136
+                                               initial_indent='# ',
137
+                                               subsequent_indent='# ',
138
+                                               break_long_words=False,
139
+                                               replace_whitespace=False))
140
+                wrapped += "#" if text == "" else text
141
+                wrapped += "\n"
142
+            lines = [wrapped]
143
+        else:
144
+            lines = ['# ' + help_text + '\n']
145
+        return lines
146
+
147
+    def _get_choice_text(self, choice):
148
+        if choice is None:
149
+            return '<None>'
150
+        elif choice == '':
151
+            return "''"
152
+        return six.text_type(choice)
153
+
154
+    def format_group(self, group_or_groupname):
155
+        """Format the description of a group header to the output file
156
+
157
+        :param group_or_groupname: a cfg.OptGroup instance or a name of group
158
+        """
159
+        if isinstance(group_or_groupname, cfg.OptGroup):
160
+            group = group_or_groupname
161
+            lines = ['[%s]\n' % group.name]
162
+            if group.help:
163
+                lines += self._format_help(group.help)
164
+        else:
165
+            groupname = group_or_groupname
166
+            lines = ['[%s]\n' % groupname]
167
+        self.writelines(lines)
168
+
169
+    def format(self, opt):
170
+        """Format a description of an option to the output file.
171
+
172
+        :param opt: a cfg.Opt instance
173
+        """
174
+        if not opt.help:
175
+            LOG.warning('"%s" is missing a help string', opt.dest)
176
+
177
+        option_type = getattr(opt, 'type', None)
178
+        opt_type = getattr(option_type, 'type_name', 'unknown value')
179
+
180
+        if opt.help:
181
+            help_text = u'%s (%s)' % (opt.help,
182
+                                      opt_type)
183
+        else:
184
+            help_text = u'(%s)' % opt_type
185
+        lines = self._format_help(help_text)
186
+
187
+        if getattr(opt.type, 'min', None) is not None:
188
+            lines.append('# Minimum value: %d\n' % opt.type.min)
189
+
190
+        if getattr(opt.type, 'max', None) is not None:
191
+            lines.append('# Maximum value: %d\n' % opt.type.max)
192
+
193
+        if getattr(opt.type, 'choices', None):
194
+            choices_text = ', '.join([self._get_choice_text(choice)
195
+                                      for choice in opt.type.choices])
196
+            lines.append('# Allowed values: %s\n' % choices_text)
197
+
198
+        try:
199
+            if opt.mutable:
200
+                lines.append(
201
+                    '# Note: This option can be changed without restarting.\n'
202
+                )
203
+        except AttributeError as err:
204
+            # NOTE(dhellmann): keystoneauth defines its own Opt class,
205
+            # and neutron (at least) returns instances of those
206
+            # classes instead of oslo_config Opt instances. The new
207
+            # mutable attribute is the first property where the API
208
+            # isn't supported in the external class, so we can use
209
+            # this failure to emit a warning. See
210
+            # https://bugs.launchpad.net/keystoneauth/+bug/1548433 for
211
+            # more details.
212
+            import warnings
213
+            if not isinstance(cfg.Opt, opt):
214
+                warnings.warn(
215
+                    'Incompatible option class for %s (%r): %s' %
216
+                    (opt.dest, opt.__class__, err),
217
+                )
218
+            else:
219
+                warnings.warn('Failed to fully format sample for %s: %s' %
220
+                              (opt.dest, err))
221
+
222
+        for d in opt.deprecated_opts:
223
+            lines.append('# Deprecated group/name - [%s]/%s\n' %
224
+                         (d.group or 'DEFAULT', d.name or opt.dest))
225
+
226
+        if opt.deprecated_for_removal:
227
+            lines.append(
228
+                '# This option is deprecated for removal.\n'
229
+                '# Its value may be silently ignored in the future.\n')
230
+            if opt.deprecated_reason:
231
+                lines.extend(
232
+                    self._format_help('Reason: ' + opt.deprecated_reason))
233
+
234
+        if hasattr(opt.type, 'format_defaults'):
235
+            defaults = opt.type.format_defaults(opt.default,
236
+                                                opt.sample_default)
237
+        else:
238
+            LOG.debug(
239
+                "The type for option %(name)s which is %(type)s is not a "
240
+                "subclass of types.ConfigType and doesn't provide a "
241
+                "'format_defaults' method. A default formatter is not "
242
+                "available so the best-effort formatter will be used.",
243
+                {'type': opt.type, 'name': opt.name})
244
+            defaults = _format_defaults(opt)
245
+        for default_str in defaults:
246
+            if default_str:
247
+                default_str = ' ' + default_str
248
+            lines.append('#%s =%s\n' % (opt.dest, default_str))
249
+
250
+        self.writelines(lines)
251
+
252
+    def write(self, s):
253
+        """Write an arbitrary string to the output file.
254
+
255
+        :param s: an arbitrary string
256
+        """
257
+        self.output_file.write(s)
258
+
259
+    def writelines(self, l):
260
+        """Write an arbitrary sequence of strings to the output file.
261
+
262
+        :param l: a list of arbitrary strings
263
+        """
264
+        self.output_file.writelines(l)
265
+
266
+
267
+def _cleanup_opts(read_opts):
268
+    """Cleanup duplicate options in namespace groups
269
+
270
+    Return a structure which removes duplicate options from a namespace group.
271
+    NOTE:(rbradfor) This does not remove duplicated options from repeating
272
+    groups in different namespaces:
273
+
274
+    :param read_opts: a list (namespace, [(group, [opt_1, opt_2])]) tuples
275
+    :returns: a list of (namespace, [(group, [opt_1, opt_2])]) tuples
276
+    """
277
+
278
+    # OrderedDict is used specifically in the three levels to maintain the
279
+    # source order of namespace/group/opt values
280
+    clean = collections.OrderedDict()
281
+    for namespace, listing in read_opts:
282
+        if namespace not in clean:
283
+            clean[namespace] = collections.OrderedDict()
284
+        for group, opts in listing:
285
+            if group not in clean[namespace]:
286
+                clean[namespace][group] = collections.OrderedDict()
287
+            for opt in opts:
288
+                clean[namespace][group][opt.dest] = opt
289
+
290
+    # recreate the list of (namespace, [(group, [opt_1, opt_2])]) tuples
291
+    # from the cleaned structure.
292
+    cleaned_opts = [
293
+        (namespace, [(group, list(clean[namespace][group].values()))
294
+                     for group in clean[namespace]])
295
+        for namespace in clean
296
+    ]
297
+
298
+    return cleaned_opts
299
+
300
+
301
+def _get_raw_opts_loaders(namespaces):
302
+    """List the options available via the given namespaces.
303
+
304
+    :param namespaces: a list of namespaces registered under 'oslo.config.opts'
305
+    :returns: a list of (namespace, [(group, [opt_1, opt_2])]) tuples
306
+    """
307
+    mgr = stevedore.named.NamedExtensionManager(
308
+        'oslo.config.opts',
309
+        names=namespaces,
310
+        on_load_failure_callback=on_load_failure_callback,
311
+        invoke_on_load=False)
312
+    return [(e.name, e.plugin) for e in mgr]
313
+
314
+
315
+def _get_opt_default_updaters(namespaces):
316
+    mgr = stevedore.named.NamedExtensionManager(
317
+        'oslo.config.opts.defaults',
318
+        names=namespaces,
319
+        on_load_failure_callback=on_load_failure_callback,
320
+        invoke_on_load=False)
321
+    return [ep.plugin for ep in mgr]
322
+
323
+
324
+def _update_defaults(namespaces):
325
+    "Let application hooks update defaults inside libraries."
326
+    for update in _get_opt_default_updaters(namespaces):
327
+        update()
328
+
329
+
330
+def _list_opts(namespaces):
331
+    """List the options available via the given namespaces.
332
+
333
+    Duplicate options from a namespace are removed.
334
+
335
+    :param namespaces: a list of namespaces registered under 'oslo.config.opts'
336
+    :returns: a list of (namespace, [(group, [opt_1, opt_2])]) tuples
337
+    """
338
+    # Load the functions to get the options.
339
+    loaders = _get_raw_opts_loaders(namespaces)
340
+    # Update defaults, which might change global settings in library
341
+    # modules.
342
+    _update_defaults(namespaces)
343
+    # Ask for the option definitions. At this point any global default
344
+    # changes made by the updaters should be in effect.
345
+    opts = [
346
+        (namespace, loader())
347
+        for namespace, loader in loaders
348
+    ]
349
+    return _cleanup_opts(opts)
350
+
351
+
352
+def on_load_failure_callback(*args, **kwargs):
353
+    raise
354
+
355
+
356
+def _output_opts(f, group, namespaces):
357
+    f.format_group(group)
358
+    for (namespace, opts) in sorted(namespaces,
359
+                                    key=operator.itemgetter(0)):
360
+        f.write('\n#\n# From %s\n#\n' % namespace)
361
+        for opt in opts:
362
+            f.write('\n')
363
+            try:
364
+                f.format(opt)
365
+            except Exception as err:
366
+                f.write('# Warning: Failed to format sample for %s\n' %
367
+                        (opt.dest,))
368
+                f.write('# %s\n' % (err,))
369
+
370
+
371
+def _append_opts_json(f, group, namespaces):
372
+    f[group] = dict()
373
+    for (namespace, opts) in sorted(namespaces,
374
+                                    key=operator.itemgetter(0)):
375
+
376
+        f[group][namespace] = dict()
377
+
378
+        for opt in opts:
379
+            f[group][namespace][opt.name] = dict()
380
+            f[group][namespace][opt.name]['help'] = opt.help or ''
381
+
382
+            f[group][namespace][opt.name]['type'] = dict()
383
+            option_type = getattr(opt, 'type', None)
384
+            opt_type = getattr(option_type, 'type_name', 'unknown type')
385
+            f[group][namespace][opt.name]['type']['name'] = opt_type
386
+
387
+            if getattr(opt.type, 'min', None) is not None:
388
+                f[group][namespace][opt.name]['type']['min'] = opt.type.min
389
+
390
+            if getattr(opt.type, 'max', None) is not None:
391
+                f[group][namespace][opt.name]['type']['max'] = opt.type.min
392
+
393
+            if getattr(opt.type, 'choices', None):
394
+                f[group][namespace][opt.name]['type'][
395
+                    'choices'] = opt.type.choices
396
+
397
+            if getattr(opt, 'mutable', None):
398
+                f[group][namespace][opt.name]['mutable'] = opt.mutable
399
+            else:
400
+                f[group][namespace][opt.name]['mutable'] = None
401
+            f[group][namespace][opt.name]['required'] = opt.required
402
+            f[group][namespace][opt.name]['secret'] = opt.secret
403
+            f[group][namespace][opt.name]['default'] = '%s' % opt.default
404
+
405
+            f[group][namespace][opt.name]['deprecated'] = []
406
+            for d in opt.deprecated_opts:
407
+                f[group][namespace][opt.name]['deprecated'].append(
408
+                 (d.group or 'DEFAULT', d.name or opt.dest))
409
+
410
+            f[group][namespace][opt.name][
411
+                'deprecated_for_removal'] = opt.deprecated_for_removal
412
+
413
+            if getattr(opt, 'deprecated_reason', None):
414
+                f[group][namespace][opt.name][
415
+                    'deprecated_reason'] = opt.deprecated_reason
416
+            else:
417
+                f[group][namespace][opt.name][
418
+                    'deprecated_reason'] = None
419
+
420
+
421
+def _get_group_name(item):
422
+    group = item[0]
423
+    # The keys of the groups dictionary could be an OptGroup. Otherwise the
424
+    # help text of an OptGroup wouldn't be part of the generated sample
425
+    # file. It could also be just a plain group name without any further
426
+    # attributes. That's the reason why we have to differentiate here.
427
+    return group.name if isinstance(group, cfg.OptGroup) else group
428
+
429
+
430
+def _get_groups(conf_ns):
431
+    groups = {'DEFAULT': []}
432
+    for namespace, listing in conf_ns:
433
+        for group, opts in listing:
434
+            if not opts:
435
+                continue
436
+            namespaces = groups.setdefault(group or 'DEFAULT', [])
437
+            namespaces.append((namespace, opts))
438
+    return groups
439
+
440
+
441
+def generate(conf):
442
+    """Generate a sample config file.
443
+
444
+    List all of the options available via the namespaces specified in the given
445
+    configuration and write a description of them to the specified output file.
446
+
447
+    :param conf: a ConfigOpts instance containing the generator's configuration
448
+    """
449
+    conf.register_opts(_generator_opts)
450
+
451
+    output_file = (open(conf.output_file, 'w')
452
+                   if conf.output_file else sys.stdout)
453
+
454
+    groups = _get_groups(_list_opts(conf.namespace))
455
+
456
+    if conf.output_format == 'json':
457
+        generate_json(conf, groups, output_file)
458
+    elif conf.output_format == 'txt':
459
+        generate_txt(conf, groups, output_file)
460
+
461
+
462
+def generate_txt(conf, groups, output_file):
463
+
464
+    formatter = _OptFormatter(output_file=output_file,
465
+                              wrap_width=conf.wrap_width)
466
+
467
+    # Output the "DEFAULT" section as the very first section
468
+    _output_opts(formatter, 'DEFAULT', groups.pop('DEFAULT'))
469
+
470
+    # output all other config sections with groups in alphabetical order
471
+    for group, namespaces in sorted(groups.items(), key=_get_group_name):
472
+        formatter.write('\n\n')
473
+        _output_opts(formatter, group, namespaces)
474
+
475
+
476
+def generate_json(conf, groups, output_file):
477
+    cfg_dict = dict()
478
+    _append_opts_json(cfg_dict, 'DEFAULT', groups.pop('DEFAULT'))
479
+
480
+    # output all other config sections with groups in alphabetical order
481
+    for group, namespaces in sorted(groups.items(), key=_get_group_name):
482
+        # for nova, below fix is required for grp conductor, vnc
483
+        if isinstance(group, cfg.OptGroup):
484
+            group = group.name
485
+
486
+        _append_opts_json(cfg_dict, group, namespaces)
487
+
488
+    import json
489
+    output_file.write(json.dumps(cfg_dict))
490
+
491
+
492
+def main(args=None):
493
+    """The main function of oslo-config-generator."""
494
+    version = pkg_resources.get_distribution('oslo.config').version
495
+    logging.basicConfig(level=logging.WARN)
496
+    conf = cfg.ConfigOpts()
497
+    register_cli_opts(conf)
498
+    conf(args, version=version)
499
+    generate(conf)
500
+
501
+
502
+if __name__ == '__main__':
503
+    main()

+ 79
- 8
namos/conductor/manager.py View File

@@ -17,6 +17,7 @@ import functools
17 17
 from oslo_config import cfg
18 18
 from oslo_context import context
19 19
 from oslo_log import log
20
+from oslo_utils import timeutils
20 21
 
21 22
 from namos.common import config
22 23
 from namos.common import exception
@@ -75,6 +76,28 @@ class ConductorManager(object):
75 76
 
76 77
         return service_worker_id
77 78
 
79
+    @request_context
80
+    def heart_beat(self, context, identification, dieing=False):
81
+        try:
82
+            sw = db_api.service_worker_get_all_by(context,
83
+                                                  pid=identification)
84
+            if len(sw) == 1:
85
+                if not dieing:
86
+                    db_api.service_worker_update(
87
+                        context,
88
+                        sw[0].id,
89
+                        dict(updated_at=timeutils.utcnow()))
90
+                    LOG.info("HEART-BEAT LIVE %s " % identification)
91
+                else:
92
+                    db_api.service_worker_delete(context,
93
+                                                 sw[0].id)
94
+                    LOG.info("HEART-BEAT STOPPED %s " % identification)
95
+            else:
96
+                LOG.error("HEART-BEAT FAILED, No service worker registered "
97
+                          "with identification %s " % identification)
98
+        except Exception as e:  # noqa
99
+            LOG.error("HEART-BEAT FAILED %s " % e)
100
+
78 101
     @request_context
79 102
     def service_perspective_get(self,
80 103
                                 context,
@@ -218,9 +241,11 @@ class ServiceProcessor(object):
218 241
                 # TODO(mrkanag) Fix the name, device driver proper !
219 242
                 dict(name='%s@%s' % (self.registration_info['pid'],
220 243
                                      service_component.name),
221
-                     pid=self.registration_info['pid'],
244
+                     pid=self.registration_info['identification'],
222 245
                      host=self.registration_info['host'],
223
-                     service_component_id=service_component.id))
246
+                     service_component_id=service_component.id,
247
+                     deleted_at=None
248
+                     ))
224 249
             LOG.info('Service Worker %s is created' % service_worker)
225 250
         except exception.AlreadyExist:
226 251
             # TODO(mrkanag) Find a way to purge the dead service worker
@@ -239,7 +264,8 @@ class ServiceProcessor(object):
239 264
                         context,
240 265
                         service_workers[0].id,
241 266
                         dict(
242
-                            pid=self.registration_info['pid'],
267
+                            deleted_at=None,
268
+                            pid=self.registration_info['identification'],
243 269
                             name='%s@%s' % (self.registration_info['pid'],
244 270
                                             service_component.name)
245 271
                         ))
@@ -253,20 +279,65 @@ class ServiceProcessor(object):
253 279
         # or per service_worker,
254 280
         for cfg_name, cfg_obj in self.registration_info[
255 281
             'config_dict'].iteritems():
256
-            cfg_obj['service_worker_id'] = service_worker.id
282
+
283
+            cfg_schs = db_api.config_schema_get_by(
284
+                context=context,
285
+                group=cfg_obj['group'],
286
+                name=cfg_obj['name']
287
+            )
288
+
289
+            if len(cfg_schs) > 1:
290
+                cfg_sche = cfg_schs[0]
291
+                LOG.info("Config Schema %s is existing and is updated" %
292
+                         cfg_sche)
293
+            else:
294
+                try:
295
+                    cfg_sche = db_api.config_schema_create(
296
+                        context,
297
+                        dict(
298
+                            namespace='UNKNOWN-NAMOS',
299
+                            default_value=cfg_obj['default_value'],
300
+                            type=cfg_obj['type'],
301
+                            help=cfg_obj['help'],
302
+                            required=cfg_obj['required'],
303
+                            secret=cfg_obj['secret'],
304
+                            mutable=False,
305
+                            group_name=cfg_obj['group'],
306
+                            name=cfg_obj['name']
307
+                        )
308
+                    )
309
+                    LOG.info("Config Schema %s is created" % cfg_sche)
310
+                except exception.AlreadyExist:
311
+                    cfg_schs = db_api.config_schema_get_by(
312
+                        context=context,
313
+                        group=cfg_obj['group'],
314
+                        name=cfg_obj['name'],
315
+                        namespace='UNKNOWN-NAMOS'
316
+                    )
317
+
318
+                    cfg_sche = cfg_schs[0]
319
+                    LOG.info("Config Schema %s is existing and is updated" %
320
+                             cfg_sche)
321
+
322
+            cfg_obj_ = dict(
323
+                service_worker_id=service_worker.id,
324
+                name="%s.%s" % (cfg_obj['group'], cfg_name),
325
+                value=cfg_obj['value'],
326
+                oslo_config_schema_id=cfg_sche.id
327
+            )
257 328
 
258 329
             try:
259
-                config = db_api.config_create(context, cfg_obj)
330
+                config = db_api.config_create(context, cfg_obj_)
260 331
                 LOG.info("Config %s is created" % config)
261 332
             except exception.AlreadyExist:
262 333
                 configs = db_api.config_get_by_name_for_service_worker(
263 334
                     context,
264
-                    service_worker_id=cfg_obj['service_worker_id'],
265
-                    name=cfg_obj['name'])
335
+                    service_worker_id=cfg_obj_['service_worker_id'],
336
+                    name=cfg_obj_['name'])
266 337
                 if len(configs) == 1:
267 338
                     config = db_api.config_update(context,
268 339
                                                   configs[0].id,
269
-                                                  cfg_obj)
340
+                                                  cfg_obj_)
270 341
                     LOG.info("Config %s is existing and is updated" % config)
271 342
 
272 343
         return service_worker.id

+ 38
- 1
namos/db/api.py View File

@@ -304,12 +304,47 @@ def service_worker_get_all(context):
304 304
     return IMPL.service_worker_get_all(context)
305 305
 
306 306
 
307
+def service_worker_get_all_by(context, **kwargs):
308
+    return IMPL.service_worker_get_all_by(context, **kwargs)
309
+
310
+
307 311
 def service_worker_delete(context, _id):
308 312
     return IMPL.service_worker_delete(context, _id)
309 313
 
310 314
 
311
-#  Config
315
+#  config schema
316
+def config_schema_create(context, values):
317
+    return IMPL.config_schema_create(context, values)
318
+
319
+
320
+def config_schema_update(context, _id, values):
321
+    return IMPL.config_schema_update(context, _id, values)
322
+
323
+
324
+def config_schema_get(context, _id):
325
+    return IMPL.config_schema_get(context, _id)
326
+
327
+
328
+def config_schema_get_by_name(context, name):
329
+    return IMPL.config_schema_get_by_name(context, name)
312 330
 
331
+
332
+def config_schema_get_by(context,
333
+                         namespace=None,
334
+                         group=None,
335
+                         name=None):
336
+    return IMPL.config_schema_get_by(context, namespace, group, name)
337
+
338
+
339
+def config_schema_get_all(context):
340
+    return IMPL.config_schema_get_all(context)
341
+
342
+
343
+def config_schema_delete(context, _id):
344
+    return IMPL.config_schema_delete(context, _id)
345
+
346
+
347
+#  Config
313 348
 def config_create(context, values):
314 349
     return IMPL.config_create(context, values)
315 350
 
@@ -344,6 +379,8 @@ def config_delete(context, _id):
344 379
     return IMPL.config_delete(context, _id)
345 380
 
346 381
 
382
+# config file
383
+
347 384
 def config_file_create(context, values):
348 385
     return IMPL.config_file_create(context, values)
349 386
 

+ 5
- 5
namos/db/sample.py View File

@@ -391,9 +391,9 @@ def _service_populate_demo_data():
391 391
         service_worker = inject_id(service_worker)
392 392
         api.service_worker_create(None, service_worker)
393 393
 
394
-    for config in CONFIG_LIST:
395
-        config = inject_id(config)
396
-        api.config_create(None, config)
394
+    # for config in CONFIG_LIST:
395
+    #     config = inject_id(config)
396
+    #     api.config_create(None, config)
397 397
 
398 398
 
399 399
 def populate_demo_data():
@@ -418,8 +418,8 @@ def _device_purge_demo_data():
418 418
 
419 419
 
420 420
 def _service_purge_demo_data():
421
-    for config in CONFIG_LIST:
422
-        api.config_delete(None, config.keys()[0])
421
+    # for config in CONFIG_LIST:
422
+    #     api.config_delete(None, config.keys()[0])
423 423
     for service_worker in SERVICE_WORKER_LIST:
424 424
         api.service_worker_delete(None, service_worker.keys()[0])
425 425
 

+ 1
- 0
namos/db/sqlalchemy/alembic/versions/48ebec3cd6f6_initial_version.py View File

@@ -179,6 +179,7 @@ def upgrade():
179 179
 def downgrade():
180 180
     op.drop_table('oslo_config_file')
181 181
     op.drop_table('oslo_config')
182
+    op.drop_table('oslo_config_schema')
182 183
     op.drop_table('device_driver')
183 184
     op.drop_table('service_worker')
184 185
     op.drop_table('service_component')

+ 56
- 0
namos/db/sqlalchemy/api.py View File

@@ -488,6 +488,10 @@ def service_worker_get_all(context):
488 488
     return _get_all(context, models.ServiceWorker)
489 489
 
490 490
 
491
+def service_worker_get_all_by(context, **kwargs):
492
+    return _service_worker_get_all_by(context, **kwargs)
493
+
494
+
491 495
 def _service_worker_get_all_by(context, **kwargs):
492 496
     return _get_all_by(context, models.ServiceWorker, **kwargs)
493 497
 
@@ -496,6 +500,58 @@ def service_worker_delete(context, _id):
496 500
     return _delete(context, models.ServiceWorker, _id)
497 501
 
498 502
 
503
+# Config Schema
504
+
505
+def config_schema_create(context, values):
506
+    return _create(context, models.OsloConfigSchema(), values)
507
+
508
+
509
+def config_schema_update(context, _id, values):
510
+    return _update(context, models.OsloConfigSchema, _id, values)
511
+
512
+
513
+def config_schema_get(context, _id):
514
+    config = _get(context, models.OsloConfigSchema, _id)
515
+    if config is None:
516
+        raise exception.ConfigNotFound(config_schema_id=_id)
517
+
518
+    return config
519
+
520
+
521
+def config_schema_get_by_name(context, name):
522
+    config = _get_by_name(context, models.OsloConfigSchema, name)
523
+    if config is None:
524
+        raise exception.ConfigSchemaNotFound(config_schema_id=name)
525
+
526
+    return config
527
+
528
+
529
+def config_schema_get_by(context,
530
+                         namespace=None,
531
+                         group=None,
532
+                         name=None):
533
+    query = _model_query(context, models.OsloConfigSchema)
534
+    if name is not None:
535
+        query = query.filter_by(name=name)
536
+    if group is not None:
537
+        query = query.filter_by(group_name=group)
538
+    if namespace is not None:
539
+        query = query.filter_by(namespace=namespace)
540
+    return query.all()
541
+
542
+
543
+def config_schema_get_all(context):
544
+    return _get_all(context, models.OsloConfigSchema)
545
+
546
+
547
+def _config_schema_get_all_by(context, **kwargs):
548
+    return _get_all_by(context, models.OsloConfigSchema, **kwargs)
549
+
550
+
551
+def config_schema_delete(context, _id):
552
+    return _delete(context, models.OsloConfigSchema, _id)
553
+
554
+
499 555
 # Config
500 556
 
501 557
 def config_create(context, values):

+ 51
- 13
namos/db/sqlalchemy/models.py View File

@@ -271,8 +271,9 @@ class ServiceWorker(BASE,
271 271
                              default=lambda: str(uuid.uuid4()))
272 272
 
273 273
     pid = sqlalchemy.Column(
274
-        sqlalchemy.String(32),
275
-        nullable=False
274
+        sqlalchemy.String(64),
275
+        nullable=False,
276
+        unique=True
276 277
     )
277 278
     host = sqlalchemy.Column(
278 279
         sqlalchemy.String(248),
@@ -284,19 +285,17 @@ class ServiceWorker(BASE,
284 285
         nullable=False)
285 286
 
286 287
 
287
-class OsloConfig(BASE,
288
-                 NamosBase,
289
-                 SoftDelete,
290
-                 Extra):
291
-    __tablename__ = 'oslo_config'
288
+class OsloConfigSchema(BASE,
289
+                       NamosBase,
290
+                       Extra):
291
+    __tablename__ = 'oslo_config_schema'
292 292
 
293
+    # TODO(mrkanag) Check whether conf is unique across all services or only
294
+    # sepcific to namespace, otherwise uniqueconstraint is name, group_name
293 295
     __table_args__ = (
294
-        UniqueConstraint("name", "service_worker_id"),
296
+        UniqueConstraint("group_name", "name", "namespace"),
295 297
     )
296 298
 
297
-    default_value = sqlalchemy.Column(
298
-        sqlalchemy.Text
299
-    )
300 299
     name = sqlalchemy.Column(sqlalchemy.String(255),
301 300
                              # unique=True,
302 301
                              nullable=False,
@@ -308,10 +307,20 @@ class OsloConfig(BASE,
308 307
         default=''
309 308
     )
310 309
     type = sqlalchemy.Column(
311
-        sqlalchemy.String(16),
310
+        sqlalchemy.String(128),
312 311
         nullable=False
313 312
     )
314
-    value = sqlalchemy.Column(
313
+    group_name = sqlalchemy.Column(
314
+        sqlalchemy.String(128),
315
+        nullable=False
316
+    )
317
+    namespace = sqlalchemy.Column(
318
+        sqlalchemy.String(128),
319
+        nullable=False
320
+    )
321
+    # TODO(mrkanag) default value is some time overriden by services, which
322
+    # osloconfig allows, so this column should have values per given service
323
+    default_value = sqlalchemy.Column(
315 324
         sqlalchemy.Text
316 325
     )
317 326
     required = sqlalchemy.Column(
@@ -322,9 +331,38 @@ class OsloConfig(BASE,
322 331
         sqlalchemy.Boolean,
323 332
         default=False
324 333
     )
334
+    mutable = sqlalchemy.Column(
335
+        sqlalchemy.Boolean,
336
+        default=False
337
+    )
338
+
339
+
340
+class OsloConfig(BASE,
341
+                 NamosBase,
342
+                 SoftDelete,
343
+                 Extra):
344
+    __tablename__ = 'oslo_config'
345
+
346
+    __table_args__ = (
347
+        UniqueConstraint("oslo_config_schema_id", "service_worker_id"),
348
+    )
349
+
350
+    name = sqlalchemy.Column(sqlalchemy.String(255),
351
+                             # unique=True,
352
+                             nullable=False,
353
+                             default=lambda: str(uuid.uuid4()))
354
+
355
+    value = sqlalchemy.Column(
356
+        sqlalchemy.Text
357
+    )
325 358
     file = sqlalchemy.Column(
326 359
         sqlalchemy.String(512)
327 360
     )
361
+    oslo_config_schema_id = sqlalchemy.Column(
362
+        Uuid,
363
+        sqlalchemy.ForeignKey('oslo_config_schema.id'),
364
+        nullable=False
365
+    )
328 366
     service_worker_id = sqlalchemy.Column(
329 367
         Uuid,
330 368
         sqlalchemy.ForeignKey('service_worker.id'),

+ 4
- 1
setup.cfg View File

@@ -46,4 +46,7 @@ output_file = namos/locale/namos.pot
46 46
 
47 47
 [entry_points]
48 48
 console_scripts =
49
-    namos-manage = namos.cmd.manage:main
49
+    namos-manage = namos.cmd.manage:main
50
+
51
+oslo.config.opts =
52
+    namos.common.config = namos.common.config:list_opts

Loading…
Cancel
Save