Browse Source

Sync charm-helpers

Change-Id: Ide6ee5ebf067dde8fc4118f371a3d759030102c0
changes/99/546599/2
Ryan Beisner 1 year ago
parent
commit
5c0e2c8c3a

+ 10
- 1
hooks/charmhelpers/contrib/network/ip.py View File

@@ -27,6 +27,7 @@ from charmhelpers.core.hookenv import (
27 27
     network_get_primary_address,
28 28
     unit_get,
29 29
     WARNING,
30
+    NoNetworkBinding,
30 31
 )
31 32
 
32 33
 from charmhelpers.core.host import (
@@ -109,7 +110,12 @@ def get_address_in_network(network, fallback=None, fatal=False):
109 110
         _validate_cidr(network)
110 111
         network = netaddr.IPNetwork(network)
111 112
         for iface in netifaces.interfaces():
112
-            addresses = netifaces.ifaddresses(iface)
113
+            try:
114
+                addresses = netifaces.ifaddresses(iface)
115
+            except ValueError:
116
+                # If an instance was deleted between
117
+                # netifaces.interfaces() run and now, its interfaces are gone
118
+                continue
113 119
             if network.version == 4 and netifaces.AF_INET in addresses:
114 120
                 for addr in addresses[netifaces.AF_INET]:
115 121
                     cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'],
@@ -578,6 +584,9 @@ def get_relation_ip(interface, cidr_network=None):
578 584
     except NotImplementedError:
579 585
         # If network-get is not available
580 586
         address = get_host_ip(unit_get('private-address'))
587
+    except NoNetworkBinding:
588
+        log("No network binding for {}".format(interface), WARNING)
589
+        address = get_host_ip(unit_get('private-address'))
581 590
 
582 591
     if config('prefer-ipv6'):
583 592
         # Currently IPv6 has priority, eventually we want IPv6 to just be

+ 2
- 2
hooks/charmhelpers/contrib/openstack/amulet/utils.py View File

@@ -92,7 +92,7 @@ class OpenStackAmuletUtils(AmuletUtils):
92 92
             return 'endpoint not found'
93 93
 
94 94
     def validate_v3_endpoint_data(self, endpoints, admin_port, internal_port,
95
-                                  public_port, expected):
95
+                                  public_port, expected, expected_num_eps=3):
96 96
         """Validate keystone v3 endpoint data.
97 97
 
98 98
         Validate the v3 endpoint data which has changed from v2.  The
@@ -138,7 +138,7 @@ class OpenStackAmuletUtils(AmuletUtils):
138 138
                 if ret:
139 139
                     return 'unexpected endpoint data - {}'.format(ret)
140 140
 
141
-        if len(found) != 3:
141
+        if len(found) != expected_num_eps:
142 142
             return 'Unexpected number of endpoints found'
143 143
 
144 144
     def validate_svc_catalog_endpoint_data(self, expected, actual):

+ 106
- 11
hooks/charmhelpers/contrib/openstack/context.py View File

@@ -617,7 +617,9 @@ class HAProxyContext(OSContextGenerator):
617 617
     """
618 618
     interfaces = ['cluster']
619 619
 
620
-    def __init__(self, singlenode_mode=False):
620
+    def __init__(self, singlenode_mode=False,
621
+                 address_types=ADDRESS_TYPES):
622
+        self.address_types = address_types
621 623
         self.singlenode_mode = singlenode_mode
622 624
 
623 625
     def __call__(self):
@@ -631,7 +633,7 @@ class HAProxyContext(OSContextGenerator):
631 633
 
632 634
         # NOTE(jamespage): build out map of configured network endpoints
633 635
         # and associated backends
634
-        for addr_type in ADDRESS_TYPES:
636
+        for addr_type in self.address_types:
635 637
             cfg_opt = 'os-{}-network'.format(addr_type)
636 638
             # NOTE(thedac) For some reason the ADDRESS_MAP uses 'int' rather
637 639
             # than 'internal'
@@ -1635,18 +1637,84 @@ class InternalEndpointContext(OSContextGenerator):
1635 1637
     endpoints by default so this allows admins to optionally use internal
1636 1638
     endpoints.
1637 1639
     """
1638
-    def __init__(self, ost_rel_check_pkg_name):
1639
-        self.ost_rel_check_pkg_name = ost_rel_check_pkg_name
1640
-
1641 1640
     def __call__(self):
1642
-        ctxt = {'use_internal_endpoints': config('use-internal-endpoints')}
1643
-        rel = os_release(self.ost_rel_check_pkg_name, base='icehouse')
1641
+        return {'use_internal_endpoints': config('use-internal-endpoints')}
1642
+
1643
+
1644
+class VolumeAPIContext(InternalEndpointContext):
1645
+    """Volume API context.
1646
+
1647
+    This context provides information regarding the volume endpoint to use
1648
+    when communicating between services. It determines which version of the
1649
+    API is appropriate for use.
1650
+
1651
+    This value will be determined in the resulting context dictionary
1652
+    returned from calling the VolumeAPIContext object. Information provided
1653
+    by this context is as follows:
1654
+
1655
+        volume_api_version: the volume api version to use, currently
1656
+            'v2' or 'v3'
1657
+        volume_catalog_info: the information to use for a cinder client
1658
+            configuration that consumes API endpoints from the keystone
1659
+            catalog. This is defined as the type:name:endpoint_type string.
1660
+    """
1661
+    # FIXME(wolsen) This implementation is based on the provider being able
1662
+    # to specify the package version to check but does not guarantee that the
1663
+    # volume service api version selected is available. In practice, it is
1664
+    # quite likely the volume service *is* providing the v3 volume service.
1665
+    # This should be resolved when the service-discovery spec is implemented.
1666
+    def __init__(self, pkg):
1667
+        """
1668
+        Creates a new VolumeAPIContext for use in determining which version
1669
+        of the Volume API should be used for communication. A package codename
1670
+        should be supplied for determining the currently installed OpenStack
1671
+        version.
1672
+
1673
+        :param pkg: the package codename to use in order to determine the
1674
+            component version (e.g. nova-common). See
1675
+            charmhelpers.contrib.openstack.utils.PACKAGE_CODENAMES for more.
1676
+        """
1677
+        super(VolumeAPIContext, self).__init__()
1678
+        self._ctxt = None
1679
+        if not pkg:
1680
+            raise ValueError('package name must be provided in order to '
1681
+                             'determine current OpenStack version.')
1682
+        self.pkg = pkg
1683
+
1684
+    @property
1685
+    def ctxt(self):
1686
+        if self._ctxt is not None:
1687
+            return self._ctxt
1688
+        self._ctxt = self._determine_ctxt()
1689
+        return self._ctxt
1690
+
1691
+    def _determine_ctxt(self):
1692
+        """Determines the Volume API endpoint information.
1693
+
1694
+        Determines the appropriate version of the API that should be used
1695
+        as well as the catalog_info string that would be supplied. Returns
1696
+        a dict containing the volume_api_version and the volume_catalog_info.
1697
+        """
1698
+        rel = os_release(self.pkg, base='icehouse')
1699
+        version = '2'
1644 1700
         if CompareOpenStackReleases(rel) >= 'pike':
1645
-            ctxt['volume_api_version'] = '3'
1646
-        else:
1647
-            ctxt['volume_api_version'] = '2'
1701
+            version = '3'
1702
+
1703
+        service_type = 'volumev{version}'.format(version=version)
1704
+        service_name = 'cinderv{version}'.format(version=version)
1705
+        endpoint_type = 'publicURL'
1706
+        if config('use-internal-endpoints'):
1707
+            endpoint_type = 'internalURL'
1708
+        catalog_info = '{type}:{name}:{endpoint}'.format(
1709
+            type=service_type, name=service_name, endpoint=endpoint_type)
1710
+
1711
+        return {
1712
+            'volume_api_version': version,
1713
+            'volume_catalog_info': catalog_info,
1714
+        }
1648 1715
 
1649
-        return ctxt
1716
+    def __call__(self):
1717
+        return self.ctxt
1650 1718
 
1651 1719
 
1652 1720
 class AppArmorContext(OSContextGenerator):
@@ -1784,3 +1852,30 @@ class MemcacheContext(OSContextGenerator):
1784 1852
                     ctxt['memcache_server_formatted'],
1785 1853
                     ctxt['memcache_port'])
1786 1854
         return ctxt
1855
+
1856
+
1857
+class EnsureDirContext(OSContextGenerator):
1858
+    '''
1859
+    Serves as a generic context to create a directory as a side-effect.
1860
+
1861
+    Useful for software that supports drop-in files (.d) in conjunction
1862
+    with config option-based templates. Examples include:
1863
+        * OpenStack oslo.policy drop-in files;
1864
+        * systemd drop-in config files;
1865
+        * other software that supports overriding defaults with .d files
1866
+
1867
+    Another use-case is when a subordinate generates a configuration for
1868
+    primary to render in a separate directory.
1869
+
1870
+    Some software requires a user to create a target directory to be
1871
+    scanned for drop-in files with a specific format. This is why this
1872
+    context is needed to do that before rendering a template.
1873
+   '''
1874
+
1875
+    def __init__(self, dirname):
1876
+        '''Used merely to ensure that a given directory exists.'''
1877
+        self.dirname = dirname
1878
+
1879
+    def __call__(self):
1880
+        mkdir(self.dirname)
1881
+        return {}

+ 71
- 24
hooks/charmhelpers/contrib/openstack/templating.py View File

@@ -93,7 +93,8 @@ class OSConfigTemplate(object):
93 93
     Associates a config file template with a list of context generators.
94 94
     Responsible for constructing a template context based on those generators.
95 95
     """
96
-    def __init__(self, config_file, contexts):
96
+
97
+    def __init__(self, config_file, contexts, config_template=None):
97 98
         self.config_file = config_file
98 99
 
99 100
         if hasattr(contexts, '__call__'):
@@ -103,6 +104,8 @@ class OSConfigTemplate(object):
103 104
 
104 105
         self._complete_contexts = []
105 106
 
107
+        self.config_template = config_template
108
+
106 109
     def context(self):
107 110
         ctxt = {}
108 111
         for context in self.contexts:
@@ -124,6 +127,11 @@ class OSConfigTemplate(object):
124 127
         self.context()
125 128
         return self._complete_contexts
126 129
 
130
+    @property
131
+    def is_string_template(self):
132
+        """:returns: Boolean if this instance is a template initialised with a string"""
133
+        return self.config_template is not None
134
+
127 135
 
128 136
 class OSConfigRenderer(object):
129 137
     """
@@ -148,6 +156,10 @@ class OSConfigRenderer(object):
148 156
                          contexts=[context.IdentityServiceContext()])
149 157
         configs.register(config_file='/etc/haproxy/haproxy.conf',
150 158
                          contexts=[context.HAProxyContext()])
159
+        configs.register(config_file='/etc/keystone/policy.d/extra.cfg',
160
+                         contexts=[context.ExtraPolicyContext()
161
+                                   context.KeystoneContext()],
162
+                         config_template=hookenv.config('extra-policy'))
151 163
         # write out a single config
152 164
         configs.write('/etc/nova/nova.conf')
153 165
         # write out all registered configs
@@ -218,14 +230,23 @@ class OSConfigRenderer(object):
218 230
             else:
219 231
                 apt_install('python3-jinja2')
220 232
 
221
-    def register(self, config_file, contexts):
233
+    def register(self, config_file, contexts, config_template=None):
222 234
         """
223 235
         Register a config file with a list of context generators to be called
224 236
         during rendering.
237
+        config_template can be used to load a template from a string instead of
238
+        using template loaders and template files.
239
+        :param config_file (str): a path where a config file will be rendered
240
+        :param contexts (list): a list of context dictionaries with kv pairs
241
+        :param config_template (str): an optional template string to use
225 242
         """
226
-        self.templates[config_file] = OSConfigTemplate(config_file=config_file,
227
-                                                       contexts=contexts)
228
-        log('Registered config file: %s' % config_file, level=INFO)
243
+        self.templates[config_file] = OSConfigTemplate(
244
+            config_file=config_file,
245
+            contexts=contexts,
246
+            config_template=config_template
247
+        )
248
+        log('Registered config file: {}'.format(config_file),
249
+            level=INFO)
229 250
 
230 251
     def _get_tmpl_env(self):
231 252
         if not self._tmpl_env:
@@ -235,32 +256,58 @@ class OSConfigRenderer(object):
235 256
     def _get_template(self, template):
236 257
         self._get_tmpl_env()
237 258
         template = self._tmpl_env.get_template(template)
238
-        log('Loaded template from %s' % template.filename, level=INFO)
259
+        log('Loaded template from {}'.format(template.filename),
260
+            level=INFO)
261
+        return template
262
+
263
+    def _get_template_from_string(self, ostmpl):
264
+        '''
265
+        Get a jinja2 template object from a string.
266
+        :param ostmpl: OSConfigTemplate to use as a data source.
267
+        '''
268
+        self._get_tmpl_env()
269
+        template = self._tmpl_env.from_string(ostmpl.config_template)
270
+        log('Loaded a template from a string for {}'.format(
271
+            ostmpl.config_file),
272
+            level=INFO)
239 273
         return template
240 274
 
241 275
     def render(self, config_file):
242 276
         if config_file not in self.templates:
243
-            log('Config not registered: %s' % config_file, level=ERROR)
277
+            log('Config not registered: {}'.format(config_file), level=ERROR)
244 278
             raise OSConfigException
245
-        ctxt = self.templates[config_file].context()
246
-
247
-        _tmpl = os.path.basename(config_file)
248
-        try:
249
-            template = self._get_template(_tmpl)
250
-        except exceptions.TemplateNotFound:
251
-            # if no template is found with basename, try looking for it
252
-            # using a munged full path, eg:
253
-            #   /etc/apache2/apache2.conf -> etc_apache2_apache2.conf
254
-            _tmpl = '_'.join(config_file.split('/')[1:])
279
+
280
+        ostmpl = self.templates[config_file]
281
+        ctxt = ostmpl.context()
282
+
283
+        if ostmpl.is_string_template:
284
+            template = self._get_template_from_string(ostmpl)
285
+            log('Rendering from a string template: '
286
+                '{}'.format(config_file),
287
+                level=INFO)
288
+        else:
289
+            _tmpl = os.path.basename(config_file)
255 290
             try:
256 291
                 template = self._get_template(_tmpl)
257
-            except exceptions.TemplateNotFound as e:
258
-                log('Could not load template from %s by %s or %s.' %
259
-                    (self.templates_dir, os.path.basename(config_file), _tmpl),
260
-                    level=ERROR)
261
-                raise e
262
-
263
-        log('Rendering from template: %s' % _tmpl, level=INFO)
292
+            except exceptions.TemplateNotFound:
293
+                # if no template is found with basename, try looking
294
+                # for it using a munged full path, eg:
295
+                # /etc/apache2/apache2.conf -> etc_apache2_apache2.conf
296
+                _tmpl = '_'.join(config_file.split('/')[1:])
297
+                try:
298
+                    template = self._get_template(_tmpl)
299
+                except exceptions.TemplateNotFound as e:
300
+                    log('Could not load template from {} by {} or {}.'
301
+                        ''.format(
302
+                            self.templates_dir,
303
+                            os.path.basename(config_file),
304
+                            _tmpl
305
+                        ),
306
+                        level=ERROR)
307
+                    raise e
308
+
309
+            log('Rendering from template: {}'.format(config_file),
310
+                level=INFO)
264 311
         return template.render(ctxt)
265 312
 
266 313
     def write(self, config_file):

+ 15
- 1
hooks/charmhelpers/core/hookenv.py View File

@@ -820,6 +820,10 @@ class Hooks(object):
820 820
         return wrapper
821 821
 
822 822
 
823
+class NoNetworkBinding(Exception):
824
+    pass
825
+
826
+
823 827
 def charm_dir():
824 828
     """Return the root directory of the current charm"""
825 829
     d = os.environ.get('JUJU_CHARM_DIR')
@@ -1106,7 +1110,17 @@ def network_get_primary_address(binding):
1106 1110
     :raise: NotImplementedError if run on Juju < 2.0
1107 1111
     '''
1108 1112
     cmd = ['network-get', '--primary-address', binding]
1109
-    return subprocess.check_output(cmd).decode('UTF-8').strip()
1113
+    try:
1114
+        response = subprocess.check_output(
1115
+            cmd,
1116
+            stderr=subprocess.STDOUT).decode('UTF-8').strip()
1117
+    except CalledProcessError as e:
1118
+        if 'no network config found for binding' in e.output.decode('UTF-8'):
1119
+            raise NoNetworkBinding("No network binding for {}"
1120
+                                   .format(binding))
1121
+        else:
1122
+            raise
1123
+    return response
1110 1124
 
1111 1125
 
1112 1126
 @translate_exc(from_exc=OSError, to_exc=NotImplementedError)

+ 18
- 9
hooks/charmhelpers/core/templating.py View File

@@ -20,7 +20,8 @@ from charmhelpers.core import hookenv
20 20
 
21 21
 
22 22
 def render(source, target, context, owner='root', group='root',
23
-           perms=0o444, templates_dir=None, encoding='UTF-8', template_loader=None):
23
+           perms=0o444, templates_dir=None, encoding='UTF-8',
24
+           template_loader=None, config_template=None):
24 25
     """
25 26
     Render a template.
26 27
 
@@ -32,6 +33,9 @@ def render(source, target, context, owner='root', group='root',
32 33
     The context should be a dict containing the values to be replaced in the
33 34
     template.
34 35
 
36
+    config_template may be provided to render from a provided template instead
37
+    of loading from a file.
38
+
35 39
     The `owner`, `group`, and `perms` options will be passed to `write_file`.
36 40
 
37 41
     If omitted, `templates_dir` defaults to the `templates` folder in the charm.
@@ -65,14 +69,19 @@ def render(source, target, context, owner='root', group='root',
65 69
         if templates_dir is None:
66 70
             templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
67 71
         template_env = Environment(loader=FileSystemLoader(templates_dir))
68
-    try:
69
-        source = source
70
-        template = template_env.get_template(source)
71
-    except exceptions.TemplateNotFound as e:
72
-        hookenv.log('Could not load template %s from %s.' %
73
-                    (source, templates_dir),
74
-                    level=hookenv.ERROR)
75
-        raise e
72
+
73
+    # load from a string if provided explicitly
74
+    if config_template is not None:
75
+        template = template_env.from_string(config_template)
76
+    else:
77
+        try:
78
+            source = source
79
+            template = template_env.get_template(source)
80
+        except exceptions.TemplateNotFound as e:
81
+            hookenv.log('Could not load template %s from %s.' %
82
+                        (source, templates_dir),
83
+                        level=hookenv.ERROR)
84
+            raise e
76 85
     content = template.render(context)
77 86
     if target is not None:
78 87
         target_dir = os.path.dirname(target)

+ 2
- 2
tests/charmhelpers/contrib/openstack/amulet/utils.py View File

@@ -92,7 +92,7 @@ class OpenStackAmuletUtils(AmuletUtils):
92 92
             return 'endpoint not found'
93 93
 
94 94
     def validate_v3_endpoint_data(self, endpoints, admin_port, internal_port,
95
-                                  public_port, expected):
95
+                                  public_port, expected, expected_num_eps=3):
96 96
         """Validate keystone v3 endpoint data.
97 97
 
98 98
         Validate the v3 endpoint data which has changed from v2.  The
@@ -138,7 +138,7 @@ class OpenStackAmuletUtils(AmuletUtils):
138 138
                 if ret:
139 139
                     return 'unexpected endpoint data - {}'.format(ret)
140 140
 
141
-        if len(found) != 3:
141
+        if len(found) != expected_num_eps:
142 142
             return 'Unexpected number of endpoints found'
143 143
 
144 144
     def validate_svc_catalog_endpoint_data(self, expected, actual):

+ 15
- 1
tests/charmhelpers/core/hookenv.py View File

@@ -820,6 +820,10 @@ class Hooks(object):
820 820
         return wrapper
821 821
 
822 822
 
823
+class NoNetworkBinding(Exception):
824
+    pass
825
+
826
+
823 827
 def charm_dir():
824 828
     """Return the root directory of the current charm"""
825 829
     d = os.environ.get('JUJU_CHARM_DIR')
@@ -1106,7 +1110,17 @@ def network_get_primary_address(binding):
1106 1110
     :raise: NotImplementedError if run on Juju < 2.0
1107 1111
     '''
1108 1112
     cmd = ['network-get', '--primary-address', binding]
1109
-    return subprocess.check_output(cmd).decode('UTF-8').strip()
1113
+    try:
1114
+        response = subprocess.check_output(
1115
+            cmd,
1116
+            stderr=subprocess.STDOUT).decode('UTF-8').strip()
1117
+    except CalledProcessError as e:
1118
+        if 'no network config found for binding' in e.output.decode('UTF-8'):
1119
+            raise NoNetworkBinding("No network binding for {}"
1120
+                                   .format(binding))
1121
+        else:
1122
+            raise
1123
+    return response
1110 1124
 
1111 1125
 
1112 1126
 @translate_exc(from_exc=OSError, to_exc=NotImplementedError)

+ 18
- 9
tests/charmhelpers/core/templating.py View File

@@ -20,7 +20,8 @@ from charmhelpers.core import hookenv
20 20
 
21 21
 
22 22
 def render(source, target, context, owner='root', group='root',
23
-           perms=0o444, templates_dir=None, encoding='UTF-8', template_loader=None):
23
+           perms=0o444, templates_dir=None, encoding='UTF-8',
24
+           template_loader=None, config_template=None):
24 25
     """
25 26
     Render a template.
26 27
 
@@ -32,6 +33,9 @@ def render(source, target, context, owner='root', group='root',
32 33
     The context should be a dict containing the values to be replaced in the
33 34
     template.
34 35
 
36
+    config_template may be provided to render from a provided template instead
37
+    of loading from a file.
38
+
35 39
     The `owner`, `group`, and `perms` options will be passed to `write_file`.
36 40
 
37 41
     If omitted, `templates_dir` defaults to the `templates` folder in the charm.
@@ -65,14 +69,19 @@ def render(source, target, context, owner='root', group='root',
65 69
         if templates_dir is None:
66 70
             templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
67 71
         template_env = Environment(loader=FileSystemLoader(templates_dir))
68
-    try:
69
-        source = source
70
-        template = template_env.get_template(source)
71
-    except exceptions.TemplateNotFound as e:
72
-        hookenv.log('Could not load template %s from %s.' %
73
-                    (source, templates_dir),
74
-                    level=hookenv.ERROR)
75
-        raise e
72
+
73
+    # load from a string if provided explicitly
74
+    if config_template is not None:
75
+        template = template_env.from_string(config_template)
76
+    else:
77
+        try:
78
+            source = source
79
+            template = template_env.get_template(source)
80
+        except exceptions.TemplateNotFound as e:
81
+            hookenv.log('Could not load template %s from %s.' %
82
+                        (source, templates_dir),
83
+                        level=hookenv.ERROR)
84
+            raise e
76 85
     content = template.render(context)
77 86
     if target is not None:
78 87
         target_dir = os.path.dirname(target)

+ 1
- 1
tox.ini View File

@@ -9,7 +9,7 @@ skipsdist = True
9 9
 setenv = VIRTUAL_ENV={envdir}
10 10
          PYTHONHASHSEED=0
11 11
          CHARM_DIR={envdir}
12
-         AMULET_SETUP_TIMEOUT=2700
12
+         AMULET_SETUP_TIMEOUT=5400
13 13
 install_command =
14 14
   pip install --allow-unverified python-apt {opts} {packages}
15 15
 commands = ostestr {posargs}

Loading…
Cancel
Save