Browse Source

Sync charm-helpers

Change-Id: Ib52cd708e1e04489f150bba62a3a3c3696f5e874
changes/32/522332/2
Ryan Beisner 1 year ago
parent
commit
87bbbe1bb0
31 changed files with 682 additions and 187 deletions
  1. 1
    1
      hooks/charmhelpers/contrib/hahelpers/apache.py
  2. 30
    0
      hooks/charmhelpers/contrib/hahelpers/cluster.py
  3. 2
    2
      hooks/charmhelpers/contrib/network/ip.py
  4. 13
    0
      hooks/charmhelpers/contrib/openstack/alternatives.py
  5. 33
    11
      hooks/charmhelpers/contrib/openstack/amulet/deployment.py
  6. 29
    7
      hooks/charmhelpers/contrib/openstack/amulet/utils.py
  7. 24
    24
      hooks/charmhelpers/contrib/openstack/context.py
  8. 1
    1
      hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh
  9. 7
    4
      hooks/charmhelpers/contrib/openstack/ha/utils.py
  10. 10
    51
      hooks/charmhelpers/contrib/openstack/neutron.py
  11. 2
    2
      hooks/charmhelpers/contrib/openstack/templates/ceph.conf
  12. 2
    0
      hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg
  13. 6
    0
      hooks/charmhelpers/contrib/openstack/templates/section-oslo-cache
  14. 2
    0
      hooks/charmhelpers/contrib/openstack/templating.py
  15. 26
    16
      hooks/charmhelpers/contrib/openstack/utils.py
  16. 29
    13
      hooks/charmhelpers/contrib/storage/linux/ceph.py
  17. 4
    4
      hooks/charmhelpers/contrib/storage/linux/lvm.py
  18. 1
    1
      hooks/charmhelpers/contrib/storage/linux/utils.py
  19. 105
    7
      hooks/charmhelpers/core/hookenv.py
  20. 72
    1
      hooks/charmhelpers/core/host.py
  21. 11
    5
      hooks/charmhelpers/core/strutils.py
  22. 1
    1
      hooks/charmhelpers/core/unitdata.py
  23. 16
    0
      hooks/charmhelpers/fetch/snap.py
  24. 1
    1
      hooks/charmhelpers/fetch/ubuntu.py
  25. 3
    3
      tests/basic_deployment.py
  26. 33
    11
      tests/charmhelpers/contrib/openstack/amulet/deployment.py
  27. 29
    7
      tests/charmhelpers/contrib/openstack/amulet/utils.py
  28. 105
    7
      tests/charmhelpers/core/hookenv.py
  29. 72
    1
      tests/charmhelpers/core/host.py
  30. 11
    5
      tests/charmhelpers/core/strutils.py
  31. 1
    1
      tests/charmhelpers/core/unitdata.py

+ 1
- 1
hooks/charmhelpers/contrib/hahelpers/apache.py View File

@@ -90,6 +90,6 @@ def install_ca_cert(ca_cert):
90 90
             log("CA cert is the same as installed version", level=INFO)
91 91
         else:
92 92
             log("Installing new CA cert", level=INFO)
93
-            with open(cert_file, 'w') as crt:
93
+            with open(cert_file, 'wb') as crt:
94 94
                 crt.write(ca_cert)
95 95
             subprocess.check_call(['update-ca-certificates', '--fresh'])

+ 30
- 0
hooks/charmhelpers/contrib/hahelpers/cluster.py View File

@@ -27,6 +27,7 @@ clustering-related helpers.
27 27
 
28 28
 import subprocess
29 29
 import os
30
+import time
30 31
 
31 32
 from socket import gethostname as get_unit_hostname
32 33
 
@@ -45,6 +46,9 @@ from charmhelpers.core.hookenv import (
45 46
     is_leader as juju_is_leader,
46 47
     status_set,
47 48
 )
49
+from charmhelpers.core.host import (
50
+    modulo_distribution,
51
+)
48 52
 from charmhelpers.core.decorators import (
49 53
     retry_on_exception,
50 54
 )
@@ -361,3 +365,29 @@ def canonical_url(configs, vip_setting='vip'):
361 365
     else:
362 366
         addr = unit_get('private-address')
363 367
     return '%s://%s' % (scheme, addr)
368
+
369
+
370
+def distributed_wait(modulo=None, wait=None, operation_name='operation'):
371
+    ''' Distribute operations by waiting based on modulo_distribution
372
+
373
+    If modulo and or wait are not set, check config_get for those values.
374
+
375
+    :param modulo: int The modulo number creates the group distribution
376
+    :param wait: int The constant time wait value
377
+    :param operation_name: string Operation name for status message
378
+                           i.e.  'restart'
379
+    :side effect: Calls config_get()
380
+    :side effect: Calls log()
381
+    :side effect: Calls status_set()
382
+    :side effect: Calls time.sleep()
383
+    '''
384
+    if modulo is None:
385
+        modulo = config_get('modulo-nodes')
386
+    if wait is None:
387
+        wait = config_get('known-wait')
388
+    calculated_wait = modulo_distribution(modulo=modulo, wait=wait)
389
+    msg = "Waiting {} seconds for {} ...".format(calculated_wait,
390
+                                                 operation_name)
391
+    log(msg, DEBUG)
392
+    status_set('maintenance', msg)
393
+    time.sleep(calculated_wait)

+ 2
- 2
hooks/charmhelpers/contrib/network/ip.py View File

@@ -490,7 +490,7 @@ def get_host_ip(hostname, fallback=None):
490 490
     if not ip_addr:
491 491
         try:
492 492
             ip_addr = socket.gethostbyname(hostname)
493
-        except:
493
+        except Exception:
494 494
             log("Failed to resolve hostname '%s'" % (hostname),
495 495
                 level=WARNING)
496 496
             return fallback
@@ -518,7 +518,7 @@ def get_hostname(address, fqdn=True):
518 518
         if not result:
519 519
             try:
520 520
                 result = socket.gethostbyaddr(address)[0]
521
-            except:
521
+            except Exception:
522 522
                 return None
523 523
     else:
524 524
         result = address

+ 13
- 0
hooks/charmhelpers/contrib/openstack/alternatives.py View File

@@ -29,3 +29,16 @@ def install_alternative(name, target, source, priority=50):
29 29
         target, name, source, str(priority)
30 30
     ]
31 31
     subprocess.check_call(cmd)
32
+
33
+
34
+def remove_alternative(name, source):
35
+    """Remove an installed alternative configuration file
36
+
37
+    :param name: string name of the alternative to remove
38
+    :param source: string full path to alternative to remove
39
+    """
40
+    cmd = [
41
+        'update-alternatives', '--remove',
42
+        name, source
43
+    ]
44
+    subprocess.check_call(cmd)

+ 33
- 11
hooks/charmhelpers/contrib/openstack/amulet/deployment.py View File

@@ -13,6 +13,7 @@
13 13
 # limitations under the License.
14 14
 
15 15
 import logging
16
+import os
16 17
 import re
17 18
 import sys
18 19
 import six
@@ -185,7 +186,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
185 186
             self.d.configure(service, config)
186 187
 
187 188
     def _auto_wait_for_status(self, message=None, exclude_services=None,
188
-                              include_only=None, timeout=1800):
189
+                              include_only=None, timeout=None):
189 190
         """Wait for all units to have a specific extended status, except
190 191
         for any defined as excluded.  Unless specified via message, any
191 192
         status containing any case of 'ready' will be considered a match.
@@ -215,7 +216,10 @@ class OpenStackAmuletDeployment(AmuletDeployment):
215 216
         :param timeout: Maximum time in seconds to wait for status match
216 217
         :returns: None.  Raises if timeout is hit.
217 218
         """
218
-        self.log.info('Waiting for extended status on units...')
219
+        if not timeout:
220
+            timeout = int(os.environ.get('AMULET_SETUP_TIMEOUT', 1800))
221
+        self.log.info('Waiting for extended status on units for {}s...'
222
+                      ''.format(timeout))
219 223
 
220 224
         all_services = self.d.services.keys()
221 225
 
@@ -250,7 +254,14 @@ class OpenStackAmuletDeployment(AmuletDeployment):
250 254
         self.log.debug('Waiting up to {}s for extended status on services: '
251 255
                        '{}'.format(timeout, services))
252 256
         service_messages = {service: message for service in services}
257
+
258
+        # Check for idleness
259
+        self.d.sentry.wait(timeout=timeout)
260
+        # Check for error states and bail early
261
+        self.d.sentry.wait_for_status(self.d.juju_env, services, timeout=timeout)
262
+        # Check for ready messages
253 263
         self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
264
+
254 265
         self.log.info('OK')
255 266
 
256 267
     def _get_openstack_release(self):
@@ -263,7 +274,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
263 274
         (self.trusty_icehouse, self.trusty_kilo, self.trusty_liberty,
264 275
          self.trusty_mitaka, self.xenial_mitaka, self.xenial_newton,
265 276
          self.yakkety_newton, self.xenial_ocata, self.zesty_ocata,
266
-         self.xenial_pike, self.artful_pike) = range(11)
277
+         self.xenial_pike, self.artful_pike, self.xenial_queens,
278
+         self.bionic_queens,) = range(13)
267 279
 
268 280
         releases = {
269 281
             ('trusty', None): self.trusty_icehouse,
@@ -274,9 +286,11 @@ class OpenStackAmuletDeployment(AmuletDeployment):
274 286
             ('xenial', 'cloud:xenial-newton'): self.xenial_newton,
275 287
             ('xenial', 'cloud:xenial-ocata'): self.xenial_ocata,
276 288
             ('xenial', 'cloud:xenial-pike'): self.xenial_pike,
289
+            ('xenial', 'cloud:xenial-queens'): self.xenial_queens,
277 290
             ('yakkety', None): self.yakkety_newton,
278 291
             ('zesty', None): self.zesty_ocata,
279 292
             ('artful', None): self.artful_pike,
293
+            ('bionic', None): self.bionic_queens,
280 294
         }
281 295
         return releases[(self.series, self.openstack)]
282 296
 
@@ -291,6 +305,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
291 305
             ('yakkety', 'newton'),
292 306
             ('zesty', 'ocata'),
293 307
             ('artful', 'pike'),
308
+            ('bionic', 'queens'),
294 309
         ])
295 310
         if self.openstack:
296 311
             os_origin = self.openstack.split(':')[1]
@@ -303,20 +318,27 @@ class OpenStackAmuletDeployment(AmuletDeployment):
303 318
         test scenario, based on OpenStack release and whether ceph radosgw
304 319
         is flagged as present or not."""
305 320
 
306
-        if self._get_openstack_release() >= self.trusty_kilo:
307
-            # Kilo or later
321
+        if self._get_openstack_release() == self.trusty_icehouse:
322
+            # Icehouse
308 323
             pools = [
324
+                'data',
325
+                'metadata',
309 326
                 'rbd',
310
-                'cinder',
327
+                'cinder-ceph',
311 328
                 'glance'
312 329
             ]
313
-        else:
314
-            # Juno or earlier
330
+        elif (self.trusty_kilo <= self._get_openstack_release() <=
331
+              self.zesty_ocata):
332
+            # Kilo through Ocata
315 333
             pools = [
316
-                'data',
317
-                'metadata',
318 334
                 'rbd',
319
-                'cinder',
335
+                'cinder-ceph',
336
+                'glance'
337
+            ]
338
+        else:
339
+            # Pike and later
340
+            pools = [
341
+                'cinder-ceph',
320 342
                 'glance'
321 343
             ]
322 344
 

+ 29
- 7
hooks/charmhelpers/contrib/openstack/amulet/utils.py View File

@@ -23,6 +23,7 @@ import urllib
23 23
 import urlparse
24 24
 
25 25
 import cinderclient.v1.client as cinder_client
26
+import cinderclient.v2.client as cinder_clientv2
26 27
 import glanceclient.v1.client as glance_client
27 28
 import heatclient.v1.client as heat_client
28 29
 from keystoneclient.v2_0 import client as keystone_client
@@ -42,7 +43,6 @@ import swiftclient
42 43
 from charmhelpers.contrib.amulet.utils import (
43 44
     AmuletUtils
44 45
 )
45
-from charmhelpers.core.decorators import retry_on_exception
46 46
 from charmhelpers.core.host import CompareHostReleases
47 47
 
48 48
 DEBUG = logging.DEBUG
@@ -310,7 +310,6 @@ class OpenStackAmuletUtils(AmuletUtils):
310 310
         self.log.debug('Checking if tenant exists ({})...'.format(tenant))
311 311
         return tenant in [t.name for t in keystone.tenants.list()]
312 312
 
313
-    @retry_on_exception(5, base_delay=10)
314 313
     def keystone_wait_for_propagation(self, sentry_relation_pairs,
315 314
                                       api_version):
316 315
         """Iterate over list of sentry and relation tuples and verify that
@@ -326,7 +325,7 @@ class OpenStackAmuletUtils(AmuletUtils):
326 325
             rel = sentry.relation('identity-service',
327 326
                                   relation_name)
328 327
             self.log.debug('keystone relation data: {}'.format(rel))
329
-            if rel['api_version'] != str(api_version):
328
+            if rel.get('api_version') != str(api_version):
330 329
                 raise Exception("api_version not propagated through relation"
331 330
                                 " data yet ('{}' != '{}')."
332 331
                                 "".format(rel['api_version'], api_version))
@@ -348,15 +347,19 @@ class OpenStackAmuletUtils(AmuletUtils):
348 347
 
349 348
         config = {'preferred-api-version': api_version}
350 349
         deployment.d.configure('keystone', config)
350
+        deployment._auto_wait_for_status()
351 351
         self.keystone_wait_for_propagation(sentry_relation_pairs, api_version)
352 352
 
353 353
     def authenticate_cinder_admin(self, keystone_sentry, username,
354
-                                  password, tenant):
354
+                                  password, tenant, api_version=2):
355 355
         """Authenticates admin user with cinder."""
356 356
         # NOTE(beisner): cinder python client doesn't accept tokens.
357 357
         keystone_ip = keystone_sentry.info['public-address']
358 358
         ept = "http://{}:5000/v2.0".format(keystone_ip.strip().decode('utf-8'))
359
-        return cinder_client.Client(username, password, tenant, ept)
359
+        _clients = {
360
+            1: cinder_client.Client,
361
+            2: cinder_clientv2.Client}
362
+        return _clients[api_version](username, password, tenant, ept)
360 363
 
361 364
     def authenticate_keystone(self, keystone_ip, username, password,
362 365
                               api_version=False, admin_port=False,
@@ -617,13 +620,25 @@ class OpenStackAmuletUtils(AmuletUtils):
617 620
             self.log.debug('Keypair ({}) already exists, '
618 621
                            'using it.'.format(keypair_name))
619 622
             return _keypair
620
-        except:
623
+        except Exception:
621 624
             self.log.debug('Keypair ({}) does not exist, '
622 625
                            'creating it.'.format(keypair_name))
623 626
 
624 627
         _keypair = nova.keypairs.create(name=keypair_name)
625 628
         return _keypair
626 629
 
630
+    def _get_cinder_obj_name(self, cinder_object):
631
+        """Retrieve name of cinder object.
632
+
633
+        :param cinder_object: cinder snapshot or volume object
634
+        :returns: str cinder object name
635
+        """
636
+        # v1 objects store name in 'display_name' attr but v2+ use 'name'
637
+        try:
638
+            return cinder_object.display_name
639
+        except AttributeError:
640
+            return cinder_object.name
641
+
627 642
     def create_cinder_volume(self, cinder, vol_name="demo-vol", vol_size=1,
628 643
                              img_id=None, src_vol_id=None, snap_id=None):
629 644
         """Create cinder volume, optionally from a glance image, OR
@@ -674,6 +689,13 @@ class OpenStackAmuletUtils(AmuletUtils):
674 689
                                             source_volid=src_vol_id,
675 690
                                             snapshot_id=snap_id)
676 691
             vol_id = vol_new.id
692
+        except TypeError:
693
+            vol_new = cinder.volumes.create(name=vol_name,
694
+                                            imageRef=img_id,
695
+                                            size=vol_size,
696
+                                            source_volid=src_vol_id,
697
+                                            snapshot_id=snap_id)
698
+            vol_id = vol_new.id
677 699
         except Exception as e:
678 700
             msg = 'Failed to create volume: {}'.format(e)
679 701
             amulet.raise_status(amulet.FAIL, msg=msg)
@@ -688,7 +710,7 @@ class OpenStackAmuletUtils(AmuletUtils):
688 710
 
689 711
         # Re-validate new volume
690 712
         self.log.debug('Validating volume attributes...')
691
-        val_vol_name = cinder.volumes.get(vol_id).display_name
713
+        val_vol_name = self._get_cinder_obj_name(cinder.volumes.get(vol_id))
692 714
         val_vol_boot = cinder.volumes.get(vol_id).bootable
693 715
         val_vol_stat = cinder.volumes.get(vol_id).status
694 716
         val_vol_size = cinder.volumes.get(vol_id).size

+ 24
- 24
hooks/charmhelpers/contrib/openstack/context.py View File

@@ -12,6 +12,7 @@
12 12
 # See the License for the specific language governing permissions and
13 13
 # limitations under the License.
14 14
 
15
+import collections
15 16
 import glob
16 17
 import json
17 18
 import math
@@ -292,7 +293,7 @@ class PostgresqlDBContext(OSContextGenerator):
292 293
 def db_ssl(rdata, ctxt, ssl_dir):
293 294
     if 'ssl_ca' in rdata and ssl_dir:
294 295
         ca_path = os.path.join(ssl_dir, 'db-client.ca')
295
-        with open(ca_path, 'w') as fh:
296
+        with open(ca_path, 'wb') as fh:
296 297
             fh.write(b64decode(rdata['ssl_ca']))
297 298
 
298 299
         ctxt['database_ssl_ca'] = ca_path
@@ -307,12 +308,12 @@ def db_ssl(rdata, ctxt, ssl_dir):
307 308
             log("Waiting 1m for ssl client cert validity", level=INFO)
308 309
             time.sleep(60)
309 310
 
310
-        with open(cert_path, 'w') as fh:
311
+        with open(cert_path, 'wb') as fh:
311 312
             fh.write(b64decode(rdata['ssl_cert']))
312 313
 
313 314
         ctxt['database_ssl_cert'] = cert_path
314 315
         key_path = os.path.join(ssl_dir, 'db-client.key')
315
-        with open(key_path, 'w') as fh:
316
+        with open(key_path, 'wb') as fh:
316 317
             fh.write(b64decode(rdata['ssl_key']))
317 318
 
318 319
         ctxt['database_ssl_key'] = key_path
@@ -458,7 +459,7 @@ class AMQPContext(OSContextGenerator):
458 459
 
459 460
                         ca_path = os.path.join(
460 461
                             self.ssl_dir, 'rabbit-client-ca.pem')
461
-                        with open(ca_path, 'w') as fh:
462
+                        with open(ca_path, 'wb') as fh:
462 463
                             fh.write(b64decode(ctxt['rabbit_ssl_ca']))
463 464
                             ctxt['rabbit_ssl_ca'] = ca_path
464 465
 
@@ -578,11 +579,14 @@ class HAProxyContext(OSContextGenerator):
578 579
             laddr = get_address_in_network(config(cfg_opt))
579 580
             if laddr:
580 581
                 netmask = get_netmask_for_address(laddr)
581
-                cluster_hosts[laddr] = {'network': "{}/{}".format(laddr,
582
-                                                                  netmask),
583
-                                        'backends': {l_unit: laddr}}
582
+                cluster_hosts[laddr] = {
583
+                    'network': "{}/{}".format(laddr,
584
+                                              netmask),
585
+                    'backends': collections.OrderedDict([(l_unit,
586
+                                                          laddr)])
587
+                }
584 588
                 for rid in relation_ids('cluster'):
585
-                    for unit in related_units(rid):
589
+                    for unit in sorted(related_units(rid)):
586 590
                         _laddr = relation_get('{}-address'.format(addr_type),
587 591
                                               rid=rid, unit=unit)
588 592
                         if _laddr:
@@ -594,10 +598,13 @@ class HAProxyContext(OSContextGenerator):
594 598
         # match in the frontend
595 599
         cluster_hosts[addr] = {}
596 600
         netmask = get_netmask_for_address(addr)
597
-        cluster_hosts[addr] = {'network': "{}/{}".format(addr, netmask),
598
-                               'backends': {l_unit: addr}}
601
+        cluster_hosts[addr] = {
602
+            'network': "{}/{}".format(addr, netmask),
603
+            'backends': collections.OrderedDict([(l_unit,
604
+                                                  addr)])
605
+        }
599 606
         for rid in relation_ids('cluster'):
600
-            for unit in related_units(rid):
607
+            for unit in sorted(related_units(rid)):
601 608
                 _laddr = relation_get('private-address',
602 609
                                       rid=rid, unit=unit)
603 610
                 if _laddr:
@@ -628,6 +635,8 @@ class HAProxyContext(OSContextGenerator):
628 635
             ctxt['local_host'] = '127.0.0.1'
629 636
             ctxt['haproxy_host'] = '0.0.0.0'
630 637
 
638
+        ctxt['ipv6_enabled'] = not is_ipv6_disabled()
639
+
631 640
         ctxt['stat_port'] = '8888'
632 641
 
633 642
         db = kv()
@@ -802,8 +811,9 @@ class ApacheSSLContext(OSContextGenerator):
802 811
         else:
803 812
             # Expect cert/key provided in config (currently assumed that ca
804 813
             # uses ip for cn)
805
-            cn = resolve_address(endpoint_type=INTERNAL)
806
-            self.configure_cert(cn)
814
+            for net_type in (INTERNAL, ADMIN, PUBLIC):
815
+                cn = resolve_address(endpoint_type=net_type)
816
+                self.configure_cert(cn)
807 817
 
808 818
         addresses = self.get_network_addresses()
809 819
         for address, endpoint in addresses:
@@ -843,15 +853,6 @@ class NeutronContext(OSContextGenerator):
843 853
         for pkgs in self.packages:
844 854
             ensure_packages(pkgs)
845 855
 
846
-    def _save_flag_file(self):
847
-        if self.network_manager == 'quantum':
848
-            _file = '/etc/nova/quantum_plugin.conf'
849
-        else:
850
-            _file = '/etc/nova/neutron_plugin.conf'
851
-
852
-        with open(_file, 'wb') as out:
853
-            out.write(self.plugin + '\n')
854
-
855 856
     def ovs_ctxt(self):
856 857
         driver = neutron_plugin_attribute(self.plugin, 'driver',
857 858
                                           self.network_manager)
@@ -996,7 +997,6 @@ class NeutronContext(OSContextGenerator):
996 997
             flags = config_flags_parser(alchemy_flags)
997 998
             ctxt['neutron_alchemy_flags'] = flags
998 999
 
999
-        self._save_flag_file()
1000 1000
         return ctxt
1001 1001
 
1002 1002
 
@@ -1176,7 +1176,7 @@ class SubordinateConfigContext(OSContextGenerator):
1176 1176
                 if sub_config and sub_config != '':
1177 1177
                     try:
1178 1178
                         sub_config = json.loads(sub_config)
1179
-                    except:
1179
+                    except Exception:
1180 1180
                         log('Could not parse JSON from '
1181 1181
                             'subordinate_configuration setting from %s'
1182 1182
                             % rid, level=ERROR)

+ 1
- 1
hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh View File

@@ -9,7 +9,7 @@
9 9
 CRITICAL=0
10 10
 NOTACTIVE=''
11 11
 LOGFILE=/var/log/nagios/check_haproxy.log
12
-AUTH=$(grep -r "stats auth" /etc/haproxy | awk 'NR=1{print $4}')
12
+AUTH=$(grep -r "stats auth" /etc/haproxy/haproxy.cfg | awk 'NR=1{print $4}')
13 13
 
14 14
 typeset -i N_INSTANCES=0
15 15
 for appserver in $(awk '/^\s+server/{print $2}' /etc/haproxy/haproxy.cfg)

+ 7
- 4
hooks/charmhelpers/contrib/openstack/ha/utils.py View File

@@ -82,15 +82,18 @@ def update_dns_ha_resource_params(resources, resource_params,
82 82
             continue
83 83
         m = re.search('os-(.+?)-hostname', setting)
84 84
         if m:
85
-            networkspace = m.group(1)
85
+            endpoint_type = m.group(1)
86
+            # resolve_address's ADDRESS_MAP uses 'int' not 'internal'
87
+            if endpoint_type == 'internal':
88
+                endpoint_type = 'int'
86 89
         else:
87 90
             msg = ('Unexpected DNS hostname setting: {}. '
88
-                   'Cannot determine network space name'
91
+                   'Cannot determine endpoint_type name'
89 92
                    ''.format(setting))
90 93
             status_set('blocked', msg)
91 94
             raise DNSHAException(msg)
92 95
 
93
-        hostname_key = 'res_{}_{}_hostname'.format(charm_name(), networkspace)
96
+        hostname_key = 'res_{}_{}_hostname'.format(charm_name(), endpoint_type)
94 97
         if hostname_key in hostname_group:
95 98
             log('DNS HA: Resource {}: {} already exists in '
96 99
                 'hostname group - skipping'.format(hostname_key, hostname),
@@ -101,7 +104,7 @@ def update_dns_ha_resource_params(resources, resource_params,
101 104
         resources[hostname_key] = crm_ocf
102 105
         resource_params[hostname_key] = (
103 106
             'params fqdn="{}" ip_address="{}" '
104
-            ''.format(hostname, resolve_address(endpoint_type=networkspace,
107
+            ''.format(hostname, resolve_address(endpoint_type=endpoint_type,
105 108
                                                 override=False)))
106 109
 
107 110
     if len(hostname_group) >= 1:

+ 10
- 51
hooks/charmhelpers/contrib/openstack/neutron.py View File

@@ -59,18 +59,13 @@ def determine_dkms_package():
59 59
 
60 60
 
61 61
 def quantum_plugins():
62
-    from charmhelpers.contrib.openstack import context
63 62
     return {
64 63
         'ovs': {
65 64
             'config': '/etc/quantum/plugins/openvswitch/'
66 65
                       'ovs_quantum_plugin.ini',
67 66
             'driver': 'quantum.plugins.openvswitch.ovs_quantum_plugin.'
68 67
                       'OVSQuantumPluginV2',
69
-            'contexts': [
70
-                context.SharedDBContext(user=config('neutron-database-user'),
71
-                                        database=config('neutron-database'),
72
-                                        relation_prefix='neutron',
73
-                                        ssl_dir=QUANTUM_CONF_DIR)],
68
+            'contexts': [],
74 69
             'services': ['quantum-plugin-openvswitch-agent'],
75 70
             'packages': [determine_dkms_package(),
76 71
                          ['quantum-plugin-openvswitch-agent']],
@@ -82,11 +77,7 @@ def quantum_plugins():
82 77
             'config': '/etc/quantum/plugins/nicira/nvp.ini',
83 78
             'driver': 'quantum.plugins.nicira.nicira_nvp_plugin.'
84 79
                       'QuantumPlugin.NvpPluginV2',
85
-            'contexts': [
86
-                context.SharedDBContext(user=config('neutron-database-user'),
87
-                                        database=config('neutron-database'),
88
-                                        relation_prefix='neutron',
89
-                                        ssl_dir=QUANTUM_CONF_DIR)],
80
+            'contexts': [],
90 81
             'services': [],
91 82
             'packages': [],
92 83
             'server_packages': ['quantum-server',
@@ -100,7 +91,6 @@ NEUTRON_CONF_DIR = '/etc/neutron'
100 91
 
101 92
 
102 93
 def neutron_plugins():
103
-    from charmhelpers.contrib.openstack import context
104 94
     release = os_release('nova-common')
105 95
     plugins = {
106 96
         'ovs': {
@@ -108,11 +98,7 @@ def neutron_plugins():
108 98
                       'ovs_neutron_plugin.ini',
109 99
             'driver': 'neutron.plugins.openvswitch.ovs_neutron_plugin.'
110 100
                       'OVSNeutronPluginV2',
111
-            'contexts': [
112
-                context.SharedDBContext(user=config('neutron-database-user'),
113
-                                        database=config('neutron-database'),
114
-                                        relation_prefix='neutron',
115
-                                        ssl_dir=NEUTRON_CONF_DIR)],
101
+            'contexts': [],
116 102
             'services': ['neutron-plugin-openvswitch-agent'],
117 103
             'packages': [determine_dkms_package(),
118 104
                          ['neutron-plugin-openvswitch-agent']],
@@ -124,11 +110,7 @@ def neutron_plugins():
124 110
             'config': '/etc/neutron/plugins/nicira/nvp.ini',
125 111
             'driver': 'neutron.plugins.nicira.nicira_nvp_plugin.'
126 112
                       'NeutronPlugin.NvpPluginV2',
127
-            'contexts': [
128
-                context.SharedDBContext(user=config('neutron-database-user'),
129
-                                        database=config('neutron-database'),
130
-                                        relation_prefix='neutron',
131
-                                        ssl_dir=NEUTRON_CONF_DIR)],
113
+            'contexts': [],
132 114
             'services': [],
133 115
             'packages': [],
134 116
             'server_packages': ['neutron-server',
@@ -138,11 +120,7 @@ def neutron_plugins():
138 120
         'nsx': {
139 121
             'config': '/etc/neutron/plugins/vmware/nsx.ini',
140 122
             'driver': 'vmware',
141
-            'contexts': [
142
-                context.SharedDBContext(user=config('neutron-database-user'),
143
-                                        database=config('neutron-database'),
144
-                                        relation_prefix='neutron',
145
-                                        ssl_dir=NEUTRON_CONF_DIR)],
123
+            'contexts': [],
146 124
             'services': [],
147 125
             'packages': [],
148 126
             'server_packages': ['neutron-server',
@@ -152,11 +130,7 @@ def neutron_plugins():
152 130
         'n1kv': {
153 131
             'config': '/etc/neutron/plugins/cisco/cisco_plugins.ini',
154 132
             'driver': 'neutron.plugins.cisco.network_plugin.PluginV2',
155
-            'contexts': [
156
-                context.SharedDBContext(user=config('neutron-database-user'),
157
-                                        database=config('neutron-database'),
158
-                                        relation_prefix='neutron',
159
-                                        ssl_dir=NEUTRON_CONF_DIR)],
133
+            'contexts': [],
160 134
             'services': [],
161 135
             'packages': [determine_dkms_package(),
162 136
                          ['neutron-plugin-cisco']],
@@ -167,11 +141,7 @@ def neutron_plugins():
167 141
         'Calico': {
168 142
             'config': '/etc/neutron/plugins/ml2/ml2_conf.ini',
169 143
             'driver': 'neutron.plugins.ml2.plugin.Ml2Plugin',
170
-            'contexts': [
171
-                context.SharedDBContext(user=config('neutron-database-user'),
172
-                                        database=config('neutron-database'),
173
-                                        relation_prefix='neutron',
174
-                                        ssl_dir=NEUTRON_CONF_DIR)],
144
+            'contexts': [],
175 145
             'services': ['calico-felix',
176 146
                          'bird',
177 147
                          'neutron-dhcp-agent',
@@ -189,11 +159,7 @@ def neutron_plugins():
189 159
         'vsp': {
190 160
             'config': '/etc/neutron/plugins/nuage/nuage_plugin.ini',
191 161
             'driver': 'neutron.plugins.nuage.plugin.NuagePlugin',
192
-            'contexts': [
193
-                context.SharedDBContext(user=config('neutron-database-user'),
194
-                                        database=config('neutron-database'),
195
-                                        relation_prefix='neutron',
196
-                                        ssl_dir=NEUTRON_CONF_DIR)],
162
+            'contexts': [],
197 163
             'services': [],
198 164
             'packages': [],
199 165
             'server_packages': ['neutron-server', 'neutron-plugin-nuage'],
@@ -203,10 +169,7 @@ def neutron_plugins():
203 169
             'config': '/etc/neutron/plugins/plumgrid/plumgrid.ini',
204 170
             'driver': ('neutron.plugins.plumgrid.plumgrid_plugin'
205 171
                        '.plumgrid_plugin.NeutronPluginPLUMgridV2'),
206
-            'contexts': [
207
-                context.SharedDBContext(user=config('database-user'),
208
-                                        database=config('database'),
209
-                                        ssl_dir=NEUTRON_CONF_DIR)],
172
+            'contexts': [],
210 173
             'services': [],
211 174
             'packages': ['plumgrid-lxc',
212 175
                          'iovisor-dkms'],
@@ -217,11 +180,7 @@ def neutron_plugins():
217 180
         'midonet': {
218 181
             'config': '/etc/neutron/plugins/midonet/midonet.ini',
219 182
             'driver': 'midonet.neutron.plugin.MidonetPluginV2',
220
-            'contexts': [
221
-                context.SharedDBContext(user=config('neutron-database-user'),
222
-                                        database=config('neutron-database'),
223
-                                        relation_prefix='neutron',
224
-                                        ssl_dir=NEUTRON_CONF_DIR)],
183
+            'contexts': [],
225 184
             'services': [],
226 185
             'packages': [determine_dkms_package()],
227 186
             'server_packages': ['neutron-server',

+ 2
- 2
hooks/charmhelpers/contrib/openstack/templates/ceph.conf View File

@@ -18,7 +18,7 @@ rbd default features = {{ rbd_features }}
18 18
 
19 19
 [client]
20 20
 {% if rbd_client_cache_settings -%}
21
-{% for key, value in rbd_client_cache_settings.iteritems() -%}
21
+{% for key, value in rbd_client_cache_settings.items() -%}
22 22
 {{ key }} = {{ value }}
23 23
 {% endfor -%}
24
-{%- endif %}
24
+{%- endif %}

+ 2
- 0
hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg View File

@@ -48,7 +48,9 @@ listen stats
48 48
 {% for service, ports in service_ports.items() -%}
49 49
 frontend tcp-in_{{ service }}
50 50
     bind *:{{ ports[0] }}
51
+    {% if ipv6_enabled -%}
51 52
     bind :::{{ ports[0] }}
53
+    {% endif -%}
52 54
     {% for frontend in frontends -%}
53 55
     acl net_{{ frontend }} dst {{ frontends[frontend]['network'] }}
54 56
     use_backend {{ service }}_{{ frontend }} if net_{{ frontend }}

+ 6
- 0
hooks/charmhelpers/contrib/openstack/templates/section-oslo-cache View File

@@ -0,0 +1,6 @@
1
+[cache]
2
+{% if memcache_url %}
3
+enabled = true
4
+backend = oslo_cache.memcache_pool
5
+memcache_servers = {{ memcache_url }}
6
+{% endif %}

+ 2
- 0
hooks/charmhelpers/contrib/openstack/templating.py View File

@@ -272,6 +272,8 @@ class OSConfigRenderer(object):
272 272
             raise OSConfigException
273 273
 
274 274
         _out = self.render(config_file)
275
+        if six.PY3:
276
+            _out = _out.encode('UTF-8')
275 277
 
276 278
         with open(config_file, 'wb') as out:
277 279
             out.write(_out)

+ 26
- 16
hooks/charmhelpers/contrib/openstack/utils.py View File

@@ -95,7 +95,7 @@ from charmhelpers.fetch import (
95 95
 from charmhelpers.fetch.snap import (
96 96
     snap_install,
97 97
     snap_refresh,
98
-    SNAP_CHANNELS,
98
+    valid_snap_channel,
99 99
 )
100 100
 
101 101
 from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
@@ -140,6 +140,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
140 140
     ('yakkety', 'newton'),
141 141
     ('zesty', 'ocata'),
142 142
     ('artful', 'pike'),
143
+    ('bionic', 'queens'),
143 144
 ])
144 145
 
145 146
 
@@ -157,6 +158,7 @@ OPENSTACK_CODENAMES = OrderedDict([
157 158
     ('2016.2', 'newton'),
158 159
     ('2017.1', 'ocata'),
159 160
     ('2017.2', 'pike'),
161
+    ('2018.1', 'queens'),
160 162
 ])
161 163
 
162 164
 # The ugly duckling - must list releases oldest to newest
@@ -187,6 +189,8 @@ SWIFT_CODENAMES = OrderedDict([
187 189
         ['2.11.0', '2.12.0', '2.13.0']),
188 190
     ('pike',
189 191
         ['2.13.0', '2.15.0']),
192
+    ('queens',
193
+        ['2.16.0']),
190 194
 ])
191 195
 
192 196
 # >= Liberty version->codename mapping
@@ -412,6 +416,8 @@ def get_os_codename_package(package, fatal=True):
412 416
         cmd = ['snap', 'list', package]
413 417
         try:
414 418
             out = subprocess.check_output(cmd)
419
+            if six.PY3:
420
+                out = out.decode('UTF-8')
415 421
         except subprocess.CalledProcessError as e:
416 422
             return None
417 423
         lines = out.split('\n')
@@ -426,7 +432,7 @@ def get_os_codename_package(package, fatal=True):
426 432
 
427 433
     try:
428 434
         pkg = cache[package]
429
-    except:
435
+    except Exception:
430 436
         if not fatal:
431 437
             return None
432 438
         # the package is unknown to the current apt cache.
@@ -579,6 +585,9 @@ def configure_installation_source(source_plus_key):
579 585
     Note that the behaviour on error is to log the error to the juju log and
580 586
     then call sys.exit(1).
581 587
     """
588
+    if source_plus_key.startswith('snap'):
589
+        # Do nothing for snap installs
590
+        return
582 591
     # extract the key if there is one, denoted by a '|' in the rel
583 592
     source, key = get_source_and_pgp_key(source_plus_key)
584 593
 
@@ -615,7 +624,7 @@ def save_script_rc(script_path="scripts/scriptrc", **env_vars):
615 624
     juju_rc_path = "%s/%s" % (charm_dir(), script_path)
616 625
     if not os.path.exists(os.path.dirname(juju_rc_path)):
617 626
         os.mkdir(os.path.dirname(juju_rc_path))
618
-    with open(juju_rc_path, 'wb') as rc_script:
627
+    with open(juju_rc_path, 'wt') as rc_script:
619 628
         rc_script.write(
620 629
             "#!/bin/bash\n")
621 630
         [rc_script.write('export %s=%s\n' % (u, p))
@@ -794,7 +803,7 @@ def git_default_repos(projects_yaml):
794 803
     service = service_name()
795 804
     core_project = service
796 805
 
797
-    for default, branch in GIT_DEFAULT_BRANCHES.iteritems():
806
+    for default, branch in six.iteritems(GIT_DEFAULT_BRANCHES):
798 807
         if projects_yaml == default:
799 808
 
800 809
             # add the requirements repo first
@@ -1615,7 +1624,7 @@ def do_action_openstack_upgrade(package, upgrade_callback, configs):
1615 1624
                     upgrade_callback(configs=configs)
1616 1625
                     action_set({'outcome': 'success, upgrade completed.'})
1617 1626
                     ret = True
1618
-                except:
1627
+                except Exception:
1619 1628
                     action_set({'outcome': 'upgrade failed, see traceback.'})
1620 1629
                     action_set({'traceback': traceback.format_exc()})
1621 1630
                     action_fail('do_openstack_upgrade resulted in an '
@@ -1720,7 +1729,7 @@ def is_unit_paused_set():
1720 1729
             kv = t[0]
1721 1730
             # transform something truth-y into a Boolean.
1722 1731
             return not(not(kv.get('unit-paused')))
1723
-    except:
1732
+    except Exception:
1724 1733
         return False
1725 1734
 
1726 1735
 
@@ -2048,7 +2057,7 @@ def update_json_file(filename, items):
2048 2057
 def snap_install_requested():
2049 2058
     """ Determine if installing from snaps
2050 2059
 
2051
-    If openstack-origin is of the form snap:channel-series-release
2060
+    If openstack-origin is of the form snap:track/channel[/branch]
2052 2061
     and channel is in SNAPS_CHANNELS return True.
2053 2062
     """
2054 2063
     origin = config('openstack-origin') or ""
@@ -2056,10 +2065,12 @@ def snap_install_requested():
2056 2065
         return False
2057 2066
 
2058 2067
     _src = origin[5:]
2059
-    channel, series, release = _src.split('-')
2060
-    if channel.lower() in SNAP_CHANNELS:
2061
-        return True
2062
-    return False
2068
+    if '/' in _src:
2069
+        channel = _src.split('/')[1]
2070
+    else:
2071
+        # Handle snap:track with no channel
2072
+        channel = 'stable'
2073
+    return valid_snap_channel(channel)
2063 2074
 
2064 2075
 
2065 2076
 def get_snaps_install_info_from_origin(snaps, src, mode='classic'):
@@ -2067,7 +2078,7 @@ def get_snaps_install_info_from_origin(snaps, src, mode='classic'):
2067 2078
 
2068 2079
     @param snaps: List of snaps
2069 2080
     @param src: String of openstack-origin or source of the form
2070
-        snap:channel-series-track
2081
+        snap:track/channel
2071 2082
     @param mode: String classic, devmode or jailmode
2072 2083
     @returns: Dictionary of snaps with channels and modes
2073 2084
     """
@@ -2077,8 +2088,7 @@ def get_snaps_install_info_from_origin(snaps, src, mode='classic'):
2077 2088
         return {}
2078 2089
 
2079 2090
     _src = src[5:]
2080
-    _channel, _series, _release = _src.split('-')
2081
-    channel = '--channel={}/{}'.format(_release, _channel)
2091
+    channel = '--channel={}'.format(_src)
2082 2092
 
2083 2093
     return {snap: {'channel': channel, 'mode': mode}
2084 2094
             for snap in snaps}
@@ -2090,8 +2100,8 @@ def install_os_snaps(snaps, refresh=False):
2090 2100
     @param snaps: Dictionary of snaps with channels and modes of the form:
2091 2101
         {'snap_name': {'channel': 'snap_channel',
2092 2102
                        'mode': 'snap_mode'}}
2093
-        Where channel a snapstore channel and mode is --classic, --devmode or
2094
-        --jailmode.
2103
+        Where channel is a snapstore channel and mode is --classic, --devmode
2104
+        or --jailmode.
2095 2105
     @param post_snap_install: Callback function to run after snaps have been
2096 2106
     installed
2097 2107
     """

+ 29
- 13
hooks/charmhelpers/contrib/storage/linux/ceph.py View File

@@ -370,9 +370,10 @@ def get_mon_map(service):
370 370
       Also raises CalledProcessError if our ceph command fails
371 371
     """
372 372
     try:
373
-        mon_status = check_output(
374
-            ['ceph', '--id', service,
375
-             'mon_status', '--format=json'])
373
+        mon_status = check_output(['ceph', '--id', service,
374
+                                   'mon_status', '--format=json'])
375
+        if six.PY3:
376
+            mon_status = mon_status.decode('UTF-8')
376 377
         try:
377 378
             return json.loads(mon_status)
378 379
         except ValueError as v:
@@ -457,7 +458,7 @@ def monitor_key_get(service, key):
457 458
     try:
458 459
         output = check_output(
459 460
             ['ceph', '--id', service,
460
-             'config-key', 'get', str(key)])
461
+             'config-key', 'get', str(key)]).decode('UTF-8')
461 462
         return output
462 463
     except CalledProcessError as e:
463 464
         log("Monitor config-key get failed with message: {}".format(
@@ -500,6 +501,8 @@ def get_erasure_profile(service, name):
500 501
         out = check_output(['ceph', '--id', service,
501 502
                             'osd', 'erasure-code-profile', 'get',
502 503
                             name, '--format=json'])
504
+        if six.PY3:
505
+            out = out.decode('UTF-8')
503 506
         return json.loads(out)
504 507
     except (CalledProcessError, OSError, ValueError):
505 508
         return None
@@ -686,7 +689,10 @@ def get_cache_mode(service, pool_name):
686 689
     """
687 690
     validator(value=service, valid_type=six.string_types)
688 691
     validator(value=pool_name, valid_type=six.string_types)
689
-    out = check_output(['ceph', '--id', service, 'osd', 'dump', '--format=json'])
692
+    out = check_output(['ceph', '--id', service,
693
+                        'osd', 'dump', '--format=json'])
694
+    if six.PY3:
695
+        out = out.decode('UTF-8')
690 696
     try:
691 697
         osd_json = json.loads(out)
692 698
         for pool in osd_json['pools']:
@@ -700,8 +706,9 @@ def get_cache_mode(service, pool_name):
700 706
 def pool_exists(service, name):
701 707
     """Check to see if a RADOS pool already exists."""
702 708
     try:
703
-        out = check_output(['rados', '--id', service,
704
-                            'lspools']).decode('UTF-8')
709
+        out = check_output(['rados', '--id', service, 'lspools'])
710
+        if six.PY3:
711
+            out = out.decode('UTF-8')
705 712
     except CalledProcessError:
706 713
         return False
707 714
 
@@ -714,9 +721,12 @@ def get_osds(service):
714 721
     """
715 722
     version = ceph_version()
716 723
     if version and version >= '0.56':
717
-        return json.loads(check_output(['ceph', '--id', service,
718
-                                        'osd', 'ls',
719
-                                        '--format=json']).decode('UTF-8'))
724
+        out = check_output(['ceph', '--id', service,
725
+                            'osd', 'ls',
726
+                            '--format=json'])
727
+        if six.PY3:
728
+            out = out.decode('UTF-8')
729
+        return json.loads(out)
720 730
 
721 731
     return None
722 732
 
@@ -734,7 +744,9 @@ def rbd_exists(service, pool, rbd_img):
734 744
     """Check to see if a RADOS block device exists."""
735 745
     try:
736 746
         out = check_output(['rbd', 'list', '--id',
737
-                            service, '--pool', pool]).decode('UTF-8')
747
+                            service, '--pool', pool])
748
+        if six.PY3:
749
+            out = out.decode('UTF-8')
738 750
     except CalledProcessError:
739 751
         return False
740 752
 
@@ -859,7 +871,9 @@ def configure(service, key, auth, use_syslog):
859 871
 def image_mapped(name):
860 872
     """Determine whether a RADOS block device is mapped locally."""
861 873
     try:
862
-        out = check_output(['rbd', 'showmapped']).decode('UTF-8')
874
+        out = check_output(['rbd', 'showmapped'])
875
+        if six.PY3:
876
+            out = out.decode('UTF-8')
863 877
     except CalledProcessError:
864 878
         return False
865 879
 
@@ -1018,7 +1032,9 @@ def ceph_version():
1018 1032
     """Retrieve the local version of ceph."""
1019 1033
     if os.path.exists('/usr/bin/ceph'):
1020 1034
         cmd = ['ceph', '-v']
1021
-        output = check_output(cmd).decode('US-ASCII')
1035
+        output = check_output(cmd)
1036
+        if six.PY3:
1037
+            output = output.decode('UTF-8')
1022 1038
         output = output.split()
1023 1039
         if len(output) > 3:
1024 1040
             return output[2]

+ 4
- 4
hooks/charmhelpers/contrib/storage/linux/lvm.py View File

@@ -74,10 +74,10 @@ def list_lvm_volume_group(block_device):
74 74
     '''
75 75
     vg = None
76 76
     pvd = check_output(['pvdisplay', block_device]).splitlines()
77
-    for l in pvd:
78
-        l = l.decode('UTF-8')
79
-        if l.strip().startswith('VG Name'):
80
-            vg = ' '.join(l.strip().split()[2:])
77
+    for lvm in pvd:
78
+        lvm = lvm.decode('UTF-8')
79
+        if lvm.strip().startswith('VG Name'):
80
+            vg = ' '.join(lvm.strip().split()[2:])
81 81
     return vg
82 82
 
83 83
 

+ 1
- 1
hooks/charmhelpers/contrib/storage/linux/utils.py View File

@@ -64,6 +64,6 @@ def is_device_mounted(device):
64 64
     '''
65 65
     try:
66 66
         out = check_output(['lsblk', '-P', device]).decode('UTF-8')
67
-    except:
67
+    except Exception:
68 68
         return False
69 69
     return bool(re.search(r'MOUNTPOINT=".+"', out))

+ 105
- 7
hooks/charmhelpers/core/hookenv.py View File

@@ -22,6 +22,7 @@ from __future__ import print_function
22 22
 import copy
23 23
 from distutils.version import LooseVersion
24 24
 from functools import wraps
25
+from collections import namedtuple
25 26
 import glob
26 27
 import os
27 28
 import json
@@ -218,6 +219,8 @@ def principal_unit():
218 219
         for rid in relation_ids(reltype):
219 220
             for unit in related_units(rid):
220 221
                 md = _metadata_unit(unit)
222
+                if not md:
223
+                    continue
221 224
                 subordinate = md.pop('subordinate', None)
222 225
                 if not subordinate:
223 226
                     return unit
@@ -511,7 +514,10 @@ def _metadata_unit(unit):
511 514
     """
512 515
     basedir = os.sep.join(charm_dir().split(os.sep)[:-2])
513 516
     unitdir = 'unit-{}'.format(unit.replace(os.sep, '-'))
514
-    with open(os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')) as md:
517
+    joineddir = os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')
518
+    if not os.path.exists(joineddir):
519
+        return None
520
+    with open(joineddir) as md:
515 521
         return yaml.safe_load(md)
516 522
 
517 523
 
@@ -639,18 +645,31 @@ def is_relation_made(relation, keys='private-address'):
639 645
     return False
640 646
 
641 647
 
648
+def _port_op(op_name, port, protocol="TCP"):
649
+    """Open or close a service network port"""
650
+    _args = [op_name]
651
+    icmp = protocol.upper() == "ICMP"
652
+    if icmp:
653
+        _args.append(protocol)
654
+    else:
655
+        _args.append('{}/{}'.format(port, protocol))
656
+    try:
657
+        subprocess.check_call(_args)
658
+    except subprocess.CalledProcessError:
659
+        # Older Juju pre 2.3 doesn't support ICMP
660
+        # so treat it as a no-op if it fails.
661
+        if not icmp:
662
+            raise
663
+
664
+
642 665
 def open_port(port, protocol="TCP"):
643 666
     """Open a service network port"""
644
-    _args = ['open-port']
645
-    _args.append('{}/{}'.format(port, protocol))
646
-    subprocess.check_call(_args)
667
+    _port_op('open-port', port, protocol)
647 668
 
648 669
 
649 670
 def close_port(port, protocol="TCP"):
650 671
     """Close a service network port"""
651
-    _args = ['close-port']
652
-    _args.append('{}/{}'.format(port, protocol))
653
-    subprocess.check_call(_args)
672
+    _port_op('close-port', port, protocol)
654 673
 
655 674
 
656 675
 def open_ports(start, end, protocol="TCP"):
@@ -667,6 +686,17 @@ def close_ports(start, end, protocol="TCP"):
667 686
     subprocess.check_call(_args)
668 687
 
669 688
 
689
+def opened_ports():
690
+    """Get the opened ports
691
+
692
+    *Note that this will only show ports opened in a previous hook*
693
+
694
+    :returns: Opened ports as a list of strings: ``['8080/tcp', '8081-8083/tcp']``
695
+    """
696
+    _args = ['opened-ports', '--format=json']
697
+    return json.loads(subprocess.check_output(_args).decode('UTF-8'))
698
+
699
+
670 700
 @cached
671 701
 def unit_get(attribute):
672 702
     """Get the unit ID for the remote unit"""
@@ -1077,6 +1107,35 @@ def network_get_primary_address(binding):
1077 1107
     return subprocess.check_output(cmd).decode('UTF-8').strip()
1078 1108
 
1079 1109
 
1110
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
1111
+def network_get(endpoint, relation_id=None):
1112
+    """
1113
+    Retrieve the network details for a relation endpoint
1114
+
1115
+    :param endpoint: string. The name of a relation endpoint
1116
+    :param relation_id: int. The ID of the relation for the current context.
1117
+    :return: dict. The loaded YAML output of the network-get query.
1118
+    :raise: NotImplementedError if run on Juju < 2.1
1119
+    """
1120
+    cmd = ['network-get', endpoint, '--format', 'yaml']
1121
+    if relation_id:
1122
+        cmd.append('-r')
1123
+        cmd.append(relation_id)
1124
+    try:
1125
+        response = subprocess.check_output(
1126
+            cmd,
1127
+            stderr=subprocess.STDOUT).decode('UTF-8').strip()
1128
+    except CalledProcessError as e:
1129
+        # Early versions of Juju 2.0.x required the --primary-address argument.
1130
+        # We catch that condition here and raise NotImplementedError since
1131
+        # the requested semantics are not available - the caller can then
1132
+        # use the network_get_primary_address() method instead.
1133
+        if '--primary-address is currently required' in e.output.decode('UTF-8'):
1134
+            raise NotImplementedError
1135
+        raise
1136
+    return yaml.safe_load(response)
1137
+
1138
+
1080 1139
 def add_metric(*args, **kwargs):
1081 1140
     """Add metric values. Values may be expressed with keyword arguments. For
1082 1141
     metric names containing dashes, these may be expressed as one or more
@@ -1106,3 +1165,42 @@ def meter_info():
1106 1165
     """Get the meter status information, if running in the meter-status-changed
1107 1166
     hook."""
1108 1167
     return os.environ.get('JUJU_METER_INFO')
1168
+
1169
+
1170
+def iter_units_for_relation_name(relation_name):
1171
+    """Iterate through all units in a relation
1172
+
1173
+    Generator that iterates through all the units in a relation and yields
1174
+    a named tuple with rid and unit field names.
1175
+
1176
+    Usage:
1177
+    data = [(u.rid, u.unit)
1178
+            for u in iter_units_for_relation_name(relation_name)]
1179
+
1180
+    :param relation_name: string relation name
1181
+    :yield: Named Tuple with rid and unit field names
1182
+    """
1183
+    RelatedUnit = namedtuple('RelatedUnit', 'rid, unit')
1184
+    for rid in relation_ids(relation_name):
1185
+        for unit in related_units(rid):
1186
+            yield RelatedUnit(rid, unit)
1187
+
1188
+
1189
+def ingress_address(rid=None, unit=None):
1190
+    """
1191
+    Retrieve the ingress-address from a relation when available. Otherwise,
1192
+    return the private-address. This function is to be used on the consuming
1193
+    side of the relation.
1194
+
1195
+    Usage:
1196
+    addresses = [ingress_address(rid=u.rid, unit=u.unit)
1197
+                 for u in iter_units_for_relation_name(relation_name)]
1198
+
1199
+    :param rid: string relation id
1200
+    :param unit: string unit name
1201
+    :side effect: calls relation_get
1202
+    :return: string IP address
1203
+    """
1204
+    settings = relation_get(rid=rid, unit=unit)
1205
+    return (settings.get('ingress-address') or
1206
+            settings.get('private-address'))

+ 72
- 1
hooks/charmhelpers/core/host.py View File

@@ -34,7 +34,7 @@ import six
34 34
 
35 35
 from contextlib import contextmanager
36 36
 from collections import OrderedDict
37
-from .hookenv import log, DEBUG
37
+from .hookenv import log, DEBUG, local_unit
38 38
 from .fstab import Fstab
39 39
 from charmhelpers.osplatform import get_platform
40 40
 
@@ -441,6 +441,49 @@ def add_user_to_group(username, group):
441 441
     subprocess.check_call(cmd)
442 442
 
443 443
 
444
+def chage(username, lastday=None, expiredate=None, inactive=None,
445
+           mindays=None, maxdays=None, root=None, warndays=None):
446
+    """Change user password expiry information
447
+
448
+    :param str username: User to update
449
+    :param str lastday: Set when password was changed in YYYY-MM-DD format
450
+    :param str expiredate: Set when user's account will no longer be
451
+                           accessible in YYYY-MM-DD format.
452
+                           -1 will remove an account expiration date.
453
+    :param str inactive: Set the number of days of inactivity after a password
454
+                         has expired before the account is locked.
455
+                         -1 will remove an account's inactivity.
456
+    :param str mindays: Set the minimum number of days between password
457
+                        changes to MIN_DAYS.
458
+                        0 indicates the password can be changed anytime.
459
+    :param str maxdays: Set the maximum number of days during which a
460
+                        password is valid.
461
+                        -1 as MAX_DAYS will remove checking maxdays
462
+    :param str root: Apply changes in the CHROOT_DIR directory
463
+    :param str warndays: Set the number of days of warning before a password
464
+                         change is required
465
+    :raises subprocess.CalledProcessError: if call to chage fails
466
+    """
467
+    cmd = ['chage']
468
+    if root:
469
+        cmd.extend(['--root', root])
470
+    if lastday:
471
+        cmd.extend(['--lastday', lastday])
472
+    if expiredate:
473
+        cmd.extend(['--expiredate', expiredate])
474
+    if inactive:
475
+        cmd.extend(['--inactive', inactive])
476
+    if mindays:
477
+        cmd.extend(['--mindays', mindays])
478
+    if maxdays:
479
+        cmd.extend(['--maxdays', maxdays])
480
+    if warndays:
481
+        cmd.extend(['--warndays', warndays])
482
+    cmd.append(username)
483
+    subprocess.check_call(cmd)
484
+
485
+remove_password_expiry = functools.partial(chage, expiredate='-1', inactive='-1', mindays='0', maxdays='-1')
486
+
444 487
 def rsync(from_path, to_path, flags='-r', options=None, timeout=None):
445 488
     """Replicate the contents of a path"""
446 489
     options = options or ['--delete', '--executability']
@@ -946,3 +989,31 @@ def updatedb(updatedb_text, new_path):
946 989
                 lines[i] = 'PRUNEPATHS="{}"'.format(' '.join(paths))
947 990
     output = "\n".join(lines)
948 991
     return output
992
+
993
+
994
+def modulo_distribution(modulo=3, wait=30):
995
+    """ Modulo distribution
996
+
997
+    This helper uses the unit number, a modulo value and a constant wait time
998
+    to produce a calculated wait time distribution. This is useful in large
999
+    scale deployments to distribute load during an expensive operation such as
1000
+    service restarts.
1001
+
1002
+    If you have 1000 nodes that need to restart 100 at a time 1 minute at a
1003
+    time:
1004
+
1005
+      time.wait(modulo_distribution(modulo=100, wait=60))
1006
+      restart()
1007
+
1008
+    If you need restarts to happen serially set modulo to the exact number of
1009
+    nodes and set a high constant wait time:
1010
+
1011
+      time.wait(modulo_distribution(modulo=10, wait=120))
1012
+      restart()
1013
+
1014
+    @param modulo: int The modulo number creates the group distribution
1015
+    @param wait: int The constant time wait value
1016
+    @return: int Calculated time to wait for unit operation
1017
+    """
1018
+    unit_number = int(local_unit().split('/')[1])
1019
+    return (unit_number % modulo) * wait

+ 11
- 5
hooks/charmhelpers/core/strutils.py View File

@@ -61,13 +61,19 @@ def bytes_from_string(value):
61 61
     if isinstance(value, six.string_types):
62 62
         value = six.text_type(value)
63 63
     else:
64
-        msg = "Unable to interpret non-string value '%s' as boolean" % (value)
64
+        msg = "Unable to interpret non-string value '%s' as bytes" % (value)
65 65
         raise ValueError(msg)
66 66
     matches = re.match("([0-9]+)([a-zA-Z]+)", value)
67
-    if not matches:
68
-        msg = "Unable to interpret string value '%s' as bytes" % (value)
69
-        raise ValueError(msg)
70
-    return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
67
+    if matches:
68
+        size = int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
69
+    else:
70
+        # Assume that value passed in is bytes
71
+        try:
72
+            size = int(value)
73
+        except ValueError:
74
+            msg = "Unable to interpret string value '%s' as bytes" % (value)
75
+            raise ValueError(msg)
76
+    return size
71 77
 
72 78
 
73 79
 class BasicStringComparator(object):

+ 1
- 1
hooks/charmhelpers/core/unitdata.py View File

@@ -358,7 +358,7 @@ class Storage(object):
358 358
         try:
359 359
             yield self.revision
360 360
             self.revision = None
361
-        except:
361
+        except Exception:
362 362
             self.flush(False)
363 363
             self.revision = None
364 364
             raise

+ 16
- 0
hooks/charmhelpers/fetch/snap.py View File

@@ -41,6 +41,10 @@ class CouldNotAcquireLockException(Exception):
41 41
     pass
42 42
 
43 43
 
44
+class InvalidSnapChannel(Exception):
45
+    pass
46
+
47
+
44 48
 def _snap_exec(commands):
45 49
     """
46 50
     Execute snap commands.
@@ -132,3 +136,15 @@ def snap_refresh(packages, *flags):
132 136
 
133 137
     log(message, level='INFO')
134 138
     return _snap_exec(['refresh'] + flags + packages)
139
+
140
+
141
+def valid_snap_channel(channel):
142
+    """ Validate snap channel exists
143
+
144
+    :raises InvalidSnapChannel: When channel does not exist
145
+    :return: Boolean
146
+    """
147
+    if channel.lower() in SNAP_CHANNELS:
148
+        return True
149
+    else:
150
+        raise InvalidSnapChannel("Invalid Snap Channel: {}".format(channel))

+ 1
- 1
hooks/charmhelpers/fetch/ubuntu.py View File

@@ -572,7 +572,7 @@ def get_upstream_version(package):
572 572
     cache = apt_cache()
573 573
     try:
574 574
         pkg = cache[package]
575
-    except:
575
+    except Exception:
576 576
         # the package is unknown to the current apt cache.
577 577
         return None
578 578
 

+ 3
- 3
tests/basic_deployment.py View File

@@ -782,17 +782,17 @@ class CinderBackupBasicDeployment(OpenStackAmuletDeployment):
782 782
 
783 783
         name = "demo-vol"
784 784
         vols = self.cinder.volumes.list()
785
-        cinder_vols = [v for v in vols if v.display_name == name]
785
+        cinder_vols = [v for v in vols if v.name == name]
786 786
         if not cinder_vols:
787 787
             # NOTE(hopem): it appears that at some point cinder-backup stopped
788 788
             # restoring volume metadata properly so revert to default name if
789 789
             # original is not found
790 790
             name = "restore_backup_%s" % (vol_backup.id)
791
-            cinder_vols = [v for v in vols if v.display_name == name]
791
+            cinder_vols = [v for v in vols if v.name == name]
792 792
 
793 793
         if not cinder_vols:
794 794
             msg = ("Could not find restore vol '%s' in %s" %
795
-                   (name, [v.display_name for v in vols]))
795
+                   (name, [v.name for v in vols]))
796 796
             u.log.error(msg)
797 797
             amulet.raise_status(amulet.FAIL, msg=msg)
798 798
 

+ 33
- 11
tests/charmhelpers/contrib/openstack/amulet/deployment.py View File

@@ -13,6 +13,7 @@
13 13
 # limitations under the License.
14 14
 
15 15
 import logging
16
+import os
16 17
 import re
17 18
 import sys
18 19
 import six
@@ -185,7 +186,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
185 186
             self.d.configure(service, config)
186 187
 
187 188
     def _auto_wait_for_status(self, message=None, exclude_services=None,
188
-                              include_only=None, timeout=1800):
189
+                              include_only=None, timeout=None):
189 190
         """Wait for all units to have a specific extended status, except
190 191
         for any defined as excluded.  Unless specified via message, any
191 192
         status containing any case of 'ready' will be considered a match.
@@ -215,7 +216,10 @@ class OpenStackAmuletDeployment(AmuletDeployment):
215 216
         :param timeout: Maximum time in seconds to wait for status match
216 217
         :returns: None.  Raises if timeout is hit.
217 218
         """
218
-        self.log.info('Waiting for extended status on units...')
219
+        if not timeout:
220
+            timeout = int(os.environ.get('AMULET_SETUP_TIMEOUT', 1800))
221
+        self.log.info('Waiting for extended status on units for {}s...'
222
+                      ''.format(timeout))
219 223
 
220 224
         all_services = self.d.services.keys()
221 225
 
@@ -250,7 +254,14 @@ class OpenStackAmuletDeployment(AmuletDeployment):
250 254
         self.log.debug('Waiting up to {}s for extended status on services: '
251 255
                        '{}'.format(timeout, services))
252 256
         service_messages = {service: message for service in services}
257
+
258
+        # Check for idleness
259
+        self.d.sentry.wait(timeout=timeout)
260
+        # Check for error states and bail early
261
+        self.d.sentry.wait_for_status(self.d.juju_env, services, timeout=timeout)
262
+        # Check for ready messages
253 263
         self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
264
+
254 265
         self.log.info('OK')
255 266
 
256 267
     def _get_openstack_release(self):
@@ -263,7 +274,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
263 274
         (self.trusty_icehouse, self.trusty_kilo, self.trusty_liberty,
264 275
          self.trusty_mitaka, self.xenial_mitaka, self.xenial_newton,
265 276
          self.yakkety_newton, self.xenial_ocata, self.zesty_ocata,
266
-         self.xenial_pike, self.artful_pike) = range(11)
277
+         self.xenial_pike, self.artful_pike, self.xenial_queens,
278
+         self.bionic_queens,) = range(13)
267 279
 
268 280
         releases = {
269 281
             ('trusty', None): self.trusty_icehouse,
@@ -274,9 +286,11 @@ class OpenStackAmuletDeployment(AmuletDeployment):
274 286
             ('xenial', 'cloud:xenial-newton'): self.xenial_newton,
275 287
             ('xenial', 'cloud:xenial-ocata'): self.xenial_ocata,
276 288
             ('xenial', 'cloud:xenial-pike'): self.xenial_pike,
289
+            ('xenial', 'cloud:xenial-queens'): self.xenial_queens,
277 290
             ('yakkety', None): self.yakkety_newton,
278 291
             ('zesty', None): self.zesty_ocata,
279 292
             ('artful', None): self.artful_pike,
293
+            ('bionic', None): self.bionic_queens,
280 294
         }
281 295
         return releases[(self.series, self.openstack)]
282 296
 
@@ -291,6 +305,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
291 305
             ('yakkety', 'newton'),
292 306
             ('zesty', 'ocata'),
293 307
             ('artful', 'pike'),
308
+            ('bionic', 'queens'),
294 309
         ])
295 310
         if self.openstack:
296 311
             os_origin = self.openstack.split(':')[1]
@@ -303,20 +318,27 @@ class OpenStackAmuletDeployment(AmuletDeployment):
303 318
         test scenario, based on OpenStack release and whether ceph radosgw
304 319
         is flagged as present or not."""
305 320
 
306
-        if self._get_openstack_release() >= self.trusty_kilo:
307
-            # Kilo or later
321
+        if self._get_openstack_release() == self.trusty_icehouse:
322
+            # Icehouse
308 323
             pools = [
324
+                'data',
325
+                'metadata',
309 326
                 'rbd',
310
-                'cinder',
327
+                'cinder-ceph',
311 328
                 'glance'
312 329
             ]
313
-        else:
314
-            # Juno or earlier
330
+        elif (self.trusty_kilo <= self._get_openstack_release() <=
331
+              self.zesty_ocata):
332
+            # Kilo through Ocata
315 333
             pools = [
316
-                'data',
317
-                'metadata',
318 334
                 'rbd',
319
-                'cinder',
335
+                'cinder-ceph',
336
+                'glance'
337
+            ]
338
+        else:
339
+            # Pike and later
340
+            pools = [
341
+                'cinder-ceph',
320 342
                 'glance'
321 343
             ]
322 344
 

+ 29
- 7
tests/charmhelpers/contrib/openstack/amulet/utils.py View File

@@ -23,6 +23,7 @@ import urllib
23 23
 import urlparse
24 24
 
25 25
 import cinderclient.v1.client as cinder_client
26
+import cinderclient.v2.client as cinder_clientv2
26 27
 import glanceclient.v1.client as glance_client
27 28
 import heatclient.v1.client as heat_client
28 29
 from keystoneclient.v2_0 import client as keystone_client
@@ -42,7 +43,6 @@ import swiftclient
42 43
 from charmhelpers.contrib.amulet.utils import (
43 44
     AmuletUtils
44 45
 )
45
-from charmhelpers.core.decorators import retry_on_exception
46 46
 from charmhelpers.core.host import CompareHostReleases
47 47
 
48 48
 DEBUG = logging.DEBUG
@@ -310,7 +310,6 @@ class OpenStackAmuletUtils(AmuletUtils):
310 310
         self.log.debug('Checking if tenant exists ({})...'.format(tenant))
311 311
         return tenant in [t.name for t in keystone.tenants.list()]
312 312
 
313
-    @retry_on_exception(5, base_delay=10)
314 313
     def keystone_wait_for_propagation(self, sentry_relation_pairs,
315 314
                                       api_version):
316 315
         """Iterate over list of sentry and relation tuples and verify that
@@ -326,7 +325,7 @@ class OpenStackAmuletUtils(AmuletUtils):
326 325
             rel = sentry.relation('identity-service',
327 326
                                   relation_name)
328 327
             self.log.debug('keystone relation data: {}'.format(rel))
329
-            if rel['api_version'] != str(api_version):
328
+            if rel.get('api_version') != str(api_version):
330 329
                 raise Exception("api_version not propagated through relation"
331 330
                                 " data yet ('{}' != '{}')."
332 331
                                 "".format(rel['api_version'], api_version))
@@ -348,15 +347,19 @@ class OpenStackAmuletUtils(AmuletUtils):
348 347
 
349 348
         config = {'preferred-api-version': api_version}
350 349
         deployment.d.configure('keystone', config)
350
+        deployment._auto_wait_for_status()
351 351
         self.keystone_wait_for_propagation(sentry_relation_pairs, api_version)
352 352
 
353 353
     def authenticate_cinder_admin(self, keystone_sentry, username,
354
-                                  password, tenant):
354
+                                  password, tenant, api_version=2):
355 355
         """Authenticates admin user with cinder."""
356 356
         # NOTE(beisner): cinder python client doesn't accept tokens.
357 357
         keystone_ip = keystone_sentry.info['public-address']
358 358
         ept = "http://{}:5000/v2.0".format(keystone_ip.strip().decode('utf-8'))
359
-        return cinder_client.Client(username, password, tenant, ept)
359
+        _clients = {
360
+            1: cinder_client.Client,
361
+            2: cinder_clientv2.Client}
362
+        return _clients[api_version](username, password, tenant, ept)
360 363
 
361 364
     def authenticate_keystone(self, keystone_ip, username, password,
362 365
                               api_version=False, admin_port=False,
@@ -617,13 +620,25 @@ class OpenStackAmuletUtils(AmuletUtils):
617 620
             self.log.debug('Keypair ({}) already exists, '
618 621
                            'using it.'.format(keypair_name))
619 622
             return _keypair
620
-        except:
623
+        except Exception:
621 624
             self.log.debug('Keypair ({}) does not exist, '
622 625
                            'creating it.'.format(keypair_name))
623 626
 
624 627
         _keypair = nova.keypairs.create(name=keypair_name)
625 628
         return _keypair
626 629
 
630
+    def _get_cinder_obj_name(self, cinder_object):
631
+        """Retrieve name of cinder object.
632
+
633
+        :param cinder_object: cinder snapshot or volume object
634
+        :returns: str cinder object name
635
+        """
636
+        # v1 objects store name in 'display_name' attr but v2+ use 'name'
637
+        try:
638
+            return cinder_object.display_name
639
+        except AttributeError:
640
+            return cinder_object.name
641
+
627 642
     def create_cinder_volume(self, cinder, vol_name="demo-vol", vol_size=1,
628 643
                              img_id=None, src_vol_id=None, snap_id=None):
629 644
         """Create cinder volume, optionally from a glance image, OR
@@ -674,6 +689,13 @@ class OpenStackAmuletUtils(AmuletUtils):
674 689
                                             source_volid=src_vol_id,
675 690
                                             snapshot_id=snap_id)
676 691
             vol_id = vol_new.id
692
+        except TypeError:
693
+            vol_new = cinder.volumes.create(name=vol_name,
694
+                                            imageRef=img_id,
695
+                                            size=vol_size,
696
+                                            source_volid=src_vol_id,
697
+                                            snapshot_id=snap_id)
698
+            vol_id = vol_new.id
677 699
         except Exception as e:
678 700
             msg = 'Failed to create volume: {}'.format(e)
679 701
             amulet.raise_status(amulet.FAIL, msg=msg)
@@ -688,7 +710,7 @@ class OpenStackAmuletUtils(AmuletUtils):
688 710
 
689 711
         # Re-validate new volume
690 712
         self.log.debug('Validating volume attributes...')
691
-        val_vol_name = cinder.volumes.get(vol_id).display_name
713
+        val_vol_name = self._get_cinder_obj_name(cinder.volumes.get(vol_id))
692 714
         val_vol_boot = cinder.volumes.get(vol_id).bootable
693 715
         val_vol_stat = cinder.volumes.get(vol_id).status
694 716
         val_vol_size = cinder.volumes.get(vol_id).size

+ 105
- 7
tests/charmhelpers/core/hookenv.py View File

@@ -22,6 +22,7 @@ from __future__ import print_function
22 22
 import copy
23 23
 from distutils.version import LooseVersion
24 24
 from functools import wraps
25
+from collections import namedtuple
25 26
 import glob
26 27
 import os
27 28
 import json
@@ -218,6 +219,8 @@ def principal_unit():
218 219
         for rid in relation_ids(reltype):
219 220
             for unit in related_units(rid):
220 221
                 md = _metadata_unit(unit)
222
+                if not md:
223
+                    continue
221 224
                 subordinate = md.pop('subordinate', None)
222 225
                 if not subordinate:
223 226
                     return unit
@@ -511,7 +514,10 @@ def _metadata_unit(unit):
511 514
     """
512 515
     basedir = os.sep.join(charm_dir().split(os.sep)[:-2])
513 516
     unitdir = 'unit-{}'.format(unit.replace(os.sep, '-'))
514
-    with open(os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')) as md:
517
+    joineddir = os.path.join(basedir, unitdir, 'charm', 'metadata.yaml')
518
+    if not os.path.exists(joineddir):
519
+        return None
520
+    with open(joineddir) as md:
515 521
         return yaml.safe_load(md)
516 522
 
517 523
 
@@ -639,18 +645,31 @@ def is_relation_made(relation, keys='private-address'):
639 645
     return False
640 646
 
641 647
 
648
+def _port_op(op_name, port, protocol="TCP"):
649
+    """Open or close a service network port"""
650
+    _args = [op_name]
651
+    icmp = protocol.upper() == "ICMP"
652
+    if icmp:
653
+        _args.append(protocol)
654
+    else:
655
+        _args.append('{}/{}'.format(port, protocol))
656
+    try:
657
+        subprocess.check_call(_args)
658
+    except subprocess.CalledProcessError:
659
+        # Older Juju pre 2.3 doesn't support ICMP
660
+        # so treat it as a no-op if it fails.
661
+        if not icmp:
662
+            raise
663
+
664
+
642 665
 def open_port(port, protocol="TCP"):
643 666
     """Open a service network port"""
644
-    _args = ['open-port']
645
-    _args.append('{}/{}'.format(port, protocol))
646
-    subprocess.check_call(_args)
667
+    _port_op('open-port', port, protocol)
647 668
 
648 669
 
649 670
 def close_port(port, protocol="TCP"):
650 671
     """Close a service network port"""
651
-    _args = ['close-port']
652
-    _args.append('{}/{}'.format(port, protocol))
653
-    subprocess.check_call(_args)
672
+    _port_op('close-port', port, protocol)
654 673
 
655 674
 
656 675
 def open_ports(start, end, protocol="TCP"):
@@ -667,6 +686,17 @@ def close_ports(start, end, protocol="TCP"):
667 686
     subprocess.check_call(_args)
668 687
 
669 688
 
689
+def opened_ports():
690
+    """Get the opened ports
691
+
692
+    *Note that this will only show ports opened in a previous hook*
693
+
694
+    :returns: Opened ports as a list of strings: ``['8080/tcp', '8081-8083/tcp']``
695
+    """
696
+    _args = ['opened-ports', '--format=json']
697
+    return json.loads(subprocess.check_output(_args).decode('UTF-8'))
698
+
699
+
670 700
 @cached
671 701
 def unit_get(attribute):
672 702
     """Get the unit ID for the remote unit"""
@@ -1077,6 +1107,35 @@ def network_get_primary_address(binding):
1077 1107
     return subprocess.check_output(cmd).decode('UTF-8').strip()
1078 1108
 
1079 1109
 
1110
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
1111
+def network_get(endpoint, relation_id=None):
1112
+    """
1113
+    Retrieve the network details for a relation endpoint
1114
+
1115
+    :param endpoint: string. The name of a relation endpoint
1116
+    :param relation_id: int. The ID of the relation for the current context.
1117
+    :return: dict. The loaded YAML output of the network-get query.
1118
+    :raise: NotImplementedError if run on Juju < 2.1
1119
+    """
1120
+    cmd = ['network-get', endpoint, '--format', 'yaml']
1121
+    if relation_id:
1122
+        cmd.append('-r')
1123
+        cmd.append(relation_id)
1124
+    try:
1125
+        response = subprocess.check_output(
1126
+            cmd,
1127
+            stderr=subprocess.STDOUT).decode('UTF-8').strip()
1128
+    except CalledProcessError as e:
1129
+        # Early versions of Juju 2.0.x required the --primary-address argument.
1130
+        # We catch that condition here and raise NotImplementedError since
1131
+        # the requested semantics are not available - the caller can then
1132
+        # use the network_get_primary_address() method instead.
1133
+        if '--primary-address is currently required' in e.output.decode('UTF-8'):
1134
+            raise NotImplementedError
1135
+        raise
1136
+    return yaml.safe_load(response)
1137
+
1138
+
1080 1139
 def add_metric(*args, **kwargs):
1081 1140
     """Add metric values. Values may be expressed with keyword arguments. For
1082 1141
     metric names containing dashes, these may be expressed as one or more
@@ -1106,3 +1165,42 @@ def meter_info():
1106 1165
     """Get the meter status information, if running in the meter-status-changed
1107 1166
     hook."""
1108 1167
     return os.environ.get('JUJU_METER_INFO')
1168
+
1169
+
1170
+def iter_units_for_relation_name(relation_name):
1171
+    """Iterate through all units in a relation
1172
+
1173
+    Generator that iterates through all the units in a relation and yields
1174
+    a named tuple with rid and unit field names.
1175
+
1176
+    Usage:
1177
+    data = [(u.rid, u.unit)
1178
+            for u in iter_units_for_relation_name(relation_name)]
1179
+
1180
+    :param relation_name: string relation name
1181
+    :yield: Named Tuple with rid and unit field names
1182
+    """
1183
+    RelatedUnit = namedtuple('RelatedUnit', 'rid, unit')
1184
+    for rid in relation_ids(relation_name):
1185
+        for unit in related_units(rid):
1186
+            yield RelatedUnit(rid, unit)
1187
+
1188
+
1189
+def ingress_address(rid=None, unit=None):
1190
+    """
1191
+    Retrieve the ingress-address from a relation when available. Otherwise,
1192
+    return the private-address. This function is to be used on the consuming
1193
+    side of the relation.
1194
+
1195
+    Usage:
1196
+    addresses = [ingress_address(rid=u.rid, unit=u.unit)
1197
+                 for u in iter_units_for_relation_name(relation_name)]
1198
+
1199
+    :param rid: string relation id
1200
+    :param unit: string unit name
1201
+    :side effect: calls relation_get
1202
+    :return: string IP address
1203
+    """
1204
+    settings = relation_get(rid=rid, unit=unit)
1205
+    return (settings.get('ingress-address') or
1206
+            settings.get('private-address'))

+ 72
- 1
tests/charmhelpers/core/host.py View File

@@ -34,7 +34,7 @@ import six
34 34
 
35 35
 from contextlib import contextmanager
36 36
 from collections import OrderedDict
37
-from .hookenv import log, DEBUG
37
+from .hookenv import log, DEBUG, local_unit
38 38
 from .fstab import Fstab
39 39
 from charmhelpers.osplatform import get_platform
40 40
 
@@ -441,6 +441,49 @@ def add_user_to_group(username, group):
441 441
     subprocess.check_call(cmd)
442 442
 
443 443
 
444
+def chage(username, lastday=None, expiredate=None, inactive=None,
445
+           mindays=None, maxdays=None, root=None, warndays=None):
446
+    """Change user password expiry information
447
+
448
+    :param str username: User to update
449
+    :param str lastday: Set when password was changed in YYYY-MM-DD format
450
+    :param str expiredate: Set when user's account will no longer be
451
+                           accessible in YYYY-MM-DD format.
452
+                           -1 will remove an account expiration date.
453
+    :param str inactive: Set the number of days of inactivity after a password
454
+                         has expired before the account is locked.
455
+                         -1 will remove an account's inactivity.
456
+    :param str mindays: Set the minimum number of days between password
457
+                        changes to MIN_DAYS.
458
+                        0 indicates the password can be changed anytime.
459
+    :param str maxdays: Set the maximum number of days during which a
460
+                        password is valid.
461
+                        -1 as MAX_DAYS will remove checking maxdays
462
+    :param str root: Apply changes in the CHROOT_DIR directory
463
+    :param str warndays: Set the number of days of warning before a password
464
+                         change is required
465
+    :raises subprocess.CalledProcessError: if call to chage fails
466
+    """
467
+    cmd = ['chage']
468
+    if root:
469
+        cmd.extend(['--root', root])
470
+    if lastday:
471
+        cmd.extend(['--lastday', lastday])
472
+    if expiredate:
473
+        cmd.extend(['--expiredate', expiredate])
474
+    if inactive:
475
+        cmd.extend(['--inactive', inactive])
476
+    if mindays:
477
+        cmd.extend(['--mindays', mindays])
478
+    if maxdays:
479
+        cmd.extend(['--maxdays', maxdays])
480
+    if warndays:
481
+        cmd.extend(['--warndays', warndays])
482
+    cmd.append(username)
483
+    subprocess.check_call(cmd)
484
+
485
+remove_password_expiry = functools.partial(chage, expiredate='-1', inactive='-1', mindays='0', maxdays='-1')
486
+
444 487
 def rsync(from_path, to_path, flags='-r', options=None, timeout=None):
445 488
     """Replicate the contents of a path"""
446 489
     options = options or ['--delete', '--executability']
@@ -946,3 +989,31 @@ def updatedb(updatedb_text, new_path):
946 989
                 lines[i] = 'PRUNEPATHS="{}"'.format(' '.join(paths))
947 990
     output = "\n".join(lines)
948 991
     return output
992
+
993
+
994
+def modulo_distribution(modulo=3, wait=30):
995
+    """ Modulo distribution
996
+
997
+    This helper uses the unit number, a modulo value and a constant wait time
998
+    to produce a calculated wait time distribution. This is useful in large
999
+    scale deployments to distribute load during an expensive operation such as
1000
+    service restarts.
1001
+
1002
+    If you have 1000 nodes that need to restart 100 at a time 1 minute at a
1003
+    time:
1004
+
1005
+      time.wait(modulo_distribution(modulo=100, wait=60))
1006
+      restart()
1007
+
1008
+    If you need restarts to happen serially set modulo to the exact number of
1009
+    nodes and set a high constant wait time:
1010
+
1011
+      time.wait(modulo_distribution(modulo=10, wait=120))
1012
+      restart()
1013
+
1014
+    @param modulo: int The modulo number creates the group distribution
1015
+    @param wait: int The constant time wait value
1016
+    @return: int Calculated time to wait for unit operation
1017
+    """
1018
+    unit_number = int(local_unit().split('/')[1])
1019
+    return (unit_number % modulo) * wait

+ 11
- 5
tests/charmhelpers/core/strutils.py View File

@@ -61,13 +61,19 @@ def bytes_from_string(value):
61 61
     if isinstance(value, six.string_types):
62 62
         value = six.text_type(value)
63 63
     else:
64
-        msg = "Unable to interpret non-string value '%s' as boolean" % (value)
64
+        msg = "Unable to interpret non-string value '%s' as bytes" % (value)
65 65
         raise ValueError(msg)
66 66
     matches = re.match("([0-9]+)([a-zA-Z]+)", value)
67
-    if not matches:
68
-        msg = "Unable to interpret string value '%s' as bytes" % (value)
69
-        raise ValueError(msg)
70
-    return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
67
+    if matches:
68
+        size = int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
69
+    else:
70
+        # Assume that value passed in is bytes
71
+        try:
72
+            size = int(value)
73
+        except ValueError:
74
+            msg = "Unable to interpret string value '%s' as bytes" % (value)
75
+            raise ValueError(msg)
76
+    return size
71 77
 
72 78
 
73 79
 class BasicStringComparator(object):

+ 1
- 1
tests/charmhelpers/core/unitdata.py View File

@@ -358,7 +358,7 @@ class Storage(object):
358 358
         try:
359 359
             yield self.revision
360 360
             self.revision = None
361
-        except:
361
+        except Exception:
362 362
             self.flush(False)
363 363
             self.revision = None
364 364
             raise

Loading…
Cancel
Save