Enable xenial-pike amulet test

Make default func27-smoke xenial-pike
Charm-helpers sync

Change-Id: I289d38e4170d204fbf9b0281b28be28c9e847e65

charmhelpers/contrib/charmsupport/nrpe.py

@@ -30,6 +30,7 @@ import yaml
 
 from charmhelpers.core.hookenv import (
     config,
+    hook_name,
     local_unit,
     log,
     relation_ids,
@@ -285,7 +286,7 @@ class NRPE(object):
         try:
             nagios_uid = pwd.getpwnam('nagios').pw_uid
             nagios_gid = grp.getgrnam('nagios').gr_gid
-        except:
+        except Exception:
             log("Nagios user not set up, nrpe checks not updated")
             return
 
@@ -302,7 +303,12 @@ class NRPE(object):
                 "command": nrpecheck.command,
             }
 
-        service('restart', 'nagios-nrpe-server')
+        # update-status hooks are configured to firing every 5 minutes by
+        # default. When nagios-nrpe-server is restarted, the nagios server
+        # reports checks failing causing unneccessary alerts. Let's not restart
+        # on update-status hooks.
+        if not hook_name() == 'update-status':
+            service('restart', 'nagios-nrpe-server')
 
         monitor_ids = relation_ids("local-monitors") + \
             relation_ids("nrpe-external-master")

charmhelpers/contrib/network/ip.py

@@ -490,7 +490,7 @@ def get_host_ip(hostname, fallback=None):
     if not ip_addr:
         try:
             ip_addr = socket.gethostbyname(hostname)
-        except:
+        except Exception:
             log("Failed to resolve hostname '%s'" % (hostname),
                 level=WARNING)
             return fallback
@@ -518,7 +518,7 @@ def get_hostname(address, fqdn=True):
         if not result:
             try:
                 result = socket.gethostbyaddr(address)[0]
-            except:
+            except Exception:
                 return None
     else:
         result = address

charmhelpers/contrib/openstack/amulet/deployment.py

@@ -250,7 +250,14 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         self.log.debug('Waiting up to {}s for extended status on services: '
                        '{}'.format(timeout, services))
         service_messages = {service: message for service in services}
+
+        # Check for idleness
+        self.d.sentry.wait()
+        # Check for error states and bail early
+        self.d.sentry.wait_for_status(self.d.juju_env, services)
+        # Check for ready messages
         self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
+
         self.log.info('OK')
 
     def _get_openstack_release(self):
@@ -303,20 +310,27 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         test scenario, based on OpenStack release and whether ceph radosgw
         is flagged as present or not."""
 
-        if self._get_openstack_release() >= self.trusty_kilo:
+        if self._get_openstack_release() <= self.trusty_juno:
-            # Kilo or later
-            pools = [
-                'rbd',
-                'cinder',
-                'glance'
-            ]
-        else:
             # Juno or earlier
             pools = [
                 'data',
                 'metadata',
                 'rbd',
-                'cinder',
+                'cinder-ceph',
+                'glance'
+            ]
+        elif (self.trust_kilo <= self._get_openstack_release() <=
+              self.zesty_ocata):
+            # Kilo through Ocata
+            pools = [
+                'rbd',
+                'cinder-ceph',
+                'glance'
+            ]
+        else:
+            # Pike and later
+            pools = [
+                'cinder-ceph',
                 'glance'
             ]
 

charmhelpers/contrib/openstack/amulet/utils.py

@@ -23,6 +23,7 @@ import urllib
 import urlparse
 
 import cinderclient.v1.client as cinder_client
+import cinderclient.v2.client as cinder_clientv2
 import glanceclient.v1.client as glance_client
 import heatclient.v1.client as heat_client
 from keystoneclient.v2_0 import client as keystone_client
@@ -42,7 +43,6 @@ import swiftclient
 from charmhelpers.contrib.amulet.utils import (
     AmuletUtils
 )
-from charmhelpers.core.decorators import retry_on_exception
 from charmhelpers.core.host import CompareHostReleases
 
 DEBUG = logging.DEBUG
@@ -310,7 +310,6 @@ class OpenStackAmuletUtils(AmuletUtils):
         self.log.debug('Checking if tenant exists ({})...'.format(tenant))
         return tenant in [t.name for t in keystone.tenants.list()]
 
-    @retry_on_exception(5, base_delay=10)
     def keystone_wait_for_propagation(self, sentry_relation_pairs,
                                       api_version):
         """Iterate over list of sentry and relation tuples and verify that
@@ -326,7 +325,7 @@ class OpenStackAmuletUtils(AmuletUtils):
             rel = sentry.relation('identity-service',
                                   relation_name)
             self.log.debug('keystone relation data: {}'.format(rel))
-            if rel['api_version'] != str(api_version):
+            if rel.get('api_version') != str(api_version):
                 raise Exception("api_version not propagated through relation"
                                 " data yet ('{}' != '{}')."
                                 "".format(rel['api_version'], api_version))
@@ -348,15 +347,19 @@ class OpenStackAmuletUtils(AmuletUtils):
 
         config = {'preferred-api-version': api_version}
         deployment.d.configure('keystone', config)
+        deployment._auto_wait_for_status()
         self.keystone_wait_for_propagation(sentry_relation_pairs, api_version)
 
     def authenticate_cinder_admin(self, keystone_sentry, username,
-                                  password, tenant):
+                                  password, tenant, api_version=2):
         """Authenticates admin user with cinder."""
         # NOTE(beisner): cinder python client doesn't accept tokens.
         keystone_ip = keystone_sentry.info['public-address']
         ept = "http://{}:5000/v2.0".format(keystone_ip.strip().decode('utf-8'))
-        return cinder_client.Client(username, password, tenant, ept)
+        _clients = {
+            1: cinder_client.Client,
+            2: cinder_clientv2.Client}
+        return _clients[api_version](username, password, tenant, ept)
 
     def authenticate_keystone(self, keystone_ip, username, password,
                               api_version=False, admin_port=False,
@@ -617,13 +620,25 @@ class OpenStackAmuletUtils(AmuletUtils):
             self.log.debug('Keypair ({}) already exists, '
                            'using it.'.format(keypair_name))
             return _keypair
-        except:
+        except Exception:
             self.log.debug('Keypair ({}) does not exist, '
                            'creating it.'.format(keypair_name))
 
         _keypair = nova.keypairs.create(name=keypair_name)
         return _keypair
 
+    def _get_cinder_obj_name(self, cinder_object):
+        """Retrieve name of cinder object.
+
+        :param cinder_object: cinder snapshot or volume object
+        :returns: str cinder object name
+        """
+        # v1 objects store name in 'display_name' attr but v2+ use 'name'
+        try:
+            return cinder_object.display_name
+        except AttributeError:
+            return cinder_object.name
+
     def create_cinder_volume(self, cinder, vol_name="demo-vol", vol_size=1,
                              img_id=None, src_vol_id=None, snap_id=None):
         """Create cinder volume, optionally from a glance image, OR
@@ -674,6 +689,13 @@ class OpenStackAmuletUtils(AmuletUtils):
                                             source_volid=src_vol_id,
                                             snapshot_id=snap_id)
             vol_id = vol_new.id
+        except TypeError:
+            vol_new = cinder.volumes.create(name=vol_name,
+                                            imageRef=img_id,
+                                            size=vol_size,
+                                            source_volid=src_vol_id,
+                                            snapshot_id=snap_id)
+            vol_id = vol_new.id
         except Exception as e:
             msg = 'Failed to create volume: {}'.format(e)
             amulet.raise_status(amulet.FAIL, msg=msg)
@@ -688,7 +710,7 @@ class OpenStackAmuletUtils(AmuletUtils):
 
         # Re-validate new volume
         self.log.debug('Validating volume attributes...')
-        val_vol_name = cinder.volumes.get(vol_id).display_name
+        val_vol_name = self._get_cinder_obj_name(cinder.volumes.get(vol_id))
         val_vol_boot = cinder.volumes.get(vol_id).bootable
         val_vol_stat = cinder.volumes.get(vol_id).status
         val_vol_size = cinder.volumes.get(vol_id).size

charmhelpers/contrib/openstack/context.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import collections
 import glob
 import json
 import math
@@ -578,11 +579,14 @@ class HAProxyContext(OSContextGenerator):
             laddr = get_address_in_network(config(cfg_opt))
             if laddr:
                 netmask = get_netmask_for_address(laddr)
-                cluster_hosts[laddr] = {'network': "{}/{}".format(laddr,
+                cluster_hosts[laddr] = {
-                                                                  netmask),
+                    'network': "{}/{}".format(laddr,
-                                        'backends': {l_unit: laddr}}
+                                              netmask),
+                    'backends': collections.OrderedDict([(l_unit,
+                                                          laddr)])
+                }
                 for rid in relation_ids('cluster'):
-                    for unit in related_units(rid):
+                    for unit in sorted(related_units(rid)):
                         _laddr = relation_get('{}-address'.format(addr_type),
                                               rid=rid, unit=unit)
                         if _laddr:
@@ -594,10 +598,13 @@ class HAProxyContext(OSContextGenerator):
         # match in the frontend
         cluster_hosts[addr] = {}
         netmask = get_netmask_for_address(addr)
-        cluster_hosts[addr] = {'network': "{}/{}".format(addr, netmask),
+        cluster_hosts[addr] = {
-                               'backends': {l_unit: addr}}
+            'network': "{}/{}".format(addr, netmask),
+            'backends': collections.OrderedDict([(l_unit,
+                                                  addr)])
+        }
         for rid in relation_ids('cluster'):
-            for unit in related_units(rid):
+            for unit in sorted(related_units(rid)):
                 _laddr = relation_get('private-address',
                                       rid=rid, unit=unit)
                 if _laddr:
@@ -628,6 +635,8 @@ class HAProxyContext(OSContextGenerator):
             ctxt['local_host'] = '127.0.0.1'
             ctxt['haproxy_host'] = '0.0.0.0'
 
+        ctxt['ipv6_enabled'] = not is_ipv6_disabled()
+
         ctxt['stat_port'] = '8888'
 
         db = kv()
@@ -844,15 +853,6 @@ class NeutronContext(OSContextGenerator):
         for pkgs in self.packages:
             ensure_packages(pkgs)
 
-    def _save_flag_file(self):
-        if self.network_manager == 'quantum':
-            _file = '/etc/nova/quantum_plugin.conf'
-        else:
-            _file = '/etc/nova/neutron_plugin.conf'
-
-        with open(_file, 'wb') as out:
-            out.write(self.plugin + '\n')
-
     def ovs_ctxt(self):
         driver = neutron_plugin_attribute(self.plugin, 'driver',
                                           self.network_manager)
@@ -997,7 +997,6 @@ class NeutronContext(OSContextGenerator):
             flags = config_flags_parser(alchemy_flags)
             ctxt['neutron_alchemy_flags'] = flags
 
-        self._save_flag_file()
         return ctxt
 
 
@@ -1177,7 +1176,7 @@ class SubordinateConfigContext(OSContextGenerator):
                 if sub_config and sub_config != '':
                     try:
                         sub_config = json.loads(sub_config)
-                    except:
+                    except Exception:
                         log('Could not parse JSON from '
                             'subordinate_configuration setting from %s'
                             % rid, level=ERROR)

charmhelpers/contrib/openstack/templates/ceph.conf

@@ -18,7 +18,7 @@ rbd default features = {{ rbd_features }}
 
 [client]
 {% if rbd_client_cache_settings -%}
-{% for key, value in rbd_client_cache_settings.iteritems() -%}
+{% for key, value in rbd_client_cache_settings.items() -%}
 {{ key }} = {{ value }}
 {% endfor -%}
-{%- endif %}
+{%- endif %}

charmhelpers/contrib/openstack/templates/haproxy.cfg

@@ -48,7 +48,9 @@ listen stats
 {% for service, ports in service_ports.items() -%}
 frontend tcp-in_{{ service }}
     bind *:{{ ports[0] }}
+    {% if ipv6_enabled -%}
     bind :::{{ ports[0] }}
+    {% endif -%}
     {% for frontend in frontends -%}
     acl net_{{ frontend }} dst {{ frontends[frontend]['network'] }}
     use_backend {{ service }}_{{ frontend }} if net_{{ frontend }}

charmhelpers/contrib/openstack/templating.py

@@ -272,6 +272,8 @@ class OSConfigRenderer(object):
             raise OSConfigException
 
         _out = self.render(config_file)
+        if six.PY3:
+            _out = _out.encode('UTF-8')
 
         with open(config_file, 'wb') as out:
             out.write(_out)

charmhelpers/contrib/openstack/utils.py

@@ -426,7 +426,7 @@ def get_os_codename_package(package, fatal=True):
 
     try:
         pkg = cache[package]
-    except:
+    except Exception:
         if not fatal:
             return None
         # the package is unknown to the current apt cache.
@@ -618,7 +618,7 @@ def save_script_rc(script_path="scripts/scriptrc", **env_vars):
     juju_rc_path = "%s/%s" % (charm_dir(), script_path)
     if not os.path.exists(os.path.dirname(juju_rc_path)):
         os.mkdir(os.path.dirname(juju_rc_path))
-    with open(juju_rc_path, 'wb') as rc_script:
+    with open(juju_rc_path, 'wt') as rc_script:
         rc_script.write(
             "#!/bin/bash\n")
         [rc_script.write('export %s=%s\n' % (u, p))
@@ -797,7 +797,7 @@ def git_default_repos(projects_yaml):
     service = service_name()
     core_project = service
 
-    for default, branch in GIT_DEFAULT_BRANCHES.iteritems():
+    for default, branch in six.iteritems(GIT_DEFAULT_BRANCHES):
         if projects_yaml == default:
 
             # add the requirements repo first
@@ -1618,7 +1618,7 @@ def do_action_openstack_upgrade(package, upgrade_callback, configs):
                     upgrade_callback(configs=configs)
                     action_set({'outcome': 'success, upgrade completed.'})
                     ret = True
-                except:
+                except Exception:
                     action_set({'outcome': 'upgrade failed, see traceback.'})
                     action_set({'traceback': traceback.format_exc()})
                     action_fail('do_openstack_upgrade resulted in an '
@@ -1723,7 +1723,7 @@ def is_unit_paused_set():
             kv = t[0]
             # transform something truth-y into a Boolean.
             return not(not(kv.get('unit-paused')))
-    except:
+    except Exception:
         return False
 
 
@@ -2051,7 +2051,7 @@ def update_json_file(filename, items):
 def snap_install_requested():
     """ Determine if installing from snaps
 
-    If openstack-origin is of the form snap:track/channel
+    If openstack-origin is of the form snap:track/channel[/branch]
     and channel is in SNAPS_CHANNELS return True.
     """
     origin = config('openstack-origin') or ""
@@ -2060,9 +2060,9 @@ def snap_install_requested():
 
     _src = origin[5:]
     if '/' in _src:
-        _track, channel = _src.split('/')
+        channel = _src.split('/')[1]
     else:
-        # Hanlde snap:track with no channel
+        # Handle snap:track with no channel
         channel = 'stable'
     return valid_snap_channel(channel)
 

charmhelpers/contrib/python/debug.py

@@ -49,6 +49,6 @@ def set_trace(addr=DEFAULT_ADDR, port=DEFAULT_PORT):
         open_port(port)
         debugger = Rpdb(addr=addr, port=port)
         debugger.set_trace(sys._getframe().f_back)
-    except:
+    except Exception:
         _error("Cannot start a remote debug session on %s:%s" % (addr,
                                                                  port))

charmhelpers/contrib/storage/linux/ceph.py

@@ -370,9 +370,10 @@ def get_mon_map(service):
       Also raises CalledProcessError if our ceph command fails
     """
     try:
-        mon_status = check_output(
+        mon_status = check_output(['ceph', '--id', service,
-            ['ceph', '--id', service,
+                                   'mon_status', '--format=json'])
-             'mon_status', '--format=json'])
+        if six.PY3:
+            mon_status = mon_status.decode('UTF-8')
         try:
             return json.loads(mon_status)
         except ValueError as v:
@@ -457,7 +458,7 @@ def monitor_key_get(service, key):
     try:
         output = check_output(
             ['ceph', '--id', service,
-             'config-key', 'get', str(key)])
+             'config-key', 'get', str(key)]).decode('UTF-8')
         return output
     except CalledProcessError as e:
         log("Monitor config-key get failed with message: {}".format(
@@ -500,6 +501,8 @@ def get_erasure_profile(service, name):
         out = check_output(['ceph', '--id', service,
                             'osd', 'erasure-code-profile', 'get',
                             name, '--format=json'])
+        if six.PY3:
+            out = out.decode('UTF-8')
         return json.loads(out)
     except (CalledProcessError, OSError, ValueError):
         return None
@@ -686,7 +689,10 @@ def get_cache_mode(service, pool_name):
     """
     validator(value=service, valid_type=six.string_types)
     validator(value=pool_name, valid_type=six.string_types)
-    out = check_output(['ceph', '--id', service, 'osd', 'dump', '--format=json'])
+    out = check_output(['ceph', '--id', service,
+                        'osd', 'dump', '--format=json'])
+    if six.PY3:
+        out = out.decode('UTF-8')
     try:
         osd_json = json.loads(out)
         for pool in osd_json['pools']:
@@ -700,8 +706,9 @@ def get_cache_mode(service, pool_name):
 def pool_exists(service, name):
     """Check to see if a RADOS pool already exists."""
     try:
-        out = check_output(['rados', '--id', service,
+        out = check_output(['rados', '--id', service, 'lspools'])
-                            'lspools']).decode('UTF-8')
+        if six.PY3:
+            out = out.decode('UTF-8')
     except CalledProcessError:
         return False
 
@@ -714,9 +721,12 @@ def get_osds(service):
     """
     version = ceph_version()
     if version and version >= '0.56':
-        return json.loads(check_output(['ceph', '--id', service,
+        out = check_output(['ceph', '--id', service,
-                                        'osd', 'ls',
+                            'osd', 'ls',
-                                        '--format=json']).decode('UTF-8'))
+                            '--format=json'])
+        if six.PY3:
+            out = out.decode('UTF-8')
+        return json.loads(out)
 
     return None
 
@@ -734,7 +744,9 @@ def rbd_exists(service, pool, rbd_img):
     """Check to see if a RADOS block device exists."""
     try:
         out = check_output(['rbd', 'list', '--id',
-                            service, '--pool', pool]).decode('UTF-8')
+                            service, '--pool', pool])
+        if six.PY3:
+            out = out.decode('UTF-8')
     except CalledProcessError:
         return False
 
@@ -859,7 +871,9 @@ def configure(service, key, auth, use_syslog):
 def image_mapped(name):
     """Determine whether a RADOS block device is mapped locally."""
     try:
-        out = check_output(['rbd', 'showmapped']).decode('UTF-8')
+        out = check_output(['rbd', 'showmapped'])
+        if six.PY3:
+            out = out.decode('UTF-8')
     except CalledProcessError:
         return False
 
@@ -1018,7 +1032,9 @@ def ceph_version():
     """Retrieve the local version of ceph."""
     if os.path.exists('/usr/bin/ceph'):
         cmd = ['ceph', '-v']
-        output = check_output(cmd).decode('US-ASCII')
+        output = check_output(cmd)
+        if six.PY3:
+            output = output.decode('UTF-8')
         output = output.split()
         if len(output) > 3:
             return output[2]

charmhelpers/contrib/storage/linux/lvm.py

@@ -74,10 +74,10 @@ def list_lvm_volume_group(block_device):
     '''
     vg = None
     pvd = check_output(['pvdisplay', block_device]).splitlines()
-    for l in pvd:
+    for lvm in pvd:
-        l = l.decode('UTF-8')
+        lvm = lvm.decode('UTF-8')
-        if l.strip().startswith('VG Name'):
+        if lvm.strip().startswith('VG Name'):
-            vg = ' '.join(l.strip().split()[2:])
+            vg = ' '.join(lvm.strip().split()[2:])
     return vg
 
 

charmhelpers/contrib/storage/linux/utils.py

@@ -64,6 +64,6 @@ def is_device_mounted(device):
     '''
     try:
         out = check_output(['lsblk', '-P', device]).decode('UTF-8')
-    except:
+    except Exception:
         return False
     return bool(re.search(r'MOUNTPOINT=".+"', out))

charmhelpers/contrib/unison/__init__.py

@@ -283,7 +283,7 @@ def sync_path_to_host(path, host, user, verbose=False, cmd=None, gid=None,
     try:
         log('Syncing local path %s to %s@%s:%s' % (path, user, host, path))
         run_as_user(user, cmd, gid)
-    except:
+    except Exception:
         log('Error syncing remote files')
         if fatal:
             raise

charmhelpers/core/hookenv.py

@@ -22,6 +22,7 @@ from __future__ import print_function
 import copy
 from distutils.version import LooseVersion
 from functools import wraps
+from collections import namedtuple
 import glob
 import os
 import json
@@ -644,18 +645,31 @@ def is_relation_made(relation, keys='private-address'):
     return False
 
 
+def _port_op(op_name, port, protocol="TCP"):
+    """Open or close a service network port"""
+    _args = [op_name]
+    icmp = protocol.upper() == "ICMP"
+    if icmp:
+        _args.append(protocol)
+    else:
+        _args.append('{}/{}'.format(port, protocol))
+    try:
+        subprocess.check_call(_args)
+    except subprocess.CalledProcessError:
+        # Older Juju pre 2.3 doesn't support ICMP
+        # so treat it as a no-op if it fails.
+        if not icmp:
+            raise
+
+
 def open_port(port, protocol="TCP"):
     """Open a service network port"""
-    _args = ['open-port']
+    _port_op('open-port', port, protocol)
-    _args.append('{}/{}'.format(port, protocol))
-    subprocess.check_call(_args)
 
 
 def close_port(port, protocol="TCP"):
     """Close a service network port"""
-    _args = ['close-port']
+    _port_op('close-port', port, protocol)
-    _args.append('{}/{}'.format(port, protocol))
-    subprocess.check_call(_args)
 
 
 def open_ports(start, end, protocol="TCP"):
@@ -1101,13 +1115,24 @@ def network_get(endpoint, relation_id=None):
     :param endpoint: string. The name of a relation endpoint
     :param relation_id: int. The ID of the relation for the current context.
     :return: dict. The loaded YAML output of the network-get query.
-    :raise: NotImplementedError if run on Juju < 2.0
+    :raise: NotImplementedError if run on Juju < 2.1
     """
     cmd = ['network-get', endpoint, '--format', 'yaml']
     if relation_id:
         cmd.append('-r')
         cmd.append(relation_id)
-    response = subprocess.check_output(cmd).decode('UTF-8').strip()
+    try:
+        response = subprocess.check_output(
+            cmd,
+            stderr=subprocess.STDOUT).decode('UTF-8').strip()
+    except CalledProcessError as e:
+        # Early versions of Juju 2.0.x required the --primary-address argument.
+        # We catch that condition here and raise NotImplementedError since
+        # the requested semantics are not available - the caller can then
+        # use the network_get_primary_address() method instead.
+        if '--primary-address is currently required' in e.output.decode('UTF-8'):
+            raise NotImplementedError
+        raise
     return yaml.safe_load(response)
 
 
@@ -1140,3 +1165,42 @@ def meter_info():
     """Get the meter status information, if running in the meter-status-changed
     hook."""
     return os.environ.get('JUJU_METER_INFO')
+
+
+def iter_units_for_relation_name(relation_name):
+    """Iterate through all units in a relation
+
+    Generator that iterates through all the units in a relation and yields
+    a named tuple with rid and unit field names.
+
+    Usage:
+    data = [(u.rid, u.unit)
+            for u in iter_units_for_relation_name(relation_name)]
+
+    :param relation_name: string relation name
+    :yield: Named Tuple with rid and unit field names
+    """
+    RelatedUnit = namedtuple('RelatedUnit', 'rid, unit')
+    for rid in relation_ids(relation_name):
+        for unit in related_units(rid):
+            yield RelatedUnit(rid, unit)
+
+
+def ingress_address(rid=None, unit=None):
+    """
+    Retrieve the ingress-address from a relation when available. Otherwise,
+    return the private-address. This function is to be used on the consuming
+    side of the relation.
+
+    Usage:
+    addresses = [ingress_address(rid=u.rid, unit=u.unit)
+                 for u in iter_units_for_relation_name(relation_name)]
+
+    :param rid: string relation id
+    :param unit: string unit name
+    :side effect: calls relation_get
+    :return: string IP address
+    """
+    settings = relation_get(rid=rid, unit=unit)
+    return (settings.get('ingress-address') or
+            settings.get('private-address'))

charmhelpers/core/strutils.py

@@ -61,13 +61,19 @@ def bytes_from_string(value):
     if isinstance(value, six.string_types):
         value = six.text_type(value)
     else:
-        msg = "Unable to interpret non-string value '%s' as boolean" % (value)
+        msg = "Unable to interpret non-string value '%s' as bytes" % (value)
         raise ValueError(msg)
     matches = re.match("([0-9]+)([a-zA-Z]+)", value)
-    if not matches:
+    if matches:
-        msg = "Unable to interpret string value '%s' as bytes" % (value)
+        size = int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
-        raise ValueError(msg)
+    else:
-    return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
+        # Assume that value passed in is bytes
+        try:
+            size = int(value)
+        except ValueError:
+            msg = "Unable to interpret string value '%s' as bytes" % (value)
+            raise ValueError(msg)
+    return size
 
 
 class BasicStringComparator(object):

charmhelpers/core/unitdata.py

@@ -358,7 +358,7 @@ class Storage(object):
         try:
             yield self.revision
             self.revision = None
-        except:
+        except Exception:
             self.flush(False)
             self.revision = None
             raise

charmhelpers/fetch/ubuntu.py

@@ -572,7 +572,7 @@ def get_upstream_version(package):
     cache = apt_cache()
     try:
         pkg = cache[package]
-    except:
+    except Exception:
         # the package is unknown to the current apt cache.
         return None
 

tests/basic_deployment.py

@@ -422,6 +422,11 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment):
     def validate_keystone_users(self, client):
         """Verify all existing roles."""
         u.log.debug('Checking keystone users...')
+
+        if self._get_openstack_release() < self.xenial_pike:
+            cinder_user = 'cinder_cinderv2'
+        else:
+            cinder_user = 'cinderv3_cinderv2'
         base = [
             {'name': 'demoUser',
              'enabled': True,
@@ -431,7 +436,7 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment):
              'enabled': True,
              'id': u.not_null,
              'email': 'juju@localhost'},
-            {'name': 'cinder_cinderv2',
+            {'name': cinder_user,
              'enabled': True,
              'id': u.not_null,
              'email': u'juju@localhost'}
@@ -609,6 +614,9 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment):
             'volume': [endpoint_check],
             'identity': [endpoint_check]
         }
+        if self._get_openstack_release() >= self.xenial_pike:
+            expected.pop('volume')
+            expected['volumev2'] = [endpoint_check]
         actual = self.keystone_v2.service_catalog.get_endpoints()
 
         ret = u.validate_svc_catalog_endpoint_data(expected, actual)
@@ -704,6 +712,8 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment):
             'service_tenant_id': u.not_null,
             'service_host': u.valid_ip
         }
+        if self._get_openstack_release() >= self.xenial_pike:
+            expected['service_username'] = 'cinderv3_cinderv2'
         for unit in self.keystone_sentries:
             ret = u.validate_relation_data(unit, relation, expected)
             if ret:
@@ -728,6 +738,22 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment):
             'cinderv2_admin_url': u.valid_url,
             'private-address': u.valid_ip,
         }
+
+        if self._get_openstack_release() >= self.xenial_pike:
+            expected.pop('cinder_region')
+            expected.pop('cinder_service')
+            expected.pop('cinder_public_url')
+            expected.pop('cinder_admin_url')
+            expected.pop('cinder_internal_url')
+            expected.update({
+                'cinderv2_region': 'RegionOne',
+                'cinderv3_region': 'RegionOne',
+                'cinderv3_service': 'cinderv3',
+                'cinderv3_region': 'RegionOne',
+                'cinderv3_public_url': u.valid_url,
+                'cinderv3_internal_url': u.valid_url,
+                'cinderv3_admin_url': u.valid_url})
+
         ret = u.validate_relation_data(unit, relation, expected)
         if ret:
             message = u.relation_error('cinder identity-service', ret)

tests/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -250,7 +250,14 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         self.log.debug('Waiting up to {}s for extended status on services: '
                        '{}'.format(timeout, services))
         service_messages = {service: message for service in services}
+
+        # Check for idleness
+        self.d.sentry.wait()
+        # Check for error states and bail early
+        self.d.sentry.wait_for_status(self.d.juju_env, services)
+        # Check for ready messages
         self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
+
         self.log.info('OK')
 
     def _get_openstack_release(self):
@@ -303,20 +310,27 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         test scenario, based on OpenStack release and whether ceph radosgw
         is flagged as present or not."""
 
-        if self._get_openstack_release() >= self.trusty_kilo:
+        if self._get_openstack_release() <= self.trusty_juno:
-            # Kilo or later
-            pools = [
-                'rbd',
-                'cinder',
-                'glance'
-            ]
-        else:
             # Juno or earlier
             pools = [
                 'data',
                 'metadata',
                 'rbd',
-                'cinder',
+                'cinder-ceph',
+                'glance'
+            ]
+        elif (self.trust_kilo <= self._get_openstack_release() <=
+              self.zesty_ocata):
+            # Kilo through Ocata
+            pools = [
+                'rbd',
+                'cinder-ceph',
+                'glance'
+            ]
+        else:
+            # Pike and later
+            pools = [
+                'cinder-ceph',
                 'glance'
             ]
 

tests/charmhelpers/contrib/openstack/amulet/utils.py

@@ -23,6 +23,7 @@ import urllib
 import urlparse
 
 import cinderclient.v1.client as cinder_client
+import cinderclient.v2.client as cinder_clientv2
 import glanceclient.v1.client as glance_client
 import heatclient.v1.client as heat_client
 from keystoneclient.v2_0 import client as keystone_client
@@ -42,7 +43,6 @@ import swiftclient
 from charmhelpers.contrib.amulet.utils import (
     AmuletUtils
 )
-from charmhelpers.core.decorators import retry_on_exception
 from charmhelpers.core.host import CompareHostReleases
 
 DEBUG = logging.DEBUG
@@ -310,7 +310,6 @@ class OpenStackAmuletUtils(AmuletUtils):
         self.log.debug('Checking if tenant exists ({})...'.format(tenant))
         return tenant in [t.name for t in keystone.tenants.list()]
 
-    @retry_on_exception(5, base_delay=10)
     def keystone_wait_for_propagation(self, sentry_relation_pairs,
                                       api_version):
         """Iterate over list of sentry and relation tuples and verify that
@@ -326,7 +325,7 @@ class OpenStackAmuletUtils(AmuletUtils):
             rel = sentry.relation('identity-service',
                                   relation_name)
             self.log.debug('keystone relation data: {}'.format(rel))
-            if rel['api_version'] != str(api_version):
+            if rel.get('api_version') != str(api_version):
                 raise Exception("api_version not propagated through relation"
                                 " data yet ('{}' != '{}')."
                                 "".format(rel['api_version'], api_version))
@@ -348,15 +347,19 @@ class OpenStackAmuletUtils(AmuletUtils):
 
         config = {'preferred-api-version': api_version}
         deployment.d.configure('keystone', config)
+        deployment._auto_wait_for_status()
         self.keystone_wait_for_propagation(sentry_relation_pairs, api_version)
 
     def authenticate_cinder_admin(self, keystone_sentry, username,
-                                  password, tenant):
+                                  password, tenant, api_version=2):
         """Authenticates admin user with cinder."""
         # NOTE(beisner): cinder python client doesn't accept tokens.
         keystone_ip = keystone_sentry.info['public-address']
         ept = "http://{}:5000/v2.0".format(keystone_ip.strip().decode('utf-8'))
-        return cinder_client.Client(username, password, tenant, ept)
+        _clients = {
+            1: cinder_client.Client,
+            2: cinder_clientv2.Client}
+        return _clients[api_version](username, password, tenant, ept)
 
     def authenticate_keystone(self, keystone_ip, username, password,
                               api_version=False, admin_port=False,
@@ -617,13 +620,25 @@ class OpenStackAmuletUtils(AmuletUtils):
             self.log.debug('Keypair ({}) already exists, '
                            'using it.'.format(keypair_name))
             return _keypair
-        except:
+        except Exception:
             self.log.debug('Keypair ({}) does not exist, '
                            'creating it.'.format(keypair_name))
 
         _keypair = nova.keypairs.create(name=keypair_name)
         return _keypair
 
+    def _get_cinder_obj_name(self, cinder_object):
+        """Retrieve name of cinder object.
+
+        :param cinder_object: cinder snapshot or volume object
+        :returns: str cinder object name
+        """
+        # v1 objects store name in 'display_name' attr but v2+ use 'name'
+        try:
+            return cinder_object.display_name
+        except AttributeError:
+            return cinder_object.name
+
     def create_cinder_volume(self, cinder, vol_name="demo-vol", vol_size=1,
                              img_id=None, src_vol_id=None, snap_id=None):
         """Create cinder volume, optionally from a glance image, OR
@@ -674,6 +689,13 @@ class OpenStackAmuletUtils(AmuletUtils):
                                             source_volid=src_vol_id,
                                             snapshot_id=snap_id)
             vol_id = vol_new.id
+        except TypeError:
+            vol_new = cinder.volumes.create(name=vol_name,
+                                            imageRef=img_id,
+                                            size=vol_size,
+                                            source_volid=src_vol_id,
+                                            snapshot_id=snap_id)
+            vol_id = vol_new.id
         except Exception as e:
             msg = 'Failed to create volume: {}'.format(e)
             amulet.raise_status(amulet.FAIL, msg=msg)
@@ -688,7 +710,7 @@ class OpenStackAmuletUtils(AmuletUtils):
 
         # Re-validate new volume
         self.log.debug('Validating volume attributes...')
-        val_vol_name = cinder.volumes.get(vol_id).display_name
+        val_vol_name = self._get_cinder_obj_name(cinder.volumes.get(vol_id))
         val_vol_boot = cinder.volumes.get(vol_id).bootable
         val_vol_stat = cinder.volumes.get(vol_id).status
         val_vol_size = cinder.volumes.get(vol_id).size

tests/charmhelpers/core/hookenv.py

@@ -22,6 +22,7 @@ from __future__ import print_function
 import copy
 from distutils.version import LooseVersion
 from functools import wraps
+from collections import namedtuple
 import glob
 import os
 import json
@@ -644,18 +645,31 @@ def is_relation_made(relation, keys='private-address'):
     return False
 
 
+def _port_op(op_name, port, protocol="TCP"):
+    """Open or close a service network port"""
+    _args = [op_name]
+    icmp = protocol.upper() == "ICMP"
+    if icmp:
+        _args.append(protocol)
+    else:
+        _args.append('{}/{}'.format(port, protocol))
+    try:
+        subprocess.check_call(_args)
+    except subprocess.CalledProcessError:
+        # Older Juju pre 2.3 doesn't support ICMP
+        # so treat it as a no-op if it fails.
+        if not icmp:
+            raise
+
+
 def open_port(port, protocol="TCP"):
     """Open a service network port"""
-    _args = ['open-port']
+    _port_op('open-port', port, protocol)
-    _args.append('{}/{}'.format(port, protocol))
-    subprocess.check_call(_args)
 
 
 def close_port(port, protocol="TCP"):
     """Close a service network port"""
-    _args = ['close-port']
+    _port_op('close-port', port, protocol)
-    _args.append('{}/{}'.format(port, protocol))
-    subprocess.check_call(_args)
 
 
 def open_ports(start, end, protocol="TCP"):
@@ -1101,13 +1115,24 @@ def network_get(endpoint, relation_id=None):
     :param endpoint: string. The name of a relation endpoint
     :param relation_id: int. The ID of the relation for the current context.
     :return: dict. The loaded YAML output of the network-get query.
-    :raise: NotImplementedError if run on Juju < 2.0
+    :raise: NotImplementedError if run on Juju < 2.1
     """
     cmd = ['network-get', endpoint, '--format', 'yaml']
     if relation_id:
         cmd.append('-r')
         cmd.append(relation_id)
-    response = subprocess.check_output(cmd).decode('UTF-8').strip()
+    try:
+        response = subprocess.check_output(
+            cmd,
+            stderr=subprocess.STDOUT).decode('UTF-8').strip()
+    except CalledProcessError as e:
+        # Early versions of Juju 2.0.x required the --primary-address argument.
+        # We catch that condition here and raise NotImplementedError since
+        # the requested semantics are not available - the caller can then
+        # use the network_get_primary_address() method instead.
+        if '--primary-address is currently required' in e.output.decode('UTF-8'):
+            raise NotImplementedError
+        raise
     return yaml.safe_load(response)
 
 
@@ -1140,3 +1165,42 @@ def meter_info():
     """Get the meter status information, if running in the meter-status-changed
     hook."""
     return os.environ.get('JUJU_METER_INFO')
+
+
+def iter_units_for_relation_name(relation_name):
+    """Iterate through all units in a relation
+
+    Generator that iterates through all the units in a relation and yields
+    a named tuple with rid and unit field names.
+
+    Usage:
+    data = [(u.rid, u.unit)
+            for u in iter_units_for_relation_name(relation_name)]
+
+    :param relation_name: string relation name
+    :yield: Named Tuple with rid and unit field names
+    """
+    RelatedUnit = namedtuple('RelatedUnit', 'rid, unit')
+    for rid in relation_ids(relation_name):
+        for unit in related_units(rid):
+            yield RelatedUnit(rid, unit)
+
+
+def ingress_address(rid=None, unit=None):
+    """
+    Retrieve the ingress-address from a relation when available. Otherwise,
+    return the private-address. This function is to be used on the consuming
+    side of the relation.
+
+    Usage:
+    addresses = [ingress_address(rid=u.rid, unit=u.unit)
+                 for u in iter_units_for_relation_name(relation_name)]
+
+    :param rid: string relation id
+    :param unit: string unit name
+    :side effect: calls relation_get
+    :return: string IP address
+    """
+    settings = relation_get(rid=rid, unit=unit)
+    return (settings.get('ingress-address') or
+            settings.get('private-address'))

tests/charmhelpers/core/strutils.py

@@ -61,13 +61,19 @@ def bytes_from_string(value):
     if isinstance(value, six.string_types):
         value = six.text_type(value)
     else:
-        msg = "Unable to interpret non-string value '%s' as boolean" % (value)
+        msg = "Unable to interpret non-string value '%s' as bytes" % (value)
         raise ValueError(msg)
     matches = re.match("([0-9]+)([a-zA-Z]+)", value)
-    if not matches:
+    if matches:
-        msg = "Unable to interpret string value '%s' as bytes" % (value)
+        size = int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
-        raise ValueError(msg)
+    else:
-    return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
+        # Assume that value passed in is bytes
+        try:
+            size = int(value)
+        except ValueError:
+            msg = "Unable to interpret string value '%s' as bytes" % (value)
+            raise ValueError(msg)
+    return size
 
 
 class BasicStringComparator(object):

tests/charmhelpers/core/unitdata.py

@@ -358,7 +358,7 @@ class Storage(object):
         try:
             yield self.revision
             self.revision = None
-        except:
+        except Exception:
             self.flush(False)
             self.revision = None
             raise

tox.ini

@@ -60,7 +60,7 @@ basepython = python2.7
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 commands =
-    bundletester -vl DEBUG -r json -o func-results.json gate-basic-xenial-mitaka --no-destroy
+    bundletester -vl DEBUG -r json -o func-results.json gate-basic-xenial-pike --no-destroy
 
 [testenv:func27-dfs]
 # Charm Functional Test

unit_tests/test_keystone_contexts.py

@@ -186,6 +186,7 @@ class TestKeystoneContexts(CharmTestCase):
              'service_ports': {'admin-port': ['12', '34'],
                                'public-port': ['12', '34']},
              'default_backend': '1.2.3.4',
+             'ipv6_enabled': True,
              'frontends': {'1.2.3.4': {
                  'network': '1.2.3.4/255.255.255.0',
                  'backends': {