Enable Ocata Amulet Tests

- Add Zesty as a supported series to metadata.yaml.
- Turn on Xenial-Ocata Amulet test definitions.
- Sync charm helpers to get Juju 2.x amulet compatibility.
- Keeping Zesty-Ocata Amulet test definitions turned off until the
  metadata.yaml changes propagate to the charm store.

Change-Id: If89406dabee66cfcf395bbb214457bdd1e576aad

charmhelpers/contrib/charmsupport/nrpe.py

@@ -227,6 +227,7 @@ class NRPE(object):
     nagios_logdir = '/var/log/nagios'
     nagios_exportdir = '/var/lib/nagios/export'
     nrpe_confdir = '/etc/nagios/nrpe.d'
+    homedir = '/var/lib/nagios'  # home dir provided by nagios-nrpe-server
 
     def __init__(self, hostname=None, primary=True):
         super(NRPE, self).__init__()
@@ -338,13 +339,14 @@ def get_nagios_unit_name(relation_name='nrpe-external-master'):
     return unit
 
 
-def add_init_service_checks(nrpe, services, unit_name):
+def add_init_service_checks(nrpe, services, unit_name, immediate_check=True):
     """
     Add checks for each service in list
 
     :param NRPE nrpe: NRPE object to add check to
     :param list services: List of services to check
     :param str unit_name: Unit name to use in check description
+    :param bool immediate_check: For sysv init, run the service check immediately
     """
     for svc in services:
         # Don't add a check for these services from neutron-gateway
@@ -368,21 +370,31 @@ def add_init_service_checks(nrpe, services, unit_name):
             )
         elif os.path.exists(sysv_init):
             cronpath = '/etc/cron.d/nagios-service-check-%s' % svc
-            cron_file = ('*/5 * * * * root '
+            checkpath = '%s/service-check-%s.txt' % (nrpe.homedir, svc)
-                         '/usr/local/lib/nagios/plugins/check_exit_status.pl '
+            croncmd = (
-                         '-s /etc/init.d/%s status > '
+                '/usr/local/lib/nagios/plugins/check_exit_status.pl '
-                         '/var/lib/nagios/service-check-%s.txt\n' % (svc,
+                '-s /etc/init.d/%s status' % svc
-                                                                     svc)
+            )
-                         )
+            cron_file = '*/5 * * * * root %s > %s\n' % (croncmd, checkpath)
             f = open(cronpath, 'w')
             f.write(cron_file)
             f.close()
             nrpe.add_check(
                 shortname=svc,
-                description='process check {%s}' % unit_name,
+                description='service check {%s}' % unit_name,
-                check_cmd='check_status_file.py -f '
+                check_cmd='check_status_file.py -f %s' % checkpath,
-                          '/var/lib/nagios/service-check-%s.txt' % svc,
             )
+            # if /var/lib/nagios doesn't exist open(checkpath, 'w') will fail
+            # (LP: #1670223).
+            if immediate_check and os.path.isdir(nrpe.homedir):
+                f = open(checkpath, 'w')
+                subprocess.call(
+                    croncmd.split(),
+                    stdout=f,
+                    stderr=subprocess.STDOUT
+                )
+                f.close()
+                os.chmod(checkpath, 0o644)
 
 
 def copy_nrpe_checks():

charmhelpers/contrib/hardening/templating.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import os
+import six
 
 from charmhelpers.core.hookenv import (
     log,
@@ -26,7 +27,10 @@ except ImportError:
     from charmhelpers.fetch import apt_install
     from charmhelpers.fetch import apt_update
     apt_update(fatal=True)
-    apt_install('python-jinja2', fatal=True)
+    if six.PY2:
+        apt_install('python-jinja2', fatal=True)
+    else:
+        apt_install('python3-jinja2', fatal=True)
     from jinja2 import FileSystemLoader, Environment
 
 

charmhelpers/contrib/network/ip.py

@@ -20,25 +20,37 @@ import socket
 
 from functools import partial
 
-from charmhelpers.core.hookenv import unit_get
 from charmhelpers.fetch import apt_install, apt_update
 from charmhelpers.core.hookenv import (
+    config,
     log,
+    network_get_primary_address,
+    unit_get,
     WARNING,
 )
 
+from charmhelpers.core.host import (
+    lsb_release,
+)
+
 try:
     import netifaces
 except ImportError:
     apt_update(fatal=True)
-    apt_install('python-netifaces', fatal=True)
+    if six.PY2:
+        apt_install('python-netifaces', fatal=True)
+    else:
+        apt_install('python3-netifaces', fatal=True)
     import netifaces
 
 try:
     import netaddr
 except ImportError:
     apt_update(fatal=True)
-    apt_install('python-netaddr', fatal=True)
+    if six.PY2:
+        apt_install('python-netaddr', fatal=True)
+    else:
+        apt_install('python3-netaddr', fatal=True)
     import netaddr
 
 
@@ -414,7 +426,10 @@ def ns_query(address):
     try:
         import dns.resolver
     except ImportError:
-        apt_install('python-dnspython', fatal=True)
+        if six.PY2:
+            apt_install('python-dnspython', fatal=True)
+        else:
+            apt_install('python3-dnspython', fatal=True)
         import dns.resolver
 
     if isinstance(address, dns.name.Name):
@@ -462,7 +477,10 @@ def get_hostname(address, fqdn=True):
         try:
             import dns.reversename
         except ImportError:
-            apt_install("python-dnspython", fatal=True)
+            if six.PY2:
+                apt_install("python-dnspython", fatal=True)
+            else:
+                apt_install("python3-dnspython", fatal=True)
             import dns.reversename
 
         rev = dns.reversename.from_address(address)
@@ -499,3 +517,40 @@ def port_has_listener(address, port):
     cmd = ['nc', '-z', address, str(port)]
     result = subprocess.call(cmd)
     return not(bool(result))
+
+
+def assert_charm_supports_ipv6():
+    """Check whether we are able to support charms ipv6."""
+    if lsb_release()['DISTRIB_CODENAME'].lower() < "trusty":
+        raise Exception("IPv6 is not supported in the charms for Ubuntu "
+                        "versions less than Trusty 14.04")
+
+
+def get_relation_ip(interface, config_override=None):
+    """Return this unit's IP for the given relation.
+
+    Allow for an arbitrary interface to use with network-get to select an IP.
+    Handle all address selection options including configuration parameter
+    override and IPv6.
+
+    Usage: get_relation_ip('amqp', config_override='access-network')
+
+    @param interface: string name of the relation.
+    @param config_override: string name of the config option for network
+           override. Supports legacy network override configuration parameters.
+    @raises Exception if prefer-ipv6 is configured but IPv6 unsupported.
+    @returns IPv6 or IPv4 address
+    """
+
+    fallback = get_host_ip(unit_get('private-address'))
+    if config('prefer-ipv6'):
+        assert_charm_supports_ipv6()
+        return get_ipv6_addr()[0]
+    elif config_override and config(config_override):
+        return get_address_in_network(config(config_override),
+                                      fallback)
+    else:
+        try:
+            return network_get_primary_address(interface)
+        except NotImplementedError:
+            return fallback

charmhelpers/contrib/openstack/amulet/utils.py

@@ -32,6 +32,7 @@ from keystoneclient.v3 import client as keystone_client_v3
 from novaclient import exceptions
 
 import novaclient.client as nova_client
+import novaclient
 import pika
 import swiftclient
 
@@ -434,9 +435,14 @@ class OpenStackAmuletUtils(AmuletUtils):
         self.log.debug('Authenticating nova user ({})...'.format(user))
         ep = keystone.service_catalog.url_for(service_type='identity',
                                               endpoint_type='publicURL')
-        return nova_client.Client(NOVA_CLIENT_VERSION,
+        if novaclient.__version__[0] >= "7":
-                                  username=user, api_key=password,
+            return nova_client.Client(NOVA_CLIENT_VERSION,
-                                  project_id=tenant, auth_url=ep)
+                                      username=user, password=password,
+                                      project_name=tenant, auth_url=ep)
+        else:
+            return nova_client.Client(NOVA_CLIENT_VERSION,
+                                      username=user, api_key=password,
+                                      project_id=tenant, auth_url=ep)
 
     def authenticate_swift_user(self, keystone, user, password, tenant):
         """Authenticates a regular user with swift api."""

charmhelpers/contrib/openstack/context.py

@@ -100,7 +100,10 @@ from charmhelpers.core.unitdata import kv
 try:
     import psutil
 except ImportError:
-    apt_install('python-psutil', fatal=True)
+    if six.PY2:
+        apt_install('python-psutil', fatal=True)
+    else:
+        apt_install('python3-psutil', fatal=True)
     import psutil
 
 CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
@@ -392,16 +395,20 @@ class AMQPContext(OSContextGenerator):
         for rid in relation_ids(self.rel_name):
             ha_vip_only = False
             self.related = True
+            transport_hosts = None
+            rabbitmq_port = '5672'
             for unit in related_units(rid):
                 if relation_get('clustered', rid=rid, unit=unit):
                     ctxt['clustered'] = True
                     vip = relation_get('vip', rid=rid, unit=unit)
                     vip = format_ipv6_addr(vip) or vip
                     ctxt['rabbitmq_host'] = vip
+                    transport_hosts = [vip]
                 else:
                     host = relation_get('private-address', rid=rid, unit=unit)
                     host = format_ipv6_addr(host) or host
                     ctxt['rabbitmq_host'] = host
+                    transport_hosts = [host]
 
                 ctxt.update({
                     'rabbitmq_user': username,
@@ -413,6 +420,7 @@ class AMQPContext(OSContextGenerator):
                 ssl_port = relation_get('ssl_port', rid=rid, unit=unit)
                 if ssl_port:
                     ctxt['rabbit_ssl_port'] = ssl_port
+                    rabbitmq_port = ssl_port
 
                 ssl_ca = relation_get('ssl_ca', rid=rid, unit=unit)
                 if ssl_ca:
@@ -450,6 +458,20 @@ class AMQPContext(OSContextGenerator):
                     rabbitmq_hosts.append(host)
 
                 ctxt['rabbitmq_hosts'] = ','.join(sorted(rabbitmq_hosts))
+                transport_hosts = rabbitmq_hosts
+
+            if transport_hosts:
+                transport_url_hosts = ''
+                for host in transport_hosts:
+                    if transport_url_hosts:
+                        format_string = ",{}:{}@{}:{}"
+                    else:
+                        format_string = "{}:{}@{}:{}"
+                    transport_url_hosts += format_string.format(
+                        ctxt['rabbitmq_user'], ctxt['rabbitmq_password'],
+                        host, rabbitmq_port)
+                ctxt['transport_url'] = "rabbit://{}/{}".format(
+                    transport_url_hosts, vhost)
 
         oslo_messaging_flags = conf.get('oslo-messaging-flags', None)
         if oslo_messaging_flags:
@@ -481,13 +503,16 @@ class CephContext(OSContextGenerator):
                     ctxt['auth'] = relation_get('auth', rid=rid, unit=unit)
                 if not ctxt.get('key'):
                     ctxt['key'] = relation_get('key', rid=rid, unit=unit)
-                ceph_pub_addr = relation_get('ceph-public-address', rid=rid,
+
+                ceph_addrs = relation_get('ceph-public-address', rid=rid,
+                                          unit=unit)
+                if ceph_addrs:
+                    for addr in ceph_addrs.split(' '):
+                        mon_hosts.append(format_ipv6_addr(addr) or addr)
+                else:
+                    priv_addr = relation_get('private-address', rid=rid,
                                              unit=unit)
-                unit_priv_addr = relation_get('private-address', rid=rid,
+                    mon_hosts.append(format_ipv6_addr(priv_addr) or priv_addr)
-                                              unit=unit)
-                ceph_addr = ceph_pub_addr or unit_priv_addr
-                ceph_addr = format_ipv6_addr(ceph_addr) or ceph_addr
-                mon_hosts.append(ceph_addr)
 
         ctxt['mon_hosts'] = ' '.join(sorted(mon_hosts))
 

charmhelpers/contrib/openstack/keystone.py

@@ -0,0 +1,178 @@
+#!/usr/bin/python
+#
+# Copyright 2017 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import six
+from charmhelpers.fetch import apt_install
+from charmhelpers.contrib.openstack.context import IdentityServiceContext
+from charmhelpers.core.hookenv import (
+    log,
+    ERROR,
+)
+
+
+def get_api_suffix(api_version):
+    """Return the formatted api suffix for the given version
+    @param api_version: version of the keystone endpoint
+    @returns the api suffix formatted according to the given api
+    version
+    """
+    return 'v2.0' if api_version in (2, "2.0") else 'v3'
+
+
+def format_endpoint(schema, addr, port, api_version):
+    """Return a formatted keystone endpoint
+    @param schema: http or https
+    @param addr: ipv4/ipv6 host of the keystone service
+    @param port: port of the keystone service
+    @param api_version: 2 or 3
+    @returns a fully formatted keystone endpoint
+    """
+    return '{}://{}:{}/{}/'.format(schema, addr, port,
+                                   get_api_suffix(api_version))
+
+
+def get_keystone_manager(endpoint, api_version, **kwargs):
+    """Return a keystonemanager for the correct API version
+
+    @param endpoint: the keystone endpoint to point client at
+    @param api_version: version of the keystone api the client should use
+    @param kwargs: token or username/tenant/password information
+    @returns keystonemanager class used for interrogating keystone
+    """
+    if api_version == 2:
+        return KeystoneManager2(endpoint, **kwargs)
+    if api_version == 3:
+        return KeystoneManager3(endpoint, **kwargs)
+    raise ValueError('No manager found for api version {}'.format(api_version))
+
+
+def get_keystone_manager_from_identity_service_context():
+    """Return a keystonmanager generated from a
+    instance of charmhelpers.contrib.openstack.context.IdentityServiceContext
+    @returns keystonamenager instance
+   """
+    context = IdentityServiceContext()()
+    if not context:
+        msg = "Identity service context cannot be generated"
+        log(msg, level=ERROR)
+        raise ValueError(msg)
+
+    endpoint = format_endpoint(context['service_protocol'],
+                               context['service_host'],
+                               context['service_port'],
+                               context['api_version'])
+
+    if context['api_version'] in (2, "2.0"):
+        api_version = 2
+    else:
+        api_version = 3
+
+    return get_keystone_manager(endpoint, api_version,
+                                username=context['admin_user'],
+                                password=context['admin_password'],
+                                tenant_name=context['admin_tenant_name'])
+
+
+class KeystoneManager(object):
+
+    def resolve_service_id(self, service_name=None, service_type=None):
+        """Find the service_id of a given service"""
+        services = [s._info for s in self.api.services.list()]
+
+        service_name = service_name.lower()
+        for s in services:
+            name = s['name'].lower()
+            if service_type and service_name:
+                if (service_name == name and service_type == s['type']):
+                    return s['id']
+            elif service_name and service_name == name:
+                return s['id']
+            elif service_type and service_type == s['type']:
+                return s['id']
+        return None
+
+    def service_exists(self, service_name=None, service_type=None):
+        """Determine if the given service exists on the service list"""
+        return self.resolve_service_id(service_name, service_type) is not None
+
+
+class KeystoneManager2(KeystoneManager):
+
+    def __init__(self, endpoint, **kwargs):
+        try:
+            from keystoneclient.v2_0 import client
+            from keystoneclient.auth.identity import v2
+            from keystoneclient import session
+        except ImportError:
+            if six.PY2:
+                apt_install(["python-keystoneclient"], fatal=True)
+            else:
+                apt_install(["python3-keystoneclient"], fatal=True)
+
+            from keystoneclient.v2_0 import client
+            from keystoneclient.auth.identity import v2
+            from keystoneclient import session
+
+        self.api_version = 2
+
+        token = kwargs.get("token", None)
+        if token:
+            api = client.Client(endpoint=endpoint, token=token)
+        else:
+            auth = v2.Password(username=kwargs.get("username"),
+                               password=kwargs.get("password"),
+                               tenant_name=kwargs.get("tenant_name"),
+                               auth_url=endpoint)
+            sess = session.Session(auth=auth)
+            api = client.Client(session=sess)
+
+        self.api = api
+
+
+class KeystoneManager3(KeystoneManager):
+
+    def __init__(self, endpoint, **kwargs):
+        try:
+            from keystoneclient.v3 import client
+            from keystoneclient.auth import token_endpoint
+            from keystoneclient import session
+            from keystoneclient.auth.identity import v3
+        except ImportError:
+            if six.PY2:
+                apt_install(["python-keystoneclient"], fatal=True)
+            else:
+                apt_install(["python3-keystoneclient"], fatal=True)
+
+            from keystoneclient.v3 import client
+            from keystoneclient.auth import token_endpoint
+            from keystoneclient import session
+            from keystoneclient.auth.identity import v3
+
+        self.api_version = 3
+
+        token = kwargs.get("token", None)
+        if token:
+            auth = token_endpoint.Token(endpoint=endpoint,
+                                        token=token)
+            sess = session.Session(auth=auth)
+        else:
+            auth = v3.Password(auth_url=endpoint,
+                               user_id=kwargs.get("username"),
+                               password=kwargs.get("password"),
+                               project_id=kwargs.get("tenant_name"))
+            sess = session.Session(auth=auth)
+
+        self.api = client.Client(session=sess)

charmhelpers/contrib/openstack/templating.py

@@ -28,7 +28,10 @@ try:
     from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
 except ImportError:
     apt_update(fatal=True)
-    apt_install('python-jinja2', fatal=True)
+    if six.PY2:
+        apt_install('python-jinja2', fatal=True)
+    else:
+        apt_install('python3-jinja2', fatal=True)
     from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
 
 
@@ -207,7 +210,10 @@ class OSConfigRenderer(object):
             # if this code is running, the object is created pre-install hook.
             # jinja2 shouldn't get touched until the module is reloaded on next
             # hook execution, with proper jinja2 bits successfully imported.
-            apt_install('python-jinja2')
+            if six.PY2:
+                apt_install('python-jinja2')
+            else:
+                apt_install('python3-jinja2')
 
     def register(self, config_file, contexts):
         """

charmhelpers/contrib/openstack/utils.py

@@ -153,7 +153,7 @@ SWIFT_CODENAMES = OrderedDict([
     ('newton',
         ['2.8.0', '2.9.0', '2.10.0']),
     ('ocata',
-        ['2.11.0', '2.12.0']),
+        ['2.11.0', '2.12.0', '2.13.0']),
 ])
 
 # >= Liberty version->codename mapping

charmhelpers/contrib/python/packages.py

@@ -16,6 +16,7 @@
 # limitations under the License.
 
 import os
+import six
 import subprocess
 import sys
 
@@ -39,7 +40,10 @@ def pip_execute(*args, **kwargs):
             from pip import main as _pip_execute
         except ImportError:
             apt_update()
-            apt_install('python-pip')
+            if six.PY2:
+                apt_install('python-pip')
+            else:
+                apt_install('python3-pip')
             from pip import main as _pip_execute
         _pip_execute(*args, **kwargs)
     finally:
@@ -136,7 +140,10 @@ def pip_list():
 
 def pip_create_virtualenv(path=None):
     """Create an isolated Python environment."""
-    apt_install('python-virtualenv')
+    if six.PY2:
+        apt_install('python-virtualenv')
+    else:
+        apt_install('python3-virtualenv')
 
     if path:
         venv_path = path

charmhelpers/core/host.py

@@ -306,6 +306,8 @@ SYSTEMD_SYSTEM = '/run/systemd/system'
 
 def init_is_systemd():
     """Return True if the host system uses systemd, False otherwise."""
+    if lsb_release()['DISTRIB_CODENAME'] == 'trusty':
+        return False
     return os.path.isdir(SYSTEMD_SYSTEM)
 
 

charmhelpers/fetch/snap.py

@@ -0,0 +1,122 @@
+# Copyright 2014-2017 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Charm helpers snap for classic charms.
+
+If writing reactive charms, use the snap layer:
+https://lists.ubuntu.com/archives/snapcraft/2016-September/001114.html
+"""
+import subprocess
+from os import environ
+from time import sleep
+from charmhelpers.core.hookenv import log
+
+__author__ = 'Joseph Borg <joseph.borg@canonical.com>'
+
+SNAP_NO_LOCK = 1  # The return code for "couldn't acquire lock" in Snap (hopefully this will be improved).
+SNAP_NO_LOCK_RETRY_DELAY = 10  # Wait X seconds between Snap lock checks.
+SNAP_NO_LOCK_RETRY_COUNT = 30  # Retry to acquire the lock X times.
+
+
+class CouldNotAcquireLockException(Exception):
+    pass
+
+
+def _snap_exec(commands):
+    """
+    Execute snap commands.
+
+    :param commands: List commands
+    :return: Integer exit code
+    """
+    assert type(commands) == list
+
+    retry_count = 0
+    return_code = None
+
+    while return_code is None or return_code == SNAP_NO_LOCK:
+        try:
+            return_code = subprocess.check_call(['snap'] + commands, env=environ)
+        except subprocess.CalledProcessError as e:
+            retry_count += + 1
+            if retry_count > SNAP_NO_LOCK_RETRY_COUNT:
+                raise CouldNotAcquireLockException('Could not aquire lock after %s attempts' % SNAP_NO_LOCK_RETRY_COUNT)
+            return_code = e.returncode
+            log('Snap failed to acquire lock, trying again in %s seconds.' % SNAP_NO_LOCK_RETRY_DELAY, level='WARN')
+            sleep(SNAP_NO_LOCK_RETRY_DELAY)
+
+    return return_code
+
+
+def snap_install(packages, *flags):
+    """
+    Install a snap package.
+
+    :param packages: String or List String package name
+    :param flags: List String flags to pass to install command
+    :return: Integer return code from snap
+    """
+    if type(packages) is not list:
+        packages = [packages]
+
+    flags = list(flags)
+
+    message = 'Installing snap(s) "%s"' % ', '.join(packages)
+    if flags:
+        message += ' with option(s) "%s"' % ', '.join(flags)
+
+    log(message, level='INFO')
+    return _snap_exec(['install'] + flags + packages)
+
+
+def snap_remove(packages, *flags):
+    """
+    Remove a snap package.
+
+    :param packages: String or List String package name
+    :param flags: List String flags to pass to remove command
+    :return: Integer return code from snap
+    """
+    if type(packages) is not list:
+        packages = [packages]
+
+    flags = list(flags)
+
+    message = 'Removing snap(s) "%s"' % ', '.join(packages)
+    if flags:
+        message += ' with options "%s"' % ', '.join(flags)
+
+    log(message, level='INFO')
+    return _snap_exec(['remove'] + flags + packages)
+
+
+def snap_refresh(packages, *flags):
+    """
+    Refresh / Update snap package.
+
+    :param packages: String or List String package name
+    :param flags: List String flags to pass to refresh command
+    :return: Integer return code from snap
+    """
+    if type(packages) is not list:
+        packages = [packages]
+
+    flags = list(flags)
+
+    message = 'Refreshing snap(s) "%s"' % ', '.join(packages)
+    if flags:
+        message += ' with options "%s"' % ', '.join(flags)
+
+    log(message, level='INFO')
+    return _snap_exec(['refresh'] + flags + packages)

charmhelpers/fetch/ubuntu.py

@@ -116,8 +116,8 @@ CLOUD_ARCHIVE_POCKETS = {
 }
 
 APT_NO_LOCK = 100  # The return code for "couldn't acquire lock" in APT.
-APT_NO_LOCK_RETRY_DELAY = 10  # Wait 10 seconds between apt lock checks.
+CMD_RETRY_DELAY = 10  # Wait 10 seconds between command retries.
-APT_NO_LOCK_RETRY_COUNT = 30  # Retry to acquire the lock X times.
+CMD_RETRY_COUNT = 30  # Retry a failing fatal command X times.
 
 
 def filter_installed_packages(packages):
@@ -249,7 +249,8 @@ def add_source(source, key=None):
         source.startswith('http') or
         source.startswith('deb ') or
             source.startswith('cloud-archive:')):
-        subprocess.check_call(['add-apt-repository', '--yes', source])
+        cmd = ['add-apt-repository', '--yes', source]
+        _run_with_retries(cmd)
     elif source.startswith('cloud:'):
         install(filter_installed_packages(['ubuntu-cloud-keyring']),
                 fatal=True)
@@ -286,41 +287,60 @@ def add_source(source, key=None):
                                    key])
 
 
-def _run_apt_command(cmd, fatal=False):
+def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,),
-    """Run an APT command.
+                      retry_message="", cmd_env=None):
-
+    """Run a command and retry until success or max_retries is reached.
-    Checks the output and retries if the fatal flag is set
-    to True.
 
     :param: cmd: str: The apt command to run.
+    :param: max_retries: int: The number of retries to attempt on a fatal
+        command. Defaults to CMD_RETRY_COUNT.
+    :param: retry_exitcodes: tuple: Optional additional exit codes to retry.
+        Defaults to retry on exit code 1.
+    :param: retry_message: str: Optional log prefix emitted during retries.
+    :param: cmd_env: dict: Environment variables to add to the command run.
+    """
+
+    env = os.environ.copy()
+    if cmd_env:
+        env.update(cmd_env)
+
+    if not retry_message:
+        retry_message = "Failed executing '{}'".format(" ".join(cmd))
+    retry_message += ". Will retry in {} seconds".format(CMD_RETRY_DELAY)
+
+    retry_count = 0
+    result = None
+
+    retry_results = (None,) + retry_exitcodes
+    while result in retry_results:
+        try:
+            result = subprocess.check_call(cmd, env=env)
+        except subprocess.CalledProcessError as e:
+            retry_count = retry_count + 1
+            if retry_count > max_retries:
+                raise
+            result = e.returncode
+            log(retry_message)
+            time.sleep(CMD_RETRY_DELAY)
+
+
+def _run_apt_command(cmd, fatal=False):
+    """Run an apt command with optional retries.
+
     :param: fatal: bool: Whether the command's output should be checked and
         retried.
     """
-    env = os.environ.copy()
+    # Provide DEBIAN_FRONTEND=noninteractive if not present in the environment.
-
+    cmd_env = {
-    if 'DEBIAN_FRONTEND' not in env:
+        'DEBIAN_FRONTEND': os.environ.get('DEBIAN_FRONTEND', 'noninteractive')}
-        env['DEBIAN_FRONTEND'] = 'noninteractive'
 
     if fatal:
-        retry_count = 0
+        _run_with_retries(
-        result = None
+            cmd, cmd_env=cmd_env, retry_exitcodes=(1, APT_NO_LOCK,),
-
+            retry_message="Couldn't acquire DPKG lock")
-        # If the command is considered "fatal", we need to retry if the apt
-        # lock was not acquired.
-
-        while result is None or result == APT_NO_LOCK:
-            try:
-                result = subprocess.check_call(cmd, env=env)
-            except subprocess.CalledProcessError as e:
-                retry_count = retry_count + 1
-                if retry_count > APT_NO_LOCK_RETRY_COUNT:
-                    raise
-                result = e.returncode
-                log("Couldn't acquire DPKG lock. Will retry in {} seconds."
-                    "".format(APT_NO_LOCK_RETRY_DELAY))
-                time.sleep(APT_NO_LOCK_RETRY_DELAY)
-
     else:
+        env = os.environ.copy()
+        env.update(cmd_env)
         subprocess.call(cmd, env=env)
 
 

metadata.yaml

@@ -11,6 +11,7 @@ tags:
   - misc
 series:
   - xenial
+  - zesty
   - trusty
   - yakkety
 extra-bindings:

tests/charmhelpers/contrib/amulet/utils.py

@@ -785,37 +785,30 @@ class AmuletUtils(object):
         generating test messages which need to be unique-ish."""
         return '[{}-{}]'.format(uuid.uuid4(), time.time())
 
-# amulet juju action helpers:
+    # amulet juju action helpers:
     def run_action(self, unit_sentry, action,
                    _check_output=subprocess.check_output,
                    params=None):
-        """Run the named action on a given unit sentry.
+        """Translate to amulet's built in run_action(). Deprecated.
+
+        Run the named action on a given unit sentry.
 
         params a dict of parameters to use
-        _check_output parameter is used for dependency injection.
+        _check_output parameter is no longer used
 
         @return action_id.
         """
-        unit_id = unit_sentry.info["unit_name"]
+        self.log.warn('charmhelpers.contrib.amulet.utils.run_action has been '
-        command = ["juju", "action", "do", "--format=json", unit_id, action]
+                      'deprecated for amulet.run_action')
-        if params is not None:
+        return unit_sentry.run_action(action, action_args=params)
-            for key, value in params.iteritems():
-                command.append("{}={}".format(key, value))
-        self.log.info("Running command: %s\n" % " ".join(command))
-        output = _check_output(command, universal_newlines=True)
-        data = json.loads(output)
-        action_id = data[u'Action queued with id']
-        return action_id
 
     def wait_on_action(self, action_id, _check_output=subprocess.check_output):
         """Wait for a given action, returning if it completed or not.
 
-        _check_output parameter is used for dependency injection.
+        action_id a string action uuid
+        _check_output parameter is no longer used
         """
-        command = ["juju", "action", "fetch", "--format=json", "--wait=0",
+        data = amulet.actions.get_action_output(action_id, full_output=True)
-                   action_id]
-        output = _check_output(command, universal_newlines=True)
-        data = json.loads(output)
         return data.get(u"status") == "completed"
 
     def status_get(self, unit):

tests/charmhelpers/contrib/openstack/amulet/utils.py

@@ -32,6 +32,7 @@ from keystoneclient.v3 import client as keystone_client_v3
 from novaclient import exceptions
 
 import novaclient.client as nova_client
+import novaclient
 import pika
 import swiftclient
 
@@ -434,9 +435,14 @@ class OpenStackAmuletUtils(AmuletUtils):
         self.log.debug('Authenticating nova user ({})...'.format(user))
         ep = keystone.service_catalog.url_for(service_type='identity',
                                               endpoint_type='publicURL')
-        return nova_client.Client(NOVA_CLIENT_VERSION,
+        if novaclient.__version__[0] >= "7":
-                                  username=user, api_key=password,
+            return nova_client.Client(NOVA_CLIENT_VERSION,
-                                  project_id=tenant, auth_url=ep)
+                                      username=user, password=password,
+                                      project_name=tenant, auth_url=ep)
+        else:
+            return nova_client.Client(NOVA_CLIENT_VERSION,
+                                      username=user, api_key=password,
+                                      project_id=tenant, auth_url=ep)
 
     def authenticate_swift_user(self, keystone, user, password, tenant):
         """Authenticates a regular user with swift api."""