[1chb1n,r=corey.bryant] Amulet test updates

hooks/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -15,6 +15,7 @@
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import six
+from collections import OrderedDict
 from charmhelpers.contrib.amulet.deployment import (
     AmuletDeployment
 )
@@ -100,12 +101,34 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            """
         (self.precise_essex, self.precise_folsom, self.precise_grizzly,
          self.precise_havana, self.precise_icehouse,
-         self.trusty_icehouse) = range(6)
+         self.trusty_icehouse, self.trusty_juno, self.trusty_kilo) = range(8)
         releases = {
             ('precise', None): self.precise_essex,
             ('precise', 'cloud:precise-folsom'): self.precise_folsom,
             ('precise', 'cloud:precise-grizzly'): self.precise_grizzly,
             ('precise', 'cloud:precise-havana'): self.precise_havana,
             ('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
-            ('trusty', None): self.trusty_icehouse}
+            ('trusty', None): self.trusty_icehouse,
+            ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
+            ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo}
         return releases[(self.series, self.openstack)]
+
+    def _get_openstack_release_string(self):
+        """Get openstack release string.
+
+           Return a string representing the openstack release.
+           """
+        releases = OrderedDict([
+            ('precise', 'essex'),
+            ('quantal', 'folsom'),
+            ('raring', 'grizzly'),
+            ('saucy', 'havana'),
+            ('trusty', 'icehouse'),
+            ('utopic', 'juno'),
+            ('vivid', 'kilo'),
+        ])
+        if self.openstack:
+            os_origin = self.openstack.split(':')[1]
+            return os_origin.split('%s-' % self.series)[1].split('/')[0]
+        else:
+            return releases[self.series]

hooks/charmhelpers/contrib/openstack/context.py

@@ -47,6 +47,7 @@ from charmhelpers.core.hookenv import (
 )
 
 from charmhelpers.core.sysctl import create as sysctl_create
+from charmhelpers.core.strutils import bool_from_string
 
 from charmhelpers.core.host import (
     list_nics,
@@ -67,6 +68,7 @@ from charmhelpers.contrib.hahelpers.apache import (
 )
 from charmhelpers.contrib.openstack.neutron import (
     neutron_plugin_attribute,
+    parse_data_port_mappings,
 )
 from charmhelpers.contrib.openstack.ip import (
     resolve_address,
@@ -82,7 +84,6 @@ from charmhelpers.contrib.network.ip import (
     is_bridge_member,
 )
 from charmhelpers.contrib.openstack.utils import get_host_ip
-
 CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
 ADDRESS_TYPES = ['admin', 'internal', 'public']
 
@@ -319,14 +320,15 @@ def db_ssl(rdata, ctxt, ssl_dir):
 
 
 class IdentityServiceContext(OSContextGenerator):
-    interfaces = ['identity-service']
 
-    def __init__(self, service=None, service_user=None):
+    def __init__(self, service=None, service_user=None, rel_name='identity-service'):
         self.service = service
         self.service_user = service_user
+        self.rel_name = rel_name
+        self.interfaces = [self.rel_name]
 
     def __call__(self):
-        log('Generating template context for identity-service', level=DEBUG)
+        log('Generating template context for ' + self.rel_name, level=DEBUG)
         ctxt = {}
 
         if self.service and self.service_user:
@@ -340,7 +342,7 @@ class IdentityServiceContext(OSContextGenerator):
 
             ctxt['signing_dir'] = cachedir
 
-        for rid in relation_ids('identity-service'):
+        for rid in relation_ids(self.rel_name):
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
                 serv_host = rdata.get('service_host')
@@ -1162,3 +1164,145 @@ class SysctlContext(OSContextGenerator):
             sysctl_create(sysctl_dict,
                           '/etc/sysctl.d/50-{0}.conf'.format(charm_name()))
         return {'sysctl': sysctl_dict}
+
+
+class NeutronAPIContext(OSContextGenerator):
+    '''
+    Inspects current neutron-plugin-api relation for neutron settings. Return
+    defaults if it is not present.
+    '''
+    interfaces = ['neutron-plugin-api']
+
+    def __call__(self):
+        self.neutron_defaults = {
+            'l2_population': {
+                'rel_key': 'l2-population',
+                'default': False,
+            },
+            'overlay_network_type': {
+                'rel_key': 'overlay-network-type',
+                'default': 'gre',
+            },
+            'neutron_security_groups': {
+                'rel_key': 'neutron-security-groups',
+                'default': False,
+            },
+            'network_device_mtu': {
+                'rel_key': 'network-device-mtu',
+                'default': None,
+            },
+            'enable_dvr': {
+                'rel_key': 'enable-dvr',
+                'default': False,
+            },
+            'enable_l3ha': {
+                'rel_key': 'enable-l3ha',
+                'default': False,
+            },
+        }
+        ctxt = self.get_neutron_options({})
+        for rid in relation_ids('neutron-plugin-api'):
+            for unit in related_units(rid):
+                rdata = relation_get(rid=rid, unit=unit)
+                if 'l2-population' in rdata:
+                    ctxt.update(self.get_neutron_options(rdata))
+
+        return ctxt
+
+    def get_neutron_options(self, rdata):
+        settings = {}
+        for nkey in self.neutron_defaults.keys():
+            defv = self.neutron_defaults[nkey]['default']
+            rkey = self.neutron_defaults[nkey]['rel_key']
+            if rkey in rdata.keys():
+                if type(defv) is bool:
+                    settings[nkey] = bool_from_string(rdata[rkey])
+                else:
+                    settings[nkey] = rdata[rkey]
+            else:
+                settings[nkey] = defv
+        return settings
+
+
+class ExternalPortContext(NeutronPortContext):
+
+    def __call__(self):
+        ctxt = {}
+        ports = config('ext-port')
+        if ports:
+            ports = [p.strip() for p in ports.split()]
+            ports = self.resolve_ports(ports)
+            if ports:
+                ctxt = {"ext_port": ports[0]}
+                napi_settings = NeutronAPIContext()()
+                mtu = napi_settings.get('network_device_mtu')
+                if mtu:
+                    ctxt['ext_port_mtu'] = mtu
+
+        return ctxt
+
+
+class DataPortContext(NeutronPortContext):
+
+    def __call__(self):
+        ports = config('data-port')
+        if ports:
+            portmap = parse_data_port_mappings(ports)
+            ports = portmap.values()
+            resolved = self.resolve_ports(ports)
+            normalized = {get_nic_hwaddr(port): port for port in resolved
+                          if port not in ports}
+            normalized.update({port: port for port in resolved
+                               if port in ports})
+            if resolved:
+                return {bridge: normalized[port] for bridge, port in
+                        six.iteritems(portmap) if port in normalized.keys()}
+
+        return None
+
+
+class PhyNICMTUContext(DataPortContext):
+
+    def __call__(self):
+        ctxt = {}
+        mappings = super(PhyNICMTUContext, self).__call__()
+        if mappings and mappings.values():
+            ports = mappings.values()
+            napi_settings = NeutronAPIContext()()
+            mtu = napi_settings.get('network_device_mtu')
+            if mtu:
+                ctxt["devs"] = '\\n'.join(ports)
+                ctxt['mtu'] = mtu
+
+        return ctxt
+
+
+class NetworkServiceContext(OSContextGenerator):
+
+    def __init__(self, rel_name='quantum-network-service'):
+        self.rel_name = rel_name
+        self.interfaces = [rel_name]
+
+    def __call__(self):
+        for rid in relation_ids(self.rel_name):
+            for unit in related_units(rid):
+                rdata = relation_get(rid=rid, unit=unit)
+                ctxt = {
+                    'keystone_host': rdata.get('keystone_host'),
+                    'service_port': rdata.get('service_port'),
+                    'auth_port': rdata.get('auth_port'),
+                    'service_tenant': rdata.get('service_tenant'),
+                    'service_username': rdata.get('service_username'),
+                    'service_password': rdata.get('service_password'),
+                    'quantum_host': rdata.get('quantum_host'),
+                    'quantum_port': rdata.get('quantum_port'),
+                    'quantum_url': rdata.get('quantum_url'),
+                    'region': rdata.get('region'),
+                    'service_protocol':
+                    rdata.get('service_protocol') or 'http',
+                    'auth_protocol':
+                    rdata.get('auth_protocol') or 'http',
+                }
+                if context_complete(ctxt):
+                    return ctxt
+        return {}

hooks/charmhelpers/contrib/openstack/templates/git.upstart

@@ -0,0 +1,13 @@
+description "{{ service_description }}"
+author "Juju {{ service_name }} Charm <juju@localhost>"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+
+exec start-stop-daemon --start --chuid {{ user_name }} \
+            --chdir {{ start_dir }} --name {{ process_name }} \
+            --exec {{ executable_name }} -- \
+            --config-file={{ config_file }} \
+            --log-file={{ log_file }}

hooks/charmhelpers/contrib/openstack/templates/section-zeromq

@@ -3,12 +3,12 @@
 rpc_backend = zmq
 rpc_zmq_host = {{ zmq_host }}
 {% if zmq_redis_address -%}
-rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_redis.MatchMakerRedis
+rpc_zmq_matchmaker = redis
 matchmaker_heartbeat_freq = 15
 matchmaker_heartbeat_ttl = 30
 [matchmaker_redis]
 host = {{ zmq_redis_address }}
 {% else -%}
-rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_ring.MatchMakerRing
+rpc_zmq_matchmaker = ring
 {% endif -%}
 {% endif -%}

hooks/charmhelpers/contrib/openstack/utils.py

@@ -30,6 +30,10 @@ import yaml
 
 from charmhelpers.contrib.network import ip
 
+from charmhelpers.core import (
+    unitdata,
+)
+
 from charmhelpers.core.hookenv import (
     config,
     log as juju_log,
@@ -330,6 +334,21 @@ def configure_installation_source(rel):
         error_out("Invalid openstack-release specified: %s" % rel)
 
 
+def config_value_changed(option):
+    """
+    Determine if config value changed since last call to this function.
+    """
+    hook_data = unitdata.HookData()
+    with hook_data():
+        db = unitdata.kv()
+        current = config(option)
+        saved = db.get(option)
+        db.set(option, current)
+        if saved is None:
+            return False
+        return current != saved
+
+
 def save_script_rc(script_path="scripts/scriptrc", **env_vars):
     """
     Write an rc file in the charm-delivered directory containing
@@ -469,82 +488,103 @@ def os_requires_version(ostack_release, pkg):
 
 
 def git_install_requested():
-    """Returns true if openstack-origin-git is specified."""
+    """
-    return config('openstack-origin-git') != "None"
+    Returns true if openstack-origin-git is specified.
+    """
+    return config('openstack-origin-git') is not None
 
 
 requirements_dir = None
 
 
-def git_clone_and_install(file_name, core_project):
+def git_clone_and_install(projects_yaml, core_project):
-    """Clone/install all OpenStack repos specified in yaml config file."""
+    """
-    global requirements_dir
+    Clone/install all specified OpenStack repositories.
 
-    if file_name == "None":
+    The expected format of projects_yaml is:
+        repositories:
+          - {name: keystone,
+             repository: 'git://git.openstack.org/openstack/keystone.git',
+             branch: 'stable/icehouse'}
+          - {name: requirements,
+             repository: 'git://git.openstack.org/openstack/requirements.git',
+             branch: 'stable/icehouse'}
+        directory: /mnt/openstack-git
+        http_proxy: http://squid.internal:3128
+        https_proxy: https://squid.internal:3128
+
+        The directory, http_proxy, and https_proxy keys are optional.
+    """
+    global requirements_dir
+    parent_dir = '/mnt/openstack-git'
+
+    if not projects_yaml:
         return
 
-    yaml_file = os.path.join(charm_dir(), file_name)
+    projects = yaml.load(projects_yaml)
+    _git_validate_projects_yaml(projects, core_project)
 
-    # clone/install the requirements project first
+    if 'http_proxy' in projects.keys():
-    installed = _git_clone_and_install_subset(yaml_file,
+        os.environ['http_proxy'] = projects['http_proxy']
-                                              whitelist=['requirements'])
-    if 'requirements' not in installed:
-        error_out('requirements git repository must be specified')
 
-    # clone/install all other projects except requirements and the core project
+    if 'https_proxy' in projects.keys():
-    blacklist = ['requirements', core_project]
+        os.environ['https_proxy'] = projects['https_proxy']
-    _git_clone_and_install_subset(yaml_file, blacklist=blacklist,
-                                  update_requirements=True)
 
-    # clone/install the core project
+    if 'directory' in projects.keys():
-    whitelist = [core_project]
+        parent_dir = projects['directory']
-    installed = _git_clone_and_install_subset(yaml_file, whitelist=whitelist,
+
-                                              update_requirements=True)
+    for p in projects['repositories']:
-    if core_project not in installed:
+        repo = p['repository']
-        error_out('{} git repository must be specified'.format(core_project))
+        branch = p['branch']
+        if p['name'] == 'requirements':
+            repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
+                                                     update_requirements=False)
+            requirements_dir = repo_dir
+        else:
+            repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
+                                                     update_requirements=True)
 
 
-def _git_clone_and_install_subset(yaml_file, whitelist=[], blacklist=[],
+def _git_validate_projects_yaml(projects, core_project):
-                                  update_requirements=False):
+    """
-    """Clone/install subset of OpenStack repos specified in yaml config file."""
+    Validate the projects yaml.
-    global requirements_dir
+    """
-    installed = []
+    _git_ensure_key_exists('repositories', projects)
 
-    with open(yaml_file, 'r') as fd:
+    for project in projects['repositories']:
-        projects = yaml.load(fd)
+        _git_ensure_key_exists('name', project.keys())
-        for proj, val in projects.items():
+        _git_ensure_key_exists('repository', project.keys())
-            # The project subset is chosen based on the following 3 rules:
+        _git_ensure_key_exists('branch', project.keys())
-            # 1) If project is in blacklist, we don't clone/install it, period.
+
-            # 2) If whitelist is empty, we clone/install everything else.
+    if projects['repositories'][0]['name'] != 'requirements':
-            # 3) If whitelist is not empty, we clone/install everything in the
+        error_out('{} git repo must be specified first'.format('requirements'))
-            #    whitelist.
+
-            if proj in blacklist:
+    if projects['repositories'][-1]['name'] != core_project:
-                continue
+        error_out('{} git repo must be specified last'.format(core_project))
-            if whitelist and proj not in whitelist:
-                continue
-            repo = val['repository']
-            branch = val['branch']
-            repo_dir = _git_clone_and_install_single(repo, branch,
-                                                     update_requirements)
-            if proj == 'requirements':
-                requirements_dir = repo_dir
-            installed.append(proj)
-    return installed
 
 
-def _git_clone_and_install_single(repo, branch, update_requirements=False):
+def _git_ensure_key_exists(key, keys):
-    """Clone and install a single git repository."""
+    """
-    dest_parent_dir = "/mnt/openstack-git/"
+    Ensure that key exists in keys.
-    dest_dir = os.path.join(dest_parent_dir, os.path.basename(repo))
+    """
+    if key not in keys:
+        error_out('openstack-origin-git key \'{}\' is missing'.format(key))
 
-    if not os.path.exists(dest_parent_dir):
+
-        juju_log('Host dir not mounted at {}. '
+def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements):
-                 'Creating directory there instead.'.format(dest_parent_dir))
+    """
-        os.mkdir(dest_parent_dir)
+    Clone and install a single git repository.
+    """
+    dest_dir = os.path.join(parent_dir, os.path.basename(repo))
+
+    if not os.path.exists(parent_dir):
+        juju_log('Directory already exists at {}. '
+                 'No need to create directory.'.format(parent_dir))
+        os.mkdir(parent_dir)
 
     if not os.path.exists(dest_dir):
         juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
-        repo_dir = install_remote(repo, dest=dest_parent_dir, branch=branch)
+        repo_dir = install_remote(repo, dest=parent_dir, branch=branch)
     else:
         repo_dir = dest_dir
 
@@ -561,16 +601,39 @@ def _git_clone_and_install_single(repo, branch, update_requirements=False):
 
 
 def _git_update_requirements(package_dir, reqs_dir):
-    """Update from global requirements.
+    """
+    Update from global requirements.
 
-       Update an OpenStack git directory's requirements.txt and
+    Update an OpenStack git directory's requirements.txt and
-       test-requirements.txt from global-requirements.txt."""
+    test-requirements.txt from global-requirements.txt.
+    """
     orig_dir = os.getcwd()
     os.chdir(reqs_dir)
-    cmd = "python update.py {}".format(package_dir)
+    cmd = ['python', 'update.py', package_dir]
     try:
-        subprocess.check_call(cmd.split(' '))
+        subprocess.check_call(cmd)
     except subprocess.CalledProcessError:
         package = os.path.basename(package_dir)
         error_out("Error updating {} from global-requirements.txt".format(package))
     os.chdir(orig_dir)
+
+
+def git_src_dir(projects_yaml, project):
+    """
+    Return the directory where the specified project's source is located.
+    """
+    parent_dir = '/mnt/openstack-git'
+
+    if not projects_yaml:
+        return
+
+    projects = yaml.load(projects_yaml)
+
+    if 'directory' in projects.keys():
+        parent_dir = projects['directory']
+
+    for p in projects['repositories']:
+        if p['name'] == project:
+            return os.path.join(parent_dir, os.path.basename(p['repository']))
+
+    return None

hooks/charmhelpers/core/hookenv.py

@@ -20,11 +20,13 @@
 # Authors:
 #  Charm Helpers Developers <juju@lists.ubuntu.com>
 
+from __future__ import print_function
 import os
 import json
 import yaml
 import subprocess
 import sys
+import errno
 from subprocess import CalledProcessError
 
 import six
@@ -87,7 +89,18 @@ def log(message, level=None):
     if not isinstance(message, six.string_types):
         message = repr(message)
     command += [message]
-    subprocess.call(command)
+    # Missing juju-log should not cause failures in unit tests
+    # Send log output to stderr
+    try:
+        subprocess.call(command)
+    except OSError as e:
+        if e.errno == errno.ENOENT:
+            if level:
+                message = "{}: {}".format(level, message)
+            message = "juju-log: {}".format(message)
+            print(message, file=sys.stderr)
+        else:
+            raise
 
 
 class Serializable(UserDict):

hooks/charmhelpers/core/unitdata.py

@@ -443,7 +443,7 @@ class HookData(object):
         data = hookenv.execution_environment()
         self.conf = conf_delta = self.kv.delta(data['conf'], 'config')
         self.rels = rels_delta = self.kv.delta(data['rels'], 'rels')
-        self.kv.set('env', data['env'])
+        self.kv.set('env', dict(data['env']))
         self.kv.set('unit', data['unit'])
         self.kv.set('relid', data.get('relid'))
         return conf_delta, rels_delta

tests/basic_deployment.py

@@ -146,14 +146,13 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment):
                  'tenantId': u.not_null,
                  'id': u.not_null,
                  'email': 'juju@localhost'}
-        user3 = {'name': 'cinder',
+        user3 = {'name': 'cinder_cinderv2',
                  'enabled': True,
                  'tenantId': u.not_null,
                  'id': u.not_null,
                  'email': u'juju@localhost'}
         expected = [user1, user2, user3]
         actual = self.keystone.users.list()
-
         ret = u.validate_user_data(expected, actual)
         if ret:
             amulet.raise_status(amulet.FAIL, msg=ret)
@@ -254,13 +253,13 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment):
             'auth_protocol': 'http',
             'private-address': u.valid_ip,
             'auth_host': u.valid_ip,
-            'service_username': 'cinder',
+            'service_username': 'cinder_cinderv2',
             'service_tenant_id': u.not_null,
             'service_host': u.valid_ip
         }
         ret = u.validate_relation_data(unit, relation, expected)
         if ret:
-            message = u.relation_error('cinder identity-service', ret)
+            message = u.relation_error('keystone identity-service', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
     def test_cinder_identity_service_relation(self):
@@ -268,12 +267,17 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment):
         unit = self.cinder_sentry
         relation = ['identity-service', 'keystone:identity-service']
         expected = {
-            'service': 'cinder',
+            'cinder_service': 'cinder',
-            'region': 'RegionOne',
+            'cinder_region': 'RegionOne',
-            'public_url': u.valid_url,
+            'cinder_public_url': u.valid_url,
-            'internal_url': u.valid_url,
+            'cinder_internal_url': u.valid_url,
+            'cinder_admin_url': u.valid_url,
+            'cinderv2_service': 'cinderv2',
+            'cinderv2_region': 'RegionOne',
+            'cinderv2_public_url': u.valid_url,
+            'cinderv2_internal_url': u.valid_url,
+            'cinderv2_admin_url': u.valid_url,
             'private-address': u.valid_ip,
-            'admin_url': u.valid_url
         }
         ret = u.validate_relation_data(unit, relation, expected)
         if ret:

tests/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -15,6 +15,7 @@
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import six
+from collections import OrderedDict
 from charmhelpers.contrib.amulet.deployment import (
     AmuletDeployment
 )
@@ -100,12 +101,34 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            """
         (self.precise_essex, self.precise_folsom, self.precise_grizzly,
          self.precise_havana, self.precise_icehouse,
-         self.trusty_icehouse) = range(6)
+         self.trusty_icehouse, self.trusty_juno, self.trusty_kilo) = range(8)
         releases = {
             ('precise', None): self.precise_essex,
             ('precise', 'cloud:precise-folsom'): self.precise_folsom,
             ('precise', 'cloud:precise-grizzly'): self.precise_grizzly,
             ('precise', 'cloud:precise-havana'): self.precise_havana,
             ('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
-            ('trusty', None): self.trusty_icehouse}
+            ('trusty', None): self.trusty_icehouse,
+            ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
+            ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo}
         return releases[(self.series, self.openstack)]
+
+    def _get_openstack_release_string(self):
+        """Get openstack release string.
+
+           Return a string representing the openstack release.
+           """
+        releases = OrderedDict([
+            ('precise', 'essex'),
+            ('quantal', 'folsom'),
+            ('raring', 'grizzly'),
+            ('saucy', 'havana'),
+            ('trusty', 'icehouse'),
+            ('utopic', 'juno'),
+            ('vivid', 'kilo'),
+        ])
+        if self.openstack:
+            os_origin = self.openstack.split(':')[1]
+            return os_origin.split('%s-' % self.series)[1].split('/')[0]
+        else:
+            return releases[self.series]