Resync charm-helpers

Also fix some mocking issues which cause test failures when not
executing on ubuntu trusty.

Change-Id: Ia72c4fc57a68b3f2c1bcec4d00d57da3b8912b96

.gitignore

@@ -4,3 +4,4 @@ bin
 .tox
 tags
 *.sw[nop]
+*.pyc

hooks/charmhelpers/contrib/network/ip.py

@@ -456,3 +456,18 @@ def get_hostname(address, fqdn=True):
             return result
     else:
         return result.split('.')[0]
+
+
+def port_has_listener(address, port):
+    """
+    Returns True if the address:port is open and being listened to,
+    else False.
+
+    @param address: an IP address or hostname
+    @param port: integer port
+
+    Note calls 'zc' via a subprocess shell
+    """
+    cmd = ['nc', '-z', address, str(port)]
+    result = subprocess.call(cmd)
+    return not(bool(result))

hooks/charmhelpers/contrib/openstack/context.py

@@ -410,6 +410,7 @@ class IdentityServiceContext(OSContextGenerator):
                 auth_host = format_ipv6_addr(auth_host) or auth_host
                 svc_protocol = rdata.get('service_protocol') or 'http'
                 auth_protocol = rdata.get('auth_protocol') or 'http'
+                api_version = rdata.get('api_version') or '2.0'
                 ctxt.update({'service_port': rdata.get('service_port'),
                              'service_host': serv_host,
                              'auth_host': auth_host,
@@ -418,7 +419,8 @@ class IdentityServiceContext(OSContextGenerator):
                              'admin_user': rdata.get('service_username'),
                              'admin_password': rdata.get('service_password'),
                              'service_protocol': svc_protocol,
-                             'auth_protocol': auth_protocol})
+                             'auth_protocol': auth_protocol,
+                             'api_version': api_version})
 
                 if self.context_complete(ctxt):
                     # NOTE(jamespage) this is required for >= icehouse
@@ -1471,6 +1473,8 @@ class NetworkServiceContext(OSContextGenerator):
                     rdata.get('service_protocol') or 'http',
                     'auth_protocol':
                     rdata.get('auth_protocol') or 'http',
+                    'api_version':
+                    rdata.get('api_version') or '2.0',
                 }
                 if self.context_complete(ctxt):
                     return ctxt

hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend

@@ -6,6 +6,8 @@ Listen {{ ext_port }}
 <VirtualHost {{ address }}:{{ ext }}>
     ServerName {{ endpoint }}
     SSLEngine on
+    SSLProtocol +TLSv1 +TLSv1.1 +TLSv1.2
+    SSLCipherSuite HIGH:!RC4:!MD5:!aNULL:!eNULL:!EXP:!LOW:!MEDIUM
     SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }}
     SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }}
     ProxyPass / http://localhost:{{ int }}/

hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf

@@ -6,6 +6,8 @@ Listen {{ ext_port }}
 <VirtualHost {{ address }}:{{ ext }}>
     ServerName {{ endpoint }}
     SSLEngine on
+    SSLProtocol +TLSv1 +TLSv1.1 +TLSv1.2
+    SSLCipherSuite HIGH:!RC4:!MD5:!aNULL:!eNULL:!EXP:!LOW:!MEDIUM
     SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }}
     SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }}
     ProxyPass / http://localhost:{{ int }}/

hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken

@@ -1,4 +1,14 @@
 {% if auth_host -%}
+{% if api_version == '3' -%}
+[keystone_authtoken]
+auth_url = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
+project_name = {{ admin_tenant_name }}
+username = {{ admin_user }}
+password = {{ admin_password }}
+project_domain_name = default
+user_domain_name = default
+auth_plugin = password
+{% else -%}
 [keystone_authtoken]
 identity_uri = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/{{ auth_admin_prefix }}
 auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/{{ service_admin_prefix }}
@@ -7,3 +17,4 @@ admin_user = {{ admin_user }}
 admin_password = {{ admin_password }}
 signing_dir = {{ signing_dir }}
 {% endif -%}
+{% endif -%}

hooks/charmhelpers/contrib/openstack/utils.py

@@ -23,6 +23,7 @@ import json
 import os
 import sys
 import re
+import itertools
 
 import six
 import tempfile
@@ -60,6 +61,7 @@ from charmhelpers.contrib.storage.linux.lvm import (
 from charmhelpers.contrib.network.ip import (
     get_ipv6_addr,
     is_ipv6,
+    port_has_listener,
 )
 
 from charmhelpers.contrib.python.packages import (
@@ -67,7 +69,7 @@ from charmhelpers.contrib.python.packages import (
     pip_install,
 )
 
-from charmhelpers.core.host import lsb_release, mounts, umount
+from charmhelpers.core.host import lsb_release, mounts, umount, service_running
 from charmhelpers.fetch import apt_install, apt_cache, install_remote
 from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
 from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
@@ -860,13 +862,23 @@ def os_workload_status(configs, required_interfaces, charm_func=None):
     return wrap
 
 
-def set_os_workload_status(configs, required_interfaces, charm_func=None):
+def set_os_workload_status(configs, required_interfaces, charm_func=None, services=None, ports=None):
     """
     Set workload status based on complete contexts.
     status-set missing or incomplete contexts
     and juju-log details of missing required data.
     charm_func is a charm specific function to run checking
     for charm specific requirements such as a VIP setting.
+
+    This function also checks for whether the services defined are ACTUALLY
+    running and that the ports they advertise are open and being listened to.
+
+    @param services - OPTIONAL: a [{'service': <string>, 'ports': [<int>]]
+                      The ports are optional.
+                      If services is a [<string>] then ports are ignored.
+    @param ports - OPTIONAL: an [<int>] representing ports that shoudl be
+                   open.
+    @returns None
     """
     incomplete_rel_data = incomplete_relation_data(configs, required_interfaces)
     state = 'active'
@@ -945,6 +957,65 @@ def set_os_workload_status(configs, required_interfaces, charm_func=None):
             else:
                 message = charm_message
 
+    # If the charm thinks the unit is active, check that the actual services
+    # really are active.
+    if services is not None and state == 'active':
+        # if we're passed the dict() then just grab the values as a list.
+        if isinstance(services, dict):
+            services = services.values()
+        # either extract the list of services from the dictionary, or if
+        # it is a simple string, use that. i.e. works with mixed lists.
+        _s = []
+        for s in services:
+            if isinstance(s, dict) and 'service' in s:
+                _s.append(s['service'])
+            if isinstance(s, str):
+                _s.append(s)
+        services_running = [service_running(s) for s in _s]
+        if not all(services_running):
+            not_running = [s for s, running in zip(_s, services_running)
+                           if not running]
+            message = ("Services not running that should be: {}"
+                       .format(", ".join(not_running)))
+            state = 'blocked'
+        # also verify that the ports that should be open are open
+        # NB, that ServiceManager objects only OPTIONALLY have ports
+        port_map = OrderedDict([(s['service'], s['ports'])
+                                for s in services if 'ports' in s])
+        if state == 'active' and port_map:
+            all_ports = list(itertools.chain(*port_map.values()))
+            ports_open = [port_has_listener('0.0.0.0', p)
+                          for p in all_ports]
+            if not all(ports_open):
+                not_opened = [p for p, opened in zip(all_ports, ports_open)
+                              if not opened]
+                map_not_open = OrderedDict()
+                for service, ports in port_map.items():
+                    closed_ports = set(ports).intersection(not_opened)
+                    if closed_ports:
+                        map_not_open[service] = closed_ports
+                # find which service has missing ports. They are in service
+                # order which makes it a bit easier.
+                message = (
+                    "Services with ports not open that should be: {}"
+                    .format(
+                        ", ".join([
+                            "{}: [{}]".format(
+                                service,
+                                ", ".join([str(v) for v in ports]))
+                            for service, ports in map_not_open.items()])))
+                state = 'blocked'
+
+    if ports is not None and state == 'active':
+        # and we can also check ports which we don't know the service for
+        ports_open = [port_has_listener('0.0.0.0', p) for p in ports]
+        if not all(ports_open):
+            message = (
+                "Ports which should be open, but are not: {}"
+                .format(", ".join([str(p) for p, v in zip(ports, ports_open)
+                                   if not v])))
+            state = 'blocked'
+
     # Set to active if all requirements have been met
     if state == 'active':
         message = "Unit is ready"

hooks/charmhelpers/contrib/storage/linux/ceph.py

@@ -120,6 +120,7 @@ class PoolCreationError(Exception):
     """
     A custom error to inform the caller that a pool creation failed.  Provides an error message
     """
+
     def __init__(self, message):
         super(PoolCreationError, self).__init__(message)
 
@@ -129,6 +130,7 @@ class Pool(object):
     An object oriented approach to Ceph pool creation. This base class is inherited by ReplicatedPool and ErasurePool.
     Do not call create() on this base class as it will not do anything.  Instantiate a child class and call create().
     """
+
     def __init__(self, service, name):
         self.service = service
         self.name = name
@@ -180,36 +182,41 @@ class Pool(object):
         :return: int.  The number of pgs to use.
         """
         validator(value=pool_size, valid_type=int)
-        osds = get_osds(self.service)
+        osd_list = get_osds(self.service)
-        if not osds:
+        if not osd_list:
             # NOTE(james-page): Default to 200 for older ceph versions
             # which don't support OSD query from cli
             return 200
 
+        osd_list_length = len(osd_list)
         # Calculate based on Ceph best practices
-        if osds < 5:
+        if osd_list_length < 5:
             return 128
-        elif 5 < osds < 10:
+        elif 5 < osd_list_length < 10:
             return 512
-        elif 10 < osds < 50:
+        elif 10 < osd_list_length < 50:
             return 4096
         else:
-            estimate = (osds * 100) / pool_size
+            estimate = (osd_list_length * 100) / pool_size
             # Return the next nearest power of 2
             index = bisect.bisect_right(powers_of_two, estimate)
             return powers_of_two[index]
 
 
 class ReplicatedPool(Pool):
-    def __init__(self, service, name, replicas=2):
+    def __init__(self, service, name, pg_num=None, replicas=2):
         super(ReplicatedPool, self).__init__(service=service, name=name)
         self.replicas = replicas
+        if pg_num is None:
+            self.pg_num = self.get_pgs(self.replicas)
+        else:
+            self.pg_num = pg_num
 
     def create(self):
         if not pool_exists(self.service, self.name):
             # Create it
-            pgs = self.get_pgs(self.replicas)
+            cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create',
-            cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', self.name, str(pgs)]
+                   self.name, str(self.pg_num)]
             try:
                 check_call(cmd)
             except CalledProcessError:
@@ -241,7 +248,7 @@ class ErasurePool(Pool):
 
             pgs = self.get_pgs(int(erasure_profile['k']) + int(erasure_profile['m']))
             # Create it
-            cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', self.name, str(pgs),
+            cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', self.name, str(pgs), str(pgs),
                    'erasure', self.erasure_code_profile]
             try:
                 check_call(cmd)
@@ -322,7 +329,8 @@ def set_pool_quota(service, pool_name, max_bytes):
     :return: None.  Can raise CalledProcessError
     """
     # Set a byte quota on a RADOS pool in ceph.
-    cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name, 'max_bytes', max_bytes]
+    cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name,
+           'max_bytes', str(max_bytes)]
     try:
         check_call(cmd)
     except CalledProcessError:
@@ -343,7 +351,25 @@ def remove_pool_quota(service, pool_name):
         raise
 
 
-def create_erasure_profile(service, profile_name, erasure_plugin_name='jerasure', failure_domain='host',
+def remove_erasure_profile(service, profile_name):
+    """
+    Create a new erasure code profile if one does not already exist for it.  Updates
+    the profile if it exists. Please see http://docs.ceph.com/docs/master/rados/operations/erasure-code-profile/
+    for more details
+    :param service: six.string_types. The Ceph user name to run the command under
+    :param profile_name: six.string_types
+    :return: None.  Can raise CalledProcessError
+    """
+    cmd = ['ceph', '--id', service, 'osd', 'erasure-code-profile', 'rm',
+           profile_name]
+    try:
+        check_call(cmd)
+    except CalledProcessError:
+        raise
+
+
+def create_erasure_profile(service, profile_name, erasure_plugin_name='jerasure',
+                           failure_domain='host',
                            data_chunks=2, coding_chunks=1,
                            locality=None, durability_estimator=None):
     """

unit_tests/test_neutron_api_hooks.py

@@ -159,15 +159,15 @@ class NeutronAPIHooksTests(CharmTestCase):
         self.open_port.assert_has_calls(_port_calls)
         self.assertTrue(self.execd_preinstall.called)
 
+    @patch.object(utils, 'get_os_codename_install_source')
     @patch.object(utils, 'git_install_requested')
-    def test_install_hook_git(self, git_requested):
+    def test_install_hook_git(self, git_requested, codename):
         git_requested.return_value = True
         _pkgs = ['foo', 'bar']
         _ports = [80, 81, 82]
         _port_calls = [call(port) for port in _ports]
         self.determine_packages.return_value = _pkgs
         self.determine_ports.return_value = _ports
-        repo = 'cloud:trusty-juno'
         openstack_origin_git = {
             'repositories': [
                 {'name': 'requirements',
@@ -179,9 +179,11 @@ class NeutronAPIHooksTests(CharmTestCase):
             ],
             'directory': '/mnt/openstack-git',
         }
+        repo = "cloud:trusty-juno"
         projects_yaml = yaml.dump(openstack_origin_git)
         self.test_config.set('openstack-origin', repo)
         self.test_config.set('openstack-origin-git', projects_yaml)
+        codename.return_value = 'juno'
         self._call_hook('install')
         self.assertTrue(self.execd_preinstall.called)
         self.configure_installation_source.assert_called_with(repo)
@@ -240,11 +242,12 @@ class NeutronAPIHooksTests(CharmTestCase):
                          'Cannot disable Router HA while ha enabled routers'
                          ' exist. Please remove any ha routers')
 
+    @patch.object(utils, 'get_os_codename_install_source')
     @patch.object(hooks, 'configure_https')
     @patch.object(hooks, 'git_install_requested')
     @patch.object(hooks, 'config_value_changed')
     def test_config_changed_git(self, config_val_changed, git_requested,
-                                configure_https):
+                                configure_https, codename):
         git_requested.return_value = True
         self.neutron_ready.return_value = True
         self.dvr_router_present.return_value = False
@@ -273,6 +276,7 @@ class NeutronAPIHooksTests(CharmTestCase):
         projects_yaml = yaml.dump(openstack_origin_git)
         self.test_config.set('openstack-origin', repo)
         self.test_config.set('openstack-origin-git', projects_yaml)
+        codename.return_value = 'juno'
         self._call_hook('config-changed')
         self.git_install.assert_called_with(projects_yaml)
         self.assertFalse(self.do_openstack_upgrade.called)