Fix alphanumeric comparisons for openstack and ubuntu releases

- sync charmhelpers with fix-alpha helpers
- fix up code where the alpha comparisons are done

Change-Id: If253c8a18886630fcc7de4cfc2a91a15c2563eb5
Related-Bug: #1659575

actions/actions.py

@@ -25,6 +25,7 @@ from charmhelpers.core.unitdata import HookData, kv
 from charmhelpers.contrib.openstack.utils import (
     get_os_codename_package,
     set_os_workload_status,
+    CompareOpenStackReleases,
 )
 from lib.swift_storage_utils import (
     assess_status,
@@ -40,7 +41,8 @@ def _get_services():
     """Return a list of services that need to be (un)paused."""
     services = SWIFT_SVCS[:]
     # Before Icehouse there was no swift-container-sync
-    if get_os_codename_package("swift-container") < "icehouse":
+    _os_release = get_os_codename_package("swift-container")
+    if CompareOpenStackReleases(_os_release) < "icehouse":
         services.remove("swift-container-sync")
     return services
 

charm-helpers-tests.yaml

@@ -4,3 +4,4 @@ include:
     - contrib.amulet
     - contrib.openstack.amulet
     - core
+    - osplatform

charmhelpers/contrib/charmsupport/nrpe.py

@@ -373,7 +373,7 @@ def add_init_service_checks(nrpe, services, unit_name, immediate_check=True):
             checkpath = '%s/service-check-%s.txt' % (nrpe.homedir, svc)
             croncmd = (
                 '/usr/local/lib/nagios/plugins/check_exit_status.pl '
-                '-s /etc/init.d/%s status' % svc
+                '-e -s /etc/init.d/%s status' % svc
             )
             cron_file = '*/5 * * * * root %s > %s\n' % (croncmd, checkpath)
             f = open(cronpath, 'w')

charmhelpers/contrib/hardening/apache/checks/config.py

@@ -26,6 +26,7 @@ from charmhelpers.contrib.hardening.audits.file import (
     DirectoryPermissionAudit,
     NoReadWriteForOther,
     TemplatedFile,
+    DeletedFile
 )
 from charmhelpers.contrib.hardening.audits.apache import DisabledModuleAudit
 from charmhelpers.contrib.hardening.apache import TEMPLATES_DIR
@@ -52,13 +53,13 @@ def get_audits():
                                    'mods-available/alias.conf'),
                       context,
                       TEMPLATES_DIR,
-                      mode=0o0755,
+                      mode=0o0640,
                       user='root',
                       service_actions=[{'service': 'apache2',
                                         'actions': ['restart']}]),
 
         TemplatedFile(os.path.join(settings['common']['apache_dir'],
-                                   'conf-enabled/hardening.conf'),
+                                   'conf-enabled/99-hardening.conf'),
                       context,
                       TEMPLATES_DIR,
                       mode=0o0640,
@@ -69,11 +70,13 @@ def get_audits():
         DirectoryPermissionAudit(settings['common']['apache_dir'],
                                  user='root',
                                  group='root',
-                                 mode=0o640),
+                                 mode=0o0750),
 
         DisabledModuleAudit(settings['hardening']['modules_to_disable']),
 
         NoReadWriteForOther(settings['common']['apache_dir']),
+
+        DeletedFile(['/var/www/html/index.html'])
     ]
 
     return audits
@@ -94,5 +97,4 @@ class ApacheConfContext(object):
         ctxt['apache_version'] = re.search(r'.+version: Apache/(.+?)\s.+',
                                            out).group(1)
         ctxt['apache_icondir'] = '/usr/share/apache2/icons/'
-        ctxt['traceenable'] = settings['hardening']['traceenable']
         return ctxt

charmhelpers/contrib/hardening/apache/templates/99-hardening.conf

@@ -4,15 +4,29 @@
 ###############################################################################
 
 <Location / >
-  <LimitExcept {{ allowed_http_methods }} > 
+  <LimitExcept {{ allowed_http_methods }} >
     # http://httpd.apache.org/docs/2.4/upgrading.html
     {% if apache_version > '2.2' -%}
     Require all granted
     {% else -%}
-    Order Allow,Deny 
+    Order Allow,Deny
-    Deny from all 
+    Deny from all
     {% endif %}
   </LimitExcept>
 </Location>
 
+<Directory />
+    Options -Indexes -FollowSymLinks
+    AllowOverride None
+</Directory>
+
+<Directory /var/www/>
+    Options -Indexes -FollowSymLinks
+    AllowOverride None
+</Directory>
+
 TraceEnable {{ traceenable }}
+ServerTokens {{ servertokens }}
+
+SSLHonorCipherOrder {{ honor_cipher_order }}
+SSLCipherSuite {{ cipher_suite }}

charmhelpers/contrib/hardening/audits/__init__.py

@@ -49,13 +49,6 @@ class BaseAudit(object):  # NO-QA
 
         # Invoke the callback if there is one.
         if hasattr(self.unless, '__call__'):
-            results = self.unless()
+            return not self.unless()
-            if results:
-                return False
-            else:
-                return True
 
-        if self.unless:
+        return not self.unless
-            return False
-        else:
-            return True

charmhelpers/contrib/hardening/defaults/apache.yaml

@@ -10,4 +10,7 @@ common:
 hardening:
     traceenable: 'off'
     allowed_http_methods: "GET POST"
-    modules_to_disable: [ cgi, cgid ]
+    modules_to_disable: [ cgi, cgid ]
+    servertokens: 'Prod'
+    honor_cipher_order: 'on'
+    cipher_suite: 'ALL:+MEDIUM:+HIGH:!LOW:!MD5:!RC4:!eNULL:!aNULL:!3DES'

charmhelpers/contrib/hardening/defaults/apache.yaml.schema

@@ -7,3 +7,6 @@ common:
 hardening:
     allowed_http_methods:
     modules_to_disable:
+    servertokens:
+    honor_cipher_order:
+    cipher_suite:

charmhelpers/contrib/hardening/defaults/os.yaml

@@ -58,6 +58,7 @@ security:
         rsync
     kernel_enable_module_loading: True  # (type:boolean)
     kernel_enable_core_dump: False  # (type:boolean)
+    ssh_tmout: 300
 
 sysctl:
     kernel_secure_sysrq: 244  # 4 + 16 + 32 + 64 + 128

charmhelpers/contrib/hardening/defaults/os.yaml.schema

@@ -34,6 +34,7 @@ security:
     packages_list:
     kernel_enable_module_loading:
     kernel_enable_core_dump:
+    ssh_tmout:
 sysctl:
     kernel_secure_sysrq:
     kernel_enable_sysrq:

charmhelpers/contrib/hardening/host/checks/profile.py

@@ -25,7 +25,6 @@ def get_audits():
     audits = []
 
     settings = utils.get_settings('os')
-
     # If core dumps are not enabled, then don't allow core dumps to be
     # created as they may contain sensitive information.
     if not settings['security']['kernel_enable_core_dump']:
@@ -33,11 +32,18 @@ def get_audits():
                                     ProfileContext(),
                                     template_dir=TEMPLATES_DIR,
                                     mode=0o0755, user='root', group='root'))
+    if settings['security']['ssh_tmout']:
+        audits.append(TemplatedFile('/etc/profile.d/99-hardening.sh',
+                                    ProfileContext(),
+                                    template_dir=TEMPLATES_DIR,
+                                    mode=0o0644, user='root', group='root'))
     return audits
 
 
 class ProfileContext(object):
 
     def __call__(self):
-        ctxt = {}
+        settings = utils.get_settings('os')
+        ctxt = {'ssh_tmout':
+                settings['security']['ssh_tmout']}
         return ctxt

charmhelpers/contrib/hardening/host/templates/99-hardening.sh

@@ -0,0 +1,5 @@
+TMOUT={{ tmout }}
+readonly TMOUT
+export TMOUT
+
+readonly HISTFILE

charmhelpers/contrib/hardening/ssh/checks/config.py

@@ -27,7 +27,10 @@ from charmhelpers.fetch import (
     apt_install,
     apt_update,
 )
-from charmhelpers.core.host import lsb_release
+from charmhelpers.core.host import (
+    lsb_release,
+    CompareHostReleases,
+)
 from charmhelpers.contrib.hardening.audits.file import (
     TemplatedFile,
     FileContentAudit,
@@ -68,7 +71,8 @@ class SSHConfigContext(object):
                    'weak': default + ',hmac-sha1'}
 
         # Use newer ciphers on Ubuntu Trusty and above
-        if lsb_release()['DISTRIB_CODENAME'].lower() >= 'trusty':
+        _release = lsb_release()['DISTRIB_CODENAME'].lower()
+        if CompareHostReleases(_release) >= 'trusty':
             log("Detected Ubuntu 14.04 or newer, using new macs", level=DEBUG)
             macs = macs_66
 
@@ -96,7 +100,8 @@ class SSHConfigContext(object):
                   'weak': weak}
 
         # Use newer kex on Ubuntu Trusty and above
-        if lsb_release()['DISTRIB_CODENAME'].lower() >= 'trusty':
+        _release = lsb_release()['DISTRIB_CODENAME'].lower()
+        if CompareHostReleases(_release) >= 'trusty':
             log('Detected Ubuntu 14.04 or newer, using new key exchange '
                 'algorithms', level=DEBUG)
             kex = kex_66
@@ -119,7 +124,8 @@ class SSHConfigContext(object):
                       'weak': default + ',aes256-cbc,aes192-cbc,aes128-cbc'}
 
         # Use newer ciphers on ubuntu Trusty and above
-        if lsb_release()['DISTRIB_CODENAME'].lower() >= 'trusty':
+        _release = lsb_release()['DISTRIB_CODENAME'].lower()
+        if CompareHostReleases(_release) >= 'trusty':
             log('Detected Ubuntu 14.04 or newer, using new ciphers',
                 level=DEBUG)
             cipher = ciphers_66
@@ -291,7 +297,8 @@ class SSHConfigFileContentAudit(FileContentAudit):
         self.fail_cases = []
         settings = utils.get_settings('ssh')
 
-        if lsb_release()['DISTRIB_CODENAME'].lower() >= 'trusty':
+        _release = lsb_release()['DISTRIB_CODENAME'].lower()
+        if CompareHostReleases(_release) >= 'trusty':
             if not settings['server']['weak_hmac']:
                 self.pass_cases.append(r'^MACs.+,hmac-ripemd160$')
             else:
@@ -364,7 +371,8 @@ class SSHDConfigFileContentAudit(FileContentAudit):
         self.fail_cases = []
         settings = utils.get_settings('ssh')
 
-        if lsb_release()['DISTRIB_CODENAME'].lower() >= 'trusty':
+        _release = lsb_release()['DISTRIB_CODENAME'].lower()
+        if CompareHostReleases(_release) >= 'trusty':
             if not settings['server']['weak_hmac']:
                 self.pass_cases.append(r'^MACs.+,hmac-ripemd160$')
             else:

charmhelpers/contrib/network/ip.py

@@ -31,6 +31,7 @@ from charmhelpers.core.hookenv import (
 
 from charmhelpers.core.host import (
     lsb_release,
+    CompareHostReleases,
 )
 
 try:
@@ -67,6 +68,24 @@ def no_ip_found_error_out(network):
     raise ValueError(errmsg)
 
 
+def _get_ipv6_network_from_address(address):
+    """Get an netaddr.IPNetwork for the given IPv6 address
+    :param address: a dict as returned by netifaces.ifaddresses
+    :returns netaddr.IPNetwork: None if the address is a link local or loopback
+    address
+    """
+    if address['addr'].startswith('fe80') or address['addr'] == "::1":
+        return None
+
+    prefix = address['netmask'].split("/")
+    if len(prefix) > 1:
+        netmask = prefix[1]
+    else:
+        netmask = address['netmask']
+    return netaddr.IPNetwork("%s/%s" % (address['addr'],
+                                        netmask))
+
+
 def get_address_in_network(network, fallback=None, fatal=False):
     """Get an IPv4 or IPv6 address within the network from the host.
 
@@ -92,19 +111,17 @@ def get_address_in_network(network, fallback=None, fatal=False):
         for iface in netifaces.interfaces():
             addresses = netifaces.ifaddresses(iface)
             if network.version == 4 and netifaces.AF_INET in addresses:
-                addr = addresses[netifaces.AF_INET][0]['addr']
+                for addr in addresses[netifaces.AF_INET]:
-                netmask = addresses[netifaces.AF_INET][0]['netmask']
+                    cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'],
-                cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask))
+                                                        addr['netmask']))
-                if cidr in network:
+                    if cidr in network:
-                    return str(cidr.ip)
+                        return str(cidr.ip)
 
             if network.version == 6 and netifaces.AF_INET6 in addresses:
                 for addr in addresses[netifaces.AF_INET6]:
-                    if not addr['addr'].startswith('fe80'):
+                    cidr = _get_ipv6_network_from_address(addr)
-                        cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'],
+                    if cidr and cidr in network:
-                                                            addr['netmask']))
+                        return str(cidr.ip)
-                        if cidr in network:
-                            return str(cidr.ip)
 
     if fallback is not None:
         return fallback
@@ -180,18 +197,18 @@ def _get_for_address(address, key):
 
         if address.version == 6 and netifaces.AF_INET6 in addresses:
             for addr in addresses[netifaces.AF_INET6]:
-                if not addr['addr'].startswith('fe80'):
+                network = _get_ipv6_network_from_address(addr)
-                    network = netaddr.IPNetwork("%s/%s" % (addr['addr'],
+                if not network:
-                                                           addr['netmask']))
+                    continue
-                    cidr = network.cidr
-                    if address in cidr:
-                        if key == 'iface':
-                            return iface
-                        elif key == 'netmask' and cidr:
-                            return str(cidr).split('/')[1]
-                        else:
-                            return addr[key]
 
+                cidr = network.cidr
+                if address in cidr:
+                    if key == 'iface':
+                        return iface
+                    elif key == 'netmask' and cidr:
+                        return str(cidr).split('/')[1]
+                    else:
+                        return addr[key]
     return None
 
 
@@ -222,6 +239,16 @@ def format_ipv6_addr(address):
     return None
 
 
+def is_ipv6_disabled():
+    try:
+        result = subprocess.check_output(
+            ['sysctl', 'net.ipv6.conf.all.disable_ipv6'],
+            stderr=subprocess.STDOUT)
+        return "net.ipv6.conf.all.disable_ipv6 = 1" in result
+    except subprocess.CalledProcessError:
+        return True
+
+
 def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False,
                    fatal=True, exc_list=None):
     """Return the assigned IP address for a given interface, if any.
@@ -521,36 +548,44 @@ def port_has_listener(address, port):
 
 def assert_charm_supports_ipv6():
     """Check whether we are able to support charms ipv6."""
-    if lsb_release()['DISTRIB_CODENAME'].lower() < "trusty":
+    release = lsb_release()['DISTRIB_CODENAME'].lower()
+    if CompareHostReleases(release) < "trusty":
         raise Exception("IPv6 is not supported in the charms for Ubuntu "
                         "versions less than Trusty 14.04")
 
 
-def get_relation_ip(interface, config_override=None):
+def get_relation_ip(interface, cidr_network=None):
-    """Return this unit's IP for the given relation.
+    """Return this unit's IP for the given interface.
 
     Allow for an arbitrary interface to use with network-get to select an IP.
-    Handle all address selection options including configuration parameter
+    Handle all address selection options including passed cidr network and
-    override and IPv6.
+    IPv6.
 
-    Usage: get_relation_ip('amqp', config_override='access-network')
+    Usage: get_relation_ip('amqp', cidr_network='10.0.0.0/8')
 
     @param interface: string name of the relation.
-    @param config_override: string name of the config option for network
+    @param cidr_network: string CIDR Network to select an address from.
-           override. Supports legacy network override configuration parameters.
     @raises Exception if prefer-ipv6 is configured but IPv6 unsupported.
     @returns IPv6 or IPv4 address
     """
+    # Select the interface address first
+    # For possible use as a fallback bellow with get_address_in_network
+    try:
+        # Get the interface specific IP
+        address = network_get_primary_address(interface)
+    except NotImplementedError:
+        # If network-get is not available
+        address = get_host_ip(unit_get('private-address'))
 
-    fallback = get_host_ip(unit_get('private-address'))
     if config('prefer-ipv6'):
+        # Currently IPv6 has priority, eventually we want IPv6 to just be
+        # another network space.
         assert_charm_supports_ipv6()
         return get_ipv6_addr()[0]
-    elif config_override and config(config_override):
+    elif cidr_network:
-        return get_address_in_network(config(config_override),
+        # If a specific CIDR network is passed get the address from that
-                                      fallback)
+        # network.
-    else:
+        return get_address_in_network(cidr_network, address)
-        try:
+
-            return network_get_primary_address(interface)
+    # Return the interface address
-        except NotImplementedError:
+    return address
-            return fallback

charmhelpers/contrib/openstack/amulet/utils.py

@@ -40,6 +40,7 @@ from charmhelpers.contrib.amulet.utils import (
     AmuletUtils
 )
 from charmhelpers.core.decorators import retry_on_exception
+from charmhelpers.core.host import CompareHostReleases
 
 DEBUG = logging.DEBUG
 ERROR = logging.ERROR
@@ -546,7 +547,7 @@ class OpenStackAmuletUtils(AmuletUtils):
         """Create the specified instance."""
         self.log.debug('Creating instance '
                        '({}|{}|{})'.format(instance_name, image_name, flavor))
-        image = nova.images.find(name=image_name)
+        image = nova.glance.find_image(image_name)
         flavor = nova.flavors.find(name=flavor)
         instance = nova.servers.create(name=instance_name, image=image,
                                        flavor=flavor)
@@ -1255,7 +1256,7 @@ class OpenStackAmuletUtils(AmuletUtils):
         contents = self.file_contents_safe(sentry_unit, '/etc/memcached.conf',
                                            fatal=True)
         ubuntu_release, _ = self.run_cmd_unit(sentry_unit, 'lsb_release -cs')
-        if ubuntu_release <= 'trusty':
+        if CompareHostReleases(ubuntu_release) <= 'trusty':
             memcache_listen_addr = 'ip6-localhost'
         else:
             memcache_listen_addr = '::1'

charmhelpers/contrib/openstack/context.py

@@ -59,6 +59,8 @@ from charmhelpers.core.host import (
     write_file,
     pwgen,
     lsb_release,
+    CompareHostReleases,
+    is_container,
 )
 from charmhelpers.contrib.hahelpers.cluster import (
     determine_apache_port,
@@ -87,6 +89,7 @@ from charmhelpers.contrib.network.ip import (
     format_ipv6_addr,
     is_address_in_network,
     is_bridge_member,
+    is_ipv6_disabled,
 )
 from charmhelpers.contrib.openstack.utils import (
     config_flags_parser,
@@ -108,6 +111,7 @@ except ImportError:
 
 CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
 ADDRESS_TYPES = ['admin', 'internal', 'public']
+HAPROXY_RUN_DIR = '/var/run/haproxy/'
 
 
 def ensure_packages(packages):
@@ -155,7 +159,8 @@ class OSContextGenerator(object):
 
         if self.missing_data:
             self.complete = False
-            log('Missing required data: %s' % ' '.join(self.missing_data), level=INFO)
+            log('Missing required data: %s' % ' '.join(self.missing_data),
+                level=INFO)
         else:
             self.complete = True
         return self.complete
@@ -213,8 +218,9 @@ class SharedDBContext(OSContextGenerator):
                 hostname_key = "{}_hostname".format(self.relation_prefix)
             else:
                 hostname_key = "hostname"
-            access_hostname = get_address_in_network(access_network,
+            access_hostname = get_address_in_network(
-                                                     unit_get('private-address'))
+                access_network,
+                unit_get('private-address'))
             set_hostname = relation_get(attribute=hostname_key,
                                         unit=local_unit())
             if set_hostname != access_hostname:
@@ -308,7 +314,10 @@ def db_ssl(rdata, ctxt, ssl_dir):
 
 class IdentityServiceContext(OSContextGenerator):
 
-    def __init__(self, service=None, service_user=None, rel_name='identity-service'):
+    def __init__(self,
+                 service=None,
+                 service_user=None,
+                 rel_name='identity-service'):
         self.service = service
         self.service_user = service_user
         self.rel_name = rel_name
@@ -457,19 +466,17 @@ class AMQPContext(OSContextGenerator):
                     host = format_ipv6_addr(host) or host
                     rabbitmq_hosts.append(host)
 
-                ctxt['rabbitmq_hosts'] = ','.join(sorted(rabbitmq_hosts))
+                rabbitmq_hosts = sorted(rabbitmq_hosts)
+                ctxt['rabbitmq_hosts'] = ','.join(rabbitmq_hosts)
                 transport_hosts = rabbitmq_hosts
 
             if transport_hosts:
-                transport_url_hosts = ''
+                transport_url_hosts = ','.join([
-                for host in transport_hosts:
+                    "{}:{}@{}:{}".format(ctxt['rabbitmq_user'],
-                    if transport_url_hosts:
+                                         ctxt['rabbitmq_password'],
-                        format_string = ",{}:{}@{}:{}"
+                                         host_,
-                    else:
+                                         rabbitmq_port)
-                        format_string = "{}:{}@{}:{}"
+                    for host_ in transport_hosts])
-                    transport_url_hosts += format_string.format(
-                        ctxt['rabbitmq_user'], ctxt['rabbitmq_password'],
-                        host, rabbitmq_port)
                 ctxt['transport_url'] = "rabbit://{}/{}".format(
                     transport_url_hosts, vhost)
 
@@ -530,6 +537,8 @@ class HAProxyContext(OSContextGenerator):
     """Provides half a context for the haproxy template, which describes
     all peers to be included in the cluster.  Each charm needs to include
     its own context generator that describes the port mapping.
+
+    :side effect: mkdir is called on HAPROXY_RUN_DIR
     """
     interfaces = ['cluster']
 
@@ -537,6 +546,8 @@ class HAProxyContext(OSContextGenerator):
         self.singlenode_mode = singlenode_mode
 
     def __call__(self):
+        if not os.path.isdir(HAPROXY_RUN_DIR):
+            mkdir(path=HAPROXY_RUN_DIR)
         if not relation_ids('cluster') and not self.singlenode_mode:
             return {}
 
@@ -1217,22 +1228,54 @@ class BindHostContext(OSContextGenerator):
             return {'bind_host': '0.0.0.0'}
 
 
+MAX_DEFAULT_WORKERS = 4
+DEFAULT_MULTIPLIER = 2
+
+
+def _calculate_workers():
+    '''
+    Determine the number of worker processes based on the CPU
+    count of the unit containing the application.
+
+    Workers will be limited to MAX_DEFAULT_WORKERS in
+    container environments where no worker-multipler configuration
+    option been set.
+
+    @returns int: number of worker processes to use
+    '''
+    multiplier = config('worker-multiplier') or DEFAULT_MULTIPLIER
+    count = int(_num_cpus() * multiplier)
+    if multiplier > 0 and count == 0:
+        count = 1
+
+    if config('worker-multiplier') is None and is_container():
+        # NOTE(jamespage): Limit unconfigured worker-multiplier
+        #                  to MAX_DEFAULT_WORKERS to avoid insane
+        #                  worker configuration in LXD containers
+        #                  on large servers
+        # Reference: https://pad.lv/1665270
+        count = min(count, MAX_DEFAULT_WORKERS)
+
+    return count
+
+
+def _num_cpus():
+    '''
+    Compatibility wrapper for calculating the number of CPU's
+    a unit has.
+
+    @returns: int: number of CPU cores detected
+    '''
+    try:
+        return psutil.cpu_count()
+    except AttributeError:
+        return psutil.NUM_CPUS
+
+
 class WorkerConfigContext(OSContextGenerator):
 
-    @property
-    def num_cpus(self):
-        # NOTE: use cpu_count if present (16.04 support)
-        if hasattr(psutil, 'cpu_count'):
-            return psutil.cpu_count()
-        else:
-            return psutil.NUM_CPUS
-
     def __call__(self):
-        multiplier = config('worker-multiplier') or 0
+        ctxt = {"workers": _calculate_workers()}
-        count = int(self.num_cpus * multiplier)
-        if multiplier > 0 and count == 0:
-            count = 1
-        ctxt = {"workers": count}
         return ctxt
 
 
@@ -1240,7 +1283,7 @@ class WSGIWorkerConfigContext(WorkerConfigContext):
 
     def __init__(self, name=None, script=None, admin_script=None,
                  public_script=None, process_weight=1.00,
-                 admin_process_weight=0.75, public_process_weight=0.25):
+                 admin_process_weight=0.25, public_process_weight=0.75):
         self.service_name = name
         self.user = name
         self.group = name
@@ -1252,8 +1295,7 @@ class WSGIWorkerConfigContext(WorkerConfigContext):
         self.public_process_weight = public_process_weight
 
     def __call__(self):
-        multiplier = config('worker-multiplier') or 1
+        total_processes = _calculate_workers()
-        total_processes = self.num_cpus * multiplier
         ctxt = {
             "service_name": self.service_name,
             "user": self.user,
@@ -1584,7 +1626,7 @@ class MemcacheContext(OSContextGenerator):
     """Memcache context
 
     This context provides options for configuring a local memcache client and
-    server
+    server for both IPv4 and IPv6
     """
 
     def __init__(self, package=None):
@@ -1601,13 +1643,25 @@ class MemcacheContext(OSContextGenerator):
         if ctxt['use_memcache']:
             # Trusty version of memcached does not support ::1 as a listen
             # address so use host file entry instead
-            if lsb_release()['DISTRIB_CODENAME'].lower() > 'trusty':
+            release = lsb_release()['DISTRIB_CODENAME'].lower()
-                ctxt['memcache_server'] = '::1'
+            if is_ipv6_disabled():
+                if CompareHostReleases(release) > 'trusty':
+                    ctxt['memcache_server'] = '127.0.0.1'
+                else:
+                    ctxt['memcache_server'] = 'localhost'
+                ctxt['memcache_server_formatted'] = '127.0.0.1'
+                ctxt['memcache_port'] = '11211'
+                ctxt['memcache_url'] = '{}:{}'.format(
+                    ctxt['memcache_server_formatted'],
+                    ctxt['memcache_port'])
             else:
-                ctxt['memcache_server'] = 'ip6-localhost'
+                if CompareHostReleases(release) > 'trusty':
-            ctxt['memcache_server_formatted'] = '[::1]'
+                    ctxt['memcache_server'] = '::1'
-            ctxt['memcache_port'] = '11211'
+                else:
-            ctxt['memcache_url'] = 'inet6:{}:{}'.format(
+                    ctxt['memcache_server'] = 'ip6-localhost'
-                ctxt['memcache_server_formatted'],
+                ctxt['memcache_server_formatted'] = '[::1]'
-                ctxt['memcache_port'])
+                ctxt['memcache_port'] = '11211'
+                ctxt['memcache_url'] = 'inet6:{}:{}'.format(
+                    ctxt['memcache_server_formatted'],
+                    ctxt['memcache_port'])
         return ctxt

charmhelpers/contrib/openstack/neutron.py

@@ -23,7 +23,10 @@ from charmhelpers.core.hookenv import (
     ERROR,
 )
 
-from charmhelpers.contrib.openstack.utils import os_release
+from charmhelpers.contrib.openstack.utils import (
+    os_release,
+    CompareOpenStackReleases,
+)
 
 
 def headers_package():
@@ -198,7 +201,8 @@ def neutron_plugins():
         },
         'plumgrid': {
             'config': '/etc/neutron/plugins/plumgrid/plumgrid.ini',
-            'driver': 'neutron.plugins.plumgrid.plumgrid_plugin.plumgrid_plugin.NeutronPluginPLUMgridV2',
+            'driver': ('neutron.plugins.plumgrid.plumgrid_plugin'
+                       '.plumgrid_plugin.NeutronPluginPLUMgridV2'),
             'contexts': [
                 context.SharedDBContext(user=config('database-user'),
                                         database=config('database'),
@@ -225,7 +229,7 @@ def neutron_plugins():
             'server_services': ['neutron-server']
         }
     }
-    if release >= 'icehouse':
+    if CompareOpenStackReleases(release) >= 'icehouse':
         # NOTE: patch in ml2 plugin for icehouse onwards
         plugins['ovs']['config'] = '/etc/neutron/plugins/ml2/ml2_conf.ini'
         plugins['ovs']['driver'] = 'neutron.plugins.ml2.plugin.Ml2Plugin'
@@ -233,10 +237,10 @@ def neutron_plugins():
                                              'neutron-plugin-ml2']
         # NOTE: patch in vmware renames nvp->nsx for icehouse onwards
         plugins['nvp'] = plugins['nsx']
-    if release >= 'kilo':
+    if CompareOpenStackReleases(release) >= 'kilo':
         plugins['midonet']['driver'] = (
             'neutron.plugins.midonet.plugin.MidonetPluginV2')
-    if release >= 'liberty':
+    if CompareOpenStackReleases(release) >= 'liberty':
         plugins['midonet']['driver'] = (
             'midonet.neutron.plugin_v1.MidonetPluginV2')
         plugins['midonet']['server_packages'].remove(
@@ -244,10 +248,11 @@ def neutron_plugins():
         plugins['midonet']['server_packages'].append(
             'python-networking-midonet')
         plugins['plumgrid']['driver'] = (
-            'networking_plumgrid.neutron.plugins.plugin.NeutronPluginPLUMgridV2')
+            'networking_plumgrid.neutron.plugins'
+            '.plugin.NeutronPluginPLUMgridV2')
         plugins['plumgrid']['server_packages'].remove(
             'neutron-plugin-plumgrid')
-    if release >= 'mitaka':
+    if CompareOpenStackReleases(release) >= 'mitaka':
         plugins['nsx']['server_packages'].remove('neutron-plugin-vmware')
         plugins['nsx']['server_packages'].append('python-vmware-nsx')
         plugins['nsx']['config'] = '/etc/neutron/nsx.ini'

charmhelpers/contrib/openstack/templates/haproxy.cfg

@@ -5,6 +5,8 @@ global
     user haproxy
     group haproxy
     spread-checks 0
+    stats socket /var/run/haproxy/admin.sock mode 600 level admin
+    stats timeout 2m
 
 defaults
     log global
@@ -58,6 +60,15 @@ frontend tcp-in_{{ service }}
 {% for frontend in frontends -%}
 backend {{ service }}_{{ frontend }}
     balance leastconn
+    {% if backend_options -%}
+    {% if backend_options[service] -%}
+    {% for option in backend_options[service] -%}
+    {% for key, value in option.items() -%}
+    {{ key }} {{ value }}
+    {% endfor -%}
+    {% endfor -%}
+    {% endif -%}
+    {% endif -%}
     {% for unit, address in frontends[frontend]['backends'].items() -%}
     server {{ unit }} {{ address }}:{{ ports[1] }} check
     {% endfor %}

charmhelpers/contrib/openstack/utils.py

@@ -33,9 +33,7 @@ import yaml
 
 from charmhelpers.contrib.network import ip
 
-from charmhelpers.core import (
+from charmhelpers.core import unitdata
-    unitdata,
-)
 
 from charmhelpers.core.hookenv import (
     action_fail,
@@ -55,6 +53,8 @@ from charmhelpers.core.hookenv import (
     application_version_set,
 )
 
+from charmhelpers.core.strutils import BasicStringComparator
+
 from charmhelpers.contrib.storage.linux.lvm import (
     deactivate_lvm_volume_group,
     is_lvm_physical_volume,
@@ -97,6 +97,22 @@ CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA'
 DISTRO_PROPOSED = ('deb http://archive.ubuntu.com/ubuntu/ %s-proposed '
                    'restricted main multiverse universe')
 
+OPENSTACK_RELEASES = (
+    'diablo',
+    'essex',
+    'folsom',
+    'grizzly',
+    'havana',
+    'icehouse',
+    'juno',
+    'kilo',
+    'liberty',
+    'mitaka',
+    'newton',
+    'ocata',
+    'pike',
+)
+
 UBUNTU_OPENSTACK_RELEASE = OrderedDict([
     ('oneiric', 'diablo'),
     ('precise', 'essex'),
@@ -238,6 +254,17 @@ GIT_DEFAULT_BRANCHES = {
 DEFAULT_LOOPBACK_SIZE = '5G'
 
 
+class CompareOpenStackReleases(BasicStringComparator):
+    """Provide comparisons of OpenStack releases.
+
+    Use in the form of
+
+    if CompareOpenStackReleases(release) > 'mitaka':
+        # do something with mitaka
+    """
+    _list = OPENSTACK_RELEASES
+
+
 def error_out(msg):
     juju_log("FATAL ERROR: %s" % msg, level='ERROR')
     sys.exit(1)
@@ -1066,7 +1093,8 @@ def git_generate_systemd_init_files(templates_dir):
 
             shutil.copyfile(init_in_source, init_source)
             with open(init_source, 'a') as outfile:
-                template = '/usr/share/openstack-pkg-tools/init-script-template'
+                template = ('/usr/share/openstack-pkg-tools/'
+                            'init-script-template')
                 with open(template) as infile:
                     outfile.write('\n\n{}'.format(infile.read()))
 
@@ -1971,9 +1999,7 @@ def enable_memcache(source=None, release=None, package=None):
     if not _release:
         _release = get_os_codename_install_source(source)
 
-    # TODO: this should be changed to a numeric comparison using a known list
+    return CompareOpenStackReleases(_release) >= 'mitaka'
-    # of releases and comparing by index.
-    return _release >= 'mitaka'
 
 
 def token_cache_pkgs(source=None, release=None):

charmhelpers/contrib/storage/linux/ceph.py

@@ -987,18 +987,20 @@ def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point,
             service_start(svc)
 
 
-def ensure_ceph_keyring(service, user=None, group=None, relation='ceph'):
+def ensure_ceph_keyring(service, user=None, group=None,
+                        relation='ceph', key=None):
     """Ensures a ceph keyring is created for a named service and optionally
     ensures user and group ownership.
 
-    Returns False if no ceph key is available in relation state.
+    @returns boolean: Flag to indicate whether a key was successfully written
+                      to disk based on either relation data or a supplied key
     """
-    key = None
+    if not key:
-    for rid in relation_ids(relation):
+        for rid in relation_ids(relation):
-        for unit in related_units(rid):
+            for unit in related_units(rid):
-            key = relation_get('key', rid=rid, unit=unit)
+                key = relation_get('key', rid=rid, unit=unit)
-            if key:
+                if key:
-                break
+                    break
 
     if not key:
         return False

charmhelpers/core/host.py

@@ -45,6 +45,7 @@ if __platform__ == "ubuntu":
         add_new_group,
         lsb_release,
         cmp_pkgrevno,
+        CompareHostReleases,
     )  # flake8: noqa -- ignore F401 for this import
 elif __platform__ == "centos":
     from charmhelpers.core.host_factory.centos import (
@@ -52,6 +53,7 @@ elif __platform__ == "centos":
         add_new_group,
         lsb_release,
         cmp_pkgrevno,
+        CompareHostReleases,
     )  # flake8: noqa -- ignore F401 for this import
 
 UPDATEDB_PATH = '/etc/updatedb.conf'
@@ -189,7 +191,7 @@ def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d",
     upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
     sysv_file = os.path.join(initd_dir, service_name)
     if init_is_systemd():
-        service('disable', service_name)
+        service('mask', service_name)
     elif os.path.exists(upstart_file):
         override_path = os.path.join(
             init_dir, '{}.override'.format(service_name))
@@ -222,7 +224,7 @@ def service_resume(service_name, init_dir="/etc/init",
     upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
     sysv_file = os.path.join(initd_dir, service_name)
     if init_is_systemd():
-        service('enable', service_name)
+        service('unmask', service_name)
     elif os.path.exists(upstart_file):
         override_path = os.path.join(
             init_dir, '{}.override'.format(service_name))

charmhelpers/core/host_factory/centos.py

@@ -2,6 +2,22 @@ import subprocess
 import yum
 import os
 
+from charmhelpers.core.strutils import BasicStringComparator
+
+
+class CompareHostReleases(BasicStringComparator):
+    """Provide comparisons of Host releases.
+
+    Use in the form of
+
+    if CompareHostReleases(release) > 'trusty':
+        # do something with mitaka
+    """
+
+    def __init__(self, item):
+        raise NotImplementedError(
+            "CompareHostReleases() is not implemented for CentOS")
+
 
 def service_available(service_name):
     # """Determine whether a system service is available."""

charmhelpers/core/host_factory/ubuntu.py

@@ -1,5 +1,37 @@
 import subprocess
 
+from charmhelpers.core.strutils import BasicStringComparator
+
+
+UBUNTU_RELEASES = (
+    'lucid',
+    'maverick',
+    'natty',
+    'oneiric',
+    'precise',
+    'quantal',
+    'raring',
+    'saucy',
+    'trusty',
+    'utopic',
+    'vivid',
+    'wily',
+    'xenial',
+    'yakkety',
+    'zesty',
+)
+
+
+class CompareHostReleases(BasicStringComparator):
+    """Provide comparisons of Ubuntu releases.
+
+    Use in the form of
+
+    if CompareHostReleases(release) > 'trusty':
+        # do something with mitaka
+    """
+    _list = UBUNTU_RELEASES
+
 
 def service_available(service_name):
     """Determine whether a system service is available"""

charmhelpers/core/strutils.py

@@ -68,3 +68,56 @@ def bytes_from_string(value):
         msg = "Unable to interpret string value '%s' as bytes" % (value)
         raise ValueError(msg)
     return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
+
+
+class BasicStringComparator(object):
+    """Provides a class that will compare strings from an iterator type object.
+    Used to provide > and < comparisons on strings that may not necessarily be
+    alphanumerically ordered.  e.g. OpenStack or Ubuntu releases AFTER the
+    z-wrap.
+    """
+
+    _list = None
+
+    def __init__(self, item):
+        if self._list is None:
+            raise Exception("Must define the _list in the class definition!")
+        try:
+            self.index = self._list.index(item)
+        except Exception:
+            raise KeyError("Item '{}' is not in list '{}'"
+                           .format(item, self._list))
+
+    def __eq__(self, other):
+        assert isinstance(other, str) or isinstance(other, self.__class__)
+        return self.index == self._list.index(other)
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __lt__(self, other):
+        assert isinstance(other, str) or isinstance(other, self.__class__)
+        return self.index < self._list.index(other)
+
+    def __ge__(self, other):
+        return not self.__lt__(other)
+
+    def __gt__(self, other):
+        assert isinstance(other, str) or isinstance(other, self.__class__)
+        return self.index > self._list.index(other)
+
+    def __le__(self, other):
+        return not self.__gt__(other)
+
+    def __str__(self):
+        """Always give back the item at the index so it can be used in
+        comparisons like:
+
+        s_mitaka = CompareOpenStack('mitaka')
+        s_newton = CompareOpenstack('newton')
+
+        assert s_newton > s_mitaka
+
+        @returns: <string>
+        """
+        return self._list[self.index]

lib/swift_storage_utils.py

@@ -35,7 +35,8 @@ from charmhelpers.core.host import (
     mount,
     fstab_add,
     service_restart,
-    lsb_release
+    lsb_release,
+    CompareHostReleases,
 )
 
 from charmhelpers.core.hookenv import (
@@ -513,7 +514,8 @@ def save_script_rc():
 
 def assert_charm_supports_ipv6():
     """Check whether we are able to support charms ipv6."""
-    if lsb_release()['DISTRIB_CODENAME'].lower() < "trusty":
+    _release = lsb_release()['DISTRIB_CODENAME'].lower()
+    if CompareHostReleases(_release) < "trusty":
         raise Exception("IPv6 is not supported in the charms for Ubuntu "
                         "versions less than Trusty 14.04")
 

tests/basic_deployment.py

@@ -324,7 +324,7 @@ class SwiftStorageBasicDeployment(OpenStackAmuletDeployment):
             'DEFAULT': {
                 'bind_ip': '0.0.0.0',
                 'bind_port': '6002',
-                'workers': '1'
+                'workers': u.not_null,
             },
             'pipeline:main': {
                 'pipeline': 'recon account-server'
@@ -353,7 +353,7 @@ class SwiftStorageBasicDeployment(OpenStackAmuletDeployment):
             'DEFAULT': {
                 'bind_ip': '0.0.0.0',
                 'bind_port': '6001',
-                'workers': '1'
+                'workers': u.not_null,
             },
             'pipeline:main': {
                 'pipeline': 'recon container-server'
@@ -383,7 +383,7 @@ class SwiftStorageBasicDeployment(OpenStackAmuletDeployment):
             'DEFAULT': {
                 'bind_ip': '0.0.0.0',
                 'bind_port': '6000',
-                'workers': '1'
+                'workers': u.not_null,
             },
             'pipeline:main': {
                 'pipeline': 'recon object-server'

tests/charmhelpers/contrib/openstack/amulet/utils.py

@@ -40,6 +40,7 @@ from charmhelpers.contrib.amulet.utils import (
     AmuletUtils
 )
 from charmhelpers.core.decorators import retry_on_exception
+from charmhelpers.core.host import CompareHostReleases
 
 DEBUG = logging.DEBUG
 ERROR = logging.ERROR
@@ -546,7 +547,7 @@ class OpenStackAmuletUtils(AmuletUtils):
         """Create the specified instance."""
         self.log.debug('Creating instance '
                        '({}|{}|{})'.format(instance_name, image_name, flavor))
-        image = nova.images.find(name=image_name)
+        image = nova.glance.find_image(image_name)
         flavor = nova.flavors.find(name=flavor)
         instance = nova.servers.create(name=instance_name, image=image,
                                        flavor=flavor)
@@ -1255,7 +1256,7 @@ class OpenStackAmuletUtils(AmuletUtils):
         contents = self.file_contents_safe(sentry_unit, '/etc/memcached.conf',
                                            fatal=True)
         ubuntu_release, _ = self.run_cmd_unit(sentry_unit, 'lsb_release -cs')
-        if ubuntu_release <= 'trusty':
+        if CompareHostReleases(ubuntu_release) <= 'trusty':
             memcache_listen_addr = 'ip6-localhost'
         else:
             memcache_listen_addr = '::1'

tests/charmhelpers/core/host.py

@@ -45,6 +45,7 @@ if __platform__ == "ubuntu":
         add_new_group,
         lsb_release,
         cmp_pkgrevno,
+        CompareHostReleases,
     )  # flake8: noqa -- ignore F401 for this import
 elif __platform__ == "centos":
     from charmhelpers.core.host_factory.centos import (
@@ -52,6 +53,7 @@ elif __platform__ == "centos":
         add_new_group,
         lsb_release,
         cmp_pkgrevno,
+        CompareHostReleases,
     )  # flake8: noqa -- ignore F401 for this import
 
 UPDATEDB_PATH = '/etc/updatedb.conf'
@@ -189,7 +191,7 @@ def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d",
     upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
     sysv_file = os.path.join(initd_dir, service_name)
     if init_is_systemd():
-        service('disable', service_name)
+        service('mask', service_name)
     elif os.path.exists(upstart_file):
         override_path = os.path.join(
             init_dir, '{}.override'.format(service_name))
@@ -222,7 +224,7 @@ def service_resume(service_name, init_dir="/etc/init",
     upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
     sysv_file = os.path.join(initd_dir, service_name)
     if init_is_systemd():
-        service('enable', service_name)
+        service('unmask', service_name)
     elif os.path.exists(upstart_file):
         override_path = os.path.join(
             init_dir, '{}.override'.format(service_name))

tests/charmhelpers/core/host_factory/centos.py

@@ -2,6 +2,22 @@ import subprocess
 import yum
 import os
 
+from charmhelpers.core.strutils import BasicStringComparator
+
+
+class CompareHostReleases(BasicStringComparator):
+    """Provide comparisons of Host releases.
+
+    Use in the form of
+
+    if CompareHostReleases(release) > 'trusty':
+        # do something with mitaka
+    """
+
+    def __init__(self, item):
+        raise NotImplementedError(
+            "CompareHostReleases() is not implemented for CentOS")
+
 
 def service_available(service_name):
     # """Determine whether a system service is available."""

tests/charmhelpers/core/host_factory/ubuntu.py

@@ -1,5 +1,37 @@
 import subprocess
 
+from charmhelpers.core.strutils import BasicStringComparator
+
+
+UBUNTU_RELEASES = (
+    'lucid',
+    'maverick',
+    'natty',
+    'oneiric',
+    'precise',
+    'quantal',
+    'raring',
+    'saucy',
+    'trusty',
+    'utopic',
+    'vivid',
+    'wily',
+    'xenial',
+    'yakkety',
+    'zesty',
+)
+
+
+class CompareHostReleases(BasicStringComparator):
+    """Provide comparisons of Ubuntu releases.
+
+    Use in the form of
+
+    if CompareHostReleases(release) > 'trusty':
+        # do something with mitaka
+    """
+    _list = UBUNTU_RELEASES
+
 
 def service_available(service_name):
     """Determine whether a system service is available"""

tests/charmhelpers/core/strutils.py

@@ -68,3 +68,56 @@ def bytes_from_string(value):
         msg = "Unable to interpret string value '%s' as bytes" % (value)
         raise ValueError(msg)
     return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
+
+
+class BasicStringComparator(object):
+    """Provides a class that will compare strings from an iterator type object.
+    Used to provide > and < comparisons on strings that may not necessarily be
+    alphanumerically ordered.  e.g. OpenStack or Ubuntu releases AFTER the
+    z-wrap.
+    """
+
+    _list = None
+
+    def __init__(self, item):
+        if self._list is None:
+            raise Exception("Must define the _list in the class definition!")
+        try:
+            self.index = self._list.index(item)
+        except Exception:
+            raise KeyError("Item '{}' is not in list '{}'"
+                           .format(item, self._list))
+
+    def __eq__(self, other):
+        assert isinstance(other, str) or isinstance(other, self.__class__)
+        return self.index == self._list.index(other)
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __lt__(self, other):
+        assert isinstance(other, str) or isinstance(other, self.__class__)
+        return self.index < self._list.index(other)
+
+    def __ge__(self, other):
+        return not self.__lt__(other)
+
+    def __gt__(self, other):
+        assert isinstance(other, str) or isinstance(other, self.__class__)
+        return self.index > self._list.index(other)
+
+    def __le__(self, other):
+        return not self.__gt__(other)
+
+    def __str__(self):
+        """Always give back the item at the index so it can be used in
+        comparisons like:
+
+        s_mitaka = CompareOpenStack('mitaka')
+        s_newton = CompareOpenstack('newton')
+
+        assert s_newton > s_mitaka
+
+        @returns: <string>
+        """
+        return self._list[self.index]

tests/charmhelpers/osplatform.py

@@ -0,0 +1,25 @@
+import platform
+
+
+def get_platform():
+    """Return the current OS platform.
+
+    For example: if current os platform is Ubuntu then a string "ubuntu"
+    will be returned (which is the name of the module).
+    This string is used to decide which platform module should be imported.
+    """
+    # linux_distribution is deprecated and will be removed in Python 3.7
+    # Warings *not* disabled, as we certainly need to fix this.
+    tuple_platform = platform.linux_distribution()
+    current_platform = tuple_platform[0]
+    if "Ubuntu" in current_platform:
+        return "ubuntu"
+    elif "CentOS" in current_platform:
+        return "centos"
+    elif "debian" in current_platform:
+        # Stock Python does not detect Ubuntu and instead returns debian.
+        # Or at least it does in some build environments like Travis CI
+        return "ubuntu"
+    else:
+        raise RuntimeError("This module is not supported on {}."
+                           .format(current_platform))

tox.ini

@@ -14,7 +14,7 @@ install_command =
   pip install --allow-unverified python-apt {opts} {packages}
 commands = ostestr {posargs}
 whitelist_externals = juju
-passenv = HOME TERM AMULET_*
+passenv = HOME TERM AMULET_* CS_API_*
 
 [testenv:py27]
 basepython = python2.7