From 5ae65ad14f6c036b5f9622ee1261f1bec4ab65b5 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 2 Mar 2016 11:08:24 +0000 Subject: [PATCH] Resync charm-helpers Change-Id: I101a9f7a03542d0e9dc466c63851848e80ac9931 --- .gitignore | 1 + hooks/charmhelpers/contrib/network/ip.py | 15 ++++ .../charmhelpers/contrib/openstack/context.py | 6 +- .../charmhelpers/contrib/openstack/neutron.py | 18 +++-- .../templates/openstack_https_frontend | 2 + .../templates/openstack_https_frontend.conf | 2 + .../templates/section-keystone-authtoken | 11 +++ hooks/charmhelpers/contrib/openstack/utils.py | 75 ++++++++++++++++++- .../contrib/storage/linux/ceph.py | 50 ++++++++++--- 9 files changed, 157 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index 38192bc8..11868ea5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ bin .testrepository tags *.sw[nop] +*.pyc diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 998f00c1..4efe7993 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -456,3 +456,18 @@ def get_hostname(address, fqdn=True): return result else: return result.split('.')[0] + + +def port_has_listener(address, port): + """ + Returns True if the address:port is open and being listened to, + else False. + + @param address: an IP address or hostname + @param port: integer port + + Note calls 'zc' via a subprocess shell + """ + cmd = ['nc', '-z', address, str(port)] + result = subprocess.call(cmd) + return not(bool(result)) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index ff597c94..a8c6ab0c 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -410,6 +410,7 @@ class IdentityServiceContext(OSContextGenerator): auth_host = format_ipv6_addr(auth_host) or auth_host svc_protocol = rdata.get('service_protocol') or 'http' auth_protocol = rdata.get('auth_protocol') or 'http' + api_version = rdata.get('api_version') or '2.0' ctxt.update({'service_port': rdata.get('service_port'), 'service_host': serv_host, 'auth_host': auth_host, @@ -418,7 +419,8 @@ class IdentityServiceContext(OSContextGenerator): 'admin_user': rdata.get('service_username'), 'admin_password': rdata.get('service_password'), 'service_protocol': svc_protocol, - 'auth_protocol': auth_protocol}) + 'auth_protocol': auth_protocol, + 'api_version': api_version}) if self.context_complete(ctxt): # NOTE(jamespage) this is required for >= icehouse @@ -1471,6 +1473,8 @@ class NetworkServiceContext(OSContextGenerator): rdata.get('service_protocol') or 'http', 'auth_protocol': rdata.get('auth_protocol') or 'http', + 'api_version': + rdata.get('api_version') or '2.0', } if self.context_complete(ctxt): return ctxt diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index dbc489ab..d057ea6e 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -237,14 +237,16 @@ def neutron_plugins(): plugins['midonet']['driver'] = ( 'neutron.plugins.midonet.plugin.MidonetPluginV2') if release >= 'liberty': - midonet_origin = config('midonet-origin') - if midonet_origin is not None and midonet_origin[4:5] == '1': - plugins['midonet']['driver'] = ( - 'midonet.neutron.plugin_v1.MidonetPluginV2') - plugins['midonet']['server_packages'].remove( - 'python-neutron-plugin-midonet') - plugins['midonet']['server_packages'].append( - 'python-networking-midonet') + plugins['midonet']['driver'] = ( + 'midonet.neutron.plugin_v1.MidonetPluginV2') + plugins['midonet']['server_packages'].remove( + 'python-neutron-plugin-midonet') + plugins['midonet']['server_packages'].append( + 'python-networking-midonet') + plugins['plumgrid']['driver'] = ( + 'networking_plumgrid.neutron.plugins.plugin.NeutronPluginPLUMgridV2') + plugins['plumgrid']['server_packages'].remove( + 'neutron-plugin-plumgrid') return plugins diff --git a/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend b/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend index ce28fa3f..6a923804 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend +++ b/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend @@ -6,6 +6,8 @@ Listen {{ ext_port }} ServerName {{ endpoint }} SSLEngine on + SSLProtocol +TLSv1 +TLSv1.1 +TLSv1.2 + SSLCipherSuite HIGH:!RC4:!MD5:!aNULL:!eNULL:!EXP:!LOW:!MEDIUM SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }} SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }} ProxyPass / http://localhost:{{ int }}/ diff --git a/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf b/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf index ce28fa3f..6a923804 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf +++ b/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf @@ -6,6 +6,8 @@ Listen {{ ext_port }} ServerName {{ endpoint }} SSLEngine on + SSLProtocol +TLSv1 +TLSv1.1 +TLSv1.2 + SSLCipherSuite HIGH:!RC4:!MD5:!aNULL:!eNULL:!EXP:!LOW:!MEDIUM SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }} SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }} ProxyPass / http://localhost:{{ int }}/ diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken index 2a37edd5..0b6da25c 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken +++ b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken @@ -1,4 +1,14 @@ {% if auth_host -%} +{% if api_version == '3' -%} +[keystone_authtoken] +auth_url = {{ service_protocol }}://{{ service_host }}:{{ service_port }} +project_name = {{ admin_tenant_name }} +username = {{ admin_user }} +password = {{ admin_password }} +project_domain_name = default +user_domain_name = default +auth_plugin = password +{% else -%} [keystone_authtoken] identity_uri = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/{{ auth_admin_prefix }} auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/{{ service_admin_prefix }} @@ -7,3 +17,4 @@ admin_user = {{ admin_user }} admin_password = {{ admin_password }} signing_dir = {{ signing_dir }} {% endif -%} +{% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 80777123..80dd2e0d 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -23,6 +23,7 @@ import json import os import sys import re +import itertools import six import tempfile @@ -60,6 +61,7 @@ from charmhelpers.contrib.storage.linux.lvm import ( from charmhelpers.contrib.network.ip import ( get_ipv6_addr, is_ipv6, + port_has_listener, ) from charmhelpers.contrib.python.packages import ( @@ -67,7 +69,7 @@ from charmhelpers.contrib.python.packages import ( pip_install, ) -from charmhelpers.core.host import lsb_release, mounts, umount +from charmhelpers.core.host import lsb_release, mounts, umount, service_running from charmhelpers.fetch import apt_install, apt_cache, install_remote from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device @@ -860,13 +862,23 @@ def os_workload_status(configs, required_interfaces, charm_func=None): return wrap -def set_os_workload_status(configs, required_interfaces, charm_func=None): +def set_os_workload_status(configs, required_interfaces, charm_func=None, services=None, ports=None): """ Set workload status based on complete contexts. status-set missing or incomplete contexts and juju-log details of missing required data. charm_func is a charm specific function to run checking for charm specific requirements such as a VIP setting. + + This function also checks for whether the services defined are ACTUALLY + running and that the ports they advertise are open and being listened to. + + @param services - OPTIONAL: a [{'service': , 'ports': []] + The ports are optional. + If services is a [] then ports are ignored. + @param ports - OPTIONAL: an [] representing ports that shoudl be + open. + @returns None """ incomplete_rel_data = incomplete_relation_data(configs, required_interfaces) state = 'active' @@ -945,6 +957,65 @@ def set_os_workload_status(configs, required_interfaces, charm_func=None): else: message = charm_message + # If the charm thinks the unit is active, check that the actual services + # really are active. + if services is not None and state == 'active': + # if we're passed the dict() then just grab the values as a list. + if isinstance(services, dict): + services = services.values() + # either extract the list of services from the dictionary, or if + # it is a simple string, use that. i.e. works with mixed lists. + _s = [] + for s in services: + if isinstance(s, dict) and 'service' in s: + _s.append(s['service']) + if isinstance(s, str): + _s.append(s) + services_running = [service_running(s) for s in _s] + if not all(services_running): + not_running = [s for s, running in zip(_s, services_running) + if not running] + message = ("Services not running that should be: {}" + .format(", ".join(not_running))) + state = 'blocked' + # also verify that the ports that should be open are open + # NB, that ServiceManager objects only OPTIONALLY have ports + port_map = OrderedDict([(s['service'], s['ports']) + for s in services if 'ports' in s]) + if state == 'active' and port_map: + all_ports = list(itertools.chain(*port_map.values())) + ports_open = [port_has_listener('0.0.0.0', p) + for p in all_ports] + if not all(ports_open): + not_opened = [p for p, opened in zip(all_ports, ports_open) + if not opened] + map_not_open = OrderedDict() + for service, ports in port_map.items(): + closed_ports = set(ports).intersection(not_opened) + if closed_ports: + map_not_open[service] = closed_ports + # find which service has missing ports. They are in service + # order which makes it a bit easier. + message = ( + "Services with ports not open that should be: {}" + .format( + ", ".join([ + "{}: [{}]".format( + service, + ", ".join([str(v) for v in ports])) + for service, ports in map_not_open.items()]))) + state = 'blocked' + + if ports is not None and state == 'active': + # and we can also check ports which we don't know the service for + ports_open = [port_has_listener('0.0.0.0', p) for p in ports] + if not all(ports_open): + message = ( + "Ports which should be open, but are not: {}" + .format(", ".join([str(p) for p, v in zip(ports, ports_open) + if not v]))) + state = 'blocked' + # Set to active if all requirements have been met if state == 'active': message = "Unit is ready" diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 60ae52b8..fb1bee34 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -120,6 +120,7 @@ class PoolCreationError(Exception): """ A custom error to inform the caller that a pool creation failed. Provides an error message """ + def __init__(self, message): super(PoolCreationError, self).__init__(message) @@ -129,6 +130,7 @@ class Pool(object): An object oriented approach to Ceph pool creation. This base class is inherited by ReplicatedPool and ErasurePool. Do not call create() on this base class as it will not do anything. Instantiate a child class and call create(). """ + def __init__(self, service, name): self.service = service self.name = name @@ -180,36 +182,41 @@ class Pool(object): :return: int. The number of pgs to use. """ validator(value=pool_size, valid_type=int) - osds = get_osds(self.service) - if not osds: + osd_list = get_osds(self.service) + if not osd_list: # NOTE(james-page): Default to 200 for older ceph versions # which don't support OSD query from cli return 200 + osd_list_length = len(osd_list) # Calculate based on Ceph best practices - if osds < 5: + if osd_list_length < 5: return 128 - elif 5 < osds < 10: + elif 5 < osd_list_length < 10: return 512 - elif 10 < osds < 50: + elif 10 < osd_list_length < 50: return 4096 else: - estimate = (osds * 100) / pool_size + estimate = (osd_list_length * 100) / pool_size # Return the next nearest power of 2 index = bisect.bisect_right(powers_of_two, estimate) return powers_of_two[index] class ReplicatedPool(Pool): - def __init__(self, service, name, replicas=2): + def __init__(self, service, name, pg_num=None, replicas=2): super(ReplicatedPool, self).__init__(service=service, name=name) self.replicas = replicas + if pg_num is None: + self.pg_num = self.get_pgs(self.replicas) + else: + self.pg_num = pg_num def create(self): if not pool_exists(self.service, self.name): # Create it - pgs = self.get_pgs(self.replicas) - cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', self.name, str(pgs)] + cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', + self.name, str(self.pg_num)] try: check_call(cmd) except CalledProcessError: @@ -241,7 +248,7 @@ class ErasurePool(Pool): pgs = self.get_pgs(int(erasure_profile['k']) + int(erasure_profile['m'])) # Create it - cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', self.name, str(pgs), + cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', self.name, str(pgs), str(pgs), 'erasure', self.erasure_code_profile] try: check_call(cmd) @@ -322,7 +329,8 @@ def set_pool_quota(service, pool_name, max_bytes): :return: None. Can raise CalledProcessError """ # Set a byte quota on a RADOS pool in ceph. - cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name, 'max_bytes', max_bytes] + cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name, + 'max_bytes', str(max_bytes)] try: check_call(cmd) except CalledProcessError: @@ -343,7 +351,25 @@ def remove_pool_quota(service, pool_name): raise -def create_erasure_profile(service, profile_name, erasure_plugin_name='jerasure', failure_domain='host', +def remove_erasure_profile(service, profile_name): + """ + Create a new erasure code profile if one does not already exist for it. Updates + the profile if it exists. Please see http://docs.ceph.com/docs/master/rados/operations/erasure-code-profile/ + for more details + :param service: six.string_types. The Ceph user name to run the command under + :param profile_name: six.string_types + :return: None. Can raise CalledProcessError + """ + cmd = ['ceph', '--id', service, 'osd', 'erasure-code-profile', 'rm', + profile_name] + try: + check_call(cmd) + except CalledProcessError: + raise + + +def create_erasure_profile(service, profile_name, erasure_plugin_name='jerasure', + failure_domain='host', data_chunks=2, coding_chunks=1, locality=None, durability_estimator=None): """