diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 91ddcecd..42316331 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -14,6 +14,7 @@ import glob import json +import math import os import re import time @@ -90,6 +91,8 @@ from charmhelpers.contrib.network.ip import ( from charmhelpers.contrib.openstack.utils import ( config_flags_parser, get_host_ip, + git_determine_usr_bin, + git_determine_python_path, enable_memcache, ) from charmhelpers.core.unitdata import kv @@ -1208,6 +1211,43 @@ class WorkerConfigContext(OSContextGenerator): return ctxt +class WSGIWorkerConfigContext(WorkerConfigContext): + + def __init__(self, name=None, script=None, admin_script=None, + public_script=None, process_weight=1.00, + admin_process_weight=0.75, public_process_weight=0.25): + self.service_name = name + self.user = name + self.group = name + self.script = script + self.admin_script = admin_script + self.public_script = public_script + self.process_weight = process_weight + self.admin_process_weight = admin_process_weight + self.public_process_weight = public_process_weight + + def __call__(self): + multiplier = config('worker-multiplier') or 1 + total_processes = self.num_cpus * multiplier + ctxt = { + "service_name": self.service_name, + "user": self.user, + "group": self.group, + "script": self.script, + "admin_script": self.admin_script, + "public_script": self.public_script, + "processes": int(math.ceil(self.process_weight * total_processes)), + "admin_processes": int(math.ceil(self.admin_process_weight * + total_processes)), + "public_processes": int(math.ceil(self.public_process_weight * + total_processes)), + "threads": 1, + "usr_bin": git_determine_usr_bin(), + "python_path": git_determine_python_path(), + } + return ctxt + + class ZeroMQContext(OSContextGenerator): interfaces = ['zeromq-configuration'] @@ -1521,9 +1561,18 @@ class MemcacheContext(OSContextGenerator): This context provides options for configuring a local memcache client and server """ + + def __init__(self, package=None): + """ + @param package: Package to examine to extrapolate OpenStack release. + Used when charms have no openstack-origin config + option (ie subordinates) + """ + self.package = package + def __call__(self): ctxt = {} - ctxt['use_memcache'] = enable_memcache(config('openstack-origin')) + ctxt['use_memcache'] = enable_memcache(package=self.package) if ctxt['use_memcache']: # Trusty version of memcached does not support ::1 as a listen # address so use host file entry instead diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 59f9f512..77d04635 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -1119,6 +1119,35 @@ def git_generate_systemd_init_files(templates_dir): shutil.copyfile(service_source, service_dest) +def git_determine_usr_bin(): + """Return the /usr/bin path for Apache2 config. + + The /usr/bin path will be located in the virtualenv if the charm + is configured to deploy from source. + """ + if git_install_requested(): + projects_yaml = config('openstack-origin-git') + projects_yaml = git_default_repos(projects_yaml) + return os.path.join(git_pip_venv_dir(projects_yaml), 'bin') + else: + return '/usr/bin' + + +def git_determine_python_path(): + """Return the python-path for Apache2 config. + + Returns 'None' unless the charm is configured to deploy from source, + in which case the path of the virtualenv's site-packages is returned. + """ + if git_install_requested(): + projects_yaml = config('openstack-origin-git') + projects_yaml = git_default_repos(projects_yaml) + return os.path.join(git_pip_venv_dir(projects_yaml), + 'lib/python2.7/site-packages') + else: + return None + + def os_workload_status(configs, required_interfaces, charm_func=None): """ Decorator to set workload status based on complete contexts @@ -1927,16 +1956,24 @@ def os_application_version_set(package): application_version_set(application_version) -def enable_memcache(source=None, release=None): +def enable_memcache(source=None, release=None, package=None): """Determine if memcache should be enabled on the local unit - @param source: source string for charm @param release: release of OpenStack currently deployed + @param package: package to derive OpenStack version deployed @returns boolean Whether memcache should be enabled """ - if not release: - release = get_os_codename_install_source(source) - return release >= 'mitaka' + _release = None + if release: + _release = release + else: + _release = os_release(package, base='icehouse') + if not _release: + _release = get_os_codename_install_source(source) + + # TODO: this should be changed to a numeric comparison using a known list + # of releases and comparing by index. + return _release >= 'mitaka' def token_cache_pkgs(source=None, release=None): diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index 94fc996c..d1cb68db 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -616,6 +616,20 @@ def close_port(port, protocol="TCP"): subprocess.check_call(_args) +def open_ports(start, end, protocol="TCP"): + """Opens a range of service network ports""" + _args = ['open-port'] + _args.append('{}-{}/{}'.format(start, end, protocol)) + subprocess.check_call(_args) + + +def close_ports(start, end, protocol="TCP"): + """Close a range of service network ports""" + _args = ['close-port'] + _args.append('{}-{}/{}'.format(start, end, protocol)) + subprocess.check_call(_args) + + @cached def unit_get(attribute): """Get the unit ID for the remote unit""" diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 04cadb3a..af5bd648 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -306,15 +306,17 @@ def add_user_to_group(username, group): subprocess.check_call(cmd) -def rsync(from_path, to_path, flags='-r', options=None): +def rsync(from_path, to_path, flags='-r', options=None, timeout=None): """Replicate the contents of a path""" options = options or ['--delete', '--executability'] cmd = ['/usr/bin/rsync', flags] + if timeout: + cmd = ['timeout', str(timeout)] + cmd cmd.extend(options) cmd.append(from_path) cmd.append(to_path) log(" ".join(cmd)) - return subprocess.check_output(cmd).decode('UTF-8').strip() + return subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode('UTF-8').strip() def symlink(source, destination): diff --git a/hooks/cinder_contexts.py b/hooks/cinder_contexts.py index 83f2437f..b1ec311c 100644 --- a/hooks/cinder_contexts.py +++ b/hooks/cinder_contexts.py @@ -88,6 +88,7 @@ class HAProxyContext(OSContextGenerator): ctxt = { 'service_ports': {'cinder_api': [haproxy_port, apache_port]}, 'osapi_volume_listen_port': api_port, + 'port': api_port, } return ctxt @@ -100,7 +101,7 @@ class ApacheSSLContext(SSLContext): def __call__(self): # late import to work around circular dependency from cinder_utils import service_enabled - if not service_enabled('cinder-api'): + if not service_enabled('cinder-common'): return {} return super(ApacheSSLContext, self).__call__() diff --git a/hooks/cinder_hooks.py b/hooks/cinder_hooks.py index d71a516f..ad6dbc03 100755 --- a/hooks/cinder_hooks.py +++ b/hooks/cinder_hooks.py @@ -24,6 +24,7 @@ from subprocess import ( from cinder_utils import ( determine_packages, + disable_package_apache_site, do_openstack_upgrade, git_install, juju_log, @@ -31,6 +32,7 @@ from cinder_utils import ( configure_lvm_storage, register_configs, restart_map, + run_in_apache, services, service_enabled, service_restart, @@ -140,6 +142,9 @@ def install(): apt_update() apt_install(determine_packages(), fatal=True) + if run_in_apache(): + disable_package_apache_site() + status_set('maintenance', 'Git install') git_install(config('openstack-origin-git')) diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index e3e81bb7..710f3b65 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -180,6 +180,8 @@ APACHE_SITE_CONF = '/etc/apache2/sites-available/openstack_https_frontend' APACHE_SITE_24_CONF = '/etc/apache2/sites-available/' \ 'openstack_https_frontend.conf' MEMCACHED_CONF = '/etc/memcached.conf' +WSGI_CINDER_API_CONF = '/etc/apache2/sites-enabled/wsgi-openstack-api.conf' +PACKAGE_CINDER_API_CONF = '/etc/apache2/sites-enabled/cinder-wsgi.conf' VERSION_PACKAGE = 'cinder-common' @@ -323,6 +325,21 @@ def resource_map(release=None): 'contexts': [context.MemcacheContext()], 'services': ['memcached']} + if run_in_apache(): + for cfile in resource_map: + svcs = resource_map[cfile]['services'] + if 'cinder-api' in svcs: + svcs.remove('cinder-api') + if 'apache2' not in svcs: + svcs.append('apache2') + wsgi_script = "/usr/bin/cinder-wsgi" + resource_map[WSGI_CINDER_API_CONF] = { + 'contexts': [context.WSGIWorkerConfigContext(name="cinder", + script=wsgi_script), + cinder_contexts.HAProxyContext()], + 'services': ['apache2'] + } + return resource_map @@ -731,6 +748,9 @@ def do_openstack_upgrade(configs): configs.set_release(openstack_release=new_os_rel) configs.write_all() + if run_in_apache(): + disable_package_apache_site() + # Stop/start services and migrate DB if leader [service_stop(s) for s in services()] if is_elected_leader(CLUSTER_RES): @@ -1053,3 +1073,18 @@ def _pause_resume_helper(f, configs): f(assess_status_func(configs), services=services(), ports=None) + + +def run_in_apache(): + """Return true if cinder API is run under apache2 with mod_wsgi in + this release. + """ + return os_release('cinder-common') >= 'ocata' + + +def disable_package_apache_site(): + """Ensure that the package-provided apache configuration is disabled to + prevent it from conflicting with the charm-provided version. + """ + if os.path.exists(PACKAGE_CINDER_API_CONF): + subprocess.check_call(['a2dissite', 'cinder-wsgi']) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index f4ab63ef..0c218aa5 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -767,10 +767,13 @@ class CinderBasicDeployment(OpenStackAmuletDeployment): # Services which are expected to restart upon config change services = { - 'cinder-api': conf_file, 'cinder-scheduler': conf_file, 'cinder-volume': conf_file } + if self._get_openstack_release() >= self.xenial_ocata: + services['apache2'] = conf_file + else: + services['cinder-api'] = conf_file # Make config change, check for service restarts u.log.debug('Making config change on {}...'.format(juju_service)) diff --git a/unit_tests/test_cinder_hooks.py b/unit_tests/test_cinder_hooks.py index a680050c..8972d52e 100644 --- a/unit_tests/test_cinder_hooks.py +++ b/unit_tests/test_cinder_hooks.py @@ -87,6 +87,7 @@ TO_PATCH = [ 'configure_installation_source', 'openstack_upgrade_available', 'os_release', + 'run_in_apache', # charmhelpers.contrib.openstack.openstack.ha.utils 'update_dns_ha_resource_params', # charmhelpers.contrib.hahelpers.cluster_utils