From b699e56699ac174a75af1bd257dfa35132231e3f Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 27 Apr 2017 09:28:27 +0100 Subject: [PATCH] Make worker-multiplier sane in container environments Resync charm-helpers to pickup the capped worker-multiplier changes when deploying in containers. Drop the default value for worker-multiplier of 2.0; this is now handled from within the codebase rather than via a default configuration value, reflecting the differing behaviours between container and non-container deployments. Change-Id: Iec5fa21b0e1b377bcb22ad5193c84aa0ae525f16 Closes-Bug: 1665270 --- config.yaml | 8 ++- .../contrib/openstack/amulet/utils.py | 2 +- .../charmhelpers/contrib/openstack/context.py | 70 ++++++++++++------- 3 files changed, 51 insertions(+), 29 deletions(-) diff --git a/config.yaml b/config.yaml index 09e67eb3..fe016e5f 100644 --- a/config.yaml +++ b/config.yaml @@ -336,11 +336,13 @@ options: normal console-log output for an instance. worker-multiplier: type: float - default: 2.0 + default: description: | The CPU core multiplier to use when configuring worker processes for - Nova and Neutron. By default, the number of workers for each daemon - is set to twice the number of CPU cores a service unit has. + this service. By default, the number of workers for each daemon is + set to twice the number of CPU cores a service unit has. When deployed + in a LXD container, this default value will be capped to 4 workers + unless this configuration option is set. cpu-allocation-ratio: type: float default: 16.0 diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index 346e6fea..bcef4cd0 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -547,7 +547,7 @@ class OpenStackAmuletUtils(AmuletUtils): """Create the specified instance.""" self.log.debug('Creating instance ' '({}|{}|{})'.format(instance_name, image_name, flavor)) - image = nova.images.find(name=image_name) + image = nova.glance.find_image(image_name) flavor = nova.flavors.find(name=flavor) instance = nova.servers.create(name=instance_name, image=image, flavor=flavor) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index c808250c..2adf2cb8 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -537,6 +537,8 @@ class HAProxyContext(OSContextGenerator): """Provides half a context for the haproxy template, which describes all peers to be included in the cluster. Each charm needs to include its own context generator that describes the port mapping. + + :side effect: mkdir is called on HAPROXY_RUN_DIR """ interfaces = ['cluster'] @@ -1230,31 +1232,50 @@ MAX_DEFAULT_WORKERS = 4 DEFAULT_MULTIPLIER = 2 +def _calculate_workers(): + ''' + Determine the number of worker processes based on the CPU + count of the unit containing the application. + + Workers will be limited to MAX_DEFAULT_WORKERS in + container environments where no worker-multipler configuration + option been set. + + @returns int: number of worker processes to use + ''' + multiplier = config('worker-multiplier') or DEFAULT_MULTIPLIER + count = int(_num_cpus() * multiplier) + if multiplier > 0 and count == 0: + count = 1 + + if config('worker-multiplier') is None and is_container(): + # NOTE(jamespage): Limit unconfigured worker-multiplier + # to MAX_DEFAULT_WORKERS to avoid insane + # worker configuration in LXD containers + # on large servers + # Reference: https://pad.lv/1665270 + count = min(count, MAX_DEFAULT_WORKERS) + + return count + + +def _num_cpus(): + ''' + Compatibility wrapper for calculating the number of CPU's + a unit has. + + @returns: int: number of CPU cores detected + ''' + try: + return psutil.cpu_count() + except AttributeError: + return psutil.NUM_CPUS + + class WorkerConfigContext(OSContextGenerator): - @property - def num_cpus(self): - # NOTE: use cpu_count if present (16.04 support) - if hasattr(psutil, 'cpu_count'): - return psutil.cpu_count() - else: - return psutil.NUM_CPUS - def __call__(self): - multiplier = config('worker-multiplier') or DEFAULT_MULTIPLIER - count = int(self.num_cpus * multiplier) - if multiplier > 0 and count == 0: - count = 1 - - if config('worker-multiplier') is None and is_container(): - # NOTE(jamespage): Limit unconfigured worker-multiplier - # to MAX_DEFAULT_WORKERS to avoid insane - # worker configuration in LXD containers - # on large servers - # Reference: https://pad.lv/1665270 - count = min(count, MAX_DEFAULT_WORKERS) - - ctxt = {"workers": count} + ctxt = {"workers": _calculate_workers()} return ctxt @@ -1262,7 +1283,7 @@ class WSGIWorkerConfigContext(WorkerConfigContext): def __init__(self, name=None, script=None, admin_script=None, public_script=None, process_weight=1.00, - admin_process_weight=0.75, public_process_weight=0.25): + admin_process_weight=0.25, public_process_weight=0.75): self.service_name = name self.user = name self.group = name @@ -1274,8 +1295,7 @@ class WSGIWorkerConfigContext(WorkerConfigContext): self.public_process_weight = public_process_weight def __call__(self): - multiplier = config('worker-multiplier') or 1 - total_processes = self.num_cpus * multiplier + total_processes = _calculate_workers() ctxt = { "service_name": self.service_name, "user": self.user,