Enable API to run under apache2 when deployed from source.

The keystone charm runs the keystone API under apache2 for liberty
and above. This patch enables the keystone API to run under apache2
when deployed from source for liberty and above.

Change-Id: I5eccf38aad9668248f4f94523d61f7bd40ed5c30

charmhelpers/contrib/openstack/amulet/deployment.py

@@ -71,7 +71,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         base_charms = {
             'mysql': ['precise', 'trusty'],
             'mongodb': ['precise', 'trusty'],
-            'nrpe': ['precise', 'trusty'],
+            'nrpe': ['precise', 'trusty', 'wily', 'xenial'],
         }
 
         for svc in other_services:
@@ -112,7 +112,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
 
         # Charms which should use the source config option
         use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
-                      'ceph-osd', 'ceph-radosgw', 'ceph-mon']
+                      'ceph-osd', 'ceph-radosgw', 'ceph-mon', 'ceph-proxy']
 
         # Charms which can not use openstack-origin, ie. many subordinates
         no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe',

charmhelpers/contrib/openstack/context.py

@@ -57,6 +57,7 @@ from charmhelpers.core.host import (
     mkdir,
     write_file,
     pwgen,
+    lsb_release,
 )
 from charmhelpers.contrib.hahelpers.cluster import (
     determine_apache_port,
@@ -1195,7 +1196,10 @@ class WorkerConfigContext(OSContextGenerator):
 
     def __call__(self):
         multiplier = config('worker-multiplier') or 0
-        ctxt = {"workers": self.num_cpus * multiplier}
+        count = int(self.num_cpus * multiplier)
+        if multiplier > 0 and count == 0:
+            count = 1
+        ctxt = {"workers": count}
         return ctxt
 
 
@@ -1436,7 +1440,8 @@ class AppArmorContext(OSContextGenerator):
         :return ctxt: Dictionary of the apparmor profile or None
         """
         if config('aa-profile-mode') in ['disable', 'enforce', 'complain']:
-            ctxt = {'aa_profile_mode': config('aa-profile-mode')}
+            ctxt = {'aa_profile_mode': config('aa-profile-mode'),
+                    'ubuntu_release': lsb_release()['DISTRIB_RELEASE']}
         else:
             ctxt = None
         return ctxt

charmhelpers/contrib/openstack/utils.py

@@ -220,7 +220,6 @@ GIT_DEFAULT_REPOS = {
 }
 
 GIT_DEFAULT_BRANCHES = {
-    'kilo': 'stable/kilo',
     'liberty': 'stable/liberty',
     'mitaka': 'stable/mitaka',
     'master': 'master',
@@ -413,7 +412,8 @@ def os_release(package, base='essex'):
     global os_rel
     if os_rel:
         return os_rel
-    os_rel = (get_os_codename_package(package, fatal=False) or
+    os_rel = (git_os_codename_install_source(config('openstack-origin-git')) or
+              get_os_codename_package(package, fatal=False) or
               get_os_codename_install_source(config('openstack-origin')) or
               base)
     return os_rel
@@ -719,7 +719,24 @@ def git_install_requested():
     return config('openstack-origin-git') is not None
 
 
-requirements_dir = None
+def git_os_codename_install_source(projects_yaml):
+    """
+    Returns OpenStack codename of release being installed from source.
+    """
+    if git_install_requested():
+        projects = _git_yaml_load(projects_yaml)
+
+        if projects in GIT_DEFAULT_BRANCHES.keys():
+            if projects == 'master':
+                return 'newton'
+            return projects
+
+        if 'release' in projects:
+            if projects['release'] == 'master':
+                return 'newton'
+            return projects['release']
+
+    return None
 
 
 def git_default_repos(projects_yaml):
@@ -740,12 +757,6 @@ def git_default_repos(projects_yaml):
             }
             repos = [repo]
 
-            # NOTE(coreycb): This is a temp work-around until the requirements
-            # repo moves from stable/kilo branch to kilo-eol tag. The core
-            # repos have already done this.
-            if default == 'kilo':
-                branch = 'kilo-eol'
-
             # neutron-* and nova-* charms require some additional repos
             if service in ['neutron-api', 'neutron-gateway',
                            'neutron-openvswitch']:
@@ -778,7 +789,7 @@ def git_default_repos(projects_yaml):
             }
             repos.append(repo)
 
-            return yaml.dump(dict(repositories=repos))
+            return yaml.dump(dict(repositories=repos, release=default))
 
     return projects_yaml
 
@@ -793,6 +804,9 @@ def _git_yaml_load(projects_yaml):
     return yaml.load(projects_yaml)
 
 
+requirements_dir = None
+
+
 def git_clone_and_install(projects_yaml, core_project):
     """
     Clone/install all specified OpenStack repositories.
@@ -856,6 +870,10 @@ def git_clone_and_install(projects_yaml, core_project):
             # upper-constraints didn't exist until after icehouse
             if not os.path.isfile(constraints):
                 constraints = None
+            # use constraints unless project yaml sets use_constraints to false
+            if 'use_constraints' in projects.keys():
+                if not projects['use_constraints']:
+                    constraints = None
         else:
             repo_dir = _git_clone_and_install_single(repo, branch, depth,
                                                      parent_dir, http_proxy,
@@ -882,6 +900,8 @@ def _git_validate_projects_yaml(projects, core_project):
     if projects['repositories'][-1]['name'] != core_project:
         error_out('{} git repo must be specified last'.format(core_project))
 
+    _git_ensure_key_exists('release', projects)
+
 
 def _git_ensure_key_exists(key, keys):
     """

charmhelpers/contrib/storage/linux/ceph.py

@@ -21,9 +21,10 @@
 #  James Page <james.page@ubuntu.com>
 #  Adam Gandelman <adamg@ubuntu.com>
 #
-import bisect
+
 import errno
 import hashlib
+import math
 import six
 
 import os
@@ -76,8 +77,16 @@ log to syslog = {use_syslog}
 err to syslog = {use_syslog}
 clog to syslog = {use_syslog}
 """
-# For 50 < osds < 240,000 OSDs (Roughly 1 Exabyte at 6T OSDs)
-powers_of_two = [8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576, 2097152, 4194304, 8388608]
+
+# The number of placement groups per OSD to target for placement group
+# calculations. This number is chosen as 100 due to the ceph PG Calc
+# documentation recommending to choose 100 for clusters which are not
+# expected to increase in the foreseeable future. Since the majority of the
+# calculations are done on deployment, target the case of non-expanding
+# clusters as the default.
+DEFAULT_PGS_PER_OSD_TARGET = 100
+DEFAULT_POOL_WEIGHT = 10.0
+LEGACY_PG_COUNT = 200
 
 
 def validator(value, valid_type, valid_range=None):
@@ -184,42 +193,106 @@ class Pool(object):
             check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove-overlay', self.name])
             check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove', self.name, cache_pool])
 
-    def get_pgs(self, pool_size):
-        """
-        :param pool_size: int. pool_size is either the number of replicas for replicated pools or the K+M sum for
-            erasure coded pools
+    def get_pgs(self, pool_size, percent_data=DEFAULT_POOL_WEIGHT):
+        """Return the number of placement groups to use when creating the pool.
+
+        Returns the number of placement groups which should be specified when
+        creating the pool. This is based upon the calculation guidelines
+        provided by the Ceph Placement Group Calculator (located online at
+        http://ceph.com/pgcalc/).
+
+        The number of placement groups are calculated using the following:
+
+            (Target PGs per OSD) * (OSD #) * (%Data)
+            ----------------------------------------
+                         (Pool size)
+
+        Per the upstream guidelines, the OSD # should really be considered
+        based on the number of OSDs which are eligible to be selected by the
+        pool. Since the pool creation doesn't specify any of CRUSH set rules,
+        the default rule will be dependent upon the type of pool being
+        created (replicated or erasure).
+
+        This code makes no attempt to determine the number of OSDs which can be
+        selected for the specific rule, rather it is left to the user to tune
+        in the form of 'expected-osd-count' config option.
+
+        :param pool_size: int. pool_size is either the number of replicas for
+            replicated pools or the K+M sum for erasure coded pools
+        :param percent_data: float. the percentage of data that is expected to
+            be contained in the pool for the specific OSD set. Default value
+            is to assume 10% of the data is for this pool, which is a
+            relatively low % of the data but allows for the pg_num to be
+            increased. NOTE: the default is primarily to handle the scenario
+            where related charms requiring pools has not been upgraded to
+            include an update to indicate their relative usage of the pools.
         :return: int.  The number of pgs to use.
         """
+
+        # Note: This calculation follows the approach that is provided
+        # by the Ceph PG Calculator located at http://ceph.com/pgcalc/.
         validator(value=pool_size, valid_type=int)
+
+        # Ensure that percent data is set to something - even with a default
+        # it can be set to None, which would wreak havoc below.
+        if percent_data is None:
+            percent_data = DEFAULT_POOL_WEIGHT
+
+        # If the expected-osd-count is specified, then use the max between
+        # the expected-osd-count and the actual osd_count
         osd_list = get_osds(self.service)
-        if not osd_list:
+        expected = config('expected-osd-count') or 0
+
+        if osd_list:
+            osd_count = max(expected, len(osd_list))
+
+            # Log a message to provide some insight if the calculations claim
+            # to be off because someone is setting the expected count and
+            # there are more OSDs in reality. Try to make a proper guess
+            # based upon the cluster itself.
+            if expected and osd_count != expected:
+                log("Found more OSDs than provided expected count. "
+                    "Using the actual count instead", INFO)
+        elif expected:
+            # Use the expected-osd-count in older ceph versions to allow for
+            # a more accurate pg calculations
+            osd_count = expected
+        else:
             # NOTE(james-page): Default to 200 for older ceph versions
             # which don't support OSD query from cli
-            return 200
+            return LEGACY_PG_COUNT
 
-        osd_list_length = len(osd_list)
-        # Calculate based on Ceph best practices
-        if osd_list_length < 5:
-            return 128
-        elif 5 < osd_list_length < 10:
-            return 512
-        elif 10 < osd_list_length < 50:
-            return 4096
+        percent_data /= 100.0
+        target_pgs_per_osd = config('pgs-per-osd') or DEFAULT_PGS_PER_OSD_TARGET
+        num_pg = (target_pgs_per_osd * osd_count * percent_data) // pool_size
+
+        # The CRUSH algorithm has a slight optimization for placement groups
+        # with powers of 2 so find the nearest power of 2. If the nearest
+        # power of 2 is more than 25% below the original value, the next
+        # highest value is used. To do this, find the nearest power of 2 such
+        # that 2^n <= num_pg, check to see if its within the 25% tolerance.
+        exponent = math.floor(math.log(num_pg, 2))
+        nearest = 2 ** exponent
+        if (num_pg - nearest) > (num_pg * 0.25):
+            # Choose the next highest power of 2 since the nearest is more
+            # than 25% below the original value.
+            return int(nearest * 2)
         else:
-            estimate = (osd_list_length * 100) / pool_size
-            # Return the next nearest power of 2
-            index = bisect.bisect_right(powers_of_two, estimate)
-            return powers_of_two[index]
+            return int(nearest)
 
 
 class ReplicatedPool(Pool):
-    def __init__(self, service, name, pg_num=None, replicas=2):
+    def __init__(self, service, name, pg_num=None, replicas=2,
+                 percent_data=10.0):
         super(ReplicatedPool, self).__init__(service=service, name=name)
         self.replicas = replicas
-        if pg_num is None:
-            self.pg_num = self.get_pgs(self.replicas)
+        if pg_num:
+            # Since the number of placement groups were specified, ensure
+            # that there aren't too many created.
+            max_pgs = self.get_pgs(self.replicas, 100.0)
+            self.pg_num = min(pg_num, max_pgs)
         else:
-            self.pg_num = pg_num
+            self.pg_num = self.get_pgs(self.replicas, percent_data)
 
     def create(self):
         if not pool_exists(self.service, self.name):
@@ -238,30 +311,39 @@ class ReplicatedPool(Pool):
 
 # Default jerasure erasure coded pool
 class ErasurePool(Pool):
-    def __init__(self, service, name, erasure_code_profile="default"):
+    def __init__(self, service, name, erasure_code_profile="default",
+                 percent_data=10.0):
         super(ErasurePool, self).__init__(service=service, name=name)
         self.erasure_code_profile = erasure_code_profile
+        self.percent_data = percent_data
 
     def create(self):
         if not pool_exists(self.service, self.name):
-            # Try to find the erasure profile information so we can properly size the pgs
-            erasure_profile = get_erasure_profile(service=self.service, name=self.erasure_code_profile)
+            # Try to find the erasure profile information in order to properly
+            # size the number of placement groups. The size of an erasure
+            # coded placement group is calculated as k+m.
+            erasure_profile = get_erasure_profile(self.service,
+                                                  self.erasure_code_profile)
 
             # Check for errors
             if erasure_profile is None:
-                log(message='Failed to discover erasure_profile named={}'.format(self.erasure_code_profile),
-                    level=ERROR)
-                raise PoolCreationError(message='unable to find erasure profile {}'.format(self.erasure_code_profile))
+                msg = ("Failed to discover erasure profile named "
+                       "{}".format(self.erasure_code_profile))
+                log(msg, level=ERROR)
+                raise PoolCreationError(msg)
             if 'k' not in erasure_profile or 'm' not in erasure_profile:
                 # Error
-                log(message='Unable to find k (data chunks) or m (coding chunks) in {}'.format(erasure_profile),
-                    level=ERROR)
-                raise PoolCreationError(
-                    message='unable to find k (data chunks) or m (coding chunks) in {}'.format(erasure_profile))
+                msg = ("Unable to find k (data chunks) or m (coding chunks) "
+                       "in erasure profile {}".format(erasure_profile))
+                log(msg, level=ERROR)
+                raise PoolCreationError(msg)
 
-            pgs = self.get_pgs(int(erasure_profile['k']) + int(erasure_profile['m']))
+            k = int(erasure_profile['k'])
+            m = int(erasure_profile['m'])
+            pgs = self.get_pgs(k + m, self.percent_data)
             # Create it
-            cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', self.name, str(pgs), str(pgs),
+            cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create',
+                   self.name, str(pgs), str(pgs),
                    'erasure', self.erasure_code_profile]
             try:
                 check_call(cmd)
@@ -955,16 +1037,22 @@ class CephBrokerRq(object):
             self.request_id = str(uuid.uuid1())
         self.ops = []
 
-    def add_op_create_pool(self, name, replica_count=3, pg_num=None):
+    def add_op_create_pool(self, name, replica_count=3, pg_num=None,
+                           weight=None):
         """Adds an operation to create a pool.
 
         @param pg_num setting:  optional setting. If not provided, this value
         will be calculated by the broker based on how many OSDs are in the
         cluster at the time of creation. Note that, if provided, this value
         will be capped at the current available maximum.
+        @param weight: the percentage of data the pool makes up
         """
+        if pg_num and weight:
+            raise ValueError('pg_num and weight are mutually exclusive')
+
         self.ops.append({'op': 'create-pool', 'name': name,
-                         'replicas': replica_count, 'pg_num': pg_num})
+                         'replicas': replica_count, 'pg_num': pg_num,
+                         'weight': weight})
 
     def set_ops(self, ops):
         """Set request ops to provided value.
@@ -982,7 +1070,7 @@ class CephBrokerRq(object):
     def _ops_equal(self, other):
         if len(self.ops) == len(other.ops):
             for req_no in range(0, len(self.ops)):
-                for key in ['replicas', 'name', 'op', 'pg_num']:
+                for key in ['replicas', 'name', 'op', 'pg_num', 'weight']:
                     if self.ops[req_no].get(key) != other.ops[req_no].get(key):
                         return False
         else:

charmhelpers/core/host.py

@@ -174,7 +174,7 @@ def init_is_systemd():
 
 
 def adduser(username, password=None, shell='/bin/bash', system_user=False,
-            primary_group=None, secondary_groups=None, uid=None):
+            primary_group=None, secondary_groups=None, uid=None, home_dir=None):
     """Add a user to the system.
 
     Will log but otherwise succeed if the user already exists.
@@ -186,6 +186,7 @@ def adduser(username, password=None, shell='/bin/bash', system_user=False,
     :param str primary_group: Primary group for user; defaults to username
     :param list secondary_groups: Optional list of additional groups
     :param int uid: UID for user being created
+    :param str home_dir: Home directory for user
 
     :returns: The password database entry struct, as returned by `pwd.getpwnam`
     """
@@ -200,6 +201,8 @@ def adduser(username, password=None, shell='/bin/bash', system_user=False,
         cmd = ['useradd']
         if uid:
             cmd.extend(['--uid', str(uid)])
+        if home_dir:
+            cmd.extend(['--home', str(home_dir)])
         if system_user or password is None:
             cmd.append('--system')
         else:

config.yaml

@@ -37,10 +37,6 @@ options:
 
       NOTE: updating this setting to a source that is known to provide
       a later version of OpenStack will trigger a software upgrade.
-
-      NOTE: when openstack-origin-git is specified, openstack specific
-      packages will be installed from source rather than from the
-      openstack-origin repository.
   openstack-origin-git:
     default:
     type: string
@@ -50,8 +46,6 @@ options:
 
       The default Openstack release name may be one of the following, where
       the corresponding OpenStack github branch will be used:
-        * icehouse
-        * kilo
         * liberty
         * mitaka
         * master
@@ -65,9 +59,7 @@ options:
         - {name: keystone,
            repository: 'git://github.com/openstack/keystone',
            branch: master}
-
-      Note that the installed config files will be determined based on
-      the OpenStack release of the openstack-origin option.
+        release: master
   config-file:
     default: "/etc/keystone/keystone.conf"
     type: string

hooks/keystone_context.py

@@ -295,6 +295,9 @@ class TokenFlushContext(context.OSContextGenerator):
 class WSGIWorkerConfigContext(context.WorkerConfigContext):
 
     def __call__(self):
+        from keystone_utils import (
+            determine_usr_bin, determine_python_path,
+        )
         multiplier = config('worker-multiplier') or 1
         total_processes = self.num_cpus * multiplier
         ctxt = {
@@ -303,5 +306,7 @@ class WSGIWorkerConfigContext(context.WorkerConfigContext):
             # Keystone install guide suggests 1 but offers no science
             "public_threads": 1,
             "admin_threads": 1,
+            "usr_bin": determine_usr_bin(),
+            "python_path": determine_python_path(),
         }
         return ctxt

hooks/keystone_hooks.py

@@ -164,10 +164,12 @@ def install():
         # NOTE: ensure that packaging provided
         #       apache configuration is disabled
         #       as it will conflict with the charm
-        #       provided version
+        #       provided version. when deployed from
+        #       source, init scripts aren't installed.
         if os.path.exists(PACKAGE_KEYSTONE_CONF):
             check_call(['a2dissite', 'keystone'])
-        service_pause('keystone')
+        if not git_install_requested():
+            service_pause('keystone')
 
     status_set('maintenance', 'Git install')
     git_install(config('openstack-origin-git'))
@@ -223,7 +225,9 @@ def config_changed_postupgrade():
         # immediatly as charm querys its local keystone before restart
         # decorator can fire
         apt_install(filter_installed_packages(determine_packages()))
-        service_pause('keystone')
+        # when deployed from source, init scripts aren't installed
+        if not git_install_requested():
+            service_pause('keystone')
         CONFIGS.write(WSGI_KEYSTONE_CONF)
         if not is_unit_paused_set():
             restart_pid_check('apache2')

hooks/keystone_utils.py

@@ -59,12 +59,12 @@ from charmhelpers.contrib.openstack.utils import (
     configure_installation_source,
     error_out,
     get_os_codename_install_source,
-    git_install_requested,
     git_clone_and_install,
     git_default_repos,
+    git_install_requested,
+    git_pip_venv_dir,
     git_src_dir,
     git_yaml_value,
-    git_pip_venv_dir,
     os_release,
     save_script_rc as _save_script_rc,
     pause_unit,
@@ -2071,6 +2071,35 @@ def is_db_ready(use_current_context=False, db_rel=None):
     return not rel_has_units
 
 
+def determine_usr_bin():
+    """Return the /usr/bin path for Apache2 vhost config.
+
+    The /usr/bin path will be located in the virtualenv if the charm
+    is configured to deploy keystone from source.
+    """
+    if git_install_requested():
+        projects_yaml = config('openstack-origin-git')
+        projects_yaml = git_default_repos(projects_yaml)
+        return os.path.join(git_pip_venv_dir(projects_yaml), 'bin')
+    else:
+        return '/usr/bin'
+
+
+def determine_python_path():
+    """Return the python-path for Apache2 vhost config.
+
+    Returns None unless the charm is configured to deploy keystone from source,
+    in which case the path of the virtualenv's site-packages is returned.
+    """
+    if git_install_requested():
+        projects_yaml = config('openstack-origin-git')
+        projects_yaml = git_default_repos(projects_yaml)
+        return os.path.join(git_pip_venv_dir(projects_yaml),
+                            'lib/python2.7/site-packages')
+    else:
+        return None
+
+
 def git_install(projects_yaml):
     """Perform setup, and install git repos specified in yaml parameter."""
     if git_install_requested():
@@ -2092,7 +2121,8 @@ def git_pre_install():
         '/var/log/keystone/keystone.log',
     ]
 
-    adduser('keystone', shell='/bin/bash', system_user=True)
+    adduser('keystone', shell='/bin/bash', system_user=True,
+            home_dir='/var/lib/keystone')
     add_group('keystone', system_group=True)
     add_user_to_group('keystone', 'keystone')
 
@@ -2138,22 +2168,24 @@ def git_post_install(projects_yaml):
     render('git/logging.conf', '/etc/keystone/logging.conf', {}, perms=0o644)
 
     bin_dir = os.path.join(git_pip_venv_dir(projects_yaml), 'bin')
-    keystone_context = {
-        'service_description': 'Keystone API server',
-        'service_name': 'Keystone',
-        'user_name': 'keystone',
-        'start_dir': '/var/lib/keystone',
-        'process_name': 'keystone',
-        'executable_name': os.path.join(bin_dir, 'keystone-all'),
-        'config_files': ['/etc/keystone/keystone.conf'],
-        'log_file': '/var/log/keystone/keystone.log',
-    }
+    # The charm runs the keystone API under apache2 for openstack liberty
+    # onward.  Prior to liberty upstart is used.
+    if os_release('keystone') < 'liberty':
+        keystone_context = {
+            'service_description': 'Keystone API server',
+            'service_name': 'Keystone',
+            'user_name': 'keystone',
+            'start_dir': '/var/lib/keystone',
+            'process_name': 'keystone',
+            'executable_name': os.path.join(bin_dir, 'keystone-all'),
+            'config_files': ['/etc/keystone/keystone.conf'],
+            'log_file': '/var/log/keystone/keystone.log',
+        }
 
-    # NOTE(coreycb): Needs systemd support
-    templates_dir = 'hooks/charmhelpers/contrib/openstack/templates'
-    templates_dir = os.path.join(charm_dir(), templates_dir)
-    render('git.upstart', '/etc/init/keystone.conf', keystone_context,
-           perms=0o644, templates_dir=templates_dir)
+        templates_dir = 'hooks/charmhelpers/contrib/openstack/templates'
+        templates_dir = os.path.join(charm_dir(), templates_dir)
+        render('git.upstart', '/etc/init/keystone.conf', keystone_context,
+               perms=0o644, templates_dir=templates_dir)
 
     # Don't restart if the unit is supposed to be paused.
     if not is_unit_paused_set():

templates/wsgi-keystone.conf

@@ -4,9 +4,13 @@ Listen {{ admin_port }}
 Listen {{ public_port }}
 
 <VirtualHost *:{{ public_port }}>
-    WSGIDaemonProcess keystone-public processes={{ public_processes }} threads={{ public_threads }} user=keystone group=keystone display-name=%{GROUP}
+    WSGIDaemonProcess keystone-public processes={{ public_processes }} threads={{ public_threads }} user=keystone group=keystone \
+{% if python_path -%}
+                      python-path={{ python_path }} \
+{% endif -%}
+                      display-name=%{GROUP}
     WSGIProcessGroup keystone-public
-    WSGIScriptAlias / /usr/bin/keystone-wsgi-public
+    WSGIScriptAlias / {{ usr_bin }}/keystone-wsgi-public
     WSGIApplicationGroup %{GLOBAL}
     WSGIPassAuthorization On
     <IfVersion >= 2.4>
@@ -15,7 +19,7 @@ Listen {{ public_port }}
     ErrorLog /var/log/apache2/keystone.log
     CustomLog /var/log/apache2/keystone_access.log combined
 
-    <Directory /usr/bin>
+    <Directory {{ usr_bin }}>
         <IfVersion >= 2.4>
             Require all granted
         </IfVersion>
@@ -27,9 +31,13 @@ Listen {{ public_port }}
 </VirtualHost>
 
 <VirtualHost *:{{ admin_port }}>
-    WSGIDaemonProcess keystone-admin processes={{ admin_processes }} threads={{ admin_threads }} user=keystone group=keystone display-name=%{GROUP}
+    WSGIDaemonProcess keystone-admin processes={{ admin_processes }} threads={{ admin_threads }} user=keystone group=keystone \
+{% if python_path -%}
+                      python-path={{ python_path }} \
+{% endif -%}
+                      display-name=%{GROUP}
     WSGIProcessGroup keystone-admin
-    WSGIScriptAlias / /usr/bin/keystone-wsgi-admin
+    WSGIScriptAlias / {{ usr_bin }}/keystone-wsgi-admin
     WSGIApplicationGroup %{GLOBAL}
     WSGIPassAuthorization On
     <IfVersion >= 2.4>
@@ -38,7 +46,7 @@ Listen {{ public_port }}
     ErrorLog /var/log/apache2/keystone.log
     CustomLog /var/log/apache2/keystone_access.log combined
 
-    <Directory /usr/bin>
+    <Directory {{ usr_bin }}>
         <IfVersion >= 2.4>
             Require all granted
         </IfVersion>

tests/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -71,7 +71,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
         base_charms = {
             'mysql': ['precise', 'trusty'],
             'mongodb': ['precise', 'trusty'],
-            'nrpe': ['precise', 'trusty'],
+            'nrpe': ['precise', 'trusty', 'wily', 'xenial'],
         }
 
         for svc in other_services:
@@ -112,7 +112,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
 
         # Charms which should use the source config option
         use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
-                      'ceph-osd', 'ceph-radosgw', 'ceph-mon']
+                      'ceph-osd', 'ceph-radosgw', 'ceph-mon', 'ceph-proxy']
 
         # Charms which can not use openstack-origin, ie. many subordinates
         no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe',

unit_tests/test_keystone_contexts.py

@@ -193,12 +193,19 @@ class TestKeystoneContexts(CharmTestCase):
         mock_is_elected_leader.return_value = True
         self.assertEqual({'token_flush': True}, ctxt())
 
+    @patch('keystone_utils.determine_usr_bin')
+    @patch('keystone_utils.determine_python_path')
     @patch('charmhelpers.contrib.openstack.context.psutil')
-    def test_wsgi_worker_config_context(self, mock_psutil):
+    def test_wsgi_worker_config_context(self, mock_psutil, python_path,
+                                        usr_bin):
         self.config.return_value = 2
+        usr_bin.return_value = '/usr/bin'
+        python_path.return_value = None
         mock_psutil.cpu_count.return_value = 40
         ctxt = context.WSGIWorkerConfigContext()
         expect = {
+            "usr_bin": "/usr/bin",
+            "python_path": None,
             "public_processes": 60,
             "admin_processes": 20,
             "public_threads": 1,

unit_tests/test_keystone_hooks.py

@@ -68,6 +68,7 @@ TO_PATCH = [
     'apt_update',
     # charmhelpers.contrib.openstack.utils
     'configure_installation_source',
+    'git_install_requested',
     # charmhelpers.contrib.openstack.ip
     'resolve_address',
     # charmhelpers.contrib.openstack.ha.utils

unit_tests/test_keystone_utils.py

@@ -35,18 +35,24 @@ TO_PATCH = [
     'get_requested_roles',
     'get_service_password',
     'get_os_codename_install_source',
+    'git_clone_and_install',
+    'git_pip_venv_dir',
+    'git_src_dir',
     'grant_role',
     'configure_installation_source',
     'is_elected_leader',
     'is_ssl_cert_master',
     'https',
+    'lsb_release',
     'peer_store_and_set',
+    'service_restart',
     'service_stop',
     'service_start',
     'relation_get',
     'relation_set',
     'relation_ids',
     'relation_id',
+    'render',
     'local_unit',
     'related_units',
     'https',
@@ -709,17 +715,15 @@ class TestKeystoneUtils(CharmTestCase):
         self.assertFalse(self.relation_set.called)
 
     @patch.object(utils, 'git_install_requested')
-    @patch.object(utils, 'git_clone_and_install')
     @patch.object(utils, 'git_post_install')
     @patch.object(utils, 'git_pre_install')
-    def test_git_install(self, git_pre, git_post, git_clone_and_install,
-                         git_requested):
+    def test_git_install(self, git_requested, git_pre, git_post):
         projects_yaml = openstack_origin_git
         git_requested.return_value = True
         utils.git_install(projects_yaml)
         self.assertTrue(git_pre.called)
-        git_clone_and_install.assert_called_with(openstack_origin_git,
-                                                 core_project='keystone')
+        self.git_clone_and_install.assert_called_with(openstack_origin_git,
+                                                      core_project='keystone')
         self.assertTrue(git_post.called)
 
     @patch.object(utils, 'mkdir')
@@ -731,7 +735,8 @@ class TestKeystoneUtils(CharmTestCase):
                              write_file, mkdir):
         utils.git_pre_install()
         adduser.assert_called_with('keystone', shell='/bin/bash',
-                                   system_user=True)
+                                   system_user=True,
+                                   home_dir='/var/lib/keystone')
         add_group.assert_called_with('keystone', system_group=True)
         add_user_to_group.assert_called_with('keystone', 'keystone')
         expected = [
@@ -747,10 +752,6 @@ class TestKeystoneUtils(CharmTestCase):
                                       '', owner='keystone', group='keystone',
                                       perms=0600)
 
-    @patch.object(utils, 'git_src_dir')
-    @patch.object(utils, 'service_restart')
-    @patch.object(utils, 'render')
-    @patch.object(utils, 'git_pip_venv_dir')
     @patch('os.path.join')
     @patch('os.path.exists')
     @patch('os.symlink')
@@ -758,11 +759,11 @@ class TestKeystoneUtils(CharmTestCase):
     @patch('shutil.rmtree')
     @patch('subprocess.check_call')
     def test_git_post_install(self, check_call, rmtree, copytree, symlink,
-                              exists, join, venv, render, service_restart,
-                              git_src_dir):
+                              exists, join):
         projects_yaml = openstack_origin_git
         join.return_value = 'joined-string'
-        venv.return_value = '/mnt/openstack-git/venv'
+        self.git_pip_venv_dir.return_value = '/mnt/openstack-git/venv'
+        self.lsb_release.return_value = {'DISTRIB_RELEASE': '15.04'}
         utils.git_post_install(projects_yaml)
         expected = [
             call('joined-string', '/etc/keystone'),
@@ -788,8 +789,8 @@ class TestKeystoneUtils(CharmTestCase):
             call('git.upstart', '/etc/init/keystone.conf',
                  keystone_context, perms=0o644, templates_dir='joined-string'),
         ]
-        self.assertEquals(render.call_args_list, expected)
-        service_restart.assert_called_with('keystone')
+        self.assertEquals(self.render.call_args_list, expected)
+        self.service_restart.assert_called_with('keystone')
 
     @patch.object(utils, 'get_manager')
     def test_is_service_present(self, KeystoneManager):