Merged trunk in + LE charmhelper sync

Makefile

@@ -2,7 +2,7 @@
 PYTHON := /usr/bin/env python
 
 lint:
-	@flake8 --exclude hooks/charmhelpers hooks unit_tests tests
+	@flake8 --exclude hooks/charmhelpers actions hooks unit_tests tests
 	@charm proof
 
 unit_test:
@@ -13,8 +13,7 @@ test:
 	@echo Starting amulet deployment tests...
 	#NOTE(beisner): can remove -v after bug 1320357 is fixed
 	#   https://bugs.launchpad.net/amulet/+bug/1320357
-	@juju test -v -p AMULET_HTTP_PROXY --timeout 900 \
+	@juju test -v -p AMULET_HTTP_PROXY,AMULET_OS_VIP --timeout 2700
-        00-setup 14-basic-precise-icehouse 15-basic-trusty-icehouse
 
 bin/charm_helpers_sync.py:
 	@mkdir -p bin

README.md

@@ -117,3 +117,98 @@ overwrite:  Whether or not to wipe local storage that of data that may prevent
 
 enabled-services:  Can be used to separate cinder services between service
                    service units (see previous section)
+
+Deploying from source
+---------------------
+
+The minimum openstack-origin-git config required to deploy from source is:
+
+    openstack-origin-git: include-file://cinder-juno.yaml
+
+    cinder-juno.yaml
+        repositories:
+        - {name: requirements,
+           repository: 'git://github.com/openstack/requirements',
+           branch: stable/juno}
+        - {name: cinder,
+           repository: 'git://github.com/openstack/cinder',
+           branch: stable/juno}
+
+Note that there are only two 'name' values the charm knows about: 'requirements'
+and 'cinder'. These repositories must correspond to these 'name' values.
+Additionally, the requirements repository must be specified first and the
+cinder repository must be specified last. All other repostories are installed
+in the order in which they are specified.
+
+The following is a full list of current tip repos (may not be up-to-date):
+
+    openstack-origin-git: include-file://cinder-master.yaml
+
+    cinder-master.yaml
+        repositories:
+        - {name: requirements,
+           repository: 'git://github.com/openstack/requirements',
+           branch: master}
+        - {name: oslo-concurrency,
+           repository: 'git://github.com/openstack/oslo.concurrency',
+           branch: master}
+        - {name: oslo-config,
+           repository: 'git://github.com/openstack/oslo.config',
+           branch: master}
+        - {name: oslo-context,
+           repository': 'git://github.com/openstack/oslo.context',
+           branch: master}
+        - {name: oslo-db,
+           repository: 'git://github.com/openstack/oslo.db',
+           branch: master}
+        - {name: oslo-i18n,
+           repository: 'git://github.com/openstack/oslo.i18n',
+           branch: master}
+        - {name: oslo-messaging,
+           repository: 'git://github.com/openstack/oslo.messaging',
+           branch: master}
+        - {name: oslo-serialization,
+           repository: 'git://github.com/openstack/oslo.serialization',
+           branch: master}
+        - {name: oslo-utils,
+           repository: 'git://github.com/openstack/oslo.utils',
+           branch: master}
+        - {name: oslo-rootwrap,
+           repository: 'git://github.com/openstack/oslo.rootwrap',
+           branch: master}
+        - {name: oslo-vmware,
+           repository: 'git://github.com/openstack/oslo.vmware',
+           branch: master}
+        - {name: osprofiler,
+           repository: 'git://github.com/stackforge/osprofiler',
+           branch: master}
+        - {name: pbr,
+           repository: 'git://github.com/openstack-dev/pbr',
+           branch: master}
+        - {name: python-barbicanclient,
+           repository: 'git://github.com/openstack/python-barbicanclient',
+           branch: master}
+        - {name: python-glanceclient,
+           repository: 'git://github.com/openstack/python-glanceclient',
+           branch: master}
+        - {name: python-novaclient,
+           repository: 'git://github.com/openstack/python-novaclient',
+           branch: master}
+        - {name: python-swiftclient:
+           repository: 'git://github.com/openstack/python-swiftclient',
+           branch: master}
+        - {name: sqlalchemy-migrate,
+           repository: 'git://github.com/stackforge/sqlalchemy-migrate',
+           branch: master}
+        - {name: stevedore,
+           repository: 'git://github.com/openstack/stevedore',
+           branch: master}
+        - {name: taskflow,
+           repository: 'git://github.com/openstack/taskflow',
+           branch: master}
+        - {name: keystonemiddleware,
+           repository: 'git://github.com/openstack/keystonemiddleware',
+           branch: master}
+        - {name: cinder,
+           repository: 'git://github.com/openstack/cinder',
+           branch: master}

actions.yaml

@@ -0,0 +1,2 @@
+git-reinstall:
+  description: Reinstall cinder from the openstack-origin-git repositories.

actions/git-reinstall

@@ -0,0 +1 @@
+git_reinstall.py

actions/git_reinstall.py

@@ -0,0 +1,45 @@
+#!/usr/bin/python
+import sys
+import traceback
+
+sys.path.append('hooks/')
+
+from charmhelpers.contrib.openstack.utils import (
+    git_install_requested,
+)
+
+from charmhelpers.core.hookenv import (
+    action_set,
+    action_fail,
+    config,
+)
+
+from cinder_utils import (
+    git_install,
+)
+
+from cinder_hooks import (
+    config_changed,
+)
+
+
+def git_reinstall():
+    """Reinstall from source and restart services.
+
+    If the openstack-origin-git config option was used to install openstack
+    from source git repositories, then this action can be used to reinstall
+    from updated git repositories, followed by a restart of services."""
+    if not git_install_requested():
+        action_fail('openstack-origin-git is not configured')
+        return
+
+    try:
+        git_install(config('openstack-origin-git'))
+        config_changed()
+    except:
+        action_set({'traceback': traceback.format_exc()})
+        action_fail('git-reinstall resulted in an unexpected error')
+
+
+if __name__ == '__main__':
+    git_reinstall()

config.yaml

@@ -15,6 +15,22 @@ options:
       the cloud:precise-folsom/updates repository instead, since Cinder
       was not available in the Ubuntu archive for Precise and is only
       available via the Ubuntu Cloud Archive.
+
+      Note that when openstack-origin-git is specified, openstack
+      specific packages will be installed from source rather than
+      from the openstack-origin repository.
+  openstack-origin-git:
+    default:
+    type: string
+    description: |
+      Specifies a YAML-formatted dictionary listing the git
+      repositories and branches from which to install OpenStack and
+      its dependencies.
+
+      Note that the installed config files will be determined based on
+      the OpenStack release of the openstack-origin option.
+
+      For more details see README.md.
   enabled-services:
     default: all
     type: string

hooks/charmhelpers/contrib/charmsupport/nrpe.py

@@ -247,7 +247,9 @@ class NRPE(object):
 
         service('restart', 'nagios-nrpe-server')
 
-        for rid in relation_ids("local-monitors"):
+        monitor_ids = relation_ids("local-monitors") + \
+            relation_ids("nrpe-external-master")
+        for rid in monitor_ids:
             relation_set(relation_id=rid, monitors=yaml.dump(monitors))
 
 

hooks/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -44,17 +44,24 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            Determine if the local branch being tested is derived from its
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
-        base_charms = ['mysql', 'mongodb', 'rabbitmq-server']
+        base_charms = ['mysql', 'mongodb']
+
+        if self.series in ['precise', 'trusty']:
+            base_series = self.series
+        else:
+            base_series = self.current_next
 
         if self.stable:
             for svc in other_services:
-                temp = 'lp:charms/{}'
+                temp = 'lp:charms/{}/{}'
-                svc['location'] = temp.format(svc['name'])
+                svc['location'] = temp.format(base_series,
+                                              svc['name'])
         else:
             for svc in other_services:
                 if svc['name'] in base_charms:
-                    temp = 'lp:charms/{}'
+                    temp = 'lp:charms/{}/{}'
-                    svc['location'] = temp.format(svc['name'])
+                    svc['location'] = temp.format(base_series,
+                                                  svc['name'])
                 else:
                     temp = 'lp:~openstack-charmers/charms/{}/{}/next'
                     svc['location'] = temp.format(self.current_next,
@@ -99,9 +106,12 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            Return an integer representing the enum value of the openstack
            release.
            """
+        # Must be ordered by OpenStack release (not by Ubuntu release):
         (self.precise_essex, self.precise_folsom, self.precise_grizzly,
          self.precise_havana, self.precise_icehouse,
-         self.trusty_icehouse, self.trusty_juno, self.trusty_kilo) = range(8)
+         self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
+         self.trusty_kilo, self.vivid_kilo) = range(10)
+
         releases = {
             ('precise', None): self.precise_essex,
             ('precise', 'cloud:precise-folsom'): self.precise_folsom,
@@ -110,7 +120,9 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
             ('trusty', None): self.trusty_icehouse,
             ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
-            ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo}
+            ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
+            ('utopic', None): self.utopic_juno,
+            ('vivid', None): self.vivid_kilo}
         return releases[(self.series, self.openstack)]
 
     def _get_openstack_release_string(self):

hooks/charmhelpers/contrib/openstack/context.py

@@ -459,6 +459,11 @@ class AMQPContext(OSContextGenerator):
 
                 ctxt['rabbitmq_hosts'] = ','.join(sorted(rabbitmq_hosts))
 
+        oslo_messaging_flags = conf.get('oslo-messaging-flags', None)
+        if oslo_messaging_flags:
+            ctxt['oslo_messaging_flags'] = config_flags_parser(
+                oslo_messaging_flags)
+
         if not context_complete(ctxt):
             return {}
 
@@ -808,6 +813,19 @@ class NeutronContext(OSContextGenerator):
 
         return ovs_ctxt
 
+    def nuage_ctxt(self):
+        driver = neutron_plugin_attribute(self.plugin, 'driver',
+                                          self.network_manager)
+        config = neutron_plugin_attribute(self.plugin, 'config',
+                                          self.network_manager)
+        nuage_ctxt = {'core_plugin': driver,
+                      'neutron_plugin': 'vsp',
+                      'neutron_security_groups': self.neutron_security_groups,
+                      'local_ip': unit_private_ip(),
+                      'config': config}
+
+        return nuage_ctxt
+
     def nvp_ctxt(self):
         driver = neutron_plugin_attribute(self.plugin, 'driver',
                                           self.network_manager)
@@ -891,6 +909,8 @@ class NeutronContext(OSContextGenerator):
             ctxt.update(self.n1kv_ctxt())
         elif self.plugin == 'Calico':
             ctxt.update(self.calico_ctxt())
+        elif self.plugin == 'vsp':
+            ctxt.update(self.nuage_ctxt())
 
         alchemy_flags = config('neutron-alchemy-flags')
         if alchemy_flags:

hooks/charmhelpers/contrib/openstack/neutron.py

@@ -180,6 +180,19 @@ def neutron_plugins():
                           'nova-api-metadata']],
             'server_packages': ['neutron-server', 'calico-control'],
             'server_services': ['neutron-server']
+        },
+        'vsp': {
+            'config': '/etc/neutron/plugins/nuage/nuage_plugin.ini',
+            'driver': 'neutron.plugins.nuage.plugin.NuagePlugin',
+            'contexts': [
+                context.SharedDBContext(user=config('neutron-database-user'),
+                                        database=config('neutron-database'),
+                                        relation_prefix='neutron',
+                                        ssl_dir=NEUTRON_CONF_DIR)],
+            'services': [],
+            'packages': [],
+            'server_packages': ['neutron-server', 'neutron-plugin-nuage'],
+            'server_services': ['neutron-server']
         }
     }
     if release >= 'icehouse':

hooks/charmhelpers/contrib/openstack/templates/git.upstart

@@ -9,5 +9,9 @@ respawn
 exec start-stop-daemon --start --chuid {{ user_name }} \
             --chdir {{ start_dir }} --name {{ process_name }} \
             --exec {{ executable_name }} -- \
+            {% for config_file in config_files -%}
             --config-file={{ config_file }} \
+            {% endfor -%}
+            {% if log_file -%}
             --log-file={{ log_file }}
+            {% endif -%}

hooks/charmhelpers/contrib/openstack/utils.py

@@ -510,8 +510,10 @@ def git_clone_and_install(projects_yaml, core_project):
              repository: 'git://git.openstack.org/openstack/requirements.git',
              branch: 'stable/icehouse'}
         directory: /mnt/openstack-git
+        http_proxy: http://squid.internal:3128
+        https_proxy: https://squid.internal:3128
 
-        The directory key is optional.
+        The directory, http_proxy, and https_proxy keys are optional.
     """
     global requirements_dir
     parent_dir = '/mnt/openstack-git'
@@ -522,6 +524,13 @@ def git_clone_and_install(projects_yaml, core_project):
     projects = yaml.load(projects_yaml)
     _git_validate_projects_yaml(projects, core_project)
 
+    old_environ = dict(os.environ)
+
+    if 'http_proxy' in projects.keys():
+        os.environ['http_proxy'] = projects['http_proxy']
+    if 'https_proxy' in projects.keys():
+        os.environ['https_proxy'] = projects['https_proxy']
+
     if 'directory' in projects.keys():
         parent_dir = projects['directory']
 
@@ -536,6 +545,8 @@ def git_clone_and_install(projects_yaml, core_project):
             repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
                                                      update_requirements=True)
 
+    os.environ = old_environ
+
 
 def _git_validate_projects_yaml(projects, core_project):
     """

hooks/charmhelpers/core/hookenv.py

@@ -20,11 +20,14 @@
 # Authors:
 #  Charm Helpers Developers <juju@lists.ubuntu.com>
 
+from __future__ import print_function
+from functools import wraps
 import os
 import json
 import yaml
 import subprocess
 import sys
+import errno
 from subprocess import CalledProcessError
 
 import six
@@ -56,15 +59,17 @@ def cached(func):
 
     will cache the result of unit_get + 'test' for future calls.
     """
+    @wraps(func)
     def wrapper(*args, **kwargs):
         global cache
         key = str((func, args, kwargs))
         try:
             return cache[key]
         except KeyError:
-            res = func(*args, **kwargs)
+            pass  # Drop out of the exception handler scope.
-            cache[key] = res
+        res = func(*args, **kwargs)
-            return res
+        cache[key] = res
+        return res
     return wrapper
 
 
@@ -87,7 +92,18 @@ def log(message, level=None):
     if not isinstance(message, six.string_types):
         message = repr(message)
     command += [message]
-    subprocess.call(command)
+    # Missing juju-log should not cause failures in unit tests
+    # Send log output to stderr
+    try:
+        subprocess.call(command)
+    except OSError as e:
+        if e.errno == errno.ENOENT:
+            if level:
+                message = "{}: {}".format(level, message)
+            message = "juju-log: {}".format(message)
+            print(message, file=sys.stderr)
+        else:
+            raise
 
 
 class Serializable(UserDict):
@@ -165,7 +181,7 @@ def local_unit():
 
 def remote_unit():
     """The remote unit for the current relation hook"""
-    return os.environ['JUJU_REMOTE_UNIT']
+    return os.environ.get('JUJU_REMOTE_UNIT', None)
 
 
 def service_name():
@@ -237,6 +253,12 @@ class Config(dict):
         except KeyError:
             return (self._prev_dict or {})[key]
 
+    def get(self, key, default=None):
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
     def keys(self):
         prev_keys = []
         if self._prev_dict is not None:
@@ -507,6 +529,11 @@ def unit_get(attribute):
         return None
 
 
+def unit_public_ip():
+    """Get this unit's public IP address"""
+    return unit_get('public-address')
+
+
 def unit_private_ip():
     """Get this unit's private IP address"""
     return unit_get('private-address')
@@ -664,3 +691,49 @@ def action_fail(message):
 
     The results set by action_set are preserved."""
     subprocess.check_call(['action-fail', message])
+
+
+def status_set(workload_state, message):
+    """Set the workload state with a message
+
+    Use status-set to set the workload state with a message which is visible
+    to the user via juju status. If the status-set command is not found then
+    assume this is juju < 1.23 and juju-log the message unstead.
+
+    workload_state -- valid juju workload state.
+    message        -- status update message
+    """
+    valid_states = ['maintenance', 'blocked', 'waiting', 'active']
+    if workload_state not in valid_states:
+        raise ValueError(
+            '{!r} is not a valid workload state'.format(workload_state)
+        )
+    cmd = ['status-set', workload_state, message]
+    try:
+        ret = subprocess.call(cmd)
+        if ret == 0:
+            return
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
+    log_message = 'status-set failed: {} {}'.format(workload_state,
+                                                    message)
+    log(log_message, level='INFO')
+
+
+def status_get():
+    """Retrieve the previously set juju workload state
+
+    If the status-set command is not found then assume this is juju < 1.23 and
+    return 'unknown'
+    """
+    cmd = ['status-get']
+    try:
+        raw_status = subprocess.check_output(cmd, universal_newlines=True)
+        status = raw_status.rstrip()
+        return status
+    except OSError as e:
+        if e.errno == errno.ENOENT:
+            return 'unknown'
+        else:
+            raise

hooks/charmhelpers/core/host.py

@@ -90,7 +90,7 @@ def service_available(service_name):
             ['service', service_name, 'status'],
             stderr=subprocess.STDOUT).decode('UTF-8')
     except subprocess.CalledProcessError as e:
-        return 'unrecognized service' not in e.output
+        return b'unrecognized service' not in e.output
     else:
         return True
 

hooks/charmhelpers/core/services/base.py

@@ -17,7 +17,7 @@
 import os
 import re
 import json
-from collections import Iterable
+from collections import Iterable, OrderedDict
 
 from charmhelpers.core import host
 from charmhelpers.core import hookenv
@@ -119,7 +119,7 @@ class ServiceManager(object):
         """
         self._ready_file = os.path.join(hookenv.charm_dir(), 'READY-SERVICES.json')
         self._ready = None
-        self.services = {}
+        self.services = OrderedDict()
         for service in services or []:
             service_name = service['service']
             self.services[service_name] = service

hooks/charmhelpers/core/strutils.py

@@ -33,9 +33,9 @@ def bool_from_string(value):
 
     value = value.strip().lower()
 
-    if value in ['y', 'yes', 'true', 't']:
+    if value in ['y', 'yes', 'true', 't', 'on']:
         return True
-    elif value in ['n', 'no', 'false', 'f']:
+    elif value in ['n', 'no', 'false', 'f', 'off']:
         return False
 
     msg = "Unable to interpret string value '%s' as boolean" % (value)

hooks/charmhelpers/fetch/__init__.py

@@ -158,7 +158,7 @@ def filter_installed_packages(packages):
 
 def apt_cache(in_memory=True):
     """Build and return an apt cache"""
-    import apt_pkg
+    from apt import apt_pkg
     apt_pkg.init()
     if in_memory:
         apt_pkg.config.set("Dir::Cache::pkgcache", "")

hooks/cinder_hooks.py

@@ -10,6 +10,7 @@ from subprocess import (
 from cinder_utils import (
     determine_packages,
     do_openstack_upgrade,
+    git_install,
     juju_log,
     migrate_database,
     configure_lvm_storage,
@@ -17,6 +18,7 @@ from cinder_utils import (
     restart_map,
     services,
     service_enabled,
+    service_restart,
     set_ceph_env_variables,
     CLUSTER_RES,
     CINDER_CONF,
@@ -53,10 +55,12 @@ from charmhelpers.core.host import (
 )
 
 from charmhelpers.contrib.openstack.utils import (
+    config_value_changed,
     configure_installation_source,
+    git_install_requested,
     openstack_upgrade_available,
     sync_db_with_multi_ipv6_addresses,
-    get_os_codename_package
+    os_release,
 )
 
 from charmhelpers.contrib.storage.linux.ceph import (
@@ -101,9 +105,12 @@ def install():
             src == 'distro'):
         src = 'cloud:precise-folsom'
     configure_installation_source(src)
+
     apt_update()
     apt_install(determine_packages(), fatal=True)
 
+    git_install(config('openstack-origin-git'))
+
 
 @hooks.hook('config-changed')
 @restart_on_change(restart_map(), stopstart=True)
@@ -123,12 +130,16 @@ def config_changed():
                               conf['overwrite'] in ['true', 'True', True],
                               conf['remove-missing'])
 
-    if openstack_upgrade_available('cinder-common'):
+    if git_install_requested():
-        do_openstack_upgrade(configs=CONFIGS)
+        if config_value_changed('openstack-origin-git'):
-        # NOTE(jamespage) tell any storage-backends we just upgraded
+            git_install(config('openstack-origin-git'))
-        for rid in relation_ids('storage-backend'):
+    else:
-            relation_set(relation_id=rid,
+        if openstack_upgrade_available('cinder-common'):
-                         upgrade_nonce=uuid.uuid4())
+            do_openstack_upgrade(configs=CONFIGS)
+            # NOTE(jamespage) tell any storage-backends we just upgraded
+            for rid in relation_ids('storage-backend'):
+                relation_set(relation_id=rid,
+                             upgrade_nonce=uuid.uuid4())
 
     CONFIGS.write_all()
     configure_https()
@@ -185,11 +196,12 @@ def db_changed():
         # acl entry has been added. So, if the db supports passing a list of
         # permitted units then check if we're in the list.
         allowed_units = relation_get('allowed_units')
-        if allowed_units and local_unit() not in allowed_units.split():
+        if allowed_units and local_unit() in allowed_units.split():
-            juju_log('Allowed_units list provided and this unit not present')
+            juju_log('Cluster leader, performing db sync')
-            return
+            migrate_database()
-        juju_log('Cluster leader, performing db sync')
+        else:
-        migrate_database()
+            juju_log('allowed_units either not presented, or local unit '
+                     'not in acl list: %s' % repr(allowed_units))
 
 
 @hooks.hook('pgsql-db-relation-changed')
@@ -255,7 +267,7 @@ def identity_joined(rid=None):
         'cinder_internal_url': internal_url,
         'cinder_admin_url': admin_url,
     }
-    if get_os_codename_package('cinder-common') >= 'icehouse':
+    if os_release('cinder-common') >= 'icehouse':
         # NOTE(jamespage) register v2 endpoint as well
         public_url = '{}:{}/v2/$(tenant_id)s'.format(
             canonical_url(CONFIGS, PUBLIC),
@@ -323,6 +335,9 @@ def ceph_changed(relation_id=None):
         set_ceph_env_variables(service=service)
         CONFIGS.write(CINDER_CONF)
         CONFIGS.write(ceph_config_file())
+        # Ensure that cinder-volume is restarted since only now can we
+        # guarantee that ceph resources are ready.
+        service_restart('cinder-volume')
     else:
         rq = CephBrokerRq()
         replicas = config('ceph-osd-replication-count')

hooks/cinder_utils.py

@@ -1,10 +1,12 @@
 import os
+import shutil
 import subprocess
 
 from collections import OrderedDict
 from copy import copy
 
 from charmhelpers.core.hookenv import (
+    charm_dir,
     config,
     relation_ids,
     log,
@@ -19,12 +21,17 @@ from charmhelpers.fetch import (
 )
 
 from charmhelpers.core.host import (
+    adduser,
+    add_group,
+    add_user_to_group,
+    lsb_release,
+    mkdir,
     mounts,
     umount,
+    service_restart,
     service_stop,
     service_start,
-    mkdir,
+    write_file,
-    lsb_release
 )
 
 from charmhelpers.contrib.openstack.alternatives import install_alternative
@@ -58,13 +65,17 @@ from charmhelpers.contrib.openstack import (
 
 from charmhelpers.contrib.openstack.utils import (
     configure_installation_source,
-    get_os_codename_package,
     get_os_codename_install_source,
+    git_install_requested,
+    git_clone_and_install,
+    git_src_dir,
+    os_release,
 )
 
 from charmhelpers.core.decorators import (
     retry_on_exception,
 )
+from charmhelpers.core.templating import render
 
 import cinder_contexts
 
@@ -73,6 +84,7 @@ COMMON_PACKAGES = [
     'cinder-common',
     'gdisk',
     'haproxy',
+    'librbd1',  # bug 1440948 vol-from-img
     'python-jinja2',
     'python-keystoneclient',
     'python-mysqldb',
@@ -84,6 +96,25 @@ API_PACKAGES = ['cinder-api']
 VOLUME_PACKAGES = ['cinder-volume']
 SCHEDULER_PACKAGES = ['cinder-scheduler']
 
+BASE_GIT_PACKAGES = [
+    'libxml2-dev',
+    'libxslt1-dev',
+    'lvm2',
+    'python-dev',
+    'python-pip',
+    'python-setuptools',
+    'zlib1g-dev',
+]
+
+# ubuntu packages that should not be installed when deploying from source
+GIT_PACKAGE_BLACKLIST = [
+    'cinder-api',
+    'cinder-common',
+    'cinder-scheduler',
+    'cinder-volume',
+    'python-keystoneclient',
+]
+
 DEFAULT_LOOPBACK_SIZE = '5G'
 
 # Cluster resource used to determine leadership when hacluster'd
@@ -169,7 +200,7 @@ def register_configs():
     # if called without anything installed (eg during install hook)
     # just default to earliest supported release. configs dont get touched
     # till post-install, anyway.
-    release = get_os_codename_package('cinder-common', fatal=False) or 'folsom'
+    release = os_release('cinder-common', base='folsom')
     configs = templating.OSConfigRenderer(templates_dir=TEMPLATES,
                                           openstack_release=release)
 
@@ -221,6 +252,13 @@ def determine_packages():
                  ('scheduler', SCHEDULER_PACKAGES)]:
         if service_enabled(s):
             pkgs += p
+
+    if git_install_requested():
+        pkgs.extend(BASE_GIT_PACKAGES)
+        # don't include packages that will be installed from git
+        for p in GIT_PACKAGE_BLACKLIST:
+            pkgs.remove(p)
+
     return pkgs
 
 
@@ -473,3 +511,142 @@ def setup_ipv6():
                    ' main')
         apt_update()
         apt_install('haproxy/trusty-backports', fatal=True)
+
+
+def git_install(projects_yaml):
+    """Perform setup, and install git repos specified in yaml parameter."""
+    if git_install_requested():
+        git_pre_install()
+        git_clone_and_install(projects_yaml, core_project='cinder')
+        git_post_install(projects_yaml)
+
+
+def git_pre_install():
+    """Perform cinder pre-install setup."""
+    dirs = [{'path': '/etc/tgt',
+             'owner': 'cinder',
+             'group': 'cinder',
+             'perms': 0750,
+             },
+            {'path': '/var/lib/cinder',
+             'owner': 'cinder',
+             'group': 'cinder',
+             'perms': 0755,
+             },
+            {'path': '/var/lib/cinder/volumes',
+             'owner': 'cinder',
+             'group': 'cinder',
+             'perms': 0750,
+             },
+            {'path': '/var/lock/cinder',
+             'owner': 'cinder',
+             'group': 'root',
+             'perms': 0750,
+             },
+            {'path': '/var/log/cinder',
+             'owner': 'cinder',
+             'group': 'cinder',
+             'perms': 0750,
+             }]
+
+    logs = [
+        '/var/log/cinder/cinder-api.log',
+        '/var/log/cinder/cinder-backup.log',
+        '/var/log/cinder/cinder-scheduler.log',
+        '/var/log/cinder/cinder-volume.log',
+    ]
+
+    adduser('cinder', shell='/bin/bash', system_user=True)
+    add_group('cinder', system_group=True)
+    add_user_to_group('cinder', 'cinder')
+
+    for d in dirs:
+        mkdir(d['path'], owner=d['owner'], group=d['group'], perms=d['perms'],
+              force=False)
+
+    for l in logs:
+        write_file(l, '', owner='cinder', group='cinder', perms=0600)
+
+
+def git_post_install(projects_yaml):
+    """Perform cinder post-install setup."""
+    src_etc = os.path.join(git_src_dir(projects_yaml, 'cinder'), 'etc/cinder')
+    configs = {
+        'src': src_etc,
+        'dest': '/etc/cinder',
+    }
+
+    if os.path.exists(configs['dest']):
+        shutil.rmtree(configs['dest'])
+    shutil.copytree(configs['src'], configs['dest'])
+
+    render('cinder.conf', '/etc/cinder/cinder.conf', {}, owner='cinder',
+           group='cinder', perms=0o644)
+    render('git/cinder_tgt.conf', '/etc/tgt/conf.d', {}, owner='cinder',
+           group='cinder', perms=0o644)
+    render('git/logging.conf', '/etc/cinder/logging.conf', {}, owner='cinder',
+           group='cinder', perms=0o644)
+    render('git/cinder_sudoers', '/etc/sudoers.d/cinder_sudoers', {},
+           owner='root', group='root', perms=0o440)
+
+    os.chmod('/etc/sudoers.d', 0o750)
+
+    cinder_api_context = {
+        'service_description': 'Cinder API server',
+        'service_name': 'Cinder',
+        'user_name': 'cinder',
+        'start_dir': '/var/lib/cinder',
+        'process_name': 'cinder-api',
+        'executable_name': '/usr/local/bin/cinder-api',
+        'config_files': ['/etc/cinder/cinder.conf'],
+        'log_file': '/var/log/cinder/cinder-api.log',
+    }
+
+    cinder_backup_context = {
+        'service_description': 'Cinder backup server',
+        'service_name': 'Cinder',
+        'user_name': 'cinder',
+        'start_dir': '/var/lib/cinder',
+        'process_name': 'cinder-backup',
+        'executable_name': '/usr/local/bin/cinder-backup',
+        'config_files': ['/etc/cinder/cinder.conf'],
+        'log_file': '/var/log/cinder/cinder-backup.log',
+    }
+
+    cinder_scheduler_context = {
+        'service_description': 'Cinder scheduler server',
+        'service_name': 'Cinder',
+        'user_name': 'cinder',
+        'start_dir': '/var/lib/cinder',
+        'process_name': 'cinder-scheduler',
+        'executable_name': '/usr/local/bin/cinder-scheduler',
+        'config_files': ['/etc/cinder/cinder.conf'],
+        'log_file': '/var/log/cinder/cinder-scheduler.log',
+    }
+
+    cinder_volume_context = {
+        'service_description': 'Cinder volume server',
+        'service_name': 'Cinder',
+        'user_name': 'cinder',
+        'start_dir': '/var/lib/cinder',
+        'process_name': 'cinder-volume',
+        'executable_name': '/usr/local/bin/cinder-volume',
+        'config_files': ['/etc/cinder/cinder.conf'],
+        'log_file': '/var/log/cinder/cinder-volume.log',
+    }
+
+    # NOTE(coreycb): Needs systemd support
+    templates_dir = 'hooks/charmhelpers/contrib/openstack/templates'
+    templates_dir = os.path.join(charm_dir(), templates_dir)
+    render('git.upstart', '/etc/init/cinder-api.conf',
+           cinder_api_context, perms=0o644, templates_dir=templates_dir)
+    render('git.upstart', '/etc/init/cinder-backup.conf',
+           cinder_backup_context, perms=0o644, templates_dir=templates_dir)
+    render('git.upstart', '/etc/init/cinder-scheduler.conf',
+           cinder_scheduler_context, perms=0o644, templates_dir=templates_dir)
+    render('git.upstart', '/etc/init/cinder-volume.conf',
+           cinder_volume_context, perms=0o644, templates_dir=templates_dir)
+
+    service_restart('tgtd')
+
+    [service_restart(s) for s in services()]

templates/git/cinder_sudoers

@@ -0,0 +1,4 @@
+Defaults:cinder !requiretty
+
+cinder ALL = (root) NOPASSWD: /usr/local/bin/cinder-rootwrap  /etc/cinder/rootwrap.conf *
+

templates/git/cinder_tgt.conf

@@ -0,0 +1 @@
+include /var/lib/cinder/volumes/*

templates/git/logging.conf

@@ -0,0 +1,76 @@
+[loggers]
+keys = root, cinder
+
+[handlers]
+keys = stderr, stdout, watchedfile, syslog, null
+
+[formatters]
+keys = legacycinder, default
+
+[logger_root]
+level = WARNING
+handlers = null
+
+[logger_cinder]
+level = INFO
+handlers = stderr
+qualname = cinder
+
+[logger_amqplib]
+level = WARNING
+handlers = stderr
+qualname = amqplib
+
+[logger_sqlalchemy]
+level = WARNING
+handlers = stderr
+qualname = sqlalchemy
+# "level = INFO" logs SQL queries.
+# "level = DEBUG" logs SQL queries and results.
+# "level = WARNING" logs neither.  (Recommended for production systems.)
+
+[logger_boto]
+level = WARNING
+handlers = stderr
+qualname = boto
+
+[logger_suds]
+level = INFO
+handlers = stderr
+qualname = suds
+
+[logger_eventletwsgi]
+level = WARNING
+handlers = stderr
+qualname = eventlet.wsgi.server
+
+[handler_stderr]
+class = StreamHandler
+args = (sys.stderr,)
+formatter = legacycinder
+
+[handler_stdout]
+class = StreamHandler
+args = (sys.stdout,)
+formatter = legacycinder
+
+[handler_watchedfile]
+class = handlers.WatchedFileHandler
+args = ('cinder.log',)
+formatter = legacycinder
+
+[handler_syslog]
+class = handlers.SysLogHandler
+args = ('/dev/log', handlers.SysLogHandler.LOG_USER)
+formatter = legacycinder
+
+[handler_null]
+class = cinder.log.NullHandler
+formatter = default
+args = ()
+
+[formatter_legacycinder]
+class = cinder.log.LegacyCinderFormatter
+
+[formatter_default]
+format = %(message)s

tests/00-setup

@@ -5,6 +5,7 @@ set -ex
 sudo add-apt-repository --yes ppa:juju/stable
 sudo apt-get update --yes
 sudo apt-get install --yes python-amulet \
-                           python-keystoneclient \
                            python-cinderclient \
-                           python-glanceclient
+                           python-glanceclient \
+                           python-keystoneclient \
+                           python-novaclient

tests/016-basic-trusty-juno

@@ -0,0 +1,11 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic Cinder deployment on trusty-juno."""
+
+from basic_deployment import CinderBasicDeployment
+
+if __name__ == '__main__':
+    deployment = CinderBasicDeployment(series='trusty',
+                                       openstack='cloud:trusty-juno',
+                                       source='cloud:trusty-updates/juno')
+    deployment.run_tests()

tests/018-basic-utopic-juno

@@ -0,0 +1,9 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic Cinder deployment on utopic-juno."""
+
+from basic_deployment import CinderBasicDeployment
+
+if __name__ == '__main__':
+    deployment = CinderBasicDeployment(series='utopic')
+    deployment.run_tests()

tests/050-basic-trusty-icehouse-git

@@ -0,0 +1,9 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic Cinder git deployment on trusty-icehouse."""
+
+from basic_deployment import CinderBasicDeployment
+
+if __name__ == '__main__':
+    deployment = CinderBasicDeployment(series='trusty', git=True)
+    deployment.run_tests()

tests/051-basic-trusty-juno-git

@@ -0,0 +1,12 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic Cinder git deployment on trusty-juno."""
+
+from basic_deployment import CinderBasicDeployment
+
+if __name__ == '__main__':
+    deployment = CinderBasicDeployment(series='trusty',
+                                       openstack='cloud:trusty-juno',
+                                       source='cloud:trusty-updates/juno',
+                                       git=True)
+    deployment.run_tests()

tests/053-basic-trusty-kilo-git

@@ -0,0 +1,12 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic cinder git deployment on trusty-kilo."""
+
+from basic_deployment import CinderBasicDeployment
+
+if __name__ == '__main__':
+    deployment = CinderBasicDeployment(series='trusty',
+                                       openstack='cloud:trusty-kilo',
+                                       source='cloud:trusty-updates/kilo',
+                                       git=True)
+    deployment.run_tests()

tests/054-basic-vivid-kilo-git

@@ -0,0 +1,9 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic Cinder git deployment on vivid-kilo."""
+
+from basic_deployment import CinderBasicDeployment
+
+if __name__ == '__main__':
+    deployment = CinderBasicDeployment(series='vivid', git=True)
+    deployment.run_tests()

tests/11-basic-precise-folsom

@@ -1,11 +0,0 @@
-#!/usr/bin/python
-
-"""Amulet tests on a basic cinder deployment on precise-folsom."""
-
-from basic_deployment import CinderBasicDeployment
-
-if __name__ == '__main__':
-    deployment = CinderBasicDeployment(series='precise',
-                                         openstack='cloud:precise-folsom',
-                                         source='cloud:precise-updates/folsom')
-    deployment.run_tests()

tests/12-basic-precise-grizzly

@@ -1,11 +0,0 @@
-#!/usr/bin/python
-
-"""Amulet tests on a basic cinder deployment on precise-grizzly."""
-
-from basic_deployment import CinderBasicDeployment
-
-if __name__ == '__main__':
-    deployment = CinderBasicDeployment(series='precise',
-                                         openstack='cloud:precise-grizzly',
-                                         source='cloud:precise-updates/grizzly')
-    deployment.run_tests()

tests/13-basic-precise-havana

@@ -1,11 +0,0 @@
-#!/usr/bin/python
-
-"""Amulet tests on a basic cinder deployment on precise-havana."""
-
-from basic_deployment import CinderBasicDeployment
-
-if __name__ == '__main__':
-    deployment = CinderBasicDeployment(series='precise',
-                                         openstack='cloud:precise-havana',
-                                         source='cloud:precise-updates/havana')
-    deployment.run_tests()

tests/basic_deployment.py

@@ -1,8 +1,10 @@
 #!/usr/bin/python
 
 import amulet
+import os
 import types
 from time import sleep
+import yaml
 import cinderclient.v1.client as cinder_client
 
 from charmhelpers.contrib.openstack.amulet.deployment import (
@@ -16,7 +18,7 @@ from charmhelpers.contrib.openstack.amulet.utils import (  # noqa
 )
 
 # Use DEBUG to turn on debug logging
-u = OpenStackAmuletUtils(ERROR)
+u = OpenStackAmuletUtils(DEBUG)
 
 
 class CinderBasicDeployment(OpenStackAmuletDeployment):
@@ -28,10 +30,12 @@ class CinderBasicDeployment(OpenStackAmuletDeployment):
     # NOTE(beisner):  Features and tests vary across Openstack releases.
     # https://wiki.openstack.org/wiki/CinderSupportMatrix
 
-    def __init__(self, series=None, openstack=None, source=None, stable=False):
+    def __init__(self, series=None, openstack=None, source=None, git=False,
+                 stable=False):
         '''Deploy the entire test environment.'''
         super(CinderBasicDeployment, self).__init__(series, openstack, source,
                                                     stable)
+        self.git = git
         self._add_services()
         self._add_relations()
         self._configure_services()
@@ -67,11 +71,31 @@ class CinderBasicDeployment(OpenStackAmuletDeployment):
 
     def _configure_services(self):
         '''Configure all of the services.'''
-        keystone_config = {'admin-password': 'openstack',
-                           'admin-token': 'ubuntutesting'}
         cinder_config = {'block-device': 'vdb',
                          'glance-api-version': '2',
                          'overwrite': 'true'}
+        if self.git:
+            branch = 'stable/' + self._get_openstack_release_string()
+            amulet_http_proxy = os.environ.get('AMULET_HTTP_PROXY')
+            openstack_origin_git = {
+                'repositories': [
+                    {'name': 'requirements',
+                     'repository':
+                     'git://git.openstack.org/openstack/requirements',
+                     'branch': branch},
+                    {'name': 'cinder',
+                     'repository': 'git://git.openstack.org/openstack/cinder',
+                     'branch': branch},
+                ],
+                'directory': '/mnt/openstack-git',
+                'http_proxy': amulet_http_proxy,
+                'https_proxy': amulet_http_proxy,
+            }
+            cinder_config['openstack-origin-git'] = \
+                yaml.dump(openstack_origin_git)
+
+        keystone_config = {'admin-password': 'openstack',
+                           'admin-token': 'ubuntutesting'}
         mysql_config = {'dataset-size': '50%'}
         configs = {'cinder': cinder_config,
                    'keystone': keystone_config,
@@ -169,9 +193,9 @@ class CinderBasicDeployment(OpenStackAmuletDeployment):
                                                           obj_count)
 
     def obj_is_status(self, obj, obj_id, stat='available',
-                      msg='openstack object status check', max_wait=60):
+                      msg='openstack object status check', max_wait=120):
         ''''Wait for an openstack object status to be as expected.
-            By default, expect an available status within 60s.  Useful
+            By default, expect an available status within 120s.  Useful
             when confirming cinder volumes, snapshots, glance images, etc.
             reach a certain state/status within a specified time.'''
         # NOTE(beisner): need to move to charmhelpers, and adjust calls here.
@@ -301,7 +325,7 @@ class CinderBasicDeployment(OpenStackAmuletDeployment):
             'auth_protocol': 'http',
             'private-address': u.valid_ip,
             'auth_host': u.valid_ip,
-            'service_username': 'cinder',
+            'service_username': 'cinder_cinderv2',
             'service_tenant_id': u.not_null,
             'service_host': u.valid_ip
         }
@@ -317,11 +341,11 @@ class CinderBasicDeployment(OpenStackAmuletDeployment):
         relation = ['identity-service',
                     'keystone:identity-service']
         expected = {
-            'service': 'cinder',
+            'cinder_service': 'cinder',
-            'region': 'RegionOne',
+            'cinder_region': 'RegionOne',
-            'public_url': u.valid_url,
+            'cinder_public_url': u.valid_url,
-            'internal_url': u.valid_url,
+            'cinder_internal_url': u.valid_url,
-            'admin_url': u.valid_url,
+            'cinder_admin_url': u.valid_url,
             'private-address': u.valid_ip
         }
         u.log.debug('')
@@ -508,7 +532,7 @@ class CinderBasicDeployment(OpenStackAmuletDeployment):
 
     def test_users(self):
         '''Verify expected users.'''
-        user0 = {'name': 'cinder',
+        user0 = {'name': 'cinder_cinderv2',
                  'enabled': True,
                  'tenantId': u.not_null,
                  'id': u.not_null,

tests/charmhelpers/contrib/amulet/utils.py

@@ -79,6 +79,9 @@ class AmuletUtils(object):
         for k, v in six.iteritems(commands):
             for cmd in v:
                 output, code = k.run(cmd)
+                self.log.debug('{} `{}` returned '
+                               '{}'.format(k.info['unit_name'],
+                                           cmd, code))
                 if code != 0:
                     return "command `{}` returned {}".format(cmd, str(code))
         return None
@@ -86,7 +89,11 @@ class AmuletUtils(object):
     def _get_config(self, unit, filename):
         """Get a ConfigParser object for parsing a unit's config file."""
         file_contents = unit.file_contents(filename)
-        config = ConfigParser.ConfigParser()
+
+        # NOTE(beisner):  by default, ConfigParser does not handle options
+        # with no value, such as the flags used in the mysql my.cnf file.
+        # https://bugs.python.org/issue7005
+        config = ConfigParser.ConfigParser(allow_no_value=True)
         config.readfp(io.StringIO(file_contents))
         return config
 
@@ -118,6 +125,9 @@ class AmuletUtils(object):
            longs, or can be a function that evaluate a variable and returns a
            bool.
            """
+        self.log.debug('actual: {}'.format(repr(actual)))
+        self.log.debug('expected: {}'.format(repr(expected)))
+
         for k, v in six.iteritems(expected):
             if k in actual:
                 if (isinstance(v, six.string_types) or
@@ -134,7 +144,6 @@ class AmuletUtils(object):
     def validate_relation_data(self, sentry_unit, relation, expected):
         """Validate actual relation data based on expected relation data."""
         actual = sentry_unit.relation(relation[0], relation[1])
-        self.log.debug('actual: {}'.format(repr(actual)))
         return self._validate_dict_data(expected, actual)
 
     def _validate_list_data(self, expected, actual):

tests/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -44,17 +44,24 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            Determine if the local branch being tested is derived from its
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
-        base_charms = ['mysql', 'mongodb', 'rabbitmq-server']
+        base_charms = ['mysql', 'mongodb']
+
+        if self.series in ['precise', 'trusty']:
+            base_series = self.series
+        else:
+            base_series = self.current_next
 
         if self.stable:
             for svc in other_services:
-                temp = 'lp:charms/{}'
+                temp = 'lp:charms/{}/{}'
-                svc['location'] = temp.format(svc['name'])
+                svc['location'] = temp.format(base_series,
+                                              svc['name'])
         else:
             for svc in other_services:
                 if svc['name'] in base_charms:
-                    temp = 'lp:charms/{}'
+                    temp = 'lp:charms/{}/{}'
-                    svc['location'] = temp.format(svc['name'])
+                    svc['location'] = temp.format(base_series,
+                                                  svc['name'])
                 else:
                     temp = 'lp:~openstack-charmers/charms/{}/{}/next'
                     svc['location'] = temp.format(self.current_next,
@@ -99,9 +106,12 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            Return an integer representing the enum value of the openstack
            release.
            """
+        # Must be ordered by OpenStack release (not by Ubuntu release):
         (self.precise_essex, self.precise_folsom, self.precise_grizzly,
          self.precise_havana, self.precise_icehouse,
-         self.trusty_icehouse, self.trusty_juno, self.trusty_kilo) = range(8)
+         self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
+         self.trusty_kilo, self.vivid_kilo) = range(10)
+
         releases = {
             ('precise', None): self.precise_essex,
             ('precise', 'cloud:precise-folsom'): self.precise_folsom,
@@ -110,7 +120,9 @@ class OpenStackAmuletDeployment(AmuletDeployment):
             ('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
             ('trusty', None): self.trusty_icehouse,
             ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
-            ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo}
+            ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
+            ('utopic', None): self.utopic_juno,
+            ('vivid', None): self.vivid_kilo}
         return releases[(self.series, self.openstack)]
 
     def _get_openstack_release_string(self):

unit_tests/__init__.py

@@ -1,2 +1,4 @@
 import sys
+
+sys.path.append('actions')
 sys.path.append('hooks')

unit_tests/test_actions_git_reinstall.py

@@ -0,0 +1,110 @@
+from mock import patch, MagicMock
+import os
+
+os.environ['JUJU_UNIT_NAME'] = 'cinder'
+
+from test_utils import RESTART_MAP
+import cinder_utils as utils
+
+# Need to do some early patching to get the module loaded.
+_restart_map = utils.restart_map
+_register_configs = utils.register_configs
+
+utils.restart_map = MagicMock()
+utils.restart_map.return_value = RESTART_MAP
+utils.register_configs = MagicMock()
+
+import git_reinstall
+
+# Unpatch it now that its loaded.
+utils.restart_map = _restart_map
+utils.register_configs = _register_configs
+
+
+from test_utils import (
+    CharmTestCase
+)
+
+TO_PATCH = [
+    'config',
+]
+
+
+openstack_origin_git = \
+    """repositories:
+         - {name: requirements,
+            repository: 'git://git.openstack.org/openstack/requirements',
+            branch: stable/juno}
+         - {name: cinder,
+            repository: 'git://git.openstack.org/openstack/cinder',
+            branch: stable/juno}"""
+
+
+class TestCinderActions(CharmTestCase):
+
+    def setUp(self):
+        super(TestCinderActions, self).setUp(git_reinstall, TO_PATCH)
+        self.config.side_effect = self.test_config.get
+
+    @patch.object(git_reinstall, 'action_set')
+    @patch.object(git_reinstall, 'action_fail')
+    @patch.object(git_reinstall, 'git_install')
+    @patch.object(git_reinstall, 'config_changed')
+    @patch('charmhelpers.contrib.openstack.utils.config')
+    def test_git_reinstall(self, _config, config_changed, git_install,
+                           action_fail, action_set):
+        _config.return_value = openstack_origin_git
+        self.test_config.set('openstack-origin-git', openstack_origin_git)
+
+        git_reinstall.git_reinstall()
+
+        git_install.assert_called_with(openstack_origin_git)
+        self.assertTrue(git_install.called)
+        self.assertTrue(config_changed.called)
+        self.assertFalse(action_set.called)
+        self.assertFalse(action_fail.called)
+
+    @patch.object(git_reinstall, 'action_set')
+    @patch.object(git_reinstall, 'action_fail')
+    @patch.object(git_reinstall, 'git_install')
+    @patch.object(git_reinstall, 'config_changed')
+    @patch('charmhelpers.contrib.openstack.utils.config')
+    def test_git_reinstall_not_configured(self, _config, config_changed,
+                                          git_install, action_fail,
+                                          action_set):
+        _config.return_value = None
+
+        git_reinstall.git_reinstall()
+
+        msg = 'openstack-origin-git is not configured'
+        action_fail.assert_called_with(msg)
+        self.assertFalse(git_install.called)
+        self.assertFalse(action_set.called)
+
+    @patch.object(git_reinstall, 'action_set')
+    @patch.object(git_reinstall, 'action_fail')
+    @patch.object(git_reinstall, 'git_install')
+    @patch.object(git_reinstall, 'config_changed')
+    @patch('traceback.format_exc')
+    @patch('charmhelpers.contrib.openstack.utils.config')
+    def test_git_reinstall_exception(self, _config, format_exc, config_changed,
+                                     git_install, action_fail, action_set):
+        _config.return_value = openstack_origin_git
+        e = OSError('something bad happened')
+        git_install.side_effect = e
+        traceback = (
+            "Traceback (most recent call last):\n"
+            "  File \"actions/git_reinstall.py\", line 37, in git_reinstall\n"
+            "    git_install(config(\'openstack-origin-git\'))\n"
+            "  File \"/usr/lib/python2.7/dist-packages/mock.py\", line 964, in __call__\n"  # noqa
+            "    return _mock_self._mock_call(*args, **kwargs)\n"
+            "  File \"/usr/lib/python2.7/dist-packages/mock.py\", line 1019, in _mock_call\n"  # noqa
+            "    raise effect\n"
+            "OSError: something bad happened\n")
+        format_exc.return_value = traceback
+
+        git_reinstall.git_reinstall()
+
+        msg = 'git-reinstall resulted in an unexpected error'
+        action_fail.assert_called_with(msg)
+        action_set.assert_called_with({'traceback': traceback})

unit_tests/test_cinder_hooks.py

@@ -4,6 +4,7 @@ from mock import (
     patch,
     call
 )
+import yaml
 
 import cinder_utils as utils
 from test_utils import (
@@ -35,6 +36,7 @@ TO_PATCH = [
     'determine_packages',
     'do_openstack_upgrade',
     'ensure_ceph_keyring',
+    'git_install',
     'juju_log',
     'log',
     'lsb_release',
@@ -63,7 +65,7 @@ TO_PATCH = [
     # charmhelpers.contrib.openstack.openstack_utils
     'configure_installation_source',
     'openstack_upgrade_available',
-    'get_os_codename_package',
+    'os_release',
     # charmhelpers.contrib.hahelpers.cluster_utils
     'canonical_url',
     'is_elected_leader',
@@ -79,17 +81,47 @@ class TestInstallHook(CharmTestCase):
 
     def setUp(self):
         super(TestInstallHook, self).setUp(hooks, TO_PATCH)
-        self.config.side_effect = self.test_config.get_all
+        self.config.side_effect = self.test_config.get
 
-    def test_install_precise_distro(self):
+    @patch.object(utils, 'git_install_requested')
+    def test_install_precise_distro(self, git_requested):
         'It redirects to cloud archive if setup to install precise+distro'
+        git_requested.return_value = False
         self.lsb_release.return_value = {'DISTRIB_CODENAME': 'precise'}
         hooks.hooks.execute(['hooks/install'])
         ca = 'cloud:precise-folsom'
         self.configure_installation_source.assert_called_with(ca)
 
-    def test_correct_install_packages(self):
+    @patch.object(utils, 'git_install_requested')
+    def test_install_git(self, git_requested):
+        git_requested.return_value = True
+        self.determine_packages.return_value = ['foo', 'bar', 'baz']
+        repo = 'cloud:trusty-juno'
+        openstack_origin_git = {
+            'repositories': [
+                {'name': 'requirements',
+                 'repository': 'git://git.openstack.org/openstack/requirements',  # noqa
+                 'branch': 'stable/juno'},
+                {'name': 'cinder',
+                 'repository': 'git://git.openstack.org/openstack/cinder',
+                 'branch': 'stable/juno'}
+            ],
+            'directory': '/mnt/openstack-git',
+        }
+        projects_yaml = yaml.dump(openstack_origin_git)
+        self.test_config.set('openstack-origin', repo)
+        self.test_config.set('openstack-origin-git', projects_yaml)
+        hooks.hooks.execute(['hooks/install'])
+        self.assertTrue(self.execd_preinstall.called)
+        self.configure_installation_source.assert_called_with(repo)
+        self.apt_update.assert_called_with()
+        self.apt_install.assert_called_with(['foo', 'bar', 'baz'], fatal=True)
+        self.git_install.assert_called_with(projects_yaml)
+
+    @patch.object(utils, 'git_install_requested')
+    def test_correct_install_packages(self, git_requested):
         'It installs the correct packages based on what is determined'
+        git_requested.return_value = False
         self.determine_packages.return_value = ['foo', 'bar', 'baz']
         hooks.hooks.execute(['hooks/install'])
         self.apt_install.assert_called_with(['foo', 'bar', 'baz'], fatal=True)
@@ -99,7 +131,7 @@ class TestChangedHooks(CharmTestCase):
 
     def setUp(self):
         super(TestChangedHooks, self).setUp(hooks, TO_PATCH)
-        self.config.side_effect = self.test_config.get_all
+        self.config.side_effect = self.test_config.get
 
     @patch.object(hooks, 'amqp_joined')
     def test_upgrade_charm_no_amqp(self, _joined):
@@ -114,8 +146,10 @@ class TestChangedHooks(CharmTestCase):
         _joined.assert_called_with(relation_id='amqp:1')
 
     @patch.object(hooks, 'configure_https')
-    def test_config_changed(self, conf_https):
+    @patch.object(hooks, 'git_install_requested')
+    def test_config_changed(self, git_requested, conf_https):
         'It writes out all config'
+        git_requested.return_value = False
         self.openstack_upgrade_available.return_value = False
         hooks.hooks.execute(['hooks/config-changed'])
         self.assertTrue(self.CONFIGS.write_all.called)
@@ -125,8 +159,10 @@ class TestChangedHooks(CharmTestCase):
                                                       False, False)
 
     @patch.object(hooks, 'configure_https')
-    def test_config_changed_block_devices(self, conf_https):
+    @patch.object(hooks, 'git_install_requested')
+    def test_config_changed_block_devices(self, git_requested, conf_https):
         'It writes out all config'
+        git_requested.return_value = False
         self.openstack_upgrade_available.return_value = False
         self.test_config.set('block-device', 'sdb /dev/sdc sde')
         self.test_config.set('volume-group', 'cinder-new')
@@ -141,12 +177,40 @@ class TestChangedHooks(CharmTestCase):
             True, True)
 
     @patch.object(hooks, 'configure_https')
-    def test_config_changed_upgrade_available(self, conf_https):
+    @patch.object(hooks, 'git_install_requested')
+    def test_config_changed_upgrade_available(self, git_requested, conf_https):
         'It writes out all config with an available OS upgrade'
+        git_requested.return_value = False
         self.openstack_upgrade_available.return_value = True
         hooks.hooks.execute(['hooks/config-changed'])
         self.do_openstack_upgrade.assert_called_with(configs=self.CONFIGS)
 
+    @patch.object(hooks, 'configure_https')
+    @patch.object(hooks, 'git_install_requested')
+    @patch.object(hooks, 'config_value_changed')
+    def test_config_changed_git_updated(self, config_val_changed,
+                                        git_requested, conf_https):
+        git_requested.return_value = True
+        repo = 'cloud:trusty-juno'
+        openstack_origin_git = {
+            'repositories': [
+                {'name': 'requirements',
+                 'repository': 'git://git.openstack.org/openstack/requirements',  # noqa
+                 'branch': 'stable/juno'},
+                {'name': 'cinder',
+                 'repository': 'git://git.openstack.org/openstack/',
+                 'branch': 'stable/juno'}
+            ],
+            'directory': '/mnt/openstack-git',
+        }
+        projects_yaml = yaml.dump(openstack_origin_git)
+        self.test_config.set('openstack-origin', repo)
+        self.test_config.set('openstack-origin-git', projects_yaml)
+        hooks.hooks.execute(['hooks/config-changed'])
+        self.git_install.assert_called_with(projects_yaml)
+        self.assertFalse(self.do_openstack_upgrade.called)
+        self.assertTrue(conf_https.called)
+
     def test_db_changed(self):
         'It writes out cinder.conf on db changed'
         self.relation_get.return_value = 'cinder/0 cinder/1'
@@ -180,6 +244,15 @@ class TestChangedHooks(CharmTestCase):
         hooks.hooks.execute(['hooks/shared-db-relation-changed'])
         self.assertFalse(self.migrate_database.called)
 
+    def test_db_changed_relation_db_missing_acls(self):
+        'No database migration is attempted when ACL list is not present'
+        self.relation_get.return_value = None
+        self.local_unit.return_value = 'cinder/0'
+        self.CONFIGS.complete_contexts.return_value = ['shared-db']
+        self.eligible_leader.return_value = True
+        hooks.hooks.execute(['hooks/shared-db-relation-changed'])
+        self.assertFalse(self.migrate_database.called)
+
     def test_pgsql_db_changed_relation_incomplete(self):
         'It does not write out cinder.conf with incomplete pgsql-db rel'
         hooks.hooks.execute(['hooks/pgsql-db-relation-changed'])
@@ -341,7 +414,7 @@ class TestJoinedHooks(CharmTestCase):
 
     def test_identity_service_joined(self):
         'It properly requests unclustered endpoint via identity-service'
-        self.get_os_codename_package.return_value = 'havana'
+        self.os_release.return_value = 'havana'
         self.unit_get.return_value = 'cindernode1'
         self.config.side_effect = self.test_config.get
         self.canonical_url.return_value = 'http://cindernode1'
@@ -363,7 +436,7 @@ class TestJoinedHooks(CharmTestCase):
 
     def test_identity_service_joined_icehouse(self):
         'It properly requests unclustered endpoint via identity-service'
-        self.get_os_codename_package.return_value = 'icehouse'
+        self.os_release.return_value = 'icehouse'
         self.unit_get.return_value = 'cindernode1'
         self.config.side_effect = self.test_config.get
         self.canonical_url.return_value = 'http://cindernode1'

unit_tests/test_cinder_utils.py

@@ -30,7 +30,7 @@ TO_PATCH = [
     'ensure_loopback_device',
     'is_block_device',
     'zap_disk',
-    'get_os_codename_package',
+    'os_release',
     'get_os_codename_install_source',
     'configure_installation_source',
     'is_elected_leader',
@@ -68,6 +68,15 @@ FDISKDISPLAY = """
 
 """
 
+openstack_origin_git = \
+    """repositories:
+         - {name: requirements,
+            repository: 'git://git.openstack.org/openstack/requirements',
+            branch: stable/juno}
+         - {name: cinder,
+            repository: 'git://git.openstack.org/openstack/cinder',
+            branch: stable/juno}"""
+
 
 class TestCinderUtils(CharmTestCase):
 
@@ -97,8 +106,10 @@ class TestCinderUtils(CharmTestCase):
         self.assertFalse(cinder_utils.service_enabled('volume'))
 
     @patch('cinder_utils.service_enabled')
-    def test_determine_packages_all(self, service_enabled):
+    @patch('cinder_utils.git_install_requested')
+    def test_determine_packages_all(self, git_requested, service_enabled):
         'It determines all packages required when all services enabled'
+        git_requested.return_value = False
         service_enabled.return_value = True
         pkgs = cinder_utils.determine_packages()
         self.assertEquals(sorted(pkgs),
@@ -108,8 +119,10 @@ class TestCinderUtils(CharmTestCase):
                                  cinder_utils.SCHEDULER_PACKAGES))
 
     @patch('cinder_utils.service_enabled')
-    def test_determine_packages_subset(self, service_enabled):
+    @patch('cinder_utils.git_install_requested')
+    def test_determine_packages_subset(self, git_requested, service_enabled):
         'It determines packages required for a subset of enabled services'
+        git_requested.return_value = False
         service_enabled.side_effect = self.svc_enabled
 
         self.test_config.set('enabled-services', 'api')
@@ -402,7 +415,7 @@ class TestCinderUtils(CharmTestCase):
     @patch('os.path.exists')
     def test_register_configs_apache(self, exists):
         exists.return_value = False
-        self.get_os_codename_package.return_value = 'grizzly'
+        self.os_release.return_value = 'grizzly'
         self.relation_ids.return_value = False
         configs = cinder_utils.register_configs()
         calls = []
@@ -419,7 +432,7 @@ class TestCinderUtils(CharmTestCase):
     @patch('os.path.exists')
     def test_register_configs_apache24(self, exists):
         exists.return_value = True
-        self.get_os_codename_package.return_value = 'grizzly'
+        self.os_release.return_value = 'grizzly'
         self.relation_ids.return_value = False
         configs = cinder_utils.register_configs()
         calls = []
@@ -438,7 +451,7 @@ class TestCinderUtils(CharmTestCase):
     def test_register_configs_ceph(self, exists, isdir):
         exists.return_value = True
         isdir.return_value = False
-        self.get_os_codename_package.return_value = 'grizzly'
+        self.os_release.return_value = 'grizzly'
         self.relation_ids.return_value = ['ceph:0']
         self.ceph_config_file.return_value = '/var/lib/charm/cinder/ceph.conf'
         configs = cinder_utils.register_configs()
@@ -504,3 +517,149 @@ class TestCinderUtils(CharmTestCase):
         self.apt_install.assert_called_with(['mypackage'], fatal=True)
         configs.set_release.assert_called_with(openstack_release='havana')
         self.assertFalse(migrate.called)
+
+    @patch.object(cinder_utils, 'git_install_requested')
+    @patch.object(cinder_utils, 'git_clone_and_install')
+    @patch.object(cinder_utils, 'git_post_install')
+    @patch.object(cinder_utils, 'git_pre_install')
+    def test_git_install(self, git_pre, git_post, git_clone_and_install,
+                         git_requested):
+        projects_yaml = openstack_origin_git
+        git_requested.return_value = True
+        cinder_utils.git_install(projects_yaml)
+        self.assertTrue(git_pre.called)
+        git_clone_and_install.assert_called_with(openstack_origin_git,
+                                                 core_project='cinder')
+        self.assertTrue(git_post.called)
+
+    @patch.object(cinder_utils, 'mkdir')
+    @patch.object(cinder_utils, 'write_file')
+    @patch.object(cinder_utils, 'add_user_to_group')
+    @patch.object(cinder_utils, 'add_group')
+    @patch.object(cinder_utils, 'adduser')
+    def test_git_pre_install(self, adduser, add_group, add_user_to_group,
+                             write_file, mkdir):
+        cinder_utils.git_pre_install()
+        adduser.assert_called_with('cinder', shell='/bin/bash',
+                                   system_user=True)
+        add_group.assert_called_with('cinder', system_group=True)
+        add_user_to_group.assert_called_with('cinder', 'cinder')
+        expected = [
+            call('/etc/tgt', owner='cinder', perms=488, force=False,
+                 group='cinder'),
+            call('/var/lib/cinder', owner='cinder', perms=493, force=False,
+                 group='cinder'),
+            call('/var/lib/cinder/volumes', owner='cinder', perms=488,
+                 force=False, group='cinder'),
+            call('/var/lock/cinder', owner='cinder', perms=488, force=False,
+                 group='root'),
+            call('/var/log/cinder', owner='cinder', perms=488, force=False,
+                 group='cinder'),
+        ]
+        self.assertEquals(mkdir.call_args_list, expected)
+        expected = [
+            call('/var/log/cinder/cinder-api.log', '', perms=0600,
+                 owner='cinder', group='cinder'),
+            call('/var/log/cinder/cinder-backup.log', '', perms=0600,
+                 owner='cinder', group='cinder'),
+            call('/var/log/cinder/cinder-scheduler.log', '', perms=0600,
+                 owner='cinder', group='cinder'),
+            call('/var/log/cinder/cinder-volume.log', '', perms=0600,
+                 owner='cinder', group='cinder'),
+        ]
+        self.assertEquals(write_file.call_args_list, expected)
+
+    @patch.object(cinder_utils, 'git_src_dir')
+    @patch.object(cinder_utils, 'service_restart')
+    @patch.object(cinder_utils, 'render')
+    @patch('os.path.join')
+    @patch('os.path.exists')
+    @patch('shutil.copytree')
+    @patch('shutil.rmtree')
+    @patch('pwd.getpwnam')
+    @patch('grp.getgrnam')
+    @patch('os.chown')
+    @patch('os.chmod')
+    def test_git_post_install(self, chmod, chown, grp, pwd, rmtree, copytree,
+                              exists, join, render, service_restart,
+                              git_src_dir):
+        projects_yaml = openstack_origin_git
+        join.return_value = 'joined-string'
+        cinder_utils.git_post_install(projects_yaml)
+        expected = [
+            call('joined-string', '/etc/cinder'),
+        ]
+        copytree.assert_has_calls(expected)
+
+        cinder_api_context = {
+            'service_description': 'Cinder API server',
+            'service_name': 'Cinder',
+            'user_name': 'cinder',
+            'start_dir': '/var/lib/cinder',
+            'process_name': 'cinder-api',
+            'executable_name': '/usr/local/bin/cinder-api',
+            'config_files': ['/etc/cinder/cinder.conf'],
+            'log_file': '/var/log/cinder/cinder-api.log',
+        }
+
+        cinder_backup_context = {
+            'service_description': 'Cinder backup server',
+            'service_name': 'Cinder',
+            'user_name': 'cinder',
+            'start_dir': '/var/lib/cinder',
+            'process_name': 'cinder-backup',
+            'executable_name': '/usr/local/bin/cinder-backup',
+            'config_files': ['/etc/cinder/cinder.conf'],
+            'log_file': '/var/log/cinder/cinder-backup.log',
+        }
+
+        cinder_scheduler_context = {
+            'service_description': 'Cinder scheduler server',
+            'service_name': 'Cinder',
+            'user_name': 'cinder',
+            'start_dir': '/var/lib/cinder',
+            'process_name': 'cinder-scheduler',
+            'executable_name': '/usr/local/bin/cinder-scheduler',
+            'config_files': ['/etc/cinder/cinder.conf'],
+            'log_file': '/var/log/cinder/cinder-scheduler.log',
+        }
+
+        cinder_volume_context = {
+            'service_description': 'Cinder volume server',
+            'service_name': 'Cinder',
+            'user_name': 'cinder',
+            'start_dir': '/var/lib/cinder',
+            'process_name': 'cinder-volume',
+            'executable_name': '/usr/local/bin/cinder-volume',
+            'config_files': ['/etc/cinder/cinder.conf'],
+            'log_file': '/var/log/cinder/cinder-volume.log',
+        }
+        expected = [
+            call('cinder.conf', '/etc/cinder/cinder.conf', {}, owner='cinder',
+                 group='cinder', perms=0o644),
+            call('git/cinder_tgt.conf', '/etc/tgt/conf.d', {}, owner='cinder',
+                 group='cinder', perms=0o644),
+            call('git/logging.conf', '/etc/cinder/logging.conf', {},
+                 owner='cinder', group='cinder', perms=0o644),
+            call('git/cinder_sudoers', '/etc/sudoers.d/cinder_sudoers', {},
+                 owner='root', group='root', perms=0o440),
+            call('git.upstart', '/etc/init/cinder-api.conf',
+                 cinder_api_context, perms=0o644,
+                 templates_dir='joined-string'),
+            call('git.upstart', '/etc/init/cinder-backup.conf',
+                 cinder_backup_context, perms=0o644,
+                 templates_dir='joined-string'),
+            call('git.upstart', '/etc/init/cinder-scheduler.conf',
+                 cinder_scheduler_context, perms=0o644,
+                 templates_dir='joined-string'),
+            call('git.upstart', '/etc/init/cinder-volume.conf',
+                 cinder_volume_context, perms=0o644,
+                 templates_dir='joined-string'),
+        ]
+        self.assertEquals(render.call_args_list, expected)
+        expected = [
+            call('tgtd'), call('haproxy'), call('apache2'),
+            call('cinder-api'), call('cinder-volume'),
+            call('cinder-scheduler'),
+        ]
+        self.assertEquals(service_restart.call_args_list, expected)