Initial support for deploying from git

README.md

@@ -23,6 +23,101 @@ This charm also supports scale out and high availability using the hacluster cha
     juju set neutron-api vip=<VIP FOR ACCESS>
     juju add-relation neutron-hacluster neutron-api
 
+# Deploying from source
+
+The minimal openstack-origin-git config required to deploy from source is:
+
+  openstack-origin-git:
+      "{'neutron':
+           {'repository': 'git://git.openstack.org/openstack/neutron.git',
+            'branch': 'stable/icehouse'}}"
+
+If you specify a 'requirements' repository, it will be used to update the
+requirements.txt files of all other git repos that it applies to, before
+they are installed:
+
+  openstack-origin-git:
+      "{'requirements':
+           {'repository': 'git://git.openstack.org/openstack/requirements.git',
+            'branch': 'master'},
+        'neutron':
+           {'repository': 'git://git.openstack.org/openstack/neutron.git',
+            'branch': 'master'}}"
+
+Note that there are only two key values the charm knows about for the outermost
+dictionary: 'neutron' and 'requirements'. These repositories must correspond to
+these keys. If the requirements repository is specified, it will be installed
+first. The neutron repository is always installed last.  All other repostories
+will be installed in between.
+
+NOTE(coreycb): The following is temporary to keep track of the full list of
+current tip repos (may not be up-to-date).
+
+  openstack-origin-git:
+      "{'requirements':
+           {'repository': 'git://git.openstack.org/openstack/requirements.git',
+            'branch': 'master'},
+        'neutron-fwaas':
+           {'repository': 'git://git.openstack.org/openstack/neutron-fwaas.git',
+            'branch': 'master'},
+        'neutron-lbaas':
+           {'repository: 'git://git.openstack.org/openstack/neutron-lbaas.git',
+            'branch': 'master'},
+        'neutron-vpnaas':
+           {'repository: 'git://git.openstack.org/openstack/neutron-vpnaas.git',
+            'branch': 'master'},
+        'keystonemiddleware:
+           {'repository': 'git://git.openstack.org/openstack/keystonemiddleware.git',
+            'branch: 'master'},
+        'oslo-concurrency':
+           {'repository': 'git://git.openstack.org/openstack/oslo.concurrency.git',
+            'branch: 'master'},
+        'oslo-config':
+           {'repository': 'git://git.openstack.org/openstack/oslo.config.git',
+            'branch: 'master'},
+        'oslo-context':
+           {'repository': 'git://git.openstack.org/openstack/oslo.context.git',
+            'branch: 'master'},
+        'oslo-db':
+           {'repository': 'git://git.openstack.org/openstack/oslo.db.git',
+            'branch: 'master'},
+        'oslo-i18n':
+           {'repository': 'git://git.openstack.org/openstack/oslo.i18n.git',
+            'branch: 'master'},
+        'oslo-messaging':
+           {'repository': 'git://git.openstack.org/openstack/oslo.messaging.git',
+            'branch: 'master'},
+        'oslo-middleware:
+           {'repository': 'git://git.openstack.org/openstack/oslo.middleware.git',
+            'branch': 'master'},
+        'oslo-rootwrap':
+           {'repository': 'git://git.openstack.org/openstack/oslo.rootwrap.git',
+            'branch: 'master'},
+        'oslo-serialization':
+           {'repository': 'git://git.openstack.org/openstack/oslo.serialization.git',
+            'branch: 'master'},
+        'oslo-utils':
+           {'repository': 'git://git.openstack.org/openstack/oslo.utils.git',
+            'branch: 'master'},
+        'pbr':
+           {'repository': 'git://git.openstack.org/openstack-dev/pbr.git',
+            'branch: 'master'},
+        'python-keystoneclient':
+           {'repository': 'git://git.openstack.org/openstack/python-keystoneclient.git',
+            'branch: 'master'},
+        'python-neutronclient':
+           {'repository': 'git://git.openstack.org/openstack/python-neutronclient.git',
+            'branch: 'master'},
+        'python-novaclient':
+           {'repository': 'git://git.openstack.org/openstack/python-novaclient.git',
+            'branch: 'master'},
+        'stevedore':
+           {'repository': 'git://git.openstack.org/openstack/stevedore.git',
+            'branch: 'master'},
+        'neutron':
+           {'repository': 'git://git.openstack.org/openstack/neutron.git',
+            'branch': 'master'}}"
+
 # Restrictions
 
 This charm only support deployment with OpenStack Icehouse or better.

charm-helpers-hooks.yaml

@@ -1,4 +1,5 @@
-branch: lp:charm-helpers
+#branch: lp:charm-helpers
+branch: /home/corey/src/charms/git/charm-helpers
 destination: hooks/charmhelpers
 include:
     - core

charm-helpers-tests.yaml

@@ -1,4 +1,5 @@
-branch: lp:charm-helpers
+#branch: lp:charm-helpers
+branch: /home/corey/src/charms/git/charm-helpers
 destination: tests/charmhelpers
 include:
     - contrib.amulet

config.yaml

@@ -14,6 +14,25 @@ options:
       Note that updating this setting to a source that is known to
       provide a later version of OpenStack will trigger a software
       upgrade.
+
+      Note that when openstack-origin-git is specified, openstack-specific
+      packages will be installed from source rather than from the
+      openstack-origin repository.
+  openstack-origin-git:
+    default: None
+    type: string
+    description: |
+      Specifies a YAML-formatted two-dimensional array listing the git
+      repositories and branches from which to install OpenStack and its
+      dependencies.
+
+      Note that the installed config files will be determined based on
+      the OpenStack release of the openstack-origin option.
+
+      Note also that this option is processed for the initial install
+      only. Setting this option after deployment is not supported.
+
+      For more details see README.md.
   rabbit-user:
     default: neutron
     type: string

hooks/neutron_api_hooks.py

@@ -30,6 +30,7 @@ from charmhelpers.fetch import (
 
 from charmhelpers.contrib.openstack.utils import (
     configure_installation_source,
+    git_install_requested,
     openstack_upgrade_available,
     sync_db_with_multi_ipv6_addresses
 )
@@ -40,6 +41,7 @@ from neutron_api_utils import (
     determine_packages,
     determine_ports,
     do_openstack_upgrade,
+    git_install,
     register_configs,
     restart_map,
     services,
@@ -103,6 +105,14 @@ def install():
     apt_update()
     apt_install(determine_packages(config('openstack-origin')),
                 fatal=True)
+
+    # NOTE(coreycb): This is temporary for sstack proxy, unless we decide
+    # we need to code proxy support into the charms.
+    os.environ["http_proxy"] = "http://squid.internal:3128"
+    os.environ["https_proxy"] = "https://squid.internal:3128"
+
+    git_install(config('openstack-origin-git'))
+
     [open_port(port) for port in determine_ports()]
 
 
@@ -119,8 +129,9 @@ def config_changed():
                                           config('database-user'))
 
     global CONFIGS
-    if openstack_upgrade_available('neutron-server'):
-        do_openstack_upgrade(CONFIGS)
+    if not git_install_requested():
+        if openstack_upgrade_available('neutron-server'):
+            do_openstack_upgrade(CONFIGS)
     configure_https()
     update_nrpe_config()
     CONFIGS.write_all()
@@ -135,6 +146,12 @@ def config_changed():
     [cluster_joined(rid) for rid in relation_ids('cluster')]
 
 
+#TODO(coreycb): For deploy from git support, need to implement action-set
+#               and action-get to trigger re-install of git-installed
+#               services.  IIUC they'd be triggered via:
+#               juju do <action> <parameters>
+
+
 @hooks.hook('amqp-relation-joined')
 def amqp_joined(relation_id=None):
     relation_set(relation_id=relation_id,

hooks/neutron_api_utils.py

@@ -1,6 +1,8 @@
 from collections import OrderedDict
 from copy import deepcopy
 import os
+import shutil
+import yaml
 from base64 import b64encode
 from charmhelpers.contrib.openstack import context, templating
 from charmhelpers.contrib.openstack.neutron import (
@@ -10,6 +12,8 @@ from charmhelpers.contrib.openstack.neutron import (
 from charmhelpers.contrib.openstack.utils import (
     os_release,
     get_os_codename_install_source,
+    git_install_requested,
+    git_clone_and_install,
     configure_installation_source,
 )
 
@@ -26,7 +30,12 @@ from charmhelpers.fetch import (
 )
 
 from charmhelpers.core.host import (
-    lsb_release
+    adduser,
+    add_group,
+    add_user_to_group,
+    mkdir,
+    lsb_release,
+    write_file,
 )
 
 import neutron_api_context
@@ -52,6 +61,27 @@ KILO_PACKAGES = [
     'python-neutron-vpnaas',
 ]
 
+BASE_GIT_PACKAGES = [
+    'libxml2-dev',
+    'libxslt1-dev',
+    'python-dev',
+    'python-pip',
+    'python-setuptools',
+    'zlib1g-dev',
+]
+
+# ubuntu packages that should not be installed when deploying from git
+GIT_PACKAGE_BLACKLIST = [
+    'neutron-server',
+    'python-keystoneclient',
+]
+
+GIT_PACKAGE_BLACKLIST_KILO = [
+    'python-neutron-lbaas',
+    'python-neutron-fwaas',
+    'python-neutron-vpnaas',
+]
+
 BASE_SERVICES = [
     'neutron-server'
 ]
@@ -110,14 +140,26 @@ def api_port(service):
 def determine_packages(source=None):
     # currently all packages match service names
     packages = [] + BASE_PACKAGES
+
     for v in resource_map().values():
         packages.extend(v['services'])
         pkgs = neutron_plugin_attribute(config('neutron-plugin'),
                                         'server_packages',
                                         'neutron')
         packages.extend(pkgs)
+
     if get_os_codename_install_source(source) >= 'kilo':
         packages.extend(KILO_PACKAGES)
+
+    if git_install_requested():
+        packages.extend(BASE_GIT_PACKAGES)
+        # don't include packages that will be installed from git
+        for p in GIT_PACKAGE_BLACKLIST:
+            packages.remove(p)
+        if get_os_codename_install_source(source) >= 'kilo':
+            for p in GIT_PACKAGE_BLACKLIST_KILO:
+                packages.remove(p)
+
     return list(set(packages))
 
 
@@ -242,3 +284,81 @@ def setup_ipv6():
                    ' main')
         apt_update()
         apt_install('haproxy/trusty-backports', fatal=True)
+
+
+def git_install(projects):
+    """Perform setup, and install git repos specified in yaml parameter."""
+    if git_install_requested():
+        git_pre_install()
+        # NOTE(coreycb): charm-helpers needs support to take array of
+        # core_projects. That would allow all neutron* projects to be
+        # installed last.
+        core = ['neutron-fwaas', 'neutron-lbaas', 'neutron-vpnaas', 'neutron']
+        git_clone_and_install(yaml.load(projects), core_projects=core)
+        git_post_install()
+
+
+def git_pre_install():
+    """Perform pre-install setup."""
+    dirs = [
+        '/etc/neutron',
+        '/etc/neutron/rootwrap.d',
+        '/etc/neutron/plugins',
+        '/var/lib/neutron',
+        '/var/lib/neutron/lock',
+        '/var/log/neutron',
+    ]
+
+    logs = [
+        '/var/log/neutron/server.log',
+    ]
+
+    adduser('neutron', shell='/bin/bash', system_user=True)
+    add_group('neutron', system_group=True)
+    add_user_to_group('neutron', 'neutron')
+
+    for d in dirs:
+        mkdir(d, owner='neutron', group='neutron', perms=0700, force=False)
+
+    for l in logs:
+        write_file(l, '', owner='neutron', group='neutron', perms=0600)
+
+
+def git_post_install():
+    """Perform post-install setup."""
+    src_etc = os.path.join(charm_dir(), '/mnt/openstack-git/neutron-api.git/etc')
+    configs = {
+        'api-paste': {
+            'src': os.path.join(src_etc, 'api-paste.ini'),
+            'dest': '/etc/neutron/api-paste.ini',
+        },
+        'debug-filters': {
+            'src': os.path.join(src_etc, 'neutron/rootwrap.d/debug.filters'),
+            'dest': '/etc/neutron/rootwrap.d/debug.filters',
+        },
+        'policy': {
+            'src': os.path.join(src_etc, 'policy.json'),
+            'dest': '/etc/neutron/policy.json',
+        },
+        'rootwrap': {
+            'src': os.path.join(src_etc, 'rootwrap.conf'),
+            'dest': '/etc/neutron/rootwrap.conf',
+        },
+    }
+
+    for conf, files in configs.iteritems():
+        shutil.copyfile(files['src'], files['dest'])
+
+    render('neutron-server.default', '/etc/default/neutron-server', {}, perms=0o440)
+    render('neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {}, perms=0o440)
+
+    neutron_api_context = {
+        'service_description': 'Neutron API server',
+        'charm_name': 'neutron-api',
+        'process_name': 'neutron-server',
+    }
+
+    render('upstart/neutron-server.upstart', '/etc/init/neutron.conf',
+           neutron_api_context, perms=0o644)
+
+    service_start('neutron-server')

templates/upstart/neutron-server.upstart

@@ -0,0 +1,22 @@
+description "{{ service_description }}"
+author "Juju {{ charm_name }} Charm <juju@localhost>"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+
+chdir /var/run
+
+pre-start script
+  mkdir -p /var/run/neutron
+  chown neutron:root /var/run/neutron
+end script
+
+script
+  [ -r /etc/default/{{ process_name }} ] && . /etc/default/{{ process_name }}
+  [ -r "$NEUTRON_PLUGIN_CONFIG" ] && CONF_ARG="--config-file $NEUTRON_PLUGIN_CONFIG"
+  exec start-stop-daemon --start --chuid neutron --exec /usr/local/bin/{{ process_name }} -- \
+    --config-file /etc/neutron/neutron.conf \
+    --log-file /var/log/neutron/server.log $CONF_ARG
+end script

tests/basic_deployment.py

@@ -65,11 +65,17 @@ class NeutronAPIBasicDeployment(OpenStackAmuletDeployment):
 
     def _configure_services(self):
         """Configure all of the services."""
+        # NOTE(coreycb): Added the following temporarily to test deploy from source
+        neutron_api_config = {'openstack-origin-git':
+                              "{'neutron':"
+                              "   {'repository': 'git://git.openstack.org/openstack/neutron.git',"
+                              "    'branch': 'stable/icehouse'}}"}
         keystone_config = {'admin-password': 'openstack',
                            'admin-token': 'ubuntutesting'}
         nova_cc_config = {'network-manager': 'Quantum',
                           'quantum-security-groups': 'yes'}
-        configs = {'keystone': keystone_config,
+        configs = {'neutron-api': neutron_api_config,
+                   'keystone': keystone_config,
                    'nova-cloud-controller': nova_cc_config}
         super(NeutronAPIBasicDeployment, self)._configure_services(configs)