Initial support for deploying from git

README.md

@@ -41,3 +41,98 @@ This charm has a configuration option to allow users to disable any per-instance
     ...
 
 These compute nodes could then be accessed by cloud users via use of host aggregates with specific flavors to target instances to hypervisors with no per-instance security.
+
+# Deploying from source
+
+The minimal openstack-origin-git config required to deploy from source is:
+
+  openstack-origin-git:
+      "{'neutron':
+           {'repository': 'git://git.openstack.org/openstack/neutron.git',
+            'branch': 'stable/icehouse'}}"
+
+If you specify a 'requirements' repository, it will be used to update the
+requirements.txt files of all other git repos that it applies to, before
+they are installed:
+
+  openstack-origin-git:
+      "{'requirements':
+           {'repository': 'git://git.openstack.org/openstack/requirements.git',
+            'branch': 'master'},
+        'neutron':
+           {'repository': 'git://git.openstack.org/openstack/neutron.git',
+            'branch': 'master'}}"
+
+Note that there are only two key values the charm knows about for the outermost
+dictionary: 'neutron' and 'requirements'. These repositories must correspond to
+these keys. If the requirements repository is specified, it will be installed
+first. The neutron repository is always installed last.  All other repostories
+will be installed in between.
+
+NOTE(coreycb): The following is temporary to keep track of the full list of
+current tip repos (may not be up-to-date).
+
+  openstack-origin-git:
+      "{'requirements':
+           {'repository': 'git://git.openstack.org/openstack/requirements.git',
+            'branch': 'master'},
+        'neutron-fwaas':
+           {'repository': 'git://git.openstack.org/openstack/neutron-fwaas.git',
+            'branch': 'master'},
+        'neutron-lbaas':
+           {'repository: 'git://git.openstack.org/openstack/neutron-lbaas.git',
+            'branch': 'master'},
+        'neutron-vpnaas':
+           {'repository: 'git://git.openstack.org/openstack/neutron-vpnaas.git',
+            'branch': 'master'},
+        'keystonemiddleware:
+           {'repository': 'git://git.openstack.org/openstack/keystonemiddleware.git',
+            'branch: 'master'},
+        'oslo-concurrency':
+           {'repository': 'git://git.openstack.org/openstack/oslo.concurrency.git',
+            'branch: 'master'},
+        'oslo-config':
+           {'repository': 'git://git.openstack.org/openstack/oslo.config.git',
+            'branch: 'master'},
+        'oslo-context':
+           {'repository': 'git://git.openstack.org/openstack/oslo.context.git',
+            'branch: 'master'},
+        'oslo-db':
+           {'repository': 'git://git.openstack.org/openstack/oslo.db.git',
+            'branch: 'master'},
+        'oslo-i18n':
+           {'repository': 'git://git.openstack.org/openstack/oslo.i18n.git',
+            'branch: 'master'},
+        'oslo-messaging':
+           {'repository': 'git://git.openstack.org/openstack/oslo.messaging.git',
+            'branch: 'master'},
+        'oslo-middleware:
+           {'repository': 'git://git.openstack.org/openstack/oslo.middleware.git',
+            'branch': 'master'},
+        'oslo-rootwrap':
+           {'repository': 'git://git.openstack.org/openstack/oslo.rootwrap.git',
+            'branch: 'master'},
+        'oslo-serialization':
+           {'repository': 'git://git.openstack.org/openstack/oslo.serialization.git',
+            'branch: 'master'},
+        'oslo-utils':
+           {'repository': 'git://git.openstack.org/openstack/oslo.utils.git',
+            'branch: 'master'},
+        'pbr':
+           {'repository': 'git://git.openstack.org/openstack-dev/pbr.git',
+            'branch: 'master'},
+        'python-keystoneclient':
+           {'repository': 'git://git.openstack.org/openstack/python-keystoneclient.git',
+            'branch: 'master'},
+        'python-neutronclient':
+           {'repository': 'git://git.openstack.org/openstack/python-neutronclient.git',
+            'branch: 'master'},
+        'python-novaclient':
+           {'repository': 'git://git.openstack.org/openstack/python-novaclient.git',
+            'branch: 'master'},
+        'stevedore':
+           {'repository': 'git://git.openstack.org/openstack/stevedore.git',
+            'branch: 'master'},
+        'neutron':
+           {'repository': 'git://git.openstack.org/openstack/neutron.git',
+            'branch': 'master'}}"

charm-helpers-sync.yaml

@@ -1,4 +1,5 @@
-branch: lp:charm-helpers
+#branch: lp:charm-helpers
+branch: /home/corey/src/charms/git/charm-helpers
 destination: hooks/charmhelpers
 include:
     - core

charm-helpers-tests.yaml

@@ -1,4 +1,5 @@
-branch: lp:charm-helpers
+#branch: lp:charm-helpers
+branch: /home/corey/src/charms/git/charm-helpers
 destination: tests/charmhelpers
 include:
     - contrib.amulet

config.yaml

@@ -1,4 +1,24 @@
 options:
+  openstack-origin-git:
+    default: None
+    type: string
+    description: |
+      Specifies a YAML-formatted two-dimensional array listing the git
+      repositories and branches from which to install OpenStack and its
+      dependencies.
+
+      When openstack-origin-git is specified, openstack-specific
+      packages will be installed from source rather than from the
+      the nova-compute charm's openstack-origin repository.
+
+      Note that the installed config files will be determined based on
+      the OpenStack release of the nova-compute charm's openstack-origin
+      option.
+
+      Note also that this option is processed for the initial install
+      only. Setting this option after deployment is not supported.
+
+      For more details see README.md.
   rabbit-user:
     default: neutron
     type: string

hooks/neutron_ovs_hooks.py

@@ -20,6 +20,7 @@ from charmhelpers.fetch import (
 
 from neutron_ovs_utils import (
     determine_packages,
+    git_install,
     register_configs,
     restart_map,
 )
@@ -35,6 +36,13 @@ def install():
     for pkg in pkgs:
         apt_install(pkg, fatal=True)
 
+    # NOTE(coreycb): This is temporary for sstack proxy, unless we decide
+    # we need to code proxy support into the charms.
+    os.environ["http_proxy"] = "http://squid.internal:3128"
+    os.environ["https_proxy"] = "https://squid.internal:3128"
+
+    git_install(config('openstack-origin-git'))
+
 
 @hooks.hook('neutron-plugin-relation-changed')
 @hooks.hook('neutron-plugin-api-relation-changed')

hooks/neutron_ovs_utils.py

@@ -1,3 +1,7 @@
+import os
+import shutil
+import yaml
+
 from charmhelpers.contrib.openstack.neutron import neutron_plugin_attribute
 from copy import deepcopy
 
@@ -8,6 +12,23 @@ from charmhelpers.contrib.openstack.utils import (
 )
 import neutron_ovs_context
 
+BASE_GIT_PACKAGES = [
+    'libxml2-dev',
+    'libxslt1-dev',
+    'openvswitch-switch',
+    'python-dev',
+    'python-pip',
+    'python-setuptools',
+    'zlib1g-dev',
+]
+
+# ubuntu packages that should not be installed when deploying from git
+GIT_PACKAGE_BLACKLIST = [
+    'neutron-server',
+    'neutron-plugin-openvswitch',
+    'neutron-plugin-openvswitch-agent',
+]
+
 NOVA_CONF_DIR = "/etc/nova"
 NEUTRON_CONF_DIR = "/etc/neutron"
 NEUTRON_CONF = '%s/neutron.conf' % NEUTRON_CONF_DIR
@@ -29,7 +50,15 @@ TEMPLATES = 'templates/'
 
 
 def determine_packages():
-    return neutron_plugin_attribute('ovs', 'packages', 'neutron')
+    pkgs = neutron_plugin_attribute('ovs', 'packages', 'neutron')
+
+    if git_install_requested():
+        pkgs.extend(BASE_GIT_PACKAGES)
+        # don't include packages that will be installed from git
+        for p in GIT_PACKAGE_BLACKLIST:
+            packages.remove(p)
+
+    return pkgs
 
 
 def register_configs(release=None):
@@ -56,3 +85,102 @@ def restart_map():
     state.
     '''
     return {k: v['services'] for k, v in resource_map().iteritems()}
+
+
+def git_install(projects):
+    """Perform setup, and install git repos specified in yaml parameter."""
+    if git_install_requested():
+        git_pre_install()
+        git_clone_and_install(yaml.load(projects), core_project='neutron')
+        git_post_install()
+
+
+def git_pre_install():
+    """Perform pre-install setup."""
+    dirs = [
+        '/etc/neutron',
+        '/etc/neutron/rootwrap.d',
+        '/etc/neutron/plugins',
+        '/var/lib/neutron',
+        '/var/lib/neutron/lock',
+        '/var/log/neutron',
+    ]
+
+    logs = [
+        '/var/log/neutron/server.log',
+    ]
+
+    adduser('neutron', shell='/bin/bash', system_user=True)
+    add_group('neutron', system_group=True)
+    add_user_to_group('neutron', 'neutron')
+
+    for d in dirs:
+        mkdir(d, owner='neutron', group='neutron', perms=0700, force=False)
+
+    for l in logs:
+        write_file(l, '', owner='neutron', group='neutron', perms=0600)
+
+
+def git_post_install():
+    """Perform post-install setup."""
+    src_etc = os.path.join(charm_dir(), '/mnt/openstack-git/neutron-api.git/etc')
+    configs = {
+        'debug-filters': {
+            'src': os.path.join(src_etc, 'neutron/rootwrap.d/debug.filters'),
+            'dest': '/etc/neutron/rootwrap.d/debug.filters',
+        },
+        'openvswitch-plugin-filters': {
+            'src': os.path.join(src_etc, 'neutron/rootwrap.d/openvswitch-plugin.filters'),
+            'dest': '/etc/neutron/rootwrap.d/openvswitch-plugin.filters',
+        },
+        'policy': {
+            'src': os.path.join(src_etc, 'policy.json'),
+            'dest': '/etc/neutron/policy.json',
+        },
+        'rootwrap': {
+            'src': os.path.join(src_etc, 'rootwrap.conf'),
+            'dest': '/etc/neutron/rootwrap.conf',
+        },
+    }
+
+    for conf, files in configs.iteritems():
+        shutil.copyfile(files['src'], files['dest'])
+
+    configs_dir = {
+        'ml2': {
+            'src': os.path.join(src_etc, 'neutron/plugins/ml2/'),
+            'dest': '/etc/neutron/plugins/ml2/',
+        },
+    }
+
+    for conf, dirs in configs_dir.iteritems():
+        shutil.copytree(dirs['src'], dirs['dest'])
+
+    #render('neutron-server.default', '/etc/default/neutron-server', {}, perms=0o440)
+    #render('neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {}, perms=0o440)
+
+    neutron_ovs_agent_context = {
+        'service_description': 'Neutron OpenvSwitch Plugin Agent',
+        'charm_name': 'neutron-openvswitch',
+        'process_name': 'neutron-openvswitch-agent',
+        'cleanup_process_name': 'neutron-ovs-cleanup',
+        'plugin_config': '/etc/neutron/plugins/ml2/ml2_conf.ini',
+        'log_file': '/var/log/neutron/openvswitch-agent.log',
+    }
+
+    neutron_ovs_cleanup_context = {
+        'service_description': 'Neutron OpenvSwitch Cleanup',
+        'charm_name': 'neutron-openvswitch',
+        'process_name': 'neutron-ovs-cleanup',
+        'log_file': '/var/log/neutron/ovs-cleanup.log',
+    }
+
+    render('upstart/neutron-plugin-openvswitch-agent.upstart',
+           '/etc/init/neutron-plugin-openvswitch-agent.conf',
+           neutron_ovs_agent_context, perms=0o644)
+
+    render('upstart/neutron-ovs-cleanup.upstart',
+           '/etc/init/neutron-ovs-cleanup.conf',
+           neutron_ovs_cleanup_context, perms=0o644)
+
+    service_start('neutron-plugin-openvswitch-agent')

templates/upstart/neutron-ovs-cleanup.upstart

@@ -0,0 +1,17 @@
+description "{{ service_description }}"
+author "Juju {{ charm_name }} Charm <juju@localhost>"
+
+start on started openvswitch-switch
+stop on runlevel [!2345]
+
+pre-start script
+  mkdir -p /var/run/neutron
+  chown neutron:root /var/run/neutron
+end script
+
+pre-start script
+  [ ! -x /usr/bin/{{ process_name }} ] && exit 0-
+  start-stop-daemon --start --chuid neutron --exec /usr/local/bin/{{ process_name }} -- \
+    --log-file /var/log/neutron/{{ log_file }} \
+    --config-file /etc/neutron/neutron.conf --verbose
+end script

templates/upstart/neutron-plugin-openvswitch-agent.upstart

@@ -0,0 +1,18 @@
+description "{{ service_description }}"
+author "Juju {{ charm_name }} Charm <juju@localhost>"
+
+start on runlevel [2345] and started {{ cleanup_process_name}}
+stop on runlevel [!2345]
+
+respawn
+
+chdir /var/run
+
+pre-start script
+  mkdir -p /var/run/neutron
+  chown neutron:root /var/run/neutron
+end script
+
+exec start-stop-daemon --start --chuid neutron --exec /usr/local/bin/{{ process_name }} -- \
+    --config-file=/etc/neutron/neutron.conf --config-file={{ plugin_config }} \
+    --log-file={{ log_file }}

tests/basic_deployment.py

@@ -61,7 +61,11 @@ class NeutronOVSBasicDeployment(OpenStackAmuletDeployment):
 
     def _configure_services(self):
         """Configure all of the services."""
-        configs = {}
+        neutron_ovs_config = {'openstack-origin-git':
+                              "{'neutron':"
+                              "   {'repository': 'git://git.openstack.org/openstack/neutron.git',"
+                              "    'branch': 'stable/icehouse'}}"}
+        configs = {'neutron-openvswitch': neutron_ovs_config}
         super(NeutronOVSBasicDeployment, self)._configure_services(configs)
 
     def _initialize_tests(self):