[james-page, r=gnuoy] Add 0mq support

hooks/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -15,6 +15,7 @@
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import six
+from collections import OrderedDict
 from charmhelpers.contrib.amulet.deployment import (
     AmuletDeployment
 )
@@ -100,12 +101,34 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            """
         (self.precise_essex, self.precise_folsom, self.precise_grizzly,
          self.precise_havana, self.precise_icehouse,
-         self.trusty_icehouse) = range(6)
+         self.trusty_icehouse, self.trusty_juno, self.trusty_kilo) = range(8)
         releases = {
             ('precise', None): self.precise_essex,
             ('precise', 'cloud:precise-folsom'): self.precise_folsom,
             ('precise', 'cloud:precise-grizzly'): self.precise_grizzly,
             ('precise', 'cloud:precise-havana'): self.precise_havana,
             ('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
-            ('trusty', None): self.trusty_icehouse}
+            ('trusty', None): self.trusty_icehouse,
+            ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
+            ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo}
         return releases[(self.series, self.openstack)]
+
+    def _get_openstack_release_string(self):
+        """Get openstack release string.
+
+           Return a string representing the openstack release.
+           """
+        releases = OrderedDict([
+            ('precise', 'essex'),
+            ('quantal', 'folsom'),
+            ('raring', 'grizzly'),
+            ('saucy', 'havana'),
+            ('trusty', 'icehouse'),
+            ('utopic', 'juno'),
+            ('vivid', 'kilo'),
+        ])
+        if self.openstack:
+            os_origin = self.openstack.split(':')[1]
+            return os_origin.split('%s-' % self.series)[1].split('/')[0]
+        else:
+            return releases[self.series]

hooks/charmhelpers/contrib/openstack/templates/git.upstart

@@ -0,0 +1,13 @@
+description "{{ service_description }}"
+author "Juju {{ service_name }} Charm <juju@localhost>"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+
+exec start-stop-daemon --start --chuid {{ user_name }} \
+            --chdir {{ start_dir }} --name {{ process_name }} \
+            --exec {{ executable_name }} -- \
+            --config-file={{ config_file }} \
+            --log-file={{ log_file }}

hooks/charmhelpers/contrib/openstack/templates/section-zeromq

@@ -3,12 +3,12 @@
 rpc_backend = zmq
 rpc_zmq_host = {{ zmq_host }}
 {% if zmq_redis_address -%}
-rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_redis.MatchMakerRedis
+rpc_zmq_matchmaker = redis
 matchmaker_heartbeat_freq = 15
 matchmaker_heartbeat_ttl = 30
 [matchmaker_redis]
 host = {{ zmq_redis_address }}
 {% else -%}
-rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_ring.MatchMakerRing
+rpc_zmq_matchmaker = ring
 {% endif -%}
 {% endif -%}

hooks/charmhelpers/contrib/openstack/utils.py

@@ -30,6 +30,10 @@ import yaml
 
 from charmhelpers.contrib.network import ip
 
+from charmhelpers.core import (
+    unitdata,
+)
+
 from charmhelpers.core.hookenv import (
     config,
     log as juju_log,
@@ -330,6 +334,21 @@ def configure_installation_source(rel):
         error_out("Invalid openstack-release specified: %s" % rel)
 
 
+def config_value_changed(option):
+    """
+    Determine if config value changed since last call to this function.
+    """
+    hook_data = unitdata.HookData()
+    with hook_data():
+        db = unitdata.kv()
+        current = config(option)
+        saved = db.get(option)
+        db.set(option, current)
+        if saved is None:
+            return False
+        return current != saved
+
+
 def save_script_rc(script_path="scripts/scriptrc", **env_vars):
     """
     Write an rc file in the charm-delivered directory containing
@@ -469,82 +488,95 @@ def os_requires_version(ostack_release, pkg):
 
 
 def git_install_requested():
-    """Returns true if openstack-origin-git is specified."""
-    return config('openstack-origin-git') != "None"
+    """
+    Returns true if openstack-origin-git is specified.
+    """
+    return config('openstack-origin-git') is not None
 
 
 requirements_dir = None
 
 
-def git_clone_and_install(file_name, core_project):
-    """Clone/install all OpenStack repos specified in yaml config file."""
-    global requirements_dir
+def git_clone_and_install(projects_yaml, core_project):
+    """
+    Clone/install all specified OpenStack repositories.
 
-    if file_name == "None":
+    The expected format of projects_yaml is:
+        repositories:
+          - {name: keystone,
+             repository: 'git://git.openstack.org/openstack/keystone.git',
+             branch: 'stable/icehouse'}
+          - {name: requirements,
+             repository: 'git://git.openstack.org/openstack/requirements.git',
+             branch: 'stable/icehouse'}
+        directory: /mnt/openstack-git
+
+        The directory key is optional.
+    """
+    global requirements_dir
+    parent_dir = '/mnt/openstack-git'
+
+    if not projects_yaml:
         return
 
-    yaml_file = os.path.join(charm_dir(), file_name)
+    projects = yaml.load(projects_yaml)
+    _git_validate_projects_yaml(projects, core_project)
 
-    # clone/install the requirements project first
-    installed = _git_clone_and_install_subset(yaml_file,
-                                              whitelist=['requirements'])
-    if 'requirements' not in installed:
-        error_out('requirements git repository must be specified')
+    if 'directory' in projects.keys():
+        parent_dir = projects['directory']
 
-    # clone/install all other projects except requirements and the core project
-    blacklist = ['requirements', core_project]
-    _git_clone_and_install_subset(yaml_file, blacklist=blacklist,
-                                  update_requirements=True)
-
-    # clone/install the core project
-    whitelist = [core_project]
-    installed = _git_clone_and_install_subset(yaml_file, whitelist=whitelist,
-                                              update_requirements=True)
-    if core_project not in installed:
-        error_out('{} git repository must be specified'.format(core_project))
+    for p in projects['repositories']:
+        repo = p['repository']
+        branch = p['branch']
+        if p['name'] == 'requirements':
+            repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
+                                                     update_requirements=False)
+            requirements_dir = repo_dir
+        else:
+            repo_dir = _git_clone_and_install_single(repo, branch, parent_dir,
+                                                     update_requirements=True)
 
 
-def _git_clone_and_install_subset(yaml_file, whitelist=[], blacklist=[],
-                                  update_requirements=False):
-    """Clone/install subset of OpenStack repos specified in yaml config file."""
-    global requirements_dir
-    installed = []
+def _git_validate_projects_yaml(projects, core_project):
+    """
+    Validate the projects yaml.
+    """
+    _git_ensure_key_exists('repositories', projects)
 
-    with open(yaml_file, 'r') as fd:
-        projects = yaml.load(fd)
-        for proj, val in projects.items():
-            # The project subset is chosen based on the following 3 rules:
-            # 1) If project is in blacklist, we don't clone/install it, period.
-            # 2) If whitelist is empty, we clone/install everything else.
-            # 3) If whitelist is not empty, we clone/install everything in the
-            #    whitelist.
-            if proj in blacklist:
-                continue
-            if whitelist and proj not in whitelist:
-                continue
-            repo = val['repository']
-            branch = val['branch']
-            repo_dir = _git_clone_and_install_single(repo, branch,
-                                                     update_requirements)
-            if proj == 'requirements':
-                requirements_dir = repo_dir
-            installed.append(proj)
-    return installed
+    for project in projects['repositories']:
+        _git_ensure_key_exists('name', project.keys())
+        _git_ensure_key_exists('repository', project.keys())
+        _git_ensure_key_exists('branch', project.keys())
+
+    if projects['repositories'][0]['name'] != 'requirements':
+        error_out('{} git repo must be specified first'.format('requirements'))
+
+    if projects['repositories'][-1]['name'] != core_project:
+        error_out('{} git repo must be specified last'.format(core_project))
 
 
-def _git_clone_and_install_single(repo, branch, update_requirements=False):
-    """Clone and install a single git repository."""
-    dest_parent_dir = "/mnt/openstack-git/"
-    dest_dir = os.path.join(dest_parent_dir, os.path.basename(repo))
+def _git_ensure_key_exists(key, keys):
+    """
+    Ensure that key exists in keys.
+    """
+    if key not in keys:
+        error_out('openstack-origin-git key \'{}\' is missing'.format(key))
 
-    if not os.path.exists(dest_parent_dir):
-        juju_log('Host dir not mounted at {}. '
-                 'Creating directory there instead.'.format(dest_parent_dir))
-        os.mkdir(dest_parent_dir)
+
+def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements):
+    """
+    Clone and install a single git repository.
+    """
+    dest_dir = os.path.join(parent_dir, os.path.basename(repo))
+
+    if not os.path.exists(parent_dir):
+        juju_log('Directory already exists at {}. '
+                 'No need to create directory.'.format(parent_dir))
+        os.mkdir(parent_dir)
 
     if not os.path.exists(dest_dir):
         juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
-        repo_dir = install_remote(repo, dest=dest_parent_dir, branch=branch)
+        repo_dir = install_remote(repo, dest=parent_dir, branch=branch)
     else:
         repo_dir = dest_dir
 
@@ -561,16 +593,39 @@ def _git_clone_and_install_single(repo, branch, update_requirements=False):
 
 
 def _git_update_requirements(package_dir, reqs_dir):
-    """Update from global requirements.
+    """
+    Update from global requirements.
 
-       Update an OpenStack git directory's requirements.txt and
-       test-requirements.txt from global-requirements.txt."""
+    Update an OpenStack git directory's requirements.txt and
+    test-requirements.txt from global-requirements.txt.
+    """
     orig_dir = os.getcwd()
     os.chdir(reqs_dir)
-    cmd = "python update.py {}".format(package_dir)
+    cmd = ['python', 'update.py', package_dir]
     try:
-        subprocess.check_call(cmd.split(' '))
+        subprocess.check_call(cmd)
     except subprocess.CalledProcessError:
         package = os.path.basename(package_dir)
         error_out("Error updating {} from global-requirements.txt".format(package))
     os.chdir(orig_dir)
+
+
+def git_src_dir(projects_yaml, project):
+    """
+    Return the directory where the specified project's source is located.
+    """
+    parent_dir = '/mnt/openstack-git'
+
+    if not projects_yaml:
+        return
+
+    projects = yaml.load(projects_yaml)
+
+    if 'directory' in projects.keys():
+        parent_dir = projects['directory']
+
+    for p in projects['repositories']:
+        if p['name'] == project:
+            return os.path.join(parent_dir, os.path.basename(p['repository']))
+
+    return None

hooks/charmhelpers/core/unitdata.py

@@ -443,7 +443,7 @@ class HookData(object):
         data = hookenv.execution_environment()
         self.conf = conf_delta = self.kv.delta(data['conf'], 'config')
         self.rels = rels_delta = self.kv.delta(data['rels'], 'rels')
-        self.kv.set('env', data['env'])
+        self.kv.set('env', dict(data['env']))
         self.kv.set('unit', data['unit'])
         self.kv.set('relid', data.get('relid'))
         return conf_delta, rels_delta

hooks/quantum_hooks.py

@@ -32,6 +32,7 @@ from charmhelpers.contrib.hahelpers.apache import(
 from charmhelpers.contrib.openstack.utils import (
     configure_installation_source,
     openstack_upgrade_available,
+    os_requires_version,
 )
 from charmhelpers.payload.execd import execd_preinstall
 from charmhelpers.core.sysctl import create as create_sysctl
@@ -48,6 +49,7 @@ from quantum_utils import (
     get_packages,
     get_early_packages,
     get_common_package,
+    get_topics,
     valid_plugin,
     configure_ovs,
     stop_services,
@@ -109,6 +111,8 @@ def config_changed():
         amqp_joined(relation_id=r_id)
     for r_id in relation_ids('amqp-nova'):
         amqp_nova_joined(relation_id=r_id)
+    for rid in relation_ids('zeromq-configuration'):
+        zeromq_configuration_relation_joined(rid)
     if valid_plugin():
         CONFIGS.write_all()
         configure_ovs()
@@ -250,6 +254,20 @@ def stop():
         cleanup_ovs_netns()
 
 
+@hooks.hook('zeromq-configuration-relation-joined')
+@os_requires_version('kilo', 'neutron-common')
+def zeromq_configuration_relation_joined(relid=None):
+    relation_set(relation_id=relid,
+                 topics=" ".join(get_topics()),
+                 users="neutron nova")
+
+
+@hooks.hook('zeromq-configuration-relation-changed')
+@restart_on_change(restart_map(), stopstart=True)
+def zeromq_configuration_relation_changed():
+    CONFIGS.write_all()
+
+
 @hooks.hook('nrpe-external-master-relation-joined',
             'nrpe-external-master-relation-changed')
 def update_nrpe_config():

hooks/quantum_utils.py

@@ -251,7 +251,9 @@ NOVA_CONFIG_FILES = {
                           context.PostgresqlDBContext(),
                           NetworkServiceContext(),
                           QuantumGatewayContext(),
-                          SyslogContext()],
+                          SyslogContext(),
+                          context.ZeroMQContext(),
+                          context.NotificationDriverContext()],
         'services': ['nova-api-metadata']
     },
 }
@@ -290,7 +292,9 @@ QUANTUM_OVS_CONFIG_FILES = {
     QUANTUM_CONF: {
         'hook_contexts': [context.AMQPContext(ssl_dir=QUANTUM_CONF_DIR),
                           QuantumGatewayContext(),
-                          SyslogContext()],
+                          SyslogContext(),
+                          context.ZeroMQContext(),
+                          context.NotificationDriverContext()],
         'services': ['quantum-l3-agent',
                      'quantum-dhcp-agent',
                      'quantum-metadata-agent',
@@ -320,7 +324,9 @@ NEUTRON_OVS_CONFIG_FILES = {
     NEUTRON_CONF: {
         'hook_contexts': [context.AMQPContext(ssl_dir=NEUTRON_CONF_DIR),
                           QuantumGatewayContext(),
-                          SyslogContext()],
+                          SyslogContext(),
+                          context.ZeroMQContext(),
+                          context.NotificationDriverContext()],
         'services': ['neutron-l3-agent',
                      'neutron-dhcp-agent',
                      'neutron-metadata-agent',
@@ -733,3 +739,24 @@ def cleanup_ovs_netns():
         subprocess.call('neutron-netns-cleanup')
     except subprocess.CalledProcessError as e:
         log('Faild to cleanup ovs and netns, %s' % e, level=ERROR)
+
+
+def get_topics():
+    # metering_agent
+    topics = []
+    if 'neutron-l3-agent' in services():
+        topics.append('l3_agent')
+    if 'neutron-dhcp-agent' in services():
+        topics.append('dhcp_agent')
+    if 'neutron-metering-agent' in services():
+        topics.append('metering_agent')
+    if 'neutron-lbaas-agent' in services():
+        topics.append('n-lbaas_agent')
+    if 'neutron-plugin-openvswitch-agent' in services():
+        topics.append('q-agent-notifier-port-update')
+        topics.append('q-agent-notifier-network-delete')
+        topics.append('q-agent-notifier-tunnel-update')
+        topics.append('q-agent-notifier-security_group-update')
+        topics.append('q-agent-notifier-dvr-update')
+    topics.append('q-agent-notifier-l2population-update')
+    return topics

hooks/zeromq-configuration-relation-changed

@@ -0,0 +1 @@
+quantum_hooks.py

hooks/zeromq-configuration-relation-joined

@@ -0,0 +1 @@
+quantum_hooks.py

metadata.yaml

@@ -30,6 +30,9 @@ requires:
     interface: rabbitmq
   amqp-nova:
     interface: rabbitmq
+  zeromq-configuration:
+    interface: zeromq-configuration
+    scope: container
   neutron-plugin-api:
     interface: neutron-plugin-api
   ha:

templates/kilo/neutron.conf

@@ -14,6 +14,8 @@ list_notifier_drivers = neutron.openstack.common.notifier.rabbit_notifier
 network_device_mtu = {{ network_device_mtu }}
 {% endif -%}
 
+{% include "section-zeromq" %}
+
 [agent]
 root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
 

templates/kilo/nova.conf

@@ -14,6 +14,9 @@ enabled_apis=metadata
 multi_host=True
 # Access to neutron API services
 network_api_class=nova.network.neutronv2.api.API
+
+{% include "section-zeromq" %}
+
 [neutron]
 auth_strategy=keystone
 url={{ quantum_url }}

tests/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -15,6 +15,7 @@
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import six
+from collections import OrderedDict
 from charmhelpers.contrib.amulet.deployment import (
     AmuletDeployment
 )
@@ -100,12 +101,34 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            """
         (self.precise_essex, self.precise_folsom, self.precise_grizzly,
          self.precise_havana, self.precise_icehouse,
-         self.trusty_icehouse) = range(6)
+         self.trusty_icehouse, self.trusty_juno, self.trusty_kilo) = range(8)
         releases = {
             ('precise', None): self.precise_essex,
             ('precise', 'cloud:precise-folsom'): self.precise_folsom,
             ('precise', 'cloud:precise-grizzly'): self.precise_grizzly,
             ('precise', 'cloud:precise-havana'): self.precise_havana,
             ('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
-            ('trusty', None): self.trusty_icehouse}
+            ('trusty', None): self.trusty_icehouse,
+            ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
+            ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo}
         return releases[(self.series, self.openstack)]
+
+    def _get_openstack_release_string(self):
+        """Get openstack release string.
+
+           Return a string representing the openstack release.
+           """
+        releases = OrderedDict([
+            ('precise', 'essex'),
+            ('quantal', 'folsom'),
+            ('raring', 'grizzly'),
+            ('saucy', 'havana'),
+            ('trusty', 'icehouse'),
+            ('utopic', 'juno'),
+            ('vivid', 'kilo'),
+        ])
+        if self.openstack:
+            os_origin = self.openstack.split(':')[1]
+            return os_origin.split('%s-' % self.series)[1].split('/')[0]
+        else:
+            return releases[self.series]

unit_tests/test_quantum_hooks.py

@@ -114,6 +114,7 @@ class TestQuantumHooks(CharmTestCase):
         _pgsql_db_joined = self.patch('pgsql_db_joined')
         _amqp_joined = self.patch('amqp_joined')
         _amqp_nova_joined = self.patch('amqp_nova_joined')
+        _zmq_joined = self.patch('zeromq_configuration_relation_joined')
         self._call_hook('config-changed')
         self.assertTrue(self.do_openstack_upgrade.called)
         self.assertTrue(self.configure_ovs.called)
@@ -121,6 +122,7 @@ class TestQuantumHooks(CharmTestCase):
         self.assertTrue(_pgsql_db_joined.called)
         self.assertTrue(_amqp_joined.called)
         self.assertTrue(_amqp_nova_joined.called)
+        self.assertTrue(_zmq_joined.called)
         self.create_sysctl.assert_called()
 
     def test_config_changed_upgrade(self):