Ocata fixes for nova-compute

Disable deprecated nova-network in ocata
libvirtd vs libvirt-bin
Amulet test fixes and enable xenial-ocata target

Change-Id: Ie08fcd333515669a2b6cdb722f2eb0ac75ee35ea

hooks/nova_compute_utils.py

@@ -310,7 +310,8 @@ REQUIRED_INTERFACES = {
 def libvirt_daemon():
     '''Resolve the correct name of the libvirt daemon service'''
     distro_codename = lsb_release()['DISTRIB_CODENAME'].lower()
-    if distro_codename >= 'yakkety':
+    if (distro_codename >= 'yakkety' or
+            os_release('nova-common') >= 'ocata'):
         return LIBVIRTD_DAEMON
     else:
         return LIBVIRT_BIN_DAEMON
@@ -331,7 +332,8 @@ def resource_map():
     # Network manager gets set late by the cloud-compute interface.
     # FlatDHCPManager only requires some extra packages.
     if (net_manager in ['flatmanager', 'flatdhcpmanager'] and
-            config('multi-host').lower() == 'yes'):
+            config('multi-host').lower() == 'yes' and
+            os_release('nova-common') < 'ocata'):
         resource_map[NOVA_CONF]['services'].extend(
             ['nova-api', 'nova-network']
         )
@@ -340,7 +342,8 @@ def resource_map():
         resource_map.pop(NOVA_NETWORK_AA_PROFILE_PATH)
 
     distro_codename = lsb_release()['DISTRIB_CODENAME'].lower()
-    if distro_codename >= 'yakkety':
+    if (distro_codename >= 'yakkety' or
+            os_release('nova-common') >= 'ocata'):
         for data in resource_map.values():
             if LIBVIRT_BIN_DAEMON in data['services']:
                 data['services'].remove(LIBVIRT_BIN_DAEMON)
@@ -420,7 +423,8 @@ def determine_packages():
 
     net_manager = network_manager()
     if (net_manager in ['flatmanager', 'flatdhcpmanager'] and
-            config('multi-host').lower() == 'yes'):
+            config('multi-host').lower() == 'yes' and
+            os_release('nova-common') < 'ocata'):
         packages.extend(['nova-api', 'nova-network'])
 
     if relation_ids('ceph'):

test-requirements.txt

@@ -11,13 +11,13 @@ requests==2.6.0
 # Liberty client lower constraints
 amulet>=1.14.3,<2.0
 bundletester>=0.6.1,<1.0
-python-ceilometerclient>=1.5.0,<2.0
-python-cinderclient>=1.4.0,<2.0
-python-glanceclient>=1.1.0,<2.0
-python-heatclient>=0.8.0,<1.0
-python-novaclient>=2.30.1,<3.0
-python-openstackclient>=1.7.0,<2.0
-python-swiftclient>=2.6.0,<3.0
+python-ceilometerclient>=1.5.0
+python-cinderclient>=1.4.0
+python-glanceclient>=1.1.0
+python-heatclient>=0.8.0
+python-novaclient>=2.30.1
+python-openstackclient>=1.7.0
+python-swiftclient>=2.6.0
 pika>=0.10.0,<1.0
 distro-info
 # END: Amulet OpenStack Charm Helper Requirements

tests/basic_deployment.py

@@ -83,6 +83,14 @@ class NovaBasicDeployment(OpenStackAmuletDeployment):
             {'name': 'glance'},
             {'name': 'percona-cluster', 'constraints': {'mem': '3072M'}},
         ]
+        if self._get_openstack_release() >= self.xenial_ocata:
+            other_ocata_services = [
+                {'name': 'neutron-gateway'},
+                {'name': 'neutron-api'},
+                {'name': 'neutron-openvswitch'},
+            ]
+            other_services += other_ocata_services
+
         super(NovaBasicDeployment, self)._add_services(this_service,
                                                        other_services)
 
@@ -104,6 +112,21 @@ class NovaBasicDeployment(OpenStackAmuletDeployment):
             'glance:shared-db': 'percona-cluster:shared-db',
             'glance:amqp': 'rabbitmq-server:amqp'
         }
+        if self._get_openstack_release() >= self.xenial_ocata:
+            ocata_relations = {
+                'neutron-gateway:amqp': 'rabbitmq-server:amqp',
+                'nova-cloud-controller:quantum-network-service':
+                'neutron-gateway:quantum-network-service',
+                'neutron-api:shared-db': 'percona-cluster:shared-db',
+                'neutron-api:amqp': 'rabbitmq-server:amqp',
+                'neutron-api:neutron-api': 'nova-cloud-controller:neutron-api',
+                'neutron-api:identity-service': 'keystone:identity-service',
+                'nova-compute:neutron-plugin': 'neutron-openvswitch:'
+                                               'neutron-plugin',
+                'rabbitmq-server:amqp': 'neutron-openvswitch:amqp',
+            }
+            relations.update(ocata_relations)
+
         super(NovaBasicDeployment, self)._add_relations(relations)
 
     def _configure_services(self):
@@ -148,6 +171,9 @@ class NovaBasicDeployment(OpenStackAmuletDeployment):
             nova_cc_config['openstack-origin-git'] = \
                 yaml.dump(openstack_origin_git)
 
+        if self._get_openstack_release() >= self.xenial_ocata:
+            nova_cc_config['network-manager'] = 'Neutron'
+
         keystone_config = {
             'admin-password': 'openstack',
             'admin-token': 'ubuntutesting',
@@ -242,6 +268,10 @@ class NovaBasicDeployment(OpenStackAmuletDeployment):
         if self._get_openstack_release() >= self.trusty_liberty:
             services[self.keystone_sentry] = ['apache2']
 
+        if self._get_openstack_release_string() >= 'ocata':
+            services[self.nova_compute_sentry].remove('nova-network')
+            services[self.nova_compute_sentry].remove('nova-api')
+
         ret = u.validate_services_by_name(services)
         if ret:
             amulet.raise_status(amulet.FAIL, msg=ret)
@@ -432,6 +462,9 @@ class NovaBasicDeployment(OpenStackAmuletDeployment):
             'restart_trigger': u.not_null
         }
 
+        if self._get_openstack_release() >= self.xenial_ocata:
+            expected['network_manager'] = 'neutron'
+
         ret = u.validate_relation_data(unit, relation, expected)
         if ret:
             message = u.relation_error('nova-cc cloud-compute', ret)
@@ -506,6 +539,17 @@ class NovaBasicDeployment(OpenStackAmuletDeployment):
                 }
             })
 
+        if self._get_openstack_release() >= self.xenial_ocata:
+            del expected['DEFAULT']['flat_interface']
+            del expected['DEFAULT']['network_manager']
+            expected['DEFAULT'].update({
+                'use_neutron': 'True',
+                'network_api_class': 'nova.network.neutronv2.api.API'})
+            expected['neutron'] = {
+                'url': u.valid_url,
+                'auth_url': u.valid_url}
+            # Add expected username?
+
         for section, pairs in expected.iteritems():
             ret = u.validate_config_data(unit, conf, section, pairs)
             if ret:
@@ -562,11 +606,13 @@ class NovaBasicDeployment(OpenStackAmuletDeployment):
         # Services which are expected to restart upon config change,
         # and corresponding config files affected by the change
         conf_file = '/etc/nova/nova.conf'
-        services = {
-            'nova-compute': conf_file,
-            'nova-api': conf_file,
-            'nova-network': conf_file
-        }
+        services = {'nova-compute': conf_file}
+
+        if self._get_openstack_release() < self.xenial_ocata:
+            services.update({
+                'nova-api': conf_file,
+                'nova-network': conf_file
+            })
 
         # Make config change, check for service restarts
         u.log.debug('Making config change on {}...'.format(juju_service))
@@ -612,10 +658,14 @@ class NovaBasicDeployment(OpenStackAmuletDeployment):
 
         services = {
             'nova-compute': '/etc/apparmor.d/usr.bin.nova-compute',
-            'nova-network': '/etc/apparmor.d/usr.bin.nova-network',
-            'nova-api': '/etc/apparmor.d/usr.bin.nova-api',
         }
 
+        if self._get_openstack_release() < self.xenial_ocata:
+            services.update({
+                'nova-network': '/etc/apparmor.d/usr.bin.nova-network',
+                'nova-api': '/etc/apparmor.d/usr.bin.nova-api',
+            })
+
         sentry = self.nova_compute_sentry
         juju_service = 'nova-compute'
         mtime = u.get_sentry_time(sentry)
@@ -642,4 +692,4 @@ class NovaBasicDeployment(OpenStackAmuletDeployment):
                                   '--complaining')
         u.log.info("Assert output of aa-status --complaining >= 3. Result: {} "
                    "Exit Code: {}".format(output, code))
-        assert int(output) >= 3
+        assert int(output) >= len(services)

tests/charmhelpers/contrib/amulet/utils.py

@@ -25,6 +25,7 @@ import uuid
 
 import amulet
 import distro_info
+from distutils.version import LooseVersion
 import six
 from six.moves import configparser
 if six.PY3:
@@ -785,7 +786,7 @@ class AmuletUtils(object):
         generating test messages which need to be unique-ish."""
         return '[{}-{}]'.format(uuid.uuid4(), time.time())
 
-# amulet juju action helpers:
+    # amulet juju action helpers:
     def run_action(self, unit_sentry, action,
                    _check_output=subprocess.check_output,
                    params=None):
@@ -797,7 +798,14 @@ class AmuletUtils(object):
         @return action_id.
         """
         unit_id = unit_sentry.info["unit_name"]
-        command = ["juju", "action", "do", "--format=json", unit_id, action]
+        # Note: There were significant CLI changes between 2.0 and 2.1
+        # This supports the 2.1 and 1.25.x only
+        # We are supporting the latest stable 1.x and 2.x
+        if self.juju_min_version("2.1"):
+            command = ["juju", "run-action", "--format=json", unit_id, action]
+        else:
+            command = ["juju", "action", "do", "--format=json",
+                       unit_id, action]
         if params is not None:
             for key, value in params.iteritems():
                 command.append("{}={}".format(key, value))
@@ -812,8 +820,15 @@ class AmuletUtils(object):
 
         _check_output parameter is used for dependency injection.
         """
-        command = ["juju", "action", "fetch", "--format=json", "--wait=0",
-                   action_id]
+        # Note: There were significant CLI changes between 2.0 and 2.1
+        # This supports the 2.1 and 1.25.x only
+        # We are supporting the latest stable 1.x and 2.x
+        if self.juju_min_version("2.1"):
+            command = ["juju", "show-action-output", "--format=json",
+                       "--wait=0", action_id]
+        else:
+            command = ["juju", "action", "fetch", "--format=json", "--wait=0",
+                       action_id]
         output = _check_output(command, universal_newlines=True)
         data = json.loads(output)
         return data.get(u"status") == "completed"
@@ -826,3 +841,20 @@ class AmuletUtils(object):
             return ("unknown", "")
         status = json.loads(raw_status)
         return (status["status"], status["message"])
+
+    def juju_version(self):
+        """Return Juju version
+
+        @reutrns string version
+        """
+        cmd = ['juju', 'version']
+        return subprocess.check_output(cmd)
+
+    def juju_min_version(self, minimum_version='2.1'):
+        """Return True if the Juju version is at least the provided version
+
+        @param minimum_version: string semantic version
+        @returns boolean
+        """
+        return (LooseVersion(self.juju_version()) >=
+                LooseVersion(minimum_version))

tests/charmhelpers/contrib/openstack/amulet/utils.py

@@ -32,6 +32,7 @@ from keystoneclient.v3 import client as keystone_client_v3
 from novaclient import exceptions
 
 import novaclient.client as nova_client
+import novaclient
 import pika
 import swiftclient
 
@@ -434,9 +435,15 @@ class OpenStackAmuletUtils(AmuletUtils):
         self.log.debug('Authenticating nova user ({})...'.format(user))
         ep = keystone.service_catalog.url_for(service_type='identity',
                                               endpoint_type='publicURL')
-        return nova_client.Client(NOVA_CLIENT_VERSION,
-                                  username=user, api_key=password,
-                                  project_id=tenant, auth_url=ep)
+        self.log.debug("~~~~~~~~~~~~~~VERSION: {} ~~~~~~~~~~~~~~~~~~~~~".format(novaclient.__version__[0]))
+        if novaclient.__version__[0] >= "7":
+            return nova_client.Client(NOVA_CLIENT_VERSION,
+                                      username=user, password=password,
+                                      project_name=tenant, auth_url=ep)
+        else:
+            return nova_client.Client(NOVA_CLIENT_VERSION,
+                                      username=user, api_key=password,
+                                      project_id=tenant, auth_url=ep)
 
     def authenticate_swift_user(self, keystone, user, password, tenant):
         """Authenticates a regular user with swift api."""

unit_tests/test_nova_compute_utils.py

@@ -85,6 +85,7 @@ class NovaComputeUtilsTests(CharmTestCase):
     @patch('platform.machine')
     def test_determine_packages_nova_network(self, machine, git_requested,
                                              net_man, en_meta):
+        self.os_release.return_value = 'icehouse'
         git_requested.return_value = False
         en_meta.return_value = (False, None)
         net_man.return_value = 'flatdhcpmanager'
@@ -98,6 +99,25 @@ class NovaComputeUtilsTests(CharmTestCase):
         ]
         self.assertEqual(ex, result)
 
+    @patch.object(utils, 'nova_metadata_requirement')
+    @patch.object(utils, 'network_manager')
+    @patch.object(utils, 'git_install_requested')
+    @patch('platform.machine')
+    def test_determine_packages_nova_network_ocata(self, machine,
+                                                   git_requested,
+                                                   net_man, en_meta):
+        self.os_release.return_value = 'ocata'
+        git_requested.return_value = False
+        en_meta.return_value = (False, None)
+        net_man.return_value = 'flatdhcpmanager'
+        machine.return_value = 'x86_64'
+        self.relation_ids.return_value = []
+        result = utils.determine_packages()
+        ex = utils.BASE_PACKAGES + [
+            'nova-compute-kvm'
+        ]
+        self.assertEqual(ex, result)
+
     @patch.object(utils, 'nova_metadata_requirement')
     @patch.object(utils, 'neutron_plugin')
     @patch.object(utils, 'network_manager')
@@ -193,6 +213,7 @@ class NovaComputeUtilsTests(CharmTestCase):
     @patch.object(utils, 'nova_metadata_requirement')
     @patch.object(utils, 'network_manager')
     def test_resource_map_nova_network_no_multihost(self, net_man, en_meta):
+        self.os_release.return_value = 'icehouse'
         self.test_config.set('multi-host', 'no')
         en_meta.return_value = (False, None)
         net_man.return_value = 'flatdhcpmanager'
@@ -243,10 +264,65 @@ class NovaComputeUtilsTests(CharmTestCase):
             self.assertEqual(set(ex[k]['services']),
                              set(result[k]['services']))
 
+    @patch.object(utils, 'nova_metadata_requirement')
+    @patch.object(utils, 'network_manager')
+    def test_resource_map_nova_network_ocata(self, net_man, en_meta):
+        self.os_release.return_value = 'ocata'
+        self.test_config.set('multi-host', 'yes')
+        en_meta.return_value = (False, None)
+        net_man.return_value = 'flatdhcpmanager'
+        result = utils.resource_map()
+        ex = {
+            '/etc/default/libvirt-bin': {
+                'contexts': [],
+                'services': ['libvirtd']
+            },
+            '/etc/libvirt/qemu.conf': {
+                'contexts': [],
+                'services': ['libvirtd']
+            },
+            '/etc/nova/nova.conf': {
+                'contexts': [],
+                'services': ['nova-compute']
+            },
+            '/etc/ceph/secret.xml': {
+                'contexts': [],
+                'services': []
+            },
+            '/var/lib/charm/nova_compute/ceph.conf': {
+                'contexts': [],
+                'services': ['nova-compute']
+            },
+            '/etc/default/qemu-kvm': {
+                'contexts': [],
+                'services': ['qemu-kvm']
+            },
+            '/etc/init/libvirt-bin.override': {
+                'contexts': [],
+                'services': ['libvirtd']
+            },
+            '/etc/libvirt/libvirtd.conf': {
+                'contexts': [],
+                'services': ['libvirtd']
+            },
+            '/etc/apparmor.d/usr.bin.nova-compute': {
+                'contexts': [],
+                'services': ['nova-compute']
+            },
+        }
+        # Mocking contexts is tricky but we can still test that
+        # the correct files are monitored and the correct services
+        # will be started
+        self.assertEqual(set(ex.keys()), set(result.keys()))
+        for k in ex.keys():
+            self.assertEqual(set(ex[k]['services']),
+                             set(result[k]['services']))
+
     @patch.object(utils, 'nova_metadata_requirement')
     @patch.object(utils, 'network_manager')
     def test_resource_map_nova_network(self, net_man, en_meta):
 
+        self.os_release.return_value = 'icehouse'
         en_meta.return_value = (False, None)
         self.test_config.set('multi-host', 'yes')
         net_man.return_value = 'flatdhcpmanager'