Process subordinate releases packages map

For principal - subordinate plugin type relations where the
principal Python payload imports code from packages managed by a
subordinate, upgrades can be problematic.

This change will allow a subordinate charm that have opted into the
feature to inform its principal about all implemented release -
packages combinations ahead of time. With this information in place
the principal can do the upgrade in one operation without risk of
charm relation RPC type processing at a critical moment.

This makes use of
https://github.com/juju/charm-helpers/pull/643

This is similar to
https://review.opendev.org/c/openstack/charm-keystone/+/781822

Also fixed broken link to charm-guide.

Change-Id: Iaf5b44be70ee108cbe88b4a26f0f15f915d507fe
Closes-Bug: #1927277
(cherry picked from commit 8fb37dc0c1)

hooks/nova_compute_utils.py

@@ -73,6 +73,8 @@ from charmhelpers.contrib.openstack.alternatives import install_alternative
 from charmhelpers.contrib.openstack.utils import (
     configure_installation_source,
     get_os_codename_install_source,
+    get_subordinate_release_packages,
+    get_subordinate_services,
     os_release,
     reset_os_release,
     is_unit_paused_set,
@@ -439,8 +441,12 @@ def restart_map():
 
 
 def services():
-    ''' Returns a list of services associated with this charm '''
+    '''
-    return list(set(chain(*restart_map().values())))
+    Returns a list of services associated with this charm and its subordinates.
+    '''
+
+    return list(set(chain(*restart_map().values()))
+                | get_subordinate_services())
 
 
 def register_configs():
@@ -537,15 +543,22 @@ def determine_packages():
         if virt_type == 'lxd':
             packages.append('python3-nova-lxd')
 
+    packages = sorted(set(packages).union(get_subordinate_release_packages(
+        release).install))
+
     return packages
 
 
 def determine_purge_packages():
     '''Return a list of packages to purge for the current OS release'''
-    cmp_os_source = CompareOpenStackReleases(os_release('nova-common'))
+    release = os_release('nova-common')
-    if cmp_os_source >= 'rocky':
+    cmp_release = CompareOpenStackReleases(release)
-        return PURGE_PACKAGES
+    packages = []
-    return []
+    if cmp_release >= 'rocky':
+        packages.extend(PURGE_PACKAGES)
+    packages = sorted(set(packages).union(get_subordinate_release_packages(
+        release).purge))
+    return packages
 
 
 def remove_old_packages():

tests/README.md

@@ -6,5 +6,5 @@ and its features, as exercised in a subset of the full OpenStack deployment
 test bundle topology.
 
 For full details on functional testing of OpenStack charms please refer to
-the [functional testing](http://docs.openstack.org/developer/charm-guide/testing.html#functional-testing)
+the [functional testing](https://docs.openstack.org/charm-guide/latest/reference/testing.html#functional-testing)
 section of the OpenStack Charm Guide.

unit_tests/test_nova_compute_utils.py

@@ -15,6 +15,8 @@
 import os
 import tempfile
 
+import charmhelpers.contrib.openstack.utils as os_utils
+
 import nova_compute_context as compute_context
 import nova_compute_utils as utils
 
@@ -84,42 +86,54 @@ class NovaComputeUtilsTests(CharmTestCase):
         self.test_kv = TestKV()
         self.kv.return_value = self.test_kv
 
+    @patch.object(utils, 'get_subordinate_release_packages')
     @patch.object(utils, 'nova_metadata_requirement')
     @patch.object(utils, 'network_manager')
     @patch('platform.machine')
-    def test_determine_packages_nova_network(self, machine,
+    def test_determine_packages_nova_network(
-                                             net_man, en_meta):
+            self, machine, net_man, en_meta,
+            mock_get_subordinate_release_packages):
         self.os_release.return_value = 'icehouse'
         en_meta.return_value = (False, None)
         net_man.return_value = 'flatdhcpmanager'
         machine.return_value = 'x86_64'
         self.relation_ids.return_value = []
+        mock_get_subordinate_release_packages.return_value = \
+            os_utils.SubordinatePackages(set(), set())
         result = utils.determine_packages()
         ex = utils.BASE_PACKAGES + [
             'nova-api',
             'nova-network',
             'nova-compute-kvm'
         ]
-        self.assertTrue(ex == result)
+        self.assertTrue(ex.sort() == result.sort())
 
+    @patch.object(utils, 'get_subordinate_release_packages')
     @patch.object(utils, 'nova_metadata_requirement')
-    def test_determine_packages_ironic(self, en_meta):
+    def test_determine_packages_ironic(self, en_meta,
+                                       mock_get_subordinate_release_packages):
         self.os_release.return_value = 'victoria'
         self.test_config.set('virt-type', 'ironic')
         en_meta.return_value = (False, None)
         self.relation_ids.return_value = []
+        mock_get_subordinate_release_packages.return_value = \
+            os_utils.SubordinatePackages(set(), set())
         result = utils.determine_packages()
         ex = utils.BASE_PACKAGES + [
             'nova-compute-ironic'
         ]
         self.assertTrue(ex.sort() == result.sort())
 
+    @patch.object(utils, 'get_subordinate_release_packages')
     @patch.object(utils, 'nova_metadata_requirement')
-    def test_determine_packages_ironic_pre_victoria(self, en_meta):
+    def test_determine_packages_ironic_pre_victoria(
+            self, en_meta, mock_get_subordinate_release_packages):
         self.os_release.return_value = 'train'
         self.test_config.set('virt-type', 'ironic')
         en_meta.return_value = (False, None)
         self.relation_ids.return_value = []
+        mock_get_subordinate_release_packages.return_value = \
+            os_utils.SubordinatePackages(set(), set())
         result = utils.determine_packages()
         ex = utils.BASE_PACKAGES + [
             'nova-compute-vmware',
@@ -127,50 +141,62 @@ class NovaComputeUtilsTests(CharmTestCase):
         ]
         self.assertTrue(ex.sort() == result.sort())
 
+    @patch.object(utils, 'get_subordinate_release_packages')
     @patch.object(utils, 'nova_metadata_requirement')
     @patch.object(utils, 'network_manager')
     @patch('platform.machine')
-    def test_determine_packages_nova_network_ocata(self, machine,
+    def test_determine_packages_nova_network_ocata(
-                                                   net_man, en_meta):
+            self, machine, net_man, en_meta,
+            mock_get_subordinate_release_packages):
         self.os_release.return_value = 'ocata'
         en_meta.return_value = (False, None)
         net_man.return_value = 'flatdhcpmanager'
         machine.return_value = 'x86_64'
         self.relation_ids.return_value = []
+        mock_get_subordinate_release_packages.return_value = \
+            os_utils.SubordinatePackages(set(), set())
         result = utils.determine_packages()
         ex = utils.BASE_PACKAGES + [
             'nova-compute-kvm'
         ]
-        self.assertTrue(ex == result)
+        self.assertTrue(ex.sort() == result.sort())
 
+    @patch.object(utils, 'get_subordinate_release_packages')
     @patch.object(utils, 'nova_metadata_requirement')
     @patch.object(utils, 'neutron_plugin')
     @patch.object(utils, 'network_manager')
     @patch('platform.machine')
-    def test_determine_packages_neutron(self, machine, net_man,
+    def test_determine_packages_neutron(
-                                        n_plugin, en_meta):
+            self, machine, net_man, n_plugin, en_meta,
+            mock_get_subordinate_release_packages):
         self.os_release.return_value = 'ocata'
         en_meta.return_value = (False, None)
         net_man.return_value = 'neutron'
         n_plugin.return_value = 'ovs'
         machine.return_value = 'x86_64'
         self.relation_ids.return_value = []
+        mock_get_subordinate_release_packages.return_value = \
+            os_utils.SubordinatePackages(set(), set())
         result = utils.determine_packages()
         ex = utils.BASE_PACKAGES + ['nova-compute-kvm']
-        self.assertTrue(ex == result)
+        self.assertTrue(ex.sort() == result.sort())
 
+    @patch.object(utils, 'get_subordinate_release_packages')
     @patch.object(utils, 'nova_metadata_requirement')
     @patch.object(utils, 'neutron_plugin')
     @patch.object(utils, 'network_manager')
     @patch('platform.machine')
-    def test_determine_packages_neutron_rocky(self, machine, net_man,
+    def test_determine_packages_neutron_rocky(
-                                              n_plugin, en_meta):
+            self, machine, net_man, n_plugin, en_meta,
+            mock_get_subordinate_release_packages):
         self.os_release.return_value = 'rocky'
         en_meta.return_value = (False, None)
         net_man.return_value = 'neutron'
         n_plugin.return_value = 'ovs'
         machine.return_value = 'x86_64'
         self.relation_ids.return_value = []
+        mock_get_subordinate_release_packages.return_value = \
+            os_utils.SubordinatePackages(set(), set())
         result = utils.determine_packages()
         ex = (
             [p for p in utils.BASE_PACKAGES
@@ -179,15 +205,16 @@ class NovaComputeUtilsTests(CharmTestCase):
             utils.PY3_PACKAGES +
             ['python3-ceilometer', 'python3-neutron', 'python3-neutron-fwaas']
         )
-        self.assertEqual(ex, result)
+        self.assertTrue(ex.sort() == result.sort())
 
+    @patch.object(utils, 'get_subordinate_release_packages')
     @patch.object(utils, 'nova_metadata_requirement')
     @patch.object(utils, 'neutron_plugin')
     @patch.object(utils, 'network_manager')
     @patch('platform.machine')
-    def test_determine_packages_neutron_aarch64_xenial(self, machine,
+    def test_determine_packages_neutron_aarch64_xenial(
-                                                       net_man, n_plugin,
+            self, machine, net_man, n_plugin, en_meta,
-                                                       en_meta):
+            mock_get_subordinate_release_packages):
         self.os_release.return_value = 'ocata'
         self.lsb_release.return_value = {
             'DISTRIB_CODENAME': 'xenial'
@@ -197,17 +224,20 @@ class NovaComputeUtilsTests(CharmTestCase):
         n_plugin.return_value = 'ovs'
         machine.return_value = 'aarch64'
         self.relation_ids.return_value = []
+        mock_get_subordinate_release_packages.return_value = \
+            os_utils.SubordinatePackages(set(), set())
         result = utils.determine_packages()
         ex = utils.BASE_PACKAGES + ['nova-compute-kvm', 'qemu-efi']
-        self.assertTrue(ex == result)
+        self.assertTrue(ex.sort() == result.sort())
 
+    @patch.object(utils, 'get_subordinate_release_packages')
     @patch.object(utils, 'nova_metadata_requirement')
     @patch.object(utils, 'neutron_plugin')
     @patch.object(utils, 'network_manager')
     @patch('platform.machine')
-    def test_determine_packages_neutron_aarch64_trusty(self, machine,
+    def test_determine_packages_neutron_aarch64_trusty(
-                                                       net_man, n_plugin,
+            self, machine, net_man, n_plugin, en_meta,
-                                                       en_meta):
+            mock_get_subordinate_release_packages):
         self.os_release.return_value = 'ocata'
         self.lsb_release.return_value = {
             'DISTRIB_CODENAME': 'trusty'
@@ -217,63 +247,81 @@ class NovaComputeUtilsTests(CharmTestCase):
         n_plugin.return_value = 'ovs'
         machine.return_value = 'aarch64'
         self.relation_ids.return_value = []
+        mock_get_subordinate_release_packages.return_value = \
+            os_utils.SubordinatePackages(set(), set())
         result = utils.determine_packages()
         ex = utils.BASE_PACKAGES + ['nova-compute-kvm']
-        self.assertEqual(ex, result)
+        self.assertTrue(ex.sort() == result.sort())
 
+    @patch.object(utils, 'get_subordinate_release_packages')
     @patch.object(utils, 'nova_metadata_requirement')
     @patch.object(utils, 'neutron_plugin')
     @patch.object(utils, 'network_manager')
     @patch('platform.machine')
-    def test_determine_packages_neutron_ceph(self, machine,
+    def test_determine_packages_neutron_ceph(
-                                             net_man, n_plugin, en_meta):
+            self, machine, net_man, n_plugin, en_meta,
+            mock_get_subordinate_release_packages):
         self.os_release.return_value = 'ocata'
         en_meta.return_value = (False, None)
         net_man.return_value = 'neutron'
         n_plugin.return_value = 'ovs'
         machine.return_value = 'x86_64'
         self.relation_ids.return_value = ['ceph:0']
+        mock_get_subordinate_release_packages.return_value = \
+            os_utils.SubordinatePackages(set(), set())
         result = utils.determine_packages()
         ex = (utils.BASE_PACKAGES + ['ceph-common', 'nova-compute-kvm'])
-        self.assertEqual(ex, result)
+        self.assertTrue(ex.sort() == result.sort())
 
+    @patch.object(utils, 'get_subordinate_release_packages')
     @patch.object(utils, 'nova_metadata_requirement')
     @patch.object(utils, 'neutron_plugin')
     @patch.object(utils, 'network_manager')
-    def test_determine_packages_metadata(self, net_man,
+    def test_determine_packages_metadata(
-                                         n_plugin, en_meta):
+            self, net_man, n_plugin, en_meta,
+            mock_get_subordinate_release_packages):
         self.os_release.return_value = 'ocata'
         en_meta.return_value = (True, None)
         net_man.return_value = 'bob'
         n_plugin.return_value = 'ovs'
         self.relation_ids.return_value = []
+        mock_get_subordinate_release_packages.return_value = \
+            os_utils.SubordinatePackages(set(), set())
         result = utils.determine_packages()
         self.assertTrue('nova-api-metadata' in result)
 
+    @patch.object(utils, 'get_subordinate_release_packages')
     @patch.object(utils, 'nova_metadata_requirement')
     @patch.object(utils, 'neutron_plugin')
     @patch.object(utils, 'network_manager')
-    def test_determine_packages_use_multipath(self, net_man,
+    def test_determine_packages_use_multipath(
-                                              n_plugin, en_meta):
+            self, net_man, n_plugin, en_meta,
+            mock_get_subordinate_release_packages):
         self.os_release.return_value = 'ocata'
         en_meta.return_value = (False, None)
         net_man.return_value = 'bob'
         self.test_config.set('use-multipath', True)
         self.relation_ids.return_value = []
+        mock_get_subordinate_release_packages.return_value = \
+            os_utils.SubordinatePackages(set(), set())
         result = utils.determine_packages()
         for pkg in utils.MULTIPATH_PACKAGES:
             self.assertTrue(pkg in result)
 
+    @patch.object(utils, 'get_subordinate_release_packages')
     @patch.object(utils, 'nova_metadata_requirement')
     @patch.object(utils, 'neutron_plugin')
     @patch.object(utils, 'network_manager')
-    def test_determine_packages_no_multipath(self, net_man,
+    def test_determine_packages_no_multipath(
-                                             n_plugin, en_meta):
+            self, net_man, n_plugin, en_meta,
+            mock_get_subordinate_release_packages):
         self.os_release.return_value = 'ocata'
         en_meta.return_value = (False, None)
         net_man.return_value = 'bob'
         self.test_config.set('use-multipath', False)
         self.relation_ids.return_value = []
+        mock_get_subordinate_release_packages.return_value = \
+            os_utils.SubordinatePackages(set(), set())
         result = utils.determine_packages()
         for pkg in utils.MULTIPATH_PACKAGES:
             self.assertFalse(pkg in result)