Necessary extending of objects to perform upgrade

The upgrade procedure should create a new cluster with the same
parameters as the original cluster. The new method was intoduced to
parameters the cluster that was created with. The return value of the
new method is compatible with the data that are expected by the create
method.

A set of hooks of the extension mechanism was extended by the
fire_callback_on_cluster_delete hook. This hooks is fired when cluster
is deleted.

Two opposite method were added: get_assigned_vips and
assign_given_vips_for_net_groups. The first method returns all assigned
VIPs of the cluster and the second complementary method receives the
same format that the first returns to assign given VIPs.

The patch has some impact on the get_admin_network_group method. This
impact consist of adding a new filter called node_group_id that allows
to filter network groups by ID of the node group when there are no any
nodes in the cluster.

Implements blueprint: nailgun-api-env-upgrade-extensions

Change-Id: I156d3c71a7e832a2eb99a1058a6ac628fab71483

nailgun/nailgun/extensions/__init__.py

@@ -23,3 +23,4 @@ from nailgun.extensions.base import fire_callback_on_node_collection_delete
 from nailgun.extensions.base import fire_callback_on_node_create
 from nailgun.extensions.base import fire_callback_on_node_update
 from nailgun.extensions.base import fire_callback_on_node_reset
+from nailgun.extensions.base import fire_callback_on_cluster_delete

nailgun/nailgun/extensions/base.py

@@ -99,6 +99,11 @@ def fire_callback_on_node_collection_delete(node_ids):
         extension.on_node_collection_delete(node_ids)
 
 
+def fire_callback_on_cluster_delete(cluster):
+    for extension in get_all_extensions():
+        extension.on_cluster_delete(cluster)
+
+
 @six.add_metaclass(abc.ABCMeta)
 class BaseExtension(object):
 
@@ -172,3 +177,8 @@ class BaseExtension(object):
     def on_node_collection_delete(cls, node_ids):
         """Callback which gets executed when node collection is deleted
         """
+
+    @classmethod
+    def on_cluster_delete(cls, cluster):
+        """Callback which gets executed when cluster is deleted
+        """

nailgun/nailgun/network/manager.py

@@ -337,6 +337,70 @@ class NetworkManager(object):
 
         return result
 
+    @classmethod
+    def _get_assigned_vips_for_net_groups(cls, cluster):
+        node_group_id = objects.Cluster.get_controllers_group_id(cluster)
+        cluster_vips = db.query(IPAddr).join(IPAddr.network_data).filter(
+            IPAddr.node.is_(None) &
+            IPAddr.vip_type.isnot(None) &
+            (NetworkGroup.group_id == node_group_id)
+        )
+        return cluster_vips
+
+    @classmethod
+    def get_assigned_vips(cls, cluster):
+        """Return assigned VIPs mapped to names of network groups.
+
+        :param cluster: Is an instance of :class:`objects.Cluster`.
+        :returns: A dict of VIPs mapped to names of network groups and
+                  they are grouped by the type.
+        """
+        cluster_vips = cls._get_assigned_vips_for_net_groups(cluster)
+        vips = defaultdict(dict)
+        for vip in cluster_vips:
+            vips[vip.network_data.name][vip.vip_type] = vip.ip_addr
+        return vips
+
+    @classmethod
+    def assign_given_vips_for_net_groups(cls, cluster, vips):
+        """Assign given VIP addresses for network groups.
+
+        This method is the opposite of the :func:`get_assigned_vips_ips`
+        and compatible with results it returns. The method primarily
+        used for the upgrading procedure of clusters to copy VIPs from
+        one cluster to the other.
+
+        :param cluster: Is an instance of :class:`objects.Cluster`.
+        :param vips: A dict of VIPs mapped to names of network groups
+                     that are grouped by the type.
+        :raises: errors.AssignIPError
+        """
+        cluster_vips = cls._get_assigned_vips_for_net_groups(cluster)
+        assigned_vips = defaultdict(dict)
+        for vip in cluster_vips:
+            assigned_vips[vip.network_data.name][vip.vip_type] = vip
+        for net_group in cluster.network_groups:
+            if net_group.name not in vips:
+                continue
+            assigned_vips_by_type = assigned_vips.get(net_group.name, {})
+            for vip_type, ip_addr in six.iteritems(vips[net_group.name]):
+                if not cls.check_ip_belongs_to_net(ip_addr, net_group):
+                    raise errors.AssignIPError(
+                        "Cannot assign VIP with the address \"{0}\" because "
+                        "it does not belong to the network \"{1}\""
+                        .format(ip_addr, net_group.name))
+                if vip_type in assigned_vips_by_type:
+                    assigned_vip = assigned_vips_by_type[vip_type]
+                    assigned_vip.ip_addr = ip_addr
+                else:
+                    vip = IPAddr(
+                        network=net_group.id,
+                        ip_addr=ip_addr,
+                        vip_type=vip_type,
+                    )
+                    db().add(vip)
+        db().flush()
+
     @classmethod
     def assign_vips_for_net_groups_for_api(cls, cluster):
         return cls.assign_vips_for_net_groups(cluster)
@@ -1201,6 +1265,14 @@ class NetworkManager(object):
         elif cluster.net_provider == 'nova_network':
             cls.create_nova_network_config(cluster)
 
+    @classmethod
+    def get_network_config_create_data(cls, cluster):
+        data = {}
+        if cluster.net_provider == consts.CLUSTER_NET_PROVIDERS.neutron:
+            data['net_l23_provider'] = cluster.network_config.net_l23_provider
+            data['net_segment_type'] = cluster.network_config.segmentation_type
+        return data
+
     @classmethod
     def get_default_gateway(cls, node_id):
         """Returns GW from Admin network if it's set, else returns Admin IP.

nailgun/nailgun/objects/cluster.py

@@ -30,6 +30,7 @@ from nailgun import consts
 from nailgun.db import db
 from nailgun.db.sqlalchemy import models
 from nailgun.errors import errors
+from nailgun.extensions import fire_callback_on_cluster_delete
 from nailgun.extensions import fire_callback_on_node_collection_delete
 from nailgun.logger import logger
 from nailgun.objects import NailgunCollection
@@ -244,6 +245,7 @@ class Cluster(NailgunObject):
             _id for (_id,) in
             db().query(models.Node.id).filter_by(cluster_id=instance.id)]
         fire_callback_on_node_collection_delete(node_ids)
+        fire_callback_on_cluster_delete(instance)
         super(Cluster, cls).delete(instance)
 
     @classmethod
@@ -875,6 +877,26 @@ class Cluster(NailgunObject):
 
         return None
 
+    @classmethod
+    def get_create_data(cls, instance):
+        """Return common parameters cluster was created with.
+
+        This method is compatible with :func:`create` and used to create
+        a new cluster with the same settings including the network
+        configuration.
+
+        :returns: a dict of key-value pairs as a cluster create data
+        """
+        data = {
+            "name": instance.name,
+            "mode": instance.mode,
+            "net_provider": instance.net_provider,
+            "release_id": instance.release.id,
+        }
+        data.update(cls.get_network_manager(instance).
+                    get_network_config_create_data(instance))
+        return data
+
     @classmethod
     def get_vmware_attributes(cls, instance):
         """Get VmwareAttributes instance from DB. Now we have

nailgun/nailgun/test/integration/test_network_manager.py

@@ -25,6 +25,7 @@ from netaddr import IPRange
 from sqlalchemy import not_
 
 import nailgun
+from nailgun.errors import errors
 from nailgun import objects
 
 from nailgun.db.sqlalchemy.models import IPAddr
@@ -39,7 +40,27 @@ from nailgun.test.base import BaseIntegrationTest
 from nailgun.test.base import fake_tasks
 
 
-class TestNetworkManager(BaseIntegrationTest):
+class BaseNetworkManagerTest(BaseIntegrationTest):
+    def _create_ip_addrs_by_rules(self, cluster, rules):
+        created_ips = []
+        for net_group in cluster.network_groups:
+            if net_group.name not in rules:
+                continue
+            vips_by_types = rules[net_group.name]
+            for vip_type, ip_addr in vips_by_types.items():
+                ip = IPAddr(
+                    network=net_group.id,
+                    ip_addr=ip_addr,
+                    vip_type=vip_type,
+                )
+                self.db.add(ip)
+                created_ips.append(ip)
+        if created_ips:
+            self.db.flush()
+        return created_ips
+
+
+class TestNetworkManager(BaseNetworkManagerTest):
 
     @fake_tasks(fake_rpc=False, mock_rpc=False)
     @patch('nailgun.rpc.cast')
@@ -346,6 +367,71 @@ class TestNetworkManager(BaseIntegrationTest):
             self.assertEqual(len(admin_ips), 1)
             self.assertEqual(admin_ips[0].ip_addr, ip)
 
+    def test_get_assigned_vips(self):
+        vips_to_create = {
+            'management': {
+                'haproxy': '192.168.0.1',
+                'vrouter': '192.168.0.2',
+            },
+            'public': {
+                'haproxy': '172.16.0.2',
+                'vrouter': '172.16.0.3',
+            },
+        }
+        cluster = self.env.create_cluster(api=False)
+        self._create_ip_addrs_by_rules(cluster, vips_to_create)
+        vips = self.env.network_manager.get_assigned_vips(cluster)
+        self.assertEqual(vips_to_create, vips)
+
+    def test_assign_given_vips_for_net_groups(self):
+        vips_to_create = {
+            'management': {
+                'haproxy': '192.168.0.1',
+            },
+            'public': {
+                'haproxy': '172.16.0.2',
+            },
+        }
+        vips_to_assign = {
+            'management': {
+                'haproxy': '192.168.0.1',
+                'vrouter': '192.168.0.2',
+            },
+            'public': {
+                'haproxy': '172.16.0.4',
+                'vrouter': '172.16.0.5',
+            },
+        }
+        cluster = self.env.create_cluster(api=False)
+        self._create_ip_addrs_by_rules(cluster, vips_to_create)
+        self.env.network_manager.assign_given_vips_for_net_groups(
+            cluster, vips_to_assign)
+        vips = self.env.network_manager.get_assigned_vips(cluster)
+        self.assertEqual(vips_to_assign, vips)
+
+    def test_assign_given_vips_for_net_groups_idempotent(self):
+        cluster = self.env.create_cluster(api=False)
+        self.env.network_manager.assign_vips_for_net_groups(cluster)
+        expected_vips = self.env.network_manager.get_assigned_vips(cluster)
+        self.env.network_manager.assign_given_vips_for_net_groups(
+            cluster, expected_vips)
+        self.env.network_manager.assign_vips_for_net_groups(cluster)
+        vips = self.env.network_manager.get_assigned_vips(cluster)
+        self.assertEqual(expected_vips, vips)
+
+    def test_assign_given_vips_for_net_groups_assign_error(self):
+        vips_to_assign = {
+            'management': {
+                'haproxy': '10.10.0.1',
+            },
+        }
+        expected_msg_regexp = '^Cannot assign VIP with the address "10.10.0.1"'
+        cluster = self.env.create_cluster(api=False)
+        with self.assertRaisesRegexp(errors.AssignIPError,
+                                     expected_msg_regexp):
+            self.env.network_manager.assign_given_vips_for_net_groups(
+                cluster, vips_to_assign)
+
     @fake_tasks(fake_rpc=False, mock_rpc=False)
     @patch('nailgun.rpc.cast')
     def test_admin_ip_cobbler(self, mocked_rpc):
@@ -511,7 +597,7 @@ class TestNeutronManager(BaseIntegrationTest):
         self.check_networks_assignment(self.env.nodes[0])
 
 
-class TestNeutronManager70(BaseIntegrationTest):
+class TestNeutronManager70(BaseNetworkManagerTest):
 
     def setUp(self):
         super(TestNeutronManager70, self).setUp()
@@ -614,3 +700,43 @@ class TestNeutronManager70(BaseIntegrationTest):
             self.assertEqual(assigned_vips[name]['node_roles'],
                              ['controller',
                               'primary-controller'])
+
+    def test_get_assigned_vips(self):
+        self.net_manager.assign_vips_for_net_groups(self.cluster)
+        vips = self.net_manager.get_assigned_vips(self.cluster)
+        expected_vips = {
+            'management': {
+                'vrouter': '192.168.0.1',
+                'management': '192.168.0.2',
+            },
+            'public': {
+                'vrouter_pub': '172.16.0.2',
+                'public': '172.16.0.3',
+            },
+        }
+        self.assertEqual(expected_vips, vips)
+
+    def test_assign_given_vips_for_net_groups(self):
+        vips_to_create = {
+            'management': {
+                'vrouter': '192.168.0.1',
+            },
+            'public': {
+                'vrouter_pub': '172.16.0.2',
+            },
+        }
+        vips_to_assign = {
+            'management': {
+                'vrouter': '192.168.0.2',
+                'management': '192.168.0.3',
+            },
+            'public': {
+                'vrouter_pub': '172.16.0.4',
+                'public': '172.16.0.5',
+            },
+        }
+        self._create_ip_addrs_by_rules(self.cluster, vips_to_create)
+        self.net_manager.assign_given_vips_for_net_groups(
+            self.cluster, vips_to_assign)
+        vips = self.net_manager.get_assigned_vips(self.cluster)
+        self.assertEqual(vips_to_assign, vips)

nailgun/nailgun/test/unit/test_extensions.py

@@ -18,6 +18,7 @@ import mock
 
 from nailgun.errors import errors
 from nailgun.extensions import BaseExtension
+from nailgun.extensions import fire_callback_on_cluster_delete
 from nailgun.extensions import fire_callback_on_node_collection_delete
 from nailgun.extensions import fire_callback_on_node_create
 from nailgun.extensions import fire_callback_on_node_delete
@@ -181,3 +182,12 @@ class TestExtensionUtils(BaseTestCase):
 
         for ext in get_m.return_value:
             ext.on_node_collection_delete.assert_called_once_with(node_ids)
+
+    @mock.patch('nailgun.extensions.base.get_all_extensions',
+                return_value=make_mock_extensions())
+    def test_fire_callback_on_cluster_deletion(self, get_m):
+        cluster = mock.MagicMock()
+        fire_callback_on_cluster_delete(cluster)
+
+        for ext in get_m.return_value:
+            ext.on_cluster_delete.assert_called_once_with(cluster)

nailgun/nailgun/test/unit/test_objects.py

@@ -536,6 +536,32 @@ class TestNodeObject(BaseIntegrationTest):
             self.assertEqual(node.ip_addrs, [])
             self.assertEqual(node.pending_roles, prev_roles)
 
+    def _assert_cluster_create_data(self, network_data):
+        release = self.env.create_release(api=False)
+        expected_data = {
+            "name": "cluster-0",
+            "mode": consts.CLUSTER_MODES.ha_compact,
+            "release_id": release.id,
+        }
+        expected_data.update(network_data)
+        cluster = self.env.create_cluster(api=False, **expected_data)
+        create_data = objects.Cluster.get_create_data(cluster)
+        self.assertEqual(expected_data, create_data)
+
+    def test_cluster_get_create_data_neutron(self):
+        network_data = {
+            "net_provider": consts.CLUSTER_NET_PROVIDERS.neutron,
+            "net_segment_type": consts.NEUTRON_SEGMENT_TYPES.vlan,
+            "net_l23_provider": consts.NEUTRON_L23_PROVIDERS.ovs,
+        }
+        self._assert_cluster_create_data(network_data)
+
+    def test_cluster_get_create_data_nova(self):
+        network_data = {
+            "net_provider": consts.CLUSTER_NET_PROVIDERS.nova_network,
+        }
+        self._assert_cluster_create_data(network_data)
+
 
 class TestTaskObject(BaseIntegrationTest):
 
@@ -858,14 +884,16 @@ class TestClusterObject(BaseTestCase):
         network_role.update(kwargs)
         return network_role
 
+    @mock.patch('nailgun.objects.cluster.fire_callback_on_cluster_delete')
     @mock.patch(
         'nailgun.objects.cluster.'
         'fire_callback_on_node_collection_delete')
-    def test_delete(self, callback_mock):
+    def test_delete(self, mock_node_coll_delete_cb, mock_cluster_delete_cb):
         cluster = self.env.clusters[0]
         ids = [node.id for node in cluster.nodes]
         objects.Cluster.delete(cluster)
-        callback_mock.assert_called_once_with(ids)
+        mock_node_coll_delete_cb.assert_called_once_with(ids)
+        mock_cluster_delete_cb.assert_called_once_with(cluster)
         self.assertEqual(self.db.query(objects.Node.model).count(), 0)
         self.assertEqual(self.db.query(objects.Cluster.model).count(), 0)