Browse Source

Merge master into stable/mitaka

Change-Id: I62b4f8d1a0a75337d617959b0d7cbc104279018b
Yuriy Taraday 2 years ago
parent
commit
40dd411fe4

+ 11
- 0
README.rst View File

@@ -0,0 +1,11 @@
1
+Fuel nailgun extenstion for cluster upgrade
2
+===========================================
3
+
4
+This extension for Nailgun provides API handlers and logic for
5
+cluster upgrading. This extension used by the fuel-octane project.
6
+
7
+Instalation
8
+-----------
9
+After installing `fuel-nailgun-extension-cluster-upgrade` package run:
10
+1) `nailgun_syncdb` - migrate database
11
+2) restart nailgun service

+ 6
- 0
bindep.txt View File

@@ -0,0 +1,6 @@
1
+libpq-dev
2
+postgresql
3
+postgresql-client
4
+# We don't use these, but mysql-prep step is in template job
5
+mysql-client
6
+mysql-server

+ 3
- 0
cluster_upgrade/extension.py View File

@@ -33,6 +33,9 @@ class ClusterUpgradeExtension(extensions.BaseExtension):
33 33
          'handler': handlers.NodeReassignHandler},
34 34
         {'uri': r'/clusters/(?P<cluster_id>\d+)/upgrade/vips/?$',
35 35
          'handler': handlers.CopyVIPsHandler},
36
+        {'uri': r'/clusters/(?P<cluster_id>\d+)/upgrade/clone_release/'
37
+                r'(?P<release_id>\d+)/?$',
38
+         'handler': handlers.CreateUpgradeReleaseHandler},
36 39
     ]
37 40
 
38 41
     @classmethod

+ 50
- 4
cluster_upgrade/handlers.py View File

@@ -29,7 +29,9 @@ class ClusterUpgradeCloneHandler(base.BaseHandler):
29 29
     single = objects.Cluster
30 30
     validator = validators.ClusterUpgradeValidator
31 31
 
32
-    @base.content
32
+    @base.handle_errors
33
+    @base.validate
34
+    @base.serialize
33 35
     def POST(self, cluster_id):
34 36
         """Initialize the upgrade of the cluster.
35 37
 
@@ -50,7 +52,7 @@ class ClusterUpgradeCloneHandler(base.BaseHandler):
50 52
         request_data = self.checked_data(cluster=orig_cluster)
51 53
         new_cluster = upgrade.UpgradeHelper.clone_cluster(orig_cluster,
52 54
                                                           request_data)
53
-        return new_cluster.to_json()
55
+        return new_cluster.to_dict()
54 56
 
55 57
 
56 58
 class NodeReassignHandler(base.BaseHandler):
@@ -67,7 +69,8 @@ class NodeReassignHandler(base.BaseHandler):
67 69
 
68 70
         self.raise_task(task)
69 71
 
70
-    @base.content
72
+    @base.handle_errors
73
+    @base.validate
71 74
     def POST(self, cluster_id):
72 75
         """Reassign node to the given cluster.
73 76
 
@@ -107,7 +110,8 @@ class CopyVIPsHandler(base.BaseHandler):
107 110
     single = objects.Cluster
108 111
     validator = validators.CopyVIPsValidator
109 112
 
110
-    @base.content
113
+    @base.handle_errors
114
+    @base.validate
111 115
     def POST(self, cluster_id):
112 116
         """Copy VIPs from original cluster to new one
113 117
 
@@ -139,3 +143,45 @@ class CopyVIPsHandler(base.BaseHandler):
139 143
 
140 144
         upgrade.UpgradeHelper.copy_vips(orig_cluster_adapter,
141 145
                                         seed_cluster_adapter)
146
+
147
+
148
+class CreateUpgradeReleaseHandler(base.BaseHandler):
149
+    @staticmethod
150
+    def merge_network_roles(base_nets, orig_nets):
151
+        """Create network metadata based on two releases.
152
+
153
+        Overwrite base default_mapping by orig default_maping values.
154
+        """
155
+        orig_network_dict = {n['id']: n for n in orig_nets}
156
+        for base_net in base_nets:
157
+            orig_net = orig_network_dict.get(base_net['id'])
158
+            if orig_net is None:
159
+                orig_net = base_net
160
+            base_net['default_mapping'] = orig_net['default_mapping']
161
+        return base_net
162
+
163
+    @base.serialize
164
+    def POST(self, cluster_id, release_id):
165
+        """Create release for upgrade purposes.
166
+
167
+        Creates a new release with network_roles_metadata based the given
168
+        release and re-use network parameters from the given cluster.
169
+
170
+        :returns: JSON representation of the created cluster
171
+        :http: * 200 (OK)
172
+               * 404 (Cluster or release not found.)
173
+        """
174
+        base_release = self.get_object_or_404(objects.Release, release_id)
175
+        orig_cluster = self.get_object_or_404(objects.Cluster, cluster_id)
176
+        orig_release = orig_cluster.release
177
+
178
+        network_metadata = self.merge_network_roles(
179
+            base_release.network_roles_metadata,
180
+            orig_release.network_roles_metadata)
181
+        data = objects.Release.to_dict(base_release)
182
+        data['network_roles_metadata'] = network_metadata
183
+        data['name'] = '{0} Upgrade ({1})'.format(
184
+            base_release.name, orig_release.id)
185
+        del data['id']
186
+        new_release = objects.Release.create(data)
187
+        return new_release.to_dict()

+ 27
- 2
cluster_upgrade/objects/adapters.py View File

@@ -14,6 +14,7 @@
14 14
 #    License for the specific language governing permissions and limitations
15 15
 #    under the License.
16 16
 
17
+from nailgun.extensions.volume_manager import extension as volume_ext
17 18
 from nailgun import objects
18 19
 
19 20
 
@@ -62,6 +63,14 @@ class NailgunClusterAdapter(object):
62 63
     def editable_attrs(self, attrs):
63 64
         self.cluster.attributes.editable = attrs
64 65
 
66
+    @property
67
+    def network_template(self):
68
+        return self.cluster.network_config.configuration_template
69
+
70
+    @network_template.setter
71
+    def network_template(self, template):
72
+        self.cluster.network_config.configuration_template = template
73
+
65 74
     def get_create_data(self):
66 75
         return objects.Cluster.get_create_data(self.cluster)
67 76
 
@@ -70,8 +79,8 @@ class NailgunClusterAdapter(object):
70 79
             instance=self.cluster)
71 80
         return NailgunNetworkManager(self.cluster, net_manager)
72 81
 
73
-    def to_json(self):
74
-        return objects.Cluster.to_json(self.cluster)
82
+    def to_dict(self):
83
+        return objects.Cluster.to_dict(self.cluster)
75 84
 
76 85
     @classmethod
77 86
     def get_by_uid(cls, cluster_id):
@@ -96,6 +105,10 @@ class NailgunReleaseAdapter(object):
96 105
             uid, fail_if_not_found=fail_if_not_found)
97 106
         return release
98 107
 
108
+    @property
109
+    def operating_system(self):
110
+        return self.release.operating_system
111
+
99 112
     @property
100 113
     def is_deployable(self):
101 114
         return objects.Release.is_deployable(self.release)
@@ -173,6 +186,10 @@ class NailgunNodeAdapter(object):
173 186
     def status(self):
174 187
         return self.node.status
175 188
 
189
+    @property
190
+    def nic_interfaces(self):
191
+        return self.node.nic_interfaces
192
+
176 193
     @property
177 194
     def error_type(self):
178 195
         return self.node.error_type
@@ -192,6 +209,14 @@ class NailgunNodeAdapter(object):
192 209
     def add_pending_change(self, change):
193 210
         objects.Node.add_pending_change(self.node, change)
194 211
 
212
+    def get_volumes(self):
213
+        return volume_ext.VolumeManagerExtension.get_node_volumes(self.node)
214
+
215
+    def set_volumes(self, volumes):
216
+        return volume_ext.VolumeManagerExtension.set_node_volumes(
217
+            self.node, volumes
218
+        )
219
+
195 220
 
196 221
 class NailgunNetworkGroupAdapter(object):
197 222
 

+ 6
- 13
cluster_upgrade/tests/test_handlers.py View File

@@ -77,12 +77,10 @@ class TestNodeReassignHandler(base.BaseIntegrationTest):
77 77
 
78 78
     @mock.patch('nailgun.task.task.rpc.cast')
79 79
     def test_node_reassign_handler(self, mcast):
80
-        self.env.create(
80
+        cluster = self.env.create(
81 81
             cluster_kwargs={'api': False},
82 82
             nodes_kwargs=[{'status': consts.NODE_STATUSES.ready}])
83
-        self.env.create_cluster()
84
-        cluster = self.env.clusters[0]
85
-        seed_cluster = self.env.clusters[1]
83
+        seed_cluster = self.env.create_cluster()
86 84
         node_id = cluster.nodes[0]['id']
87 85
 
88 86
         resp = self.app.post(
@@ -144,9 +142,7 @@ class TestNodeReassignHandler(base.BaseIntegrationTest):
144 142
         self.assertEqual(node.roles, ['compute'])
145 143
 
146 144
     def test_node_reassign_handler_no_node(self):
147
-        self.env.create_cluster()
148
-
149
-        cluster = self.env.clusters[0]
145
+        cluster = self.env.create_cluster()
150 146
 
151 147
         resp = self.app.post(
152 148
             reverse('NodeReassignHandler',
@@ -159,10 +155,9 @@ class TestNodeReassignHandler(base.BaseIntegrationTest):
159 155
                          resp.json_body['message'])
160 156
 
161 157
     def test_node_reassing_handler_wrong_status(self):
162
-        self.env.create(
158
+        cluster = self.env.create(
163 159
             cluster_kwargs={'api': False},
164 160
             nodes_kwargs=[{'status': 'discover'}])
165
-        cluster = self.env.clusters[0]
166 161
 
167 162
         resp = self.app.post(
168 163
             reverse('NodeReassignHandler',
@@ -175,11 +170,10 @@ class TestNodeReassignHandler(base.BaseIntegrationTest):
175 170
                                  "^Node should be in one of statuses:")
176 171
 
177 172
     def test_node_reassing_handler_wrong_error_type(self):
178
-        self.env.create(
173
+        cluster = self.env.create(
179 174
             cluster_kwargs={'api': False},
180 175
             nodes_kwargs=[{'status': 'error',
181 176
                            'error_type': 'provision'}])
182
-        cluster = self.env.clusters[0]
183 177
 
184 178
         resp = self.app.post(
185 179
             reverse('NodeReassignHandler',
@@ -192,10 +186,9 @@ class TestNodeReassignHandler(base.BaseIntegrationTest):
192 186
                                  "^Node should be in error state")
193 187
 
194 188
     def test_node_reassign_handler_to_the_same_cluster(self):
195
-        self.env.create(
189
+        cluster = self.env.create(
196 190
             cluster_kwargs={'api': False},
197 191
             nodes_kwargs=[{'status': 'ready'}])
198
-        cluster = self.env.clusters[0]
199 192
 
200 193
         cluster_id = cluster['id']
201 194
         node_id = cluster.nodes[0]['id']

+ 221
- 0
cluster_upgrade/tests/test_transformations.py View File

@@ -0,0 +1,221 @@
1
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+# not use this file except in compliance with the License. You may obtain
3
+# a copy of the License at
4
+#
5
+#      http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+# Unless required by applicable law or agreed to in writing, software
8
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+# License for the specific language governing permissions and limitations
11
+# under the License.
12
+
13
+from distutils import version
14
+
15
+import mock
16
+from nailgun.test import base as nailgun_test_base
17
+import six
18
+
19
+from .. import transformations
20
+from ..transformations import cluster
21
+
22
+
23
+class TestTransformations(nailgun_test_base.BaseUnitTest):
24
+    def test_get_config(self):
25
+        config = object()
26
+
27
+        class Manager(transformations.Manager):
28
+            default_config = config
29
+
30
+        self.assertIs(config, Manager.get_config('testname'))
31
+
32
+    def setup_extension_manager(self, extensions):
33
+        p = mock.patch("stevedore.ExtensionManager", spec=['__call__'])
34
+        mock_extman = p.start()
35
+        self.addCleanup(p.stop)
36
+
37
+        def extman(namespace, *args, **kwargs):
38
+            instance = mock.MagicMock(name=namespace)
39
+            ext_results = {}
40
+            for ver, exts in six.iteritems(extensions):
41
+                if namespace.endswith(ver):
42
+                    ext_results = {name: mock.Mock(name=name, plugin=ext)
43
+                                   for name, ext in six.iteritems(exts)}
44
+                    break
45
+            else:
46
+                self.fail("Called with unexpected version in namespace: {}, "
47
+                          "expected versions: {}".format(
48
+                              namespace, list(extensions)))
49
+            instance.__getitem__.side_effect = ext_results.__getitem__
50
+            return instance
51
+
52
+        mock_extman.side_effect = extman
53
+        return mock_extman
54
+
55
+    def test_load_transformers(self):
56
+        config = {'9.0': ['a', 'b']}
57
+        extensions = {'9.0': {
58
+            'a': mock.Mock(name='a'),
59
+            'b': mock.Mock(name='b'),
60
+        }}
61
+        mock_extman = self.setup_extension_manager(extensions)
62
+
63
+        res = transformations.Manager.load_transformers('testname', config)
64
+
65
+        self.assertEqual(res, [(version.StrictVersion('9.0'), [
66
+            extensions['9.0']['a'],
67
+            extensions['9.0']['b'],
68
+        ])])
69
+        callback = transformations.reraise_endpoint_load_failure
70
+        self.assertEqual(mock_extman.mock_calls, [
71
+            mock.call(
72
+                'nailgun.cluster_upgrade.transformations.testname.9.0',
73
+                on_load_failure_callback=callback,
74
+            ),
75
+        ])
76
+
77
+    def test_load_transformers_empty(self):
78
+        config = {}
79
+        extensions = {'9.0': {
80
+            'a': mock.Mock(name='a'),
81
+            'b': mock.Mock(name='b'),
82
+        }}
83
+        mock_extman = self.setup_extension_manager(extensions)
84
+
85
+        res = transformations.Manager.load_transformers('testname', config)
86
+
87
+        self.assertEqual(res, [])
88
+        self.assertEqual(mock_extman.mock_calls, [])
89
+
90
+    def test_load_transformers_sorted(self):
91
+        config = {'9.0': ['a', 'b'], '8.0': ['c']}
92
+        extensions = {
93
+            '9.0': {
94
+                'a': mock.Mock(name='a'),
95
+                'b': mock.Mock(name='b'),
96
+            },
97
+            '8.0': {
98
+                'c': mock.Mock(name='c'),
99
+                'd': mock.Mock(name='d'),
100
+            },
101
+        }
102
+        mock_extman = self.setup_extension_manager(extensions)
103
+
104
+        orig_iteritems = six.iteritems
105
+        iteritems_patch = mock.patch('six.iteritems')
106
+        mock_iteritems = iteritems_patch.start()
107
+        self.addCleanup(iteritems_patch.stop)
108
+
109
+        def sorted_iteritems(d):
110
+            return sorted(orig_iteritems(d), reverse=True)
111
+
112
+        mock_iteritems.side_effect = sorted_iteritems
113
+
114
+        res = transformations.Manager.load_transformers('testname', config)
115
+
116
+        self.assertEqual(res, [
117
+            (version.StrictVersion('8.0'), [
118
+                extensions['8.0']['c'],
119
+            ]),
120
+            (version.StrictVersion('9.0'), [
121
+                extensions['9.0']['a'],
122
+                extensions['9.0']['b'],
123
+            ]),
124
+        ])
125
+        callback = transformations.reraise_endpoint_load_failure
126
+        self.assertItemsEqual(mock_extman.mock_calls, [
127
+            mock.call(
128
+                'nailgun.cluster_upgrade.transformations.testname.9.0',
129
+                on_load_failure_callback=callback,
130
+            ),
131
+            mock.call(
132
+                'nailgun.cluster_upgrade.transformations.testname.8.0',
133
+                on_load_failure_callback=callback,
134
+            ),
135
+        ])
136
+
137
+    def test_load_transformers_keyerror(self):
138
+        config = {'9.0': ['a', 'b', 'c']}
139
+        extensions = {'9.0': {
140
+            'a': mock.Mock(name='a'),
141
+            'b': mock.Mock(name='b'),
142
+        }}
143
+        mock_extman = self.setup_extension_manager(extensions)
144
+
145
+        with self.assertRaisesRegexp(KeyError, 'c'):
146
+            transformations.Manager.load_transformers('testname', config)
147
+
148
+        callback = transformations.reraise_endpoint_load_failure
149
+        self.assertEqual(mock_extman.mock_calls, [
150
+            mock.call(
151
+                'nailgun.cluster_upgrade.transformations.testname.9.0',
152
+                on_load_failure_callback=callback,
153
+            ),
154
+        ])
155
+
156
+    @mock.patch.object(transformations.Manager, 'load_transformers')
157
+    def test_apply(self, mock_load):
158
+        mock_trans = mock.Mock()
159
+        mock_load.return_value = [
160
+            (version.StrictVersion('7.0'), [mock_trans.a, mock_trans.b]),
161
+            (version.StrictVersion('8.0'), [mock_trans.c, mock_trans.d]),
162
+            (version.StrictVersion('9.0'), [mock_trans.e, mock_trans.f]),
163
+        ]
164
+        man = transformations.Manager()
165
+        res = man.apply('7.0', '9.0', {})
166
+        self.assertEqual(res, mock_trans.f.return_value)
167
+        self.assertEqual(mock_trans.mock_calls, [
168
+            mock.call.c({}),
169
+            mock.call.d(mock_trans.c.return_value),
170
+            mock.call.e(mock_trans.d.return_value),
171
+            mock.call.f(mock_trans.e.return_value),
172
+        ])
173
+
174
+
175
+class TestLazy(nailgun_test_base.BaseUnitTest):
176
+    def test_lazy(self):
177
+        mgr_cls_mock = mock.Mock()
178
+        lazy_obj = transformations.Lazy(mgr_cls_mock)
179
+        lazy_obj.apply()
180
+        self.assertEqual(lazy_obj.apply, mgr_cls_mock.return_value.apply)
181
+
182
+
183
+class TestClusterTransformers(nailgun_test_base.BaseUnitTest):
184
+    def setUp(self):
185
+        self.data = {
186
+            'editable': {
187
+                'external_dns': {
188
+                    'dns_list': {'type': 'text', 'value': 'a,b,\nc, d'}},
189
+                'external_ntp': {
190
+                    'ntp_list': {'type': 'text', 'value': 'a,b,\nc, d'}},
191
+            },
192
+            'generated': {
193
+                'provision': {},
194
+            },
195
+        }
196
+
197
+    def test_dns_list(self):
198
+        res = cluster.transform_dns_list(self.data)
199
+        self.assertEqual(
200
+            res['editable']['external_dns']['dns_list'],
201
+            {'type': 'text_list', 'value': ['a', 'b', 'c', 'd']},
202
+        )
203
+
204
+    def test_ntp_list(self):
205
+        res = cluster.transform_ntp_list(self.data)
206
+        self.assertEqual(
207
+            res['editable']['external_ntp']['ntp_list'],
208
+            {'type': 'text_list', 'value': ['a', 'b', 'c', 'd']},
209
+        )
210
+
211
+    def test_provision(self):
212
+        res = cluster.drop_generated_provision(self.data)
213
+        self.assertNotIn('provision', res['generated'])
214
+
215
+    def test_manager(self):
216
+        man = cluster.Manager()  # verify default config and entry points
217
+        self.assertEqual(man.transformers, [(version.StrictVersion('9.0'), [
218
+            cluster.transform_dns_list,
219
+            cluster.transform_ntp_list,
220
+            cluster.drop_generated_provision,
221
+        ])])

+ 43
- 4
cluster_upgrade/tests/test_upgrade.py View File

@@ -19,6 +19,7 @@ import six
19 19
 
20 20
 from nailgun import consts
21 21
 from nailgun.objects.serializers import network_configuration
22
+from nailgun.test.base import fake_tasks
22 23
 
23 24
 from .. import upgrade
24 25
 from . import base as base_tests
@@ -49,7 +50,7 @@ class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest):
49 50
                 {"metadata": "src_fake",
50 51
                  "key":
51 52
                      {"type": "text",
52
-                      "value": "fake1, fake2,fake3 , fake4"},
53
+                      "value": "fake"},
53 54
                  "src_key": "src_data"
54 55
                  },
55 56
             "repo_setup": "src_data"
@@ -68,9 +69,6 @@ class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest):
68 69
         result = upgrade.merge_attributes(
69 70
             src_editable_attrs, new_editable_attrs
70 71
         )
71
-        new_editable_attrs["test"]["key"]["value"] = [
72
-            "fake1", "fake2", "fake3", "fake4"
73
-        ]
74 72
         self.assertEqual(result, new_editable_attrs)
75 73
 
76 74
     def test_create_cluster_clone(self):
@@ -238,3 +236,44 @@ class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest):
238 236
         self.helper.change_env_settings(self.src_cluster, new_cluster)
239 237
         self.assertEqual('image',
240 238
                          attrs['editable']['provision']['method']['value'])
239
+
240
+    def get_assigned_nets(self, node):
241
+        assigned_nets = {}
242
+        for iface in node.nic_interfaces:
243
+            nets = [net.name for net in iface.assigned_networks_list]
244
+            assigned_nets[iface.name] = nets
245
+        return assigned_nets
246
+
247
+    @fake_tasks()
248
+    def assign_node_to_cluster(self, template=None):
249
+        new_cluster = self.helper.clone_cluster(self.src_cluster, self.data)
250
+        node = adapters.NailgunNodeAdapter(self.src_cluster.cluster.nodes[0])
251
+
252
+        orig_assigned_nets = self.get_assigned_nets(node)
253
+
254
+        if template:
255
+            net_template = self.env.read_fixtures(['network_template_80'])[0]
256
+            new_cluster.network_template = net_template
257
+            orig_assigned_nets = {
258
+                'eth0': ['fuelweb_admin'], 'eth1': ['public', 'management']
259
+            }
260
+
261
+        self.helper.assign_node_to_cluster(node, new_cluster, node.roles, [])
262
+        self.db.refresh(new_cluster.cluster)
263
+
264
+        self.assertEqual(node.cluster_id, new_cluster.id)
265
+
266
+        self.env.clusters.append(new_cluster.cluster)
267
+        task = self.env.launch_provisioning_selected(cluster_id=new_cluster.id)
268
+        self.assertEqual(task.status, consts.TASK_STATUSES.ready)
269
+        for n in new_cluster.cluster.nodes:
270
+            self.assertEqual(consts.NODE_STATUSES.provisioned, n.status)
271
+
272
+        new_assigned_nets = self.get_assigned_nets(node)
273
+        self.assertEqual(orig_assigned_nets, new_assigned_nets)
274
+
275
+    def test_assign_node_to_cluster(self):
276
+        self.assign_node_to_cluster()
277
+
278
+    def test_assign_node_to_cluster_with_template(self):
279
+        self.assign_node_to_cluster(template=True)

+ 14
- 2
cluster_upgrade/tests/test_validators.py View File

@@ -58,6 +58,14 @@ class TestClusterUpgradeValidator(tests_base.BaseCloneClusterTest):
58 58
             self.validator.validate_release_upgrade(self.dst_release,
59 59
                                                     self.src_release)
60 60
 
61
+    def test_validate_release_upgrade_to_different_os(self):
62
+        self.dst_release.operating_system = consts.RELEASE_OS.centos
63
+        msg = "^Changing of operating system is not possible during upgrade " \
64
+              "\(from {0} to {1}\).$".format("Ubuntu", "CentOS")
65
+        with self.assertRaisesRegexp(errors.InvalidData, msg):
66
+            self.validator.validate_release_upgrade(self.src_release,
67
+                                                    self.dst_release)
68
+
61 69
     def test_validate_cluster_name(self):
62 70
         self.validator.validate_cluster_name("cluster-42")
63 71
 
@@ -187,10 +195,14 @@ class TestNodeReassignNoReinstallValidator(tests_base.BaseCloneClusterTest):
187 195
             "reprovision": False,
188 196
             "roles": ['controller', 'compute'],
189 197
         })
190
-        msg = "^Role 'controller' in conflict with role 'compute'.$"
191
-        with self.assertRaisesRegexp(errors.InvalidData, msg):
198
+        with self.assertRaises(errors.InvalidData) as exc:
192 199
             self.validator.validate(data, self.dst_cluster)
193 200
 
201
+        self.assertEqual(
202
+            exc.exception.message,
203
+            "Role 'controller' in conflict with role 'compute'."
204
+        )
205
+
194 206
 
195 207
 class TestCopyVIPsValidator(base.BaseTestCase):
196 208
     validator = validators.CopyVIPsValidator

+ 94
- 0
cluster_upgrade/transformations/__init__.py View File

@@ -0,0 +1,94 @@
1
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+# not use this file except in compliance with the License. You may obtain
3
+# a copy of the License at
4
+#
5
+#      http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+# Unless required by applicable law or agreed to in writing, software
8
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+# License for the specific language governing permissions and limitations
11
+# under the License.
12
+
13
+import copy
14
+import distutils.version
15
+import logging
16
+import threading
17
+
18
+import six
19
+
20
+import stevedore
21
+
22
+LOG = logging.getLogger(__name__)
23
+
24
+
25
+def reraise_endpoint_load_failure(manager, endpoint, exc):
26
+    LOG.error('Failed to load %s: %s', endpoint.name, exc)
27
+    raise  # Avoid unexpectedly skipped steps
28
+
29
+
30
+class Manager(object):
31
+    default_config = None
32
+    name = None
33
+
34
+    def __init__(self):
35
+        self.config = self.get_config(self.name)
36
+        self.transformers = self.load_transformers(self.name, self.config)
37
+
38
+    @classmethod
39
+    def get_config(cls, name):
40
+        # TODO(yorik-sar): merge actual config with defaults
41
+        return cls.default_config
42
+
43
+    @staticmethod
44
+    def load_transformers(name, config):
45
+        transformers = []
46
+        for version, names in six.iteritems(config):
47
+            extension_manager = stevedore.ExtensionManager(
48
+                'nailgun.cluster_upgrade.transformations.{}.{}'.format(
49
+                    name, version),
50
+                on_load_failure_callback=reraise_endpoint_load_failure,
51
+            )
52
+            try:
53
+                sorted_extensions = [extension_manager[n].plugin
54
+                                     for n in names]
55
+            except KeyError as exc:
56
+                LOG.error('%s transformer %s not found for version %s',
57
+                          name, exc, version)
58
+                raise
59
+            strict_version = distutils.version.StrictVersion(version)
60
+            transformers.append((strict_version, sorted_extensions))
61
+        transformers.sort()
62
+        return transformers
63
+
64
+    def apply(self, from_version, to_version, data):
65
+        strict_from = distutils.version.StrictVersion(from_version)
66
+        strict_to = distutils.version.StrictVersion(to_version)
67
+        assert strict_from <= strict_to, \
68
+            "from_version must not be greater than to_version"
69
+        data = copy.deepcopy(data)
70
+        for version, transformers in self.transformers:
71
+            if version <= strict_from:
72
+                continue
73
+            if version > strict_to:
74
+                break
75
+            for transformer in transformers:
76
+                LOG.debug("Applying %s transformer %s",
77
+                          self.name, transformer)
78
+                data = transformer(data)
79
+        return data
80
+
81
+
82
+class Lazy(object):
83
+    def __init__(self, mgr_cls):
84
+        self.mgr_cls = mgr_cls
85
+        self.mgr = None
86
+        self.lock = threading.Lock()
87
+
88
+    def apply(self, *args, **kwargs):
89
+        if self.mgr is None:
90
+            with self.lock:
91
+                if self.mgr is None:
92
+                    self.mgr = self.mgr_cls()
93
+                    self.apply = self.mgr.apply
94
+        return self.mgr.apply(*args, **kwargs)

+ 52
- 0
cluster_upgrade/transformations/cluster.py View File

@@ -0,0 +1,52 @@
1
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+# not use this file except in compliance with the License. You may obtain
3
+# a copy of the License at
4
+#
5
+#      http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+# Unless required by applicable law or agreed to in writing, software
8
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+# License for the specific language governing permissions and limitations
11
+# under the License.
12
+
13
+from cluster_upgrade import transformations
14
+
15
+# NOTE: In the mitaka-9.0 release types of values dns_list and
16
+# ntp_list were changed from 'text'
17
+# (a string of comma-separated IP-addresses)
18
+# to 'text_list' (a list of strings of IP-addresses).
19
+
20
+
21
+def transform_to_text_list(data):
22
+    if data['type'] == 'text':
23
+        data['type'] = 'text_list'
24
+        data['value'] = [
25
+            part.strip() for part in data['value'].split(',')
26
+        ]
27
+
28
+    return data
29
+
30
+
31
+def transform_dns_list(data):
32
+    dns_list = data['editable']['external_dns']['dns_list']
33
+    transform_to_text_list(dns_list)
34
+    return data
35
+
36
+
37
+def transform_ntp_list(data):
38
+    ntp_list = data['editable']['external_ntp']['ntp_list']
39
+    transform_to_text_list(ntp_list)
40
+    return data
41
+
42
+
43
+def drop_generated_provision(data):
44
+    data['generated'].pop('provision', None)
45
+    return data
46
+
47
+
48
+class Manager(transformations.Manager):
49
+    default_config = {
50
+        '9.0': ['dns_list', 'ntp_list', 'drop_provision'],
51
+    }
52
+    name = 'cluster'

+ 62
- 0
cluster_upgrade/transformations/vip.py View File

@@ -0,0 +1,62 @@
1
+# coding: utf-8
2
+
3
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+#    not use this file except in compliance with the License. You may obtain
5
+#    a copy of the License at
6
+#
7
+#         http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+#    Unless required by applicable law or agreed to in writing, software
10
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+#    License for the specific language governing permissions and limitations
13
+#    under the License.
14
+
15
+import collections
16
+
17
+from cluster_upgrade import transformations
18
+
19
+
20
+def transform_vips(data):
21
+    """Rename or remove types of VIPs for 7.0 network groups.
22
+
23
+    This method renames types of VIPs from older releases (<7.0) to
24
+    be compatible with network groups of the 7.0 release according
25
+    to the rules:
26
+
27
+        management: haproxy -> management
28
+        public: haproxy -> public
29
+        public: vrouter -> vrouter_pub
30
+
31
+    Note, that in the result VIPs are present only those IPs that
32
+    correspond to the given rules.
33
+    """
34
+    rename_vip_rules = {
35
+        "management": {
36
+            "haproxy": "management",
37
+            "vrouter": "vrouter",
38
+        },
39
+        "public": {
40
+            "haproxy": "public",
41
+            "vrouter": "vrouter_pub",
42
+        },
43
+    }
44
+    renamed_vips = collections.defaultdict(dict)
45
+    for ng_name, vips_obj in data.items():
46
+
47
+        ng_vip_rules = rename_vip_rules[ng_name]
48
+        for vip_name, vip_addr in vips_obj.items():
49
+            if vip_name not in ng_vip_rules:
50
+                continue
51
+
52
+            new_vip_name = ng_vip_rules[vip_name]
53
+            renamed_vips[ng_name][new_vip_name] = vip_addr
54
+
55
+    return renamed_vips
56
+
57
+
58
+class Manager(transformations.Manager):
59
+    default_config = {
60
+        '7.0': ['transform_vips']
61
+    }
62
+    name = 'vip'

+ 53
- 0
cluster_upgrade/transformations/volumes.py View File

@@ -0,0 +1,53 @@
1
+# coding: utf-8
2
+
3
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
4
+#    not use this file except in compliance with the License. You may obtain
5
+#    a copy of the License at
6
+#
7
+#         http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+#    Unless required by applicable law or agreed to in writing, software
10
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12
+#    License for the specific language governing permissions and limitations
13
+#    under the License.
14
+
15
+from cluster_upgrade import transformations
16
+
17
+
18
+def transform_node_volumes(volumes):
19
+    try:
20
+        os_vg = next(vol for vol in volumes
21
+                     if 'id' in vol and vol['id'] == 'os')
22
+    except StopIteration:
23
+        return volumes
24
+
25
+    other_volumes = [vol for vol in volumes
26
+                     if 'id' not in vol or vol['id'] != 'os']
27
+
28
+    for disk in other_volumes:
29
+        disk_volumes = disk['volumes']
30
+        disk['volumes'] = []
31
+
32
+        for v in disk_volumes:
33
+            if v['type'] == 'pv' and v['vg'] == 'os' and v['size'] > 0:
34
+                for vv in os_vg['volumes']:
35
+                    partition = {'name': vv['name'],
36
+                                 'size': vv['size'],
37
+                                 'type': 'partition',
38
+                                 'mount': vv['mount'],
39
+                                 'file_system': vv['file_system']}
40
+                    disk['volumes'].append(partition)
41
+            else:
42
+                if v['type'] == 'lvm_meta_pool' or v['type'] == 'boot':
43
+                    v['size'] = 0
44
+                disk['volumes'].append(v)
45
+
46
+    return volumes
47
+
48
+
49
+class Manager(transformations.Manager):
50
+    default_config = {
51
+        '6.1': ['node_volumes']
52
+    }
53
+    name = 'volumes'

+ 47
- 80
cluster_upgrade/upgrade.py View File

@@ -14,9 +14,7 @@
14 14
 #    License for the specific language governing permissions and limitations
15 15
 #    under the License.
16 16
 
17
-import collections
18 17
 import copy
19
-from distutils import version
20 18
 import six
21 19
 
22 20
 from nailgun import consts
@@ -24,7 +22,11 @@ from nailgun import objects
24 22
 from nailgun.objects.serializers import network_configuration
25 23
 from nailgun import utils
26 24
 
25
+from . import transformations  # That's weird, but that's how hacking likes
27 26
 from .objects import adapters
27
+from .transformations import cluster as cluster_trs
28
+from .transformations import vip
29
+from .transformations import volumes as volumes_trs
28 30
 
29 31
 
30 32
 def merge_attributes(a, b):
@@ -41,25 +43,9 @@ def merge_attributes(a, b):
41 43
         for key, values in six.iteritems(pairs):
42 44
             if key != "metadata" and key in a_values:
43 45
                 values["value"] = a_values[key]["value"]
44
-                # NOTE: In the mitaka-9.0 release types of values dns_list and
45
-                # ntp_list were changed from 'text'
46
-                # (a string of comma-separated IP-addresses)
47
-                # to 'text_list' (a list of strings of IP-addresses).
48
-                if a_values[key]['type'] == 'text' and \
49
-                        values['type'] == 'text_list':
50
-                    values["value"] = [
51
-                        value.strip() for value in values['value'].split(',')
52
-                    ]
53 46
     return attrs
54 47
 
55 48
 
56
-def merge_generated_attrs(new_attrs, orig_attrs):
57
-    # skip attributes that should be generated for new cluster
58
-    attrs = copy.deepcopy(orig_attrs)
59
-    attrs.pop('provision', None)
60
-    return utils.dict_merge(new_attrs, attrs)
61
-
62
-
63 49
 def merge_nets(a, b):
64 50
     new_settings = copy.deepcopy(b)
65 51
     source_networks = dict((n["name"], n) for n in a["networks"])
@@ -87,6 +73,9 @@ class UpgradeHelper(object):
87 73
         consts.CLUSTER_NET_PROVIDERS.nova_network:
88 74
         network_configuration.NovaNetworkConfigurationSerializer,
89 75
     }
76
+    cluster_transformations = transformations.Lazy(cluster_trs.Manager)
77
+    vip_transformations = transformations.Lazy(vip.Manager)
78
+    volumes_transformations = transformations.Lazy(volumes_trs.Manager)
90 79
 
91 80
     @classmethod
92 81
     def clone_cluster(cls, orig_cluster, data):
@@ -110,61 +99,30 @@ class UpgradeHelper(object):
110 99
 
111 100
     @classmethod
112 101
     def copy_attributes(cls, orig_cluster, new_cluster):
113
-        # TODO(akscram): Attributes should be copied including
114
-        #                borderline cases when some parameters are
115
-        #                renamed or moved into plugins. Also, we should
116
-        #                to keep special steps in copying of parameters
117
-        #                that know how to translate parameters from one
118
-        #                version to another. A set of this kind of steps
119
-        #                should define an upgrade path of a particular
120
-        #                cluster.
121
-        new_cluster.generated_attrs = merge_generated_attrs(
102
+        attrs = cls.cluster_transformations.apply(
103
+            orig_cluster.release.environment_version,
104
+            new_cluster.release.environment_version,
105
+            {
106
+                'editable': orig_cluster.editable_attrs,
107
+                'generated': orig_cluster.generated_attrs,
108
+            },
109
+        )
110
+
111
+        new_cluster.generated_attrs = utils.dict_merge(
122 112
             new_cluster.generated_attrs,
123
-            orig_cluster.generated_attrs)
113
+            attrs['generated'],
114
+        )
115
+
124 116
         new_cluster.editable_attrs = merge_attributes(
125
-            orig_cluster.editable_attrs,
126
-            new_cluster.editable_attrs)
117
+            attrs['editable'],
118
+            new_cluster.editable_attrs,
119
+        )
127 120
 
128 121
     @classmethod
129 122
     def change_env_settings(cls, orig_cluster, new_cluster):
130 123
         attrs = new_cluster.attributes
131 124
         attrs['editable']['provision']['method']['value'] = 'image'
132 125
 
133
-    @classmethod
134
-    def transform_vips_for_net_groups_70(cls, vips):
135
-        """Rename or remove types of VIPs for 7.0 network groups.
136
-
137
-        This method renames types of VIPs from older releases (<7.0) to
138
-        be compatible with network groups of the 7.0 release according
139
-        to the rules:
140
-
141
-            management: haproxy -> management
142
-            public: haproxy -> public
143
-            public: vrouter -> vrouter_pub
144
-
145
-        Note, that in the result VIPs are present only those IPs that
146
-        correspond to the given rules.
147
-        """
148
-        rename_vip_rules = {
149
-            "management": {
150
-                "haproxy": "management",
151
-                "vrouter": "vrouter",
152
-            },
153
-            "public": {
154
-                "haproxy": "public",
155
-                "vrouter": "vrouter_pub",
156
-            },
157
-        }
158
-        renamed_vips = collections.defaultdict(dict)
159
-        for ng_name, vips in six.iteritems(vips):
160
-            ng_vip_rules = rename_vip_rules[ng_name]
161
-            for vip_name, vip_addr in six.iteritems(vips):
162
-                if vip_name not in ng_vip_rules:
163
-                    continue
164
-                new_vip_name = ng_vip_rules[vip_name]
165
-                renamed_vips[ng_name][new_vip_name] = vip_addr
166
-        return renamed_vips
167
-
168 126
     @classmethod
169 127
     def copy_network_config(cls, orig_cluster, new_cluster):
170 128
         nets_serializer = cls.network_serializers[orig_cluster.net_provider]
@@ -181,17 +139,16 @@ class UpgradeHelper(object):
181 139
         orig_net_manager = orig_cluster.get_network_manager()
182 140
         new_net_manager = new_cluster.get_network_manager()
183 141
 
184
-        vips = orig_net_manager.get_assigned_vips()
185
-        for ng_name in vips:
186
-            if ng_name not in (consts.NETWORKS.public,
187
-                               consts.NETWORKS.management):
188
-                vips.pop(ng_name)
189
-        # NOTE(akscram): In the 7.0 release was introduced networking
190
-        #                templates that use the vip_name column as
191
-        #                unique names of VIPs.
192
-        if version.LooseVersion(orig_cluster.release.environment_version) < \
193
-                version.LooseVersion("7.0"):
194
-            vips = cls.transform_vips_for_net_groups_70(vips)
142
+        vips = {}
143
+        assigned_vips = orig_net_manager.get_assigned_vips()
144
+        for ng_name in (consts.NETWORKS.public, consts.NETWORKS.management):
145
+            vips[ng_name] = assigned_vips[ng_name]
146
+
147
+        vips = cls.vip_transformations.apply(
148
+            orig_cluster.release.environment_version,
149
+            new_cluster.release.environment_version,
150
+            vips
151
+        )
195 152
         new_net_manager.assign_given_vips_for_net_groups(vips)
196 153
         new_net_manager.assign_vips_for_net_groups()
197 154
 
@@ -224,6 +181,13 @@ class UpgradeHelper(object):
224 181
         orig_cluster = adapters.NailgunClusterAdapter.get_by_uid(
225 182
             node.cluster_id)
226 183
 
184
+        volumes = cls.volumes_transformations.apply(
185
+            orig_cluster.release.environment_version,
186
+            seed_cluster.release.environment_version,
187
+            node.get_volumes(),
188
+        )
189
+        node.set_volumes(volumes)
190
+
227 191
         orig_manager = orig_cluster.get_network_manager()
228 192
 
229 193
         netgroups_id_mapping = cls.get_netgroups_id_mapping(
@@ -231,10 +195,13 @@ class UpgradeHelper(object):
231 195
 
232 196
         node.update_cluster_assignment(seed_cluster, roles, pending_roles)
233 197
         objects.Node.set_netgroups_ids(node, netgroups_id_mapping)
234
-        orig_manager.set_nic_assignment_netgroups_ids(
235
-            node, netgroups_id_mapping)
236
-        orig_manager.set_bond_assignment_netgroups_ids(
237
-            node, netgroups_id_mapping)
198
+
199
+        if not seed_cluster.network_template:
200
+            orig_manager.set_nic_assignment_netgroups_ids(
201
+                node, netgroups_id_mapping)
202
+            orig_manager.set_bond_assignment_netgroups_ids(
203
+                node, netgroups_id_mapping)
204
+
238 205
         node.add_pending_change(consts.CLUSTER_CHANGES.interfaces)
239 206
 
240 207
     @classmethod

+ 6
- 0
cluster_upgrade/validators.py View File

@@ -62,6 +62,12 @@ class ClusterUpgradeValidator(base.BasicValidator):
62 62
                 "this release is equal or lower than the release of the "
63 63
                 "original cluster.".format(new_release.id),
64 64
                 log_message=True)
65
+        if orig_release.operating_system != new_release.operating_system:
66
+            raise errors.InvalidData(
67
+                "Changing of operating system is not possible during upgrade "
68
+                "(from {0} to {1}).".format(orig_release.operating_system,
69
+                                            new_release.operating_system),
70
+                log_message=True)
65 71
 
66 72
     @classmethod
67 73
     def validate_cluster_name(cls, cluster_name):

+ 9
- 0
setup.cfg View File

@@ -1,6 +1,7 @@
1 1
 [metadata]
2 2
 name = fuel-nailgun-extension-cluster-upgrade
3 3
 summary = Cluster upgrade extension for Fuel
4
+description-file = README.rst
4 5
 author = Mirantis Inc.
5 6
 author-email = product@mirantis.com
6 7
 home-page = http://mirantis.com
@@ -24,3 +25,11 @@ packages =
24 25
 [entry_points]
25 26
 nailgun.extensions =
26 27
     cluster_upgrade = cluster_upgrade.extension:ClusterUpgradeExtension
28
+nailgun.cluster_upgrade.transformations.volumes.6.1 =
29
+    node_volumes = cluster_upgrade.transformations.volumes:transform_node_volumes
30
+nailgun.cluster_upgrade.transformations.cluster.9.0 =
31
+    dns_list = cluster_upgrade.transformations.cluster:transform_dns_list
32
+    ntp_list = cluster_upgrade.transformations.cluster:transform_ntp_list
33
+    drop_provision = cluster_upgrade.transformations.cluster:drop_generated_provision
34
+nailgun.cluster_upgrade.transformations.vip.7.0 =
35
+    transform_vips = cluster_upgrade.transformations.vip:transform_vips

Loading…
Cancel
Save