Browse Source

Add pluggable transformations for data migration

This change introduces new transformation mechanism:
- all available transformations are listed in setuptools entry points
  under namespace like this (for cluster transformations):

    nailgun.cluster_upgrade.transformations.cluster.9.0 =
        dns_list = ...
        ntp_list = ...
    nailgun.cluster_upgrade.transformations.cluster.8.0 =
        ...
    <etc>

- config file will include section that specifies enabled
  transformations like this:

    CLUSTER_UPGRADE_TRANSFORMATIONS:
      cluster:
        9.0: dns_list ntp_list ...
        8.0: ...
        7.0: ...

  (only default values are implemented here, actual config support will
  follow)

- when transformations are applied to clone cluster from version X to
  version Y, first transformations for version X+1 are applied, then
  X+2, and so on ending with transformations for version Y.

Since Nailgun doesn't provide any special extension initialization
callback, a Lazy wrapper is implemented to facilitate transformations
manager usage in extension.

Change-Id: I8ee75b54180106ad46c1df67f8d5937d6bd810a1
Yuriy Taraday 2 years ago
parent
commit
163ce243fb

+ 179
- 0
cluster_upgrade/tests/test_transformations.py View File

@@ -0,0 +1,179 @@
1
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+# not use this file except in compliance with the License. You may obtain
3
+# a copy of the License at
4
+#
5
+#      http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+# Unless required by applicable law or agreed to in writing, software
8
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+# License for the specific language governing permissions and limitations
11
+# under the License.
12
+
13
+from distutils import version
14
+
15
+import mock
16
+from nailgun.test import base as nailgun_test_base
17
+import six
18
+
19
+from .. import transformations
20
+
21
+
22
+class TestTransformations(nailgun_test_base.BaseUnitTest):
23
+    def test_get_config(self):
24
+        config = object()
25
+
26
+        class Manager(transformations.Manager):
27
+            default_config = config
28
+
29
+        self.assertIs(config, Manager.get_config('testname'))
30
+
31
+    def setup_extension_manager(self, extensions):
32
+        p = mock.patch("stevedore.ExtensionManager", spec=['__call__'])
33
+        mock_extman = p.start()
34
+        self.addCleanup(p.stop)
35
+
36
+        def extman(namespace, *args, **kwargs):
37
+            instance = mock.MagicMock(name=namespace)
38
+            ext_results = {}
39
+            for ver, exts in six.iteritems(extensions):
40
+                if namespace.endswith(ver):
41
+                    ext_results = {name: mock.Mock(name=name, plugin=ext)
42
+                                   for name, ext in six.iteritems(exts)}
43
+                    break
44
+            else:
45
+                self.fail("Called with unexpected version in namespace: {}, "
46
+                          "expected versions: {}".format(
47
+                              namespace, list(extensions)))
48
+            instance.__getitem__.side_effect = ext_results.__getitem__
49
+            return instance
50
+
51
+        mock_extman.side_effect = extman
52
+        return mock_extman
53
+
54
+    def test_load_transformers(self):
55
+        config = {'9.0': ['a', 'b']}
56
+        extensions = {'9.0': {
57
+            'a': mock.Mock(name='a'),
58
+            'b': mock.Mock(name='b'),
59
+        }}
60
+        mock_extman = self.setup_extension_manager(extensions)
61
+
62
+        res = transformations.Manager.load_transformers('testname', config)
63
+
64
+        self.assertEqual(res, [(version.StrictVersion('9.0'), [
65
+            extensions['9.0']['a'],
66
+            extensions['9.0']['b'],
67
+        ])])
68
+        callback = transformations.reraise_endpoint_load_failure
69
+        self.assertEqual(mock_extman.mock_calls, [
70
+            mock.call(
71
+                'nailgun.cluster_upgrade.transformations.testname.9.0',
72
+                on_load_failure_callback=callback,
73
+            ),
74
+        ])
75
+
76
+    def test_load_transformers_empty(self):
77
+        config = {}
78
+        extensions = {'9.0': {
79
+            'a': mock.Mock(name='a'),
80
+            'b': mock.Mock(name='b'),
81
+        }}
82
+        mock_extman = self.setup_extension_manager(extensions)
83
+
84
+        res = transformations.Manager.load_transformers('testname', config)
85
+
86
+        self.assertEqual(res, [])
87
+        self.assertEqual(mock_extman.mock_calls, [])
88
+
89
+    def test_load_transformers_sorted(self):
90
+        config = {'9.0': ['a', 'b'], '8.0': ['c']}
91
+        extensions = {
92
+            '9.0': {
93
+                'a': mock.Mock(name='a'),
94
+                'b': mock.Mock(name='b'),
95
+            },
96
+            '8.0': {
97
+                'c': mock.Mock(name='c'),
98
+                'd': mock.Mock(name='d'),
99
+            },
100
+        }
101
+        mock_extman = self.setup_extension_manager(extensions)
102
+
103
+        orig_iteritems = six.iteritems
104
+        iteritems_patch = mock.patch('six.iteritems')
105
+        mock_iteritems = iteritems_patch.start()
106
+        self.addCleanup(iteritems_patch.stop)
107
+
108
+        def sorted_iteritems(d):
109
+            return sorted(orig_iteritems(d), reverse=True)
110
+
111
+        mock_iteritems.side_effect = sorted_iteritems
112
+
113
+        res = transformations.Manager.load_transformers('testname', config)
114
+
115
+        self.assertEqual(res, [
116
+            (version.StrictVersion('8.0'), [
117
+                extensions['8.0']['c'],
118
+            ]),
119
+            (version.StrictVersion('9.0'), [
120
+                extensions['9.0']['a'],
121
+                extensions['9.0']['b'],
122
+            ]),
123
+        ])
124
+        callback = transformations.reraise_endpoint_load_failure
125
+        self.assertItemsEqual(mock_extman.mock_calls, [
126
+            mock.call(
127
+                'nailgun.cluster_upgrade.transformations.testname.9.0',
128
+                on_load_failure_callback=callback,
129
+            ),
130
+            mock.call(
131
+                'nailgun.cluster_upgrade.transformations.testname.8.0',
132
+                on_load_failure_callback=callback,
133
+            ),
134
+        ])
135
+
136
+    def test_load_transformers_keyerror(self):
137
+        config = {'9.0': ['a', 'b', 'c']}
138
+        extensions = {'9.0': {
139
+            'a': mock.Mock(name='a'),
140
+            'b': mock.Mock(name='b'),
141
+        }}
142
+        mock_extman = self.setup_extension_manager(extensions)
143
+
144
+        with self.assertRaisesRegexp(KeyError, 'c'):
145
+            transformations.Manager.load_transformers('testname', config)
146
+
147
+        callback = transformations.reraise_endpoint_load_failure
148
+        self.assertEqual(mock_extman.mock_calls, [
149
+            mock.call(
150
+                'nailgun.cluster_upgrade.transformations.testname.9.0',
151
+                on_load_failure_callback=callback,
152
+            ),
153
+        ])
154
+
155
+    @mock.patch.object(transformations.Manager, 'load_transformers')
156
+    def test_apply(self, mock_load):
157
+        mock_trans = mock.Mock()
158
+        mock_load.return_value = [
159
+            (version.StrictVersion('7.0'), [mock_trans.a, mock_trans.b]),
160
+            (version.StrictVersion('8.0'), [mock_trans.c, mock_trans.d]),
161
+            (version.StrictVersion('9.0'), [mock_trans.e, mock_trans.f]),
162
+        ]
163
+        man = transformations.Manager()
164
+        res = man.apply('7.0', '9.0', {})
165
+        self.assertEqual(res, mock_trans.f.return_value)
166
+        self.assertEqual(mock_trans.mock_calls, [
167
+            mock.call.c({}),
168
+            mock.call.d(mock_trans.c.return_value),
169
+            mock.call.e(mock_trans.d.return_value),
170
+            mock.call.f(mock_trans.e.return_value),
171
+        ])
172
+
173
+
174
+class TestLazy(nailgun_test_base.BaseUnitTest):
175
+    def test_lazy(self):
176
+        mgr_cls_mock = mock.Mock()
177
+        lazy_obj = transformations.Lazy(mgr_cls_mock)
178
+        lazy_obj.apply()
179
+        self.assertEqual(lazy_obj.apply, mgr_cls_mock.return_value.apply)

+ 94
- 0
cluster_upgrade/transformations/__init__.py View File

@@ -0,0 +1,94 @@
1
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
2
+# not use this file except in compliance with the License. You may obtain
3
+# a copy of the License at
4
+#
5
+#      http://www.apache.org/licenses/LICENSE-2.0
6
+#
7
+# Unless required by applicable law or agreed to in writing, software
8
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10
+# License for the specific language governing permissions and limitations
11
+# under the License.
12
+
13
+import copy
14
+import distutils.version
15
+import logging
16
+import threading
17
+
18
+import six
19
+
20
+import stevedore
21
+
22
+LOG = logging.getLogger(__name__)
23
+
24
+
25
+def reraise_endpoint_load_failure(manager, endpoint, exc):
26
+    LOG.error('Failed to load %s: %s', endpoint.name, exc)
27
+    raise  # Avoid unexpectedly skipped steps
28
+
29
+
30
+class Manager(object):
31
+    default_config = None
32
+    name = None
33
+
34
+    def __init__(self):
35
+        self.config = self.get_config(self.name)
36
+        self.transformers = self.load_transformers(self.name, self.config)
37
+
38
+    @classmethod
39
+    def get_config(cls, name):
40
+        # TODO(yorik-sar): merge actual config with defaults
41
+        return cls.default_config
42
+
43
+    @staticmethod
44
+    def load_transformers(name, config):
45
+        transformers = []
46
+        for version, names in six.iteritems(config):
47
+            extension_manager = stevedore.ExtensionManager(
48
+                'nailgun.cluster_upgrade.transformations.{}.{}'.format(
49
+                    name, version),
50
+                on_load_failure_callback=reraise_endpoint_load_failure,
51
+            )
52
+            try:
53
+                sorted_extensions = [extension_manager[n].plugin
54
+                                     for n in names]
55
+            except KeyError as exc:
56
+                LOG.error('%s transformer %s not found for version %s',
57
+                          name, exc, version)
58
+                raise
59
+            strict_version = distutils.version.StrictVersion(version)
60
+            transformers.append((strict_version, sorted_extensions))
61
+        transformers.sort()
62
+        return transformers
63
+
64
+    def apply(self, from_version, to_version, data):
65
+        strict_from = distutils.version.StrictVersion(from_version)
66
+        strict_to = distutils.version.StrictVersion(to_version)
67
+        assert strict_from < strict_to, \
68
+            "from_version must be smaller than to_version"
69
+        data = copy.deepcopy(data)
70
+        for version, transformers in self.transformers:
71
+            if version <= strict_from:
72
+                continue
73
+            if version > strict_to:
74
+                break
75
+            for transformer in transformers:
76
+                LOG.debug("Applying %s transformer %s",
77
+                          self.name, transformer)
78
+                data = transformer(data)
79
+        return data
80
+
81
+
82
+class Lazy(object):
83
+    def __init__(self, mgr_cls):
84
+        self.mgr_cls = mgr_cls
85
+        self.mgr = None
86
+        self.lock = threading.Lock()
87
+
88
+    def apply(self, *args, **kwargs):
89
+        if self.mgr is None:
90
+            with self.lock:
91
+                if self.mgr is None:
92
+                    self.mgr = self.mgr_cls()
93
+                    self.apply = self.mgr.apply
94
+        return self.mgr.apply(*args, **kwargs)

Loading…
Cancel
Save