Browse Source

Merge "Lift too strict restrictions on cross-deps role name"

Jenkins 3 years ago
parent
commit
6dc5c22149

+ 4
- 0
fuel_plugin_builder/errors.py View File

@@ -33,6 +33,10 @@ class ValidationError(FuelPluginException):
33 33
     pass
34 34
 
35 35
 
36
+class TaskFieldError(ValidationError):
37
+    pass
38
+
39
+
36 40
 class FileIsEmpty(ValidationError):
37 41
     def __init__(self, file_path):
38 42
         super(FileIsEmpty, self).__init__(

+ 3
- 2
fuel_plugin_builder/tests/test_base_validator.py View File

@@ -13,7 +13,7 @@
13 13
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14 14
 #    License for the specific language governing permissions and limitations
15 15
 #    under the License.
16
-
16
+import jsonschema
17 17
 import mock
18 18
 
19 19
 from fuel_plugin_builder import errors
@@ -37,6 +37,7 @@ class TestBaseValidator(BaseTestCase):
37 37
         self.validator = NewValidator(self.plugin_path)
38 38
         self.data = {'data': 'data1'}
39 39
         self.schema = self.make_schema(['data'], {'data': {'type': 'string'}})
40
+        self.format_checker = jsonschema.FormatChecker
40 41
 
41 42
     @classmethod
42 43
     def make_schema(cls, required, properties):
@@ -54,7 +55,7 @@ class TestBaseValidator(BaseTestCase):
54 55
             'file_path')
55 56
         schema_mock.validate.assert_called_once_with(
56 57
             self.data,
57
-            self.schema)
58
+            self.schema, format_checker=self.format_checker)
58 59
 
59 60
     def test_validate_schema_raises_error(self):
60 61
         schema = self.make_schema(['key'], {'key': {'type': 'string'}})

+ 78
- 9
fuel_plugin_builder/tests/test_validator_v4.py View File

@@ -290,7 +290,7 @@ class TestValidatorV4(TestValidatorV3):
290 290
         mock_data = [{
291 291
             'id': 'plugin_task',
292 292
             'type': 'puppet',
293
-            'group': ['controller'],
293
+            'groups': ['controller'],
294 294
             'reexecute_on': ['bla']}]
295 295
         err_msg = "File '/tmp/plugin_path/deployment_tasks.yaml', " \
296 296
                   "'bla' is not one of"
@@ -333,7 +333,8 @@ class TestValidatorV4(TestValidatorV3):
333 333
         ]
334 334
 
335 335
         err_msg = "File '/tmp/plugin_path/deployment_tasks.yaml', " \
336
-                  "'role' is a required property, value path '0'"
336
+                  "'(role|groups|roles)' is " \
337
+                  "a required property, value path '0'"
337 338
         for task in deployment_tasks_data:
338 339
             self.check_raised_exception(
339 340
                 utils_mock, [task],
@@ -342,21 +343,89 @@ class TestValidatorV4(TestValidatorV3):
342 343
     # This is the section of tests inherited from the v3 validator
343 344
     # where decorators is re-defined for module v4
344 345
 
345
-    @mock.patch('fuel_plugin_builder.validators.validator_v4.utils')
346
-    def test_check_deployment_task_role(self, utils_mock, *args):
347
-        super(TestValidatorV4, self).test_check_deployment_task_role(
348
-            utils_mock)
349
-
350 346
     @mock.patch('fuel_plugin_builder.validators.validator_v4.utils')
351 347
     @mock.patch('fuel_plugin_builder.validators.base.utils.exists')
352 348
     def test_check_tasks_no_file(self, exists_mock, utils_mock, *args):
353 349
         super(TestValidatorV4, self).test_check_deployment_task_role(
354 350
             exists_mock, utils_mock)
355 351
 
352
+    @mock.patch('fuel_plugin_builder.validators.validator_v4.utils')
353
+    def test_check_deployment_task_role(self, utils_mock, *args):
354
+        utils_mock.parse_yaml.return_value = [
355
+            {'id': 'plugin_name', 'type': 'group', 'groups': ['a', 'b']},
356
+            {'id': 'plugin_name', 'type': 'group', 'groups': '*'},
357
+            {'id': 'plugin_name', 'type': 'puppet', 'role': ['a', 'b']},
358
+            {'id': 'plugin_name', 'type': 'puppet', 'role': '*'},
359
+            {'id': 'plugin_name', 'type': 'shell', 'roles': ['a', 'b']},
360
+            {'id': 'plugin_name', 'type': 'shell', 'roles': '*'},
361
+            {'id': 'plugin_name', 'type': 'skipped', 'role': '/test/'},
362
+            {'id': 'plugin_name', 'type': 'stage'},
363
+            {'id': 'plugin_name', 'type': 'reboot', 'groups': 'contrail'},
364
+            {
365
+                'id': 'plugin_name',
366
+                'type': 'copy_files',
367
+                'role': '*',
368
+                'parameters': {
369
+                    'files': [
370
+                        {'src': 'some_source', 'dst': 'some_destination'}]}
371
+            },
372
+            {
373
+                'id': 'plugin_name',
374
+                'type': 'sync',
375
+                'role': 'plugin_name',
376
+                'parameters': {
377
+                    'src': 'some_source', 'dst': 'some_destination'}
378
+            },
379
+            {
380
+                'id': 'plugin_name',
381
+                'type': 'upload_file',
382
+                'role': '/^.*plugin\w+name$/',
383
+                'parameters': {
384
+                    'path': 'some_path', 'data': 'some_data'}
385
+            },
386
+        ]
387
+
388
+        self.validator.check_deployment_tasks()
389
+
356 390
     @mock.patch('fuel_plugin_builder.validators.validator_v4.utils')
357 391
     def test_check_deployment_task_role_failed(self, utils_mock, *args):
358
-        super(TestValidatorV4, self).test_check_deployment_task_role_failed(
359
-            utils_mock)
392
+        mock_data = [{
393
+            'id': 'plugin_name',
394
+            'type': 'group',
395
+            'role': ['plugin_n@me']}]
396
+        err_msg = "field should"
397
+        self.check_raised_exception(
398
+            utils_mock, mock_data,
399
+            err_msg, self.validator.check_deployment_tasks)
400
+
401
+    @mock.patch('fuel_plugin_builder.validators.validator_v4.utils')
402
+    def test_check_deployment_task_required_missing(self, utils_mock, *args):
403
+        mock_data = [{
404
+            'id': 'plugin_name',
405
+            'type': 'puppet'}]
406
+        err_msg = "is a required property"
407
+        self.check_raised_exception(
408
+            utils_mock, mock_data,
409
+            err_msg, self.validator.check_deployment_tasks)
410
+
411
+    @mock.patch('fuel_plugin_builder.validators.validator_v4.utils')
412
+    def test_check_deployment_task_required_roles_missing_is_ok(
413
+            self, utils_mock, *args):
414
+        utils_mock.parse_yaml.return_value = [{
415
+            'id': 'plugin_name',
416
+            'type': 'stage'}]
417
+        self.validator.check_deployment_tasks()
418
+
419
+    @mock.patch('fuel_plugin_builder.validators.validator_v4.utils')
420
+    def test_check_deployment_task_role_regexp_failed(self, utils_mock, *args):
421
+        mock_data = [{
422
+            'id': 'plugin_name',
423
+            'type': 'group',
424
+            'role': '/[0-9]++/'}]
425
+        err_msg = "field should.*multiple repeat"
426
+        self.check_raised_exception(
427
+            utils_mock, mock_data,
428
+            err_msg, self.validator.check_deployment_tasks)
360 429
 
361 430
     @mock.patch('fuel_plugin_builder.validators.validator_v4.utils')
362 431
     def test_check_group_type_deployment_task_does_not_contain_manifests(

+ 7
- 3
fuel_plugin_builder/validators/base.py View File

@@ -35,15 +35,16 @@ class BaseValidator(object):
35 35
     def basic_version(self):
36 36
         pass
37 37
 
38
-    def __init__(self, plugin_path):
38
+    def __init__(self, plugin_path, format_checker=jsonschema.FormatChecker):
39 39
         self.plugin_path = plugin_path
40
+        self.format_checker = format_checker
40 41
 
41 42
     def validate_schema(self, data, schema, file_path, value_path=None):
42 43
         logger.debug(
43 44
             'Start schema validation for %s file, %s', file_path, schema)
44
-
45 45
         try:
46
-            jsonschema.validate(data, schema)
46
+            jsonschema.validate(data, schema,
47
+                                format_checker=self.format_checker)
47 48
         except jsonschema.exceptions.ValidationError as exc:
48 49
             raise errors.ValidationError(
49 50
                 self._make_error_message(exc, file_path, value_path))
@@ -104,6 +105,7 @@ class BaseValidator(object):
104 105
     @abc.abstractmethod
105 106
     def validate(self):
106 107
         """Performs validation
108
+
107 109
         """
108 110
 
109 111
     def check_schemas(self):
@@ -169,9 +171,11 @@ class BaseValidator(object):
169 171
 
170 172
     def check_compatibility(self):
171 173
         """Json schema doesn't have any conditions, so we have
174
+
172 175
         to make sure here, that this validation schema can be used
173 176
         for described fuel releases
174 177
         """
178
+
175 179
         meta = utils.parse_yaml(self.meta_path)
176 180
         for fuel_release in meta['fuel_version']:
177 181
             if StrictVersion(fuel_release) < StrictVersion(self.basic_version):

+ 53
- 0
fuel_plugin_builder/validators/formatchecker.py View File

@@ -0,0 +1,53 @@
1
+# -*- coding: utf-8 -*-
2
+
3
+#    Copyright 2016 Mirantis, Inc.
4
+#
5
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
6
+#    not use this file except in compliance with the License. You may obtain
7
+#    a copy of the License at
8
+#
9
+#         http://www.apache.org/licenses/LICENSE-2.0
10
+#
11
+#    Unless required by applicable law or agreed to in writing, software
12
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14
+#    License for the specific language governing permissions and limitations
15
+#    under the License.
16
+
17
+import re
18
+from sre_constants import error as sre_error
19
+
20
+import jsonschema
21
+import six
22
+
23
+from fuel_plugin_builder import errors
24
+
25
+
26
+class FormatChecker(jsonschema.FormatChecker):
27
+
28
+    def __init__(self, role_patterns=(), *args, **kwargs):
29
+        super(FormatChecker, self).__init__()
30
+
31
+        @self.checks('fuel_task_role_format')
32
+        def _validate_role(instance):
33
+            sre_msg = None
34
+            if isinstance(instance, six.string_types):
35
+                if instance.startswith('/') and instance.endswith('/'):
36
+                    try:
37
+                        if re.compile(instance[1:-1]):
38
+                            return True
39
+                    except sre_error as e:
40
+                        sre_msg = str(e)
41
+                else:
42
+                    for role_pattern in role_patterns:
43
+                        if re.match(role_pattern, instance):
44
+                            return True
45
+                err_msg = "Role field should be either a valid " \
46
+                          "regexp enclosed by " \
47
+                          "slashes or a string of '{0}' or an " \
48
+                          "array of those. " \
49
+                          "Got '{1}' instead.".format(", ".join(role_patterns),
50
+                                                      instance)
51
+                if sre_msg:
52
+                    err_msg += "SRE error: \"{0}\"".format(sre_msg)
53
+                raise errors.TaskFieldError(err_msg)

+ 35
- 13
fuel_plugin_builder/validators/schemas/v4.py View File

@@ -27,16 +27,23 @@ COMPATIBLE_COMPONENT_NAME_PATTERN = \
27 27
     '^({0}):([0-9a-z_-]+:)*([0-9a-z_-]+|(\*)?)$'.format(COMPONENTS_TYPES_STR)
28 28
 
29 29
 
30
-TASK_NAME_PATTERN = TASK_ROLE_PATTERN = '^[0-9a-zA-Z_-]+$'
30
+TASK_NAME_PATTERN = TASK_ROLE_PATTERN = '^[0-9a-zA-Z_-]+$|^\*$'
31 31
 NETWORK_ROLE_PATTERN = '^[0-9a-z_-]+$'
32 32
 FILE_PERMISSIONS_PATTERN = '^[0-7]{4}$'
33 33
 TASK_VERSION_PATTERN = '^\d+.\d+.\d+$'
34 34
 STAGE_PATTERN = '^(post_deployment|pre_deployment)' \
35 35
                 '(/[-+]?([0-9]*\.[0-9]+|[0-9]+))?$'
36 36
 
37
+DEFAULT_TASK_ROLE_GROUP_ASSIGNMENT = ('roles', 'groups', 'role')
38
+TASK_OBLIGATORY_FIELDS = ['id', 'type']
39
+
37 40
 
38 41
 class SchemaV4(SchemaV3):
39 42
 
43
+    def __init__(self):
44
+        super(SchemaV4, self).__init__()
45
+        self.role_pattern = TASK_ROLE_PATTERN
46
+
40 47
     @property
41 48
     def _task_relation(self):
42 49
         return {
@@ -58,13 +65,13 @@ class SchemaV4(SchemaV3):
58 65
             'oneOf': [
59 66
                 {
60 67
                     'type': 'string',
61
-                    'enum': ['*', 'master', 'self']
68
+                    'format': 'fuel_task_role_format'
62 69
                 },
63 70
                 {
64 71
                     'type': 'array',
65 72
                     'items': {
66 73
                         'type': 'string',
67
-                        'pattern': TASK_ROLE_PATTERN
74
+                        'format': 'fuel_task_role_format'
68 75
                     }
69 76
                 }
70 77
             ]
@@ -95,7 +102,9 @@ class SchemaV4(SchemaV3):
95 102
             }
96 103
         }
97 104
 
98
-    def _gen_task_schema(self, task_types, required=None, parameters=None):
105
+    def _gen_task_schema(self, task_types, required=None,
106
+                         parameters=None,
107
+                         required_any=DEFAULT_TASK_ROLE_GROUP_ASSIGNMENT):
99 108
         """Generate deployment task schema using prototype.
100 109
 
101 110
         :param task_types: task types
@@ -119,11 +128,21 @@ class SchemaV4(SchemaV3):
119 128
         }
120 129
         parameters.setdefault("properties", {})
121 130
         parameters["properties"].setdefault("strategy", self._task_strategy)
122
-
131
+        task_specific_req_fields = list(set(TASK_OBLIGATORY_FIELDS +
132
+                                            (required or [])))
133
+        required_fields = []
134
+
135
+        # Some tasks are ephemeral, so we can leave it as is without target
136
+        # groups|role|roles, others must have at least one such field
137
+        if required_any:
138
+            for group_name_alias in DEFAULT_TASK_ROLE_GROUP_ASSIGNMENT:
139
+                required_fields.append({"required": task_specific_req_fields
140
+                                        + [group_name_alias]})
141
+        else:
142
+            required_fields.append({"required": task_specific_req_fields})
123 143
         return {
124 144
             '$schema': 'http://json-schema.org/draft-04/schema#',
125 145
             'type': 'object',
126
-            'required': list(set(['id', 'type'] + (required or []))),
127 146
             'properties': {
128 147
                 'type': {'enum': task_types},
129 148
                 'id': {
@@ -132,6 +151,8 @@ class SchemaV4(SchemaV3):
132 151
                 'version': {
133 152
                     'type': 'string', "pattern": TASK_VERSION_PATTERN},
134 153
                 'role': self._task_role,
154
+                'groups': self._task_role,
155
+                'roles': self._task_role,
135 156
                 'required_for': self.task_group,
136 157
                 'requires': self.task_group,
137 158
                 'cross-depends': {
@@ -148,7 +169,8 @@ class SchemaV4(SchemaV3):
148 169
                         'pattern': TASK_ROLE_PATTERN}},
149 170
                 'reexecute_on': self._task_reexecute,
150 171
                 'parameters': parameters or {},
151
-            }
172
+            },
173
+            'anyOf': required_fields
152 174
         }
153 175
 
154 176
     @property
@@ -180,7 +202,7 @@ class SchemaV4(SchemaV3):
180 202
     def copy_files_task(self):
181 203
         return self._gen_task_schema(
182 204
             "copy_files",
183
-            ['role', 'parameters'],
205
+            ['parameters'],
184 206
             {
185 207
                 'type': 'object',
186 208
                 'required': ['files'],
@@ -203,7 +225,7 @@ class SchemaV4(SchemaV3):
203 225
 
204 226
     @property
205 227
     def group_task(self):
206
-        return self._gen_task_schema("group", ['role'])
228
+        return self._gen_task_schema("group", [])
207 229
 
208 230
     @property
209 231
     def puppet_task(self):
@@ -242,7 +264,7 @@ class SchemaV4(SchemaV3):
242 264
     def shell_task(self):
243 265
         return self._gen_task_schema(
244 266
             "shell",
245
-            ['role'],
267
+            [],
246 268
             {
247 269
                 'type': 'object',
248 270
                 'required': ['cmd'],
@@ -265,13 +287,13 @@ class SchemaV4(SchemaV3):
265 287
 
266 288
     @property
267 289
     def stage_task(self):
268
-        return self._gen_task_schema("stage")
290
+        return self._gen_task_schema("stage", required_any=())
269 291
 
270 292
     @property
271 293
     def sync_task(self):
272 294
         return self._gen_task_schema(
273 295
             "sync",
274
-            ['role', 'parameters'],
296
+            ['parameters'],
275 297
             {
276 298
                 'type': 'object',
277 299
                 'required': ['src', 'dst'],
@@ -287,7 +309,7 @@ class SchemaV4(SchemaV3):
287 309
     def upload_file_task(self):
288 310
         return self._gen_task_schema(
289 311
             "upload_file",
290
-            ['role', 'parameters'],
312
+            ['parameters'],
291 313
             {
292 314
                 'type': 'object',
293 315
                 'required': ['path', 'data'],

+ 4
- 1
fuel_plugin_builder/validators/validator_v4.py View File

@@ -19,9 +19,11 @@ from os.path import join as join_path
19 19
 
20 20
 from fuel_plugin_builder import errors
21 21
 from fuel_plugin_builder import utils
22
+from fuel_plugin_builder.validators.formatchecker import FormatChecker
22 23
 from fuel_plugin_builder.validators.schemas import SchemaV4
23 24
 from fuel_plugin_builder.validators import ValidatorV3
24 25
 
26
+
25 27
 logger = logging.getLogger(__name__)
26 28
 
27 29
 
@@ -30,7 +32,8 @@ class ValidatorV4(ValidatorV3):
30 32
     schema = SchemaV4()
31 33
 
32 34
     def __init__(self, *args, **kwargs):
33
-        super(ValidatorV4, self).__init__(*args, **kwargs)
35
+        super(ValidatorV4, self).__init__(format_checker=FormatChecker(
36
+            role_patterns=[self.schema.role_pattern]), *args, **kwargs)
34 37
         self.components_path = join_path(self.plugin_path, 'components.yaml')
35 38
 
36 39
     @property

Loading…
Cancel
Save