openstack
/
fuel-plugins
Code Issues Proposed changes

Added pre-post tasks validation to plugins builder 

- The plugin deployment tasks now could contain
  cross-depends and cross-depended-by properties
  inroduced by task based deploment feature.

Change-Id: Icb351e4475ba6e78e7c8ef22cefc255116d44ba8
Related-Bug: #1527325
This commit is contained in:
Ilya Kutukov 2015-12-21 03:26:58 +03:00
parent c41143cba2
commit 4ca6c5950e
6 changed files with 503 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
fuel_plugin_builder/templates/v4/plugin_data/deployment_tasks.yaml.mako Normal file
View File

@ -0,0 +1,56 @@
# These tasks will be merged into deployment graph. Here you
# can specify new tasks for any roles, even built-in ones.
- id: ${plugin_name}
type: group
role: [${plugin_name}]
parameters:
strategy:
type: parallel
- id: ${plugin_name}-deployment-puppet
version: 2.0.0 # task format version
type: puppet
role: [${plugin_name}]
required_for: [deploy_end] # for version 1.0 compatibility
requires: [deploy_start] # for version 1.0 compatibility
cross-depended-by:
- name: deploy_end
cross-depends: # version 2.0 dependency
- name: deploy_start
parameters:
puppet_manifest: "deploy.pp"
puppet_modules: "."
timeout: 3600
#- id: ${plugin_name}-post-deployment-sh
# version: 2.0.0
# type: shell
# role: [${plugin_name}]
# required_for: [post_deployment_end]
# requires: [post_deployment_start]
# cross-depended-by:
# - name: post_deployment_end
# cross-depends:
# - name: post_deployment_start
# parameters:
# cmd: echo post_deployment_task_executed > /tmp/post_deployment
# retries: 3
# interval: 20
# timeout: 180
#- id: ${plugin_name}-pre-deployment-sh
# version: 2.0.0
# type: shell
# role: [${plugin_name}]
# required_for: [pre_deployment_end]
# requires: [pre_deployment_start]
# cross-depended-by:
# - name: pre_deployment_end
# cross-depends:
# - name: pre_deployment_start
# parameters:
# cmd: echo pre_deployment_task_executed > /tmp/pre_deployment
# retries: 3
# interval: 20
# timeout: 180

46
fuel_plugin_builder/tests/test_validator_v3.py
View File

@ -38,7 +38,8 @@ class TestValidatorV3(BaseValidator):
]
self.check_validate(mocked_methods)
def test_check_tasks_schema_validation_failed(self):
@mock.patch('fuel_plugin_builder.validators.validator_v3.utils')
def test_check_tasks_schema_validation_failed(self, utils_mock, *args):
data_sets = [
{
'type': 'shell',
@ -110,14 +111,13 @@ class TestValidatorV3(BaseValidator):
}
]
with mock.patch('fuel_plugin_builder.validators.validator_v3.utils') \
as mock_utils:
for data in data_sets:
mock_utils.parse_yaml.return_value = [data]
self.assertRaises(errors.ValidationError,
self.validator.check_deployment_tasks)
for data in data_sets:
utils_mock.parse_yaml.return_value = [data]
self.assertRaises(errors.ValidationError,
self.validator.check_deployment_tasks)
def test_check_tasks_schema_validation_passed(self):
@mock.patch('fuel_plugin_builder.validators.validator_v3.utils')
def test_check_tasks_schema_validation_passed(self, utils_mock, *args):
data_sets = [
[
{
@ -227,15 +227,13 @@ class TestValidatorV3(BaseValidator):
]
]
with mock.patch('fuel_plugin_builder.validators.validator_v3.utils') \
as mock_utils:
for data in data_sets:
mock_utils.parse_yaml.return_value = data
self.validator.check_deployment_tasks()
for data in data_sets:
utils_mock.parse_yaml.return_value = data
self.validator.check_deployment_tasks()
@mock.patch('fuel_plugin_builder.validators.validator_v3.utils')
@mock.patch('fuel_plugin_builder.validators.base.utils.exists')
def test_check_tasks_no_file(self, exists_mock, utils_mock):
def test_check_tasks_no_file(self, exists_mock, utils_mock, *args):
mocked_methods = ['validate_schema']
self.mock_methods(self.validator, mocked_methods)
exists_mock.return_value = False
@ -293,7 +291,7 @@ class TestValidatorV3(BaseValidator):
@mock.patch('fuel_plugin_builder.validators.validator_v3.utils')
def test_role_attribute_is_required_for_deployment_task_types(
self, utils_mock):
self, utils_mock, *args):
deployment_task_types = [
'group', 'shell', 'copy_files', 'sync', 'upload_file']
@ -309,7 +307,7 @@ class TestValidatorV3(BaseValidator):
@mock.patch('fuel_plugin_builder.validators.validator_v3.utils')
def test_parameters_attribute_is_required_for_deployment_task_types(
self, utils_mock):
self, utils_mock, *args):
deployment_task_types = ['copy_files', 'sync', 'upload_file']
for task_type in deployment_task_types:
@ -325,7 +323,7 @@ class TestValidatorV3(BaseValidator):
@mock.patch('fuel_plugin_builder.validators.validator_v3.utils')
def test_files_attribute_is_required_for_copy_files_task_type(
self, utils_mock):
self, utils_mock, *args):
mock_data = [{
'id': 'plugin_name',
'type': 'copy_files',
@ -340,7 +338,7 @@ class TestValidatorV3(BaseValidator):
@mock.patch('fuel_plugin_builder.validators.validator_v3.utils')
def test_files_should_contain_at_least_one_item_for_copy_files_task_type(
self, utils_mock):
self, utils_mock, *args):
mock_data = [{
'id': 'plugin_name',
'type': 'copy_files',
@ -354,7 +352,7 @@ class TestValidatorV3(BaseValidator):
@mock.patch('fuel_plugin_builder.validators.validator_v3.utils')
def test_src_and_dst_attributes_are_required_for_copy_files_task_type(
self, utils_mock):
self, utils_mock, *args):
data_to_check = [
([{
'id': 'plugin_name',
@ -381,7 +379,7 @@ class TestValidatorV3(BaseValidator):
@mock.patch('fuel_plugin_builder.validators.validator_v3.utils')
def test_src_and_dst_attributes_are_required_for_sync_task_type(
self, utils_mock):
self, utils_mock, *args):
data_to_check = [
([{
'id': 'plugin_name',
@ -406,7 +404,7 @@ class TestValidatorV3(BaseValidator):
@mock.patch('fuel_plugin_builder.validators.validator_v3.utils')
def test_path_and_data_attributes_are_required_for_upload_file_task_type(
self, utils_mock):
self, utils_mock, *args):
data_to_check = [
([{
'id': 'plugin_name',
@ -431,7 +429,7 @@ class TestValidatorV3(BaseValidator):
@mock.patch('fuel_plugin_builder.validators.validator_v3.utils')
def test_check_group_type_deployment_task_does_not_contain_manifests(
self, utils_mock):
self, utils_mock, *args):
utils_mock.parse_yaml.return_value = [{
'id': 'plugin_name',
'type': 'group',
@ -441,7 +439,7 @@ class TestValidatorV3(BaseValidator):
self.validator.check_deployment_tasks()
@mock.patch('fuel_plugin_builder.validators.validator_v3.utils')
def test_check_deployment_task_role_failed(self, utils_mock):
def test_check_deployment_task_role_failed(self, utils_mock, *args):
mock_data = [{
'id': 'plugin_name',
'type': 'group',
@ -453,7 +451,7 @@ class TestValidatorV3(BaseValidator):
err_msg, self.validator.check_deployment_tasks)
@mock.patch('fuel_plugin_builder.validators.validator_v3.utils')
def test_check_deployment_task_role(self, utils_mock):
def test_check_deployment_task_role(self, utils_mock, *args):
utils_mock.parse_yaml.return_value = [
{'id': 'plugin_name', 'type': 'group', 'role': []},
{'id': 'plugin_name', 'type': 'group', 'role': ['a', 'b']},

127
fuel_plugin_builder/tests/test_validator_v4.py
View File

@ -259,3 +259,130 @@ class TestValidatorV4(TestValidatorV3):
self.check_raised_exception(
utils_mock, mock_data,
err_msg, self.validator.check_deployment_tasks_schema)
@mock.patch('fuel_plugin_builder.validators.validator_v4.utils')
def test_role_attribute_is_required_for_deployment_task_types(
self, utils_mock, *args):
deployment_tasks_data = [
{
'id': 'plugin_name',
'type': 'group'
},
{
'id': 'plugin_name',
'type': 'shell'
},
{
'id': 'plugin_name',
'type': 'copy_files',
'parameters': {
'files': [{'src': '/dev/null', 'dst': '/dev/null'}]
}
},
{
'id': 'plugin_name',
'type': 'sync',
'parameters': {'src': '/dev/null', 'dst': '/dev/null'}
},
{
'id': 'plugin_name',
'type': 'upload_file',
'parameters': {
'path': 'http://test.com',
'data': 'VGVzdERhdGE='
}
}
]
err_msg = "File '/tmp/plugin_path/deployment_tasks.yaml', " \
"'role' is a required property, value path '0'"
for task in deployment_tasks_data:
self.check_raised_exception(
utils_mock, [task],
err_msg, self.validator.check_deployment_tasks)
# This is the section of tests inherited from the v3 validator
# where decorators is re-defined for module v4
@mock.patch('fuel_plugin_builder.validators.validator_v4.utils')
def test_check_deployment_task_role(self, utils_mock, *args):
super(TestValidatorV4, self).test_check_deployment_task_role(
utils_mock)
@mock.patch('fuel_plugin_builder.validators.validator_v4.utils')
@mock.patch('fuel_plugin_builder.validators.base.utils.exists')
def test_check_tasks_no_file(self, exists_mock, utils_mock, *args):
super(TestValidatorV4, self).test_check_deployment_task_role(
exists_mock, utils_mock)
@mock.patch('fuel_plugin_builder.validators.validator_v4.utils')
def test_check_deployment_task_role_failed(self, utils_mock, *args):
super(TestValidatorV4, self).test_check_deployment_task_role_failed(
utils_mock)
@mock.patch('fuel_plugin_builder.validators.validator_v4.utils')
def test_check_group_type_deployment_task_does_not_contain_manifests(
self, utils_mock, *args):
super(
TestValidatorV4, self
).test_check_group_type_deployment_task_does_not_contain_manifests(
utils_mock)
@mock.patch('fuel_plugin_builder.validators.validator_v4.utils')
def test_files_attribute_is_required_for_copy_files_task_type(
self, utils_mock, *args):
super(
TestValidatorV4, self
).test_files_attribute_is_required_for_copy_files_task_type(
utils_mock)
@mock.patch('fuel_plugin_builder.validators.validator_v4.utils')
def test_files_should_contain_at_least_one_item_for_copy_files_task_type(
self, utils_mock, *args):
super(
TestValidatorV4, self
).test_files_should_contain_at_least_one_item_for_copy_files_task_type(
utils_mock)
@mock.patch('fuel_plugin_builder.validators.validator_v4.utils')
def test_parameters_attribute_is_required_for_deployment_task_types(
self, utils_mock, *args):
super(
TestValidatorV4, self
).test_parameters_attribute_is_required_for_deployment_task_types(
utils_mock)
@mock.patch('fuel_plugin_builder.validators.validator_v4.utils')
def test_path_and_data_attributes_are_required_for_upload_file_task_type(
self, utils_mock, *args):
super(
TestValidatorV4, self
).test_path_and_data_attributes_are_required_for_upload_file_task_type(
utils_mock)
@mock.patch('fuel_plugin_builder.validators.validator_v4.utils')
def test_src_and_dst_attributes_are_required_for_copy_files_task_type(
self, utils_mock, *args):
super(
TestValidatorV4, self
).test_src_and_dst_attributes_are_required_for_copy_files_task_type(
utils_mock)
@mock.patch('fuel_plugin_builder.validators.validator_v4.utils')
def test_src_and_dst_attributes_are_required_for_sync_task_type(
self, utils_mock, *args):
super(
TestValidatorV4, self
).test_src_and_dst_attributes_are_required_for_sync_task_type(
utils_mock)
# todo(ikutukov): validation for old-style tasks.yaml without
# id and normal dependencies. Have to find out what to do with them.
@mock.patch('fuel_plugin_builder.validators.validator_v4.utils')
def test_check_tasks_schema_validation_failed(self, utils_mock, *args):
pass
@mock.patch('fuel_plugin_builder.validators.validator_v4.utils')
def test_check_tasks_schema_validation_passed(self, utils_mock, *args):
pass

267
fuel_plugin_builder/validators/schemas/v4.py
View File

@ -14,6 +14,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import six
from fuel_plugin_builder.validators.schemas import SchemaV3
@ -25,14 +27,66 @@ COMPATIBLE_COMPONENT_NAME_PATTERN = \
'^({0}):([0-9a-z_-]+:)*([0-9a-z_-]+|(\*)?)$'.format(COMPONENTS_TYPES_STR)
TASK_NAME_PATTERN = TASK_ROLE_PATTERN = '^[0-9a-zA-Z_-]+$'
NETWORK_ROLE_PATTERN = '^[0-9a-z_-]+$'
FILE_PERMISSIONS_PATTERN = '^[0-7]{4}$'
TASK_VERSION_PATTERN = '^\d+.\d+.\d+$'
STAGE_PATTERN = '^(post_deployment|pre_deployment)' \
'(/[-+]?([0-9]*\.[0-9]+|[0-9]+))?$'
class SchemaV4(SchemaV3):
@property
def package_version(self):
return {'enum': ['4.0.0']}
def _task_relation(self):
return {
'type': 'object',
'required': ['name'],
'properties': {
'name': {'type': 'string'},
'role': self._task_role,
'policy': {
'type': 'string',
'enum': ['all', 'any']
}
}
}
@property
def task_reexecute(self):
def _task_role(self):
return {
'oneOf': [
{
'type': 'string',
'enum': ['*', 'master', 'self']
},
{
'type': 'array',
'items': {
'type': 'string',
'pattern': TASK_ROLE_PATTERN
}
}
]
}
@property
def _task_strategy(self):
return {
'type': 'object',
'properties': {
'type': {
'enum': ['parallel', 'one_by_one']
}
}
}
@property
def _task_stage(self):
return {'type': 'string', 'pattern': STAGE_PATTERN}
@property
def _task_reexecute(self):
return {
'type': 'array',
'items': {
@ -41,11 +95,212 @@ class SchemaV4(SchemaV3):
}
}
def _gen_task_schema(self, task_types, required=None, parameters=None):
"""Generate deployment task schema using prototype.
:param task_types: task types
:type task_types: str|list
:param required: new required fields
:type required: list
:param parameters: new properties dict
:type parameters: dict
:return:
:rtype: dict
"""
if not task_types:
raise ValueError('Task type should not be empty')
if isinstance(task_types, six.string_types):
task_types = [task_types]
# patch strategy parameter
parameters = parameters or {
"type": "object",
}
parameters.setdefault("properties", {})
parameters["properties"].setdefault("strategy", self._task_strategy)
return {
'$schema': 'http://json-schema.org/draft-04/schema#',
'type': 'object',
'required': list(set(['id', 'type'] + (required or []))),
'properties': {
'type': {'enum': task_types},
'id': {
'type': 'string',
'pattern': TASK_NAME_PATTERN},
'version': {
'type': 'string', "pattern": TASK_VERSION_PATTERN},
'role': self._task_role,
'required_for': self.task_group,
'requires': self.task_group,
'cross-depends': {
'type': 'array',
'items': self._task_relation},
'cross-depended-by': {
'type': 'array',
'items': self._task_relation},
'stage': self._task_stage,
'tasks': { # used only for 'group' tasks
'type': 'array',
'items': {
'type': 'string',
'pattern': TASK_ROLE_PATTERN}},
'reexecute_on': self._task_reexecute,
'parameters': parameters or {},
}
}
@property
def deployment_task_schema(self):
schema = super(SchemaV4, self).deployment_task_schema
schema['items']['properties']['reexecute_on'] = self.task_reexecute
return schema
return {
'$schema': 'http://json-schema.org/draft-04/schema#',
'type': 'array',
'items': {
"$ref": "#/definitions/anyTask"
},
"definitions": {
"anyTask": self._gen_task_schema(
[
'copy_files',
'group',
'reboot',
'shell',
'skipped',
'stage',
'sync',
'puppet',
'upload_file',
]
)
}
}
@property
def copy_files_task(self):
return self._gen_task_schema(
"copy_files",
['role', 'parameters'],
{
'type': 'object',
'required': ['files'],
'properties': {
'files': {
'type': 'array',
'minItems': 1,
'items': {
'type': 'object',
'required': ['src', 'dst'],
'properties': {
'src': {'type': 'string'},
'dst': {'type': 'string'}}}},
'permissions': {
'type': 'string',
'pattern': FILE_PERMISSIONS_PATTERN},
'dir_permissions': {
'type': 'string',
'pattern': FILE_PERMISSIONS_PATTERN}}})
@property
def group_task(self):
return self._gen_task_schema("group", ['role'])
@property
def puppet_task(self):
return self._gen_task_schema(
"puppet",
[],
{
'type': 'object',
'required': [
'puppet_manifest', 'puppet_modules', 'timeout'],
'properties': {
'puppet_manifest': {
'type': 'string', 'minLength': 1},
'puppet_modules': {
'type': 'string', 'minLength': 1},
'timeout': {'type': 'integer'},
'retries': {'type': 'integer'}
}
}
)
@property
def reboot_task(self):
return self._gen_task_schema(
"reboot",
[],
{
'type': 'object',
'properties': {
'timeout': {'type': 'integer'}
}
}
)
@property
def shell_task(self):
return self._gen_task_schema(
"shell",
['role'],
{
'type': 'object',
'required': ['cmd'],
'properties': {
'cmd': {
'type': 'string'},
'retries': {
'type': 'integer'},
'interval': {
'type': 'integer'},
'timeout': {
'type': 'integer'}
}
}
)
@property
def skipped_task(self):
return self._gen_task_schema("skipped")
@property
def stage_task(self):
return self._gen_task_schema("stage")
@property
def sync_task(self):
return self._gen_task_schema(
"sync",
['role', 'parameters'],
{
'type': 'object',
'required': ['src', 'dst'],
'properties': {
'src': {'type': 'string'},
'dst': {'type': 'string'},
'timeout': {'type': 'integer'}
}
}
)
@property
def upload_file_task(self):
return self._gen_task_schema(
"upload_file",
['role', 'parameters'],
{
'type': 'object',
'required': ['path', 'data'],
'properties': {
'path': {'type': 'string'},
'data': {'type': 'string'}
}
}
)
@property
def package_version(self):
return {'enum': ['4.0.0']}
@property
def metadata_schema(self):

4
fuel_plugin_builder/validators/validator_v3.py
View File

@ -105,6 +105,10 @@ class ValidatorV3(ValidatorV2):
'reboot': self.schema.reboot}
for idx, deployment_task in enumerate(deployment_tasks):
if deployment_task['type'] not in schemas:
error_msg = 'There is no such task type:' \
'{0}'.format(deployment_task['type'])
raise errors.ValidationError(error_msg)
self.validate_schema(
deployment_task,
schemas[deployment_task['type']],

33
fuel_plugin_builder/validators/validator_v4.py
View File

@ -14,11 +14,16 @@
# License for the specific language governing permissions and limitations
# under the License.
import logging
from os.path import join as join_path
from fuel_plugin_builder import errors
from fuel_plugin_builder import utils
from fuel_plugin_builder.validators.schemas import SchemaV4
from fuel_plugin_builder.validators import ValidatorV3
logger = logging.getLogger(__name__)
class ValidatorV4(ValidatorV3):
@ -46,3 +51,31 @@ class ValidatorV4(ValidatorV3):
self.validate_file_by_schema(self.schema.components_schema,
self.components_path,
check_file_exists=False)
def check_deployment_tasks(self):
logger.debug(
'Start deployment tasks checking "%s"',
self.deployment_tasks_path)
deployment_tasks = utils.parse_yaml(self.deployment_tasks_path)
schemas = {
'puppet': self.schema.puppet_task,
'shell': self.schema.shell_task,
'group': self.schema.group_task,
'skipped': self.schema.skipped_task,
'copy_files': self.schema.copy_files_task,
'sync': self.schema.sync_task,
'upload_file': self.schema.upload_file_task,
'stage': self.schema.stage_task,
'reboot': self.schema.reboot_task}
for idx, deployment_task in enumerate(deployment_tasks):
if deployment_task['type'] not in schemas:
error_msg = 'There is no such task type:' \
'{0}'.format(deployment_task['type'])
raise errors.ValidationError(error_msg)
self.validate_schema(
deployment_task,
schemas[deployment_task['type']],
self.deployment_tasks_path,
value_path=[idx])