Example plugins and templates is updated accordingly to package version v4.0.0

Task-based deployment is intoduced as experimental orchestrator in Fuel v8.0.0
Plugins v4.0.0 could use tasks v2.0.0 in deployment_tasks.yaml

Also small problem with existing but empty legacy tasks.yaml is fixed.

Change-Id: Ibf8d65833b74c79e0c63ae9c51995c953b83d301
Closes-Bug: #1533771
Related-Bug: #1534235

examples/fuel_plugin_example_v4/.gitignore

@@ -1,3 +0,0 @@
-.tox
-.build
-*.pyc

examples/fuel_plugin_example_v4/components.yaml

@@ -1,3 +1,9 @@
+# This file contains wizard components descriptions that are pretty similar to
+# the `environment_config.yaml`.
+# Please, take a look at following link for the details:
+# - https://blueprints.launchpad.net/fuel/+spec/component-registry
+# - https://specs.openstack.org/openstack/fuel-specs/specs/8.0/component-registry.html
+
 - name: additional_service:service_plugin_v4_component
   label: "Title for service_plugin_v4_component component."
   description: "Description for service_plugin_v4_component component."

examples/fuel_plugin_example_v4/deployment_scripts/deploy.pp

@@ -1,3 +1,5 @@
+# This puppet manifest creates example file in /tmp folder.
+
 notice('PLUGIN: fuel_plugin_example_v4 - deploy.pp')
 
 class fuel_plugin_example_v4 {

examples/fuel_plugin_example_v4/environment_config.yaml

@@ -1,3 +1,9 @@
+# This file describes additional attributes that will appear on the Settings
+# tab of the Fuel web UI. When the environment is deployed, these attributes
+# are passed to the task executor so that the data is available
+# in the /etc/astute.yaml file on each target node and can be accessed from
+# your bash or puppet scripts.
+
 attributes:
   metadata:
     group: 'other'

examples/fuel_plugin_example_v4/network_roles.yaml

@@ -1,5 +1,8 @@
+# `network_roles.yaml` file is used to declare VIP addresses and link
+# them with networks and nodes.
+
 # Unique network role name
-- id: "fuel_plugin_example_v4"
+- id: "fuel_plugin_example_v4_network_role"
   # Role mapping to network
   default_mapping: "public"
   properties:
@@ -18,4 +21,4 @@
          # Optional node role list to map VIP to (defaults to
          # primary-controller and controller)
          node_roles:
-           - "fuel_plugin_example_v4"
+           - "fuel_plugin_example_v4_role"

examples/fuel_plugin_example_v4/tasks.yaml

@@ -1,3 +1,6 @@
+# WARNING: `tasks.yaml` will be deprecated in further releases.
+# Please, use `deployment_tasks.yaml` to describe tasks instead.
+
 # This tasks will be applied on controller nodes
 - role: ['primary-controller', 'controller']
   stage: post_deployment

examples/fuel_plugin_example_v4/volumes.yaml

@@ -1,6 +1,6 @@
 # Set here new volumes for your role
 volumes: []
 volumes_roles_mapping:
-  fuel_plugin_example_v4:
+  fuel_plugin_example_v4_role:
     # Default role mapping
     - {allocate_size: "min", id: "os"}

fuel_plugin_builder/templates/v4/plugin_data/components.yaml.mako

@@ -1,3 +1,9 @@
+# This file contains wizard components descriptions that are pretty similar to
+# the `environment_config.yaml`.
+# Please, take a look at following link for the details:
+# - https://blueprints.launchpad.net/fuel/+spec/component-registry
+# - https://specs.openstack.org/openstack/fuel-specs/specs/8.0/component-registry.html
+
 - name: additional_service:${plugin_name}
   compatible: []
   requires: []

fuel_plugin_builder/templates/v4/plugin_data/deployment_tasks.yaml.mako

@@ -1,23 +1,32 @@
 # These tasks will be merged into deployment graph. Here you
 # can specify new tasks for any roles, even built-in ones.
 
-- id: ${plugin_name}
+- id: ${plugin_name}_role
   type: group
-  role: [${plugin_name}]
+  role: [${plugin_name}_role]
   parameters:
     strategy:
       type: parallel
 
 - id: ${plugin_name}-deployment-puppet
-  version: 2.0.0              # task format version
   type: puppet
-  role: [${plugin_name}]
-  required_for: [deploy_end]  # for version 1.0 compatibility
-  requires: [deploy_start]    # for version 1.0 compatibility
-  cross-depended-by:
-    - name: deploy_end
-  cross-depends:              # version 2.0 dependency
-    - name: deploy_start
+  role: [${plugin_name}_role]
+  requires: [deploy_start]
+  required_for: [deploy_end]
+
+## If you want to use task-based deployment that is introduced as experimental
+## in fuel v8.0 uncomment code section below this comment and uncomment similar
+## sections in tasks below.
+## See the example:
+## https://github.com/openstack/fuel-plugins/tree/master/examples/fuel_plugin_example_v4_experimental
+## for more details.
+
+#  version: 2.0.0              # tasks v2.0.0 is supporting task-based deployment
+#   cross-depends:
+#     - name: deploy_start
+#   cross-depended-by:
+#     - name: deploy_end
+
   parameters:
     puppet_manifest: "deploy.pp"
     puppet_modules: "."
@@ -26,13 +35,14 @@
 #- id: ${plugin_name}-post-deployment-sh
 #  version: 2.0.0
 #  type: shell
-#  role: [${plugin_name}]
-#  required_for: [post_deployment_end]
+#  role: [${plugin_name}_role]
 #  requires: [post_deployment_start]
-#  cross-depended-by:
-#    - name: post_deployment_end
-#  cross-depends:
-#    - name: post_deployment_start
+#  required_for: [post_deployment_end]
+##  version: 2.0.0
+##  cross-depends:
+##    - name: post_deployment_start
+##  cross-depended-by:
+##    - name: post_deployment_end
 #  parameters:
 #    cmd: echo post_deployment_task_executed > /tmp/post_deployment
 #    retries: 3
@@ -42,13 +52,14 @@
 #- id: ${plugin_name}-pre-deployment-sh
 #  version: 2.0.0
 #  type: shell
-#  role: [${plugin_name}]
-#  required_for: [pre_deployment_end]
+#  role: [${plugin_name}_role]
 #  requires: [pre_deployment_start]
-#  cross-depended-by:
-#    - name: pre_deployment_end
-#  cross-depends:
-#    - name: pre_deployment_start
+#  required_for: [pre_deployment_end]
+##  version: 2.0.0
+##  cross-depends:
+##    - name: pre_deployment_start
+##  cross-depended-by:
+##    - name: pre_deployment_end
 #  parameters:
 #    cmd: echo pre_deployment_task_executed > /tmp/pre_deployment
 #    retries: 3

fuel_plugin_builder/templates/v4/plugin_data/node_roles.yaml.mako

@@ -0,0 +1,13 @@
+${plugin_name}_role:
+  # Role name
+  name: "Set here the name for the role. This name will be displayed in the Fuel web UI"
+  # Role description
+  description: "Write description for your role"
+  # If primary then during orchestration this role will be
+  # separated into primary-role and role
+  has_primary: false
+  # Assign public IP to node if true
+  public_ip_required: false
+  # Weight that will be used to sort out the
+  # roles on the Fuel web UI
+  weight: 1000

fuel_plugin_builder/templates/v4/plugin_data/tasks.yaml

@@ -0,0 +1,26 @@
+# WARNING: `tasks.yaml` will be deprecated in further releases.
+# Please, use `deployment_tasks.yaml` to describe tasks istead.
+
+# This tasks will be applied on controller nodes,
+# here you can also specify several roles, for example
+# ['cinder', 'compute'] will be applied only on
+# cinder and compute nodes
+- role: ['controller']
+  stage: post_deployment
+  type: shell
+  parameters:
+    cmd: bash deploy.sh
+    timeout: 42
+# Task is applied for all roles
+- role: '*'
+  stage: pre_deployment
+  type: shell
+  parameters:
+    cmd: echo all > /tmp/plugin.all
+    timeout: 42
+# "reboot" task reboots the nodes and waits until they get back online
+# - role: '*'
+#   stage: pre_deployment
+#   type: reboot
+#   parameters:
+#     timeout: 600

fuel_plugin_builder/templates/v4/plugin_data/volumes.yaml.mako

@@ -0,0 +1,7 @@
+volumes_roles_mapping:
+  # Default role mapping
+  ${plugin_name}_role:
+    - {allocate_size: "min", id: "os"}
+
+# Set here new volumes for your role
+volumes: []

fuel_plugin_builder/tests/test_base_validator.py

@@ -109,15 +109,28 @@ class TestBaseValidator(BaseTestCase):
         validate_mock(self.data, self.schema, self.plugin_path)
 
     @mock.patch('fuel_plugin_builder.validators.base.utils')
-    @mock.patch('fuel_plugin_builder.validators.base.utils.exists')
-    def test_error_message_on_empty_file(self, exists_mock, utils_mock):
+    @mock.patch(
+        'fuel_plugin_builder.validators.base.BaseValidator.validate_schema')
+    def test_validate_file_by_schema_empty_file_passes(
+            self, validate_mock, utils_mock):
         utils_mock.parse_yaml.return_value = None
-        exists_mock.return_value = True
-        with self.assertRaisesRegexp(
-                errors.ValidationError,
-                "File '/tmp/plugin_path' is empty"):
+        self.validator.validate_file_by_schema(
+            self.schema,
+            self.plugin_path,
+            allow_empty=True)
+        utils_mock.parse_yaml.assert_called_once_with(self.plugin_path)
+
+    @mock.patch('fuel_plugin_builder.validators.base.utils')
+    @mock.patch(
+        'fuel_plugin_builder.validators.base.BaseValidator.validate_schema')
+    def test_validate_file_by_schema_empty_file_fails(
+            self, validate_mock, utils_mock):
+        utils_mock.parse_yaml.return_value = None
+        with self.assertRaises(errors.FileIsEmpty):
             self.validator.validate_file_by_schema(
-                self.schema, self.plugin_path)
+                self.schema,
+                self.plugin_path,
+                allow_empty=False)
 
     def test_validate_schema_with_subschemas(self):
         schema_object = {

fuel_plugin_builder/tests/test_validator_v3.py

@@ -256,7 +256,7 @@ class TestValidatorV3(BaseValidator):
             [mock.call(self.schema_class().metadata_schema,
                        self.validator.meta_path),
              mock.call(self.schema_class().tasks_schema,
-                       self.validator.tasks_path, check_file_exists=False)],
+                       self.validator.tasks_path, allow_not_exists=True)],
             self.validator.validate_file_by_schema.call_args_list)
 
         for method in mocked_methods:

fuel_plugin_builder/tests/test_validator_v4.py

@@ -30,7 +30,9 @@ class TestValidatorV4(TestValidatorV3):
 
     def test_check_schemas(self):
         mocked_methods = [
+            'check_metadata_schema',
             'check_env_config_attrs',
+            'check_tasks_schema',
             'check_deployment_tasks_schema',
             'check_network_roles_schema',
             'check_node_roles_schema',
@@ -41,13 +43,6 @@ class TestValidatorV4(TestValidatorV3):
         self.mock_methods(self.validator, ['validate_file_by_schema'])
         self.validator.check_schemas()
 
-        self.assertEqual(
-            [mock.call(self.schema_class().metadata_schema,
-                       self.validator.meta_path),
-             mock.call(self.schema_class().tasks_schema,
-                       self.validator.tasks_path, check_file_exists=False)],
-            self.validator.validate_file_by_schema.call_args_list)
-
         for method in mocked_methods:
             getattr(self.validator, method).assert_called_once_with()
 

fuel_plugin_builder/validators/base.py

@@ -74,20 +74,32 @@ class BaseValidator(object):
         return error_msg
 
     def validate_file_by_schema(self, schema, file_path,
-                                check_file_exists=True):
-        if not check_file_exists and not utils.exists(file_path):
-            logger.debug('No file "%s". Skipping check.', file_path)
-            return
+                                allow_not_exists=False, allow_empty=False):
+        """Validate file with given JSON schema.
 
+        :param schema: object dict
+        :type schema: object
+        :param file_path: path to the file
+        :type file_path: basestring
+        :param allow_not_exists: if true don't raise error on missing file
+        :type allow_not_exists: bool
+        :param allow_empty: allow file to contain no json
+        :type allow_empty: bool
+        :return:
+        """
         if not utils.exists(file_path):
-            raise errors.FileDoesNotExist(file_path)
+            if allow_not_exists:
+                logger.debug('No file "%s". Skipping check.', file_path)
+                return
+            else:
+                raise errors.FileDoesNotExist(file_path)
 
         data = utils.parse_yaml(file_path)
-
-        if data is None:
-            raise errors.FileIsEmpty(file_path)
-
-        self.validate_schema(data, schema, file_path)
+        if data is not None:
+            self.validate_schema(data, schema, file_path)
+        else:
+            if not allow_empty:
+                raise errors.FileIsEmpty(file_path)
 
     @abc.abstractmethod
     def validate(self):

fuel_plugin_builder/validators/validator_v3.py

@@ -56,7 +56,7 @@ class ValidatorV3(ValidatorV2):
         self.validate_file_by_schema(
             self.schema.tasks_schema,
             self.tasks_path,
-            check_file_exists=False
+            allow_not_exists=True
         )
         self.check_env_config_attrs()
         self.check_deployment_tasks_schema()
@@ -73,19 +73,19 @@ class ValidatorV3(ValidatorV2):
         self.validate_file_by_schema(
             self.schema.network_roles_schema,
             self.network_roles_path,
-            check_file_exists=False)
+            allow_not_exists=True)
 
     def check_node_roles_schema(self):
         self.validate_file_by_schema(
             self.schema.node_roles_schema,
             self.node_roles_path,
-            check_file_exists=False)
+            allow_not_exists=True)
 
     def check_volumes_schema(self):
         self.validate_file_by_schema(
             self.schema.volume_schema,
             self.volumes_path,
-            check_file_exists=False)
+            allow_not_exists=True)
 
     def check_deployment_tasks(self):
         logger.debug(

fuel_plugin_builder/validators/validator_v4.py

@@ -41,16 +41,30 @@ class ValidatorV4(ValidatorV3):
         self.validate_file_by_schema(
             self.schema.metadata_schema,
             self.meta_path,
-            check_file_exists=False)
+            allow_not_exists=True)
+
+    def check_tasks_schema(self):
+        self.validate_file_by_schema(
+            self.schema.tasks_schema,
+            self.tasks_path,
+            allow_empty=True
+        )
 
     def check_schemas(self):
-        super(ValidatorV4, self).check_schemas()
+        logger.debug('Start schema checking "%s"', self.plugin_path)
+        self.check_metadata_schema()
+        self.check_tasks_schema()
+        self.check_env_config_attrs()
+        self.check_deployment_tasks_schema()
+        self.check_network_roles_schema()
+        self.check_node_roles_schema()
+        self.check_volumes_schema()
         self.check_components_schema()
 
     def check_components_schema(self):
         self.validate_file_by_schema(self.schema.components_schema,
                                      self.components_path,
-                                     check_file_exists=False)
+                                     allow_not_exists=True)
 
     def check_deployment_tasks(self):
         logger.debug(
@@ -79,3 +93,26 @@ class ValidatorV4(ValidatorV3):
                 schemas[deployment_task['type']],
                 self.deployment_tasks_path,
                 value_path=[idx])
+
+    def check_tasks(self):
+        """Check legacy tasks.yaml."""
+        logger.debug('Start tasks checking "%s"', self.tasks_path)
+        if utils.exists(self.tasks_path):
+            # todo(ikutukov): remove self._check_tasks
+            tasks = utils.parse_yaml(self.tasks_path)
+            if tasks is None:
+                return
+
+            schemas = {
+                'puppet': self.schema.puppet_parameters,
+                'shell': self.schema.shell_parameters,
+                'reboot': self.schema.reboot_parameters}
+
+            for idx, task in enumerate(tasks):
+                self.validate_schema(
+                    task.get('parameters'),
+                    schemas[task['type']],
+                    self.tasks_path,
+                    value_path=[idx, 'parameters'])
+        else:
+            logger.debug('File "%s" doesn\'t exist', self.tasks_path)