'File exists' validator and Utils class for validators

Change-Id: I43451ae5a79618cd7d8b60a98e9904052b242276

rally-jobs/rally-murano.yaml

@@ -67,7 +67,7 @@
   MuranoPackages.import_and_list_packages:
     -
       args:
-        package: "/home/jenkins/.rally/extra/murano/applications/HelloReporter/io.murano.apps.HelloReporter/"
+        package: "~/.rally/extra/murano/applications/HelloReporter/io.murano.apps.HelloReporter/"
       runner:
         type: "constant"
         times: 10
@@ -81,7 +81,7 @@
           max: 0
     -
       args:
-        package: "/home/jenkins/.rally/extra/murano/applications/HelloReporter/io.murano.apps.HelloReporter.zip"
+        package: "~/.rally/extra/murano/applications/HelloReporter/io.murano.apps.HelloReporter.zip"
       runner:
         type: "constant"
         times: 1
@@ -97,7 +97,7 @@
   MuranoPackages.import_and_delete_package:
     -
       args:
-        package: "/home/jenkins/.rally/extra/murano/applications/HelloReporter/io.murano.apps.HelloReporter/"
+        package: "~/.rally/extra/murano/applications/HelloReporter/io.murano.apps.HelloReporter/"
       runner:
         type: "constant"
         times: 10
@@ -113,7 +113,7 @@
   MuranoPackages.import_and_filter_applications:
     -
       args:
-        package: "/home/jenkins/.rally/extra/murano/applications/HelloReporter/io.murano.apps.HelloReporter/"
+        package: "~/.rally/extra/murano/applications/HelloReporter/io.murano.apps.HelloReporter/"
         filter_query: {"category" : "Web"}
       runner:
         type: "constant"
@@ -130,7 +130,7 @@
   MuranoPackages.package_lifecycle:
     -
       args:
-        package: "/home/jenkins/.rally/extra/murano/applications/HelloReporter/io.murano.apps.HelloReporter/"
+        package: "~/.rally/extra/murano/applications/HelloReporter/io.murano.apps.HelloReporter/"
         body: {"categories": ["Web"]}
         operation: "add"
       runner:

rally-jobs/rally-neutron.yaml

@@ -737,7 +737,7 @@
       context:
         image_command_customizer:
           command:
-            local_path: "/home/jenkins/.rally/extra/install_benchmark.sh"
+            local_path: "~/.rally/extra/install_benchmark.sh"
             remote_path: "./install_benchmark.sh"
           flavor:
             name: "m1.tiny"

rally-jobs/self-rally.yaml

@@ -255,7 +255,7 @@
           hooks:
             - name: sys_call
               description: Run script
-              args: sh /home/jenkins/.rally/extra/hook_example_script.sh
+              args: sh ~/.rally/extra/hook_example_script.sh
               trigger:
                 name: event
                 args:

rally/plugins/common/validators.py

@@ -1,4 +1,3 @@
-# Copyright 2017: Mirantis Inc.
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -14,6 +13,7 @@
 #    under the License.
 
 import inspect
+import os
 
 import jsonschema
 import six
@@ -24,6 +24,23 @@ from rally.common import validation
 LOG = logging.getLogger(__name__)
 
 
+class ValidatorUtils(object):
+
+    @staticmethod
+    def _file_access_ok(filename, mode, param_name, required=True):
+        if not filename:
+            return validation.ValidationResult(
+                not required,
+                "Parameter %s required" % param_name)
+        if not os.access(os.path.expanduser(filename), mode):
+            return validation.ValidationResult(
+                False, "Could not open %(filename)s with mode %(mode)s "
+                       "for parameter %(param_name)s"
+                       % {"filename": filename, "mode": mode,
+                          "param_name": param_name})
+        return validation.ValidationResult(True)
+
+
 @validation.configure(name="jsonschema")
 class JsonSchemaValidator(validation.Validator):
     """JSON schema validator"""
@@ -294,8 +311,7 @@ class RequiredContextsValidator(validation.Validator):
             return self.fail(msg)
 
 
-@validation.configure(name="required_param_or_context",
+@validation.configure(name="required_param_or_context")
-                      namespace="openstack")
 class RequiredParamOrContextValidator(validation.Validator):
 
     def __init__(self, param_name, ctx_name):
@@ -318,3 +334,36 @@ class RequiredParamOrContextValidator(validation.Validator):
         if self.param_name in config.get("args", {}):
             return
         return self.fail(msg)
+
+
+@validation.configure(name="file_exists")
+class FileExistsValidator(validation.Validator):
+
+    def __init__(self, param_name, mode=os.R_OK, required=True):
+        """Validator checks parameter is proper path to file with proper mode.
+
+        Ensure a file exists and can be accessed with the specified mode.
+        Note that path to file will be expanded before access checking.
+
+        :param param_name: Name of parameter to validate
+        :param mode: Access mode to test for. This should be one of:
+            * os.F_OK (file exists)
+            * os.R_OK (file is readable)
+            * os.W_OK (file is writable)
+            * os.X_OK (file is executable)
+
+            If multiple modes are required they can be added, eg:
+                mode=os.R_OK+os.W_OK
+        :param required: Boolean indicating whether this argument is required.
+        """
+        super(FileExistsValidator, self).__init__()
+
+        self.param_name = param_name
+        self.mode = mode
+        self.required = required
+
+    def validate(self, config, credentials, plugin_cls, plugin_cfg):
+
+        return ValidatorUtils._file_access_ok(
+            config.get("args", {}).get(self.param_name),
+            self.mode, self.param_name, self.required)

rally/plugins/openstack/scenarios/heat/utils.py

@@ -13,7 +13,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-
 from oslo_config import cfg
 import requests
 

rally/plugins/openstack/scenarios/mistral/executions.py

@@ -51,7 +51,7 @@ class ListExecutions(utils.MistralScenario):
                               sort_keys=sort_keys, sort_dirs=sort_dirs)
 
 
-@validation.file_exists("definition")
+@validation.add("file_exists", param_name="definition")
 @types.convert(definition={"type": "file"})
 @types.convert(params={"type": "file"})
 @types.convert(wf_input={"type": "file"})

rally/plugins/openstack/scenarios/mistral/workbooks.py

@@ -38,7 +38,7 @@ class ListWorkbooks(utils.MistralScenario):
         self._list_workbooks()
 
 
-@validation.file_exists("definition")
+@validation.add("file_exists", param_name="definition")
 @types.convert(definition={"type": "file"})
 @validation.add("required_platform", platform="openstack", users=True)
 @validation.add("required_services",

rally/plugins/openstack/scenarios/murano/packages.py

@@ -25,7 +25,7 @@ from rally.task import validation
 """Scenarios for Murano packages."""
 
 
-@validation.file_exists(param_name="package", mode=os.F_OK)
+@validation.add("file_exists", param_name="package", mode=os.F_OK)
 @validation.add("required_services", services=[consts.Service.MURANO])
 @validation.add("required_platform", platform="openstack", users=True)
 @scenario.configure(context={"cleanup": ["murano.packages"]},
@@ -55,7 +55,7 @@ class ImportAndListPackages(utils.MuranoScenario):
             os.remove(package_path)
 
 
-@validation.file_exists(param_name="package", mode=os.F_OK)
+@validation.add("file_exists", param_name="package", mode=os.F_OK)
 @validation.add("required_services", services=[consts.Service.MURANO])
 @validation.add("required_platform", platform="openstack", users=True)
 @scenario.configure(context={"cleanup": ["murano.packages"]},
@@ -82,7 +82,7 @@ class ImportAndDeletePackage(utils.MuranoScenario):
             os.remove(package_path)
 
 
-@validation.file_exists(param_name="package", mode=os.F_OK)
+@validation.add("file_exists", param_name="package", mode=os.F_OK)
 @validation.add("required_services", services=[consts.Service.MURANO])
 @validation.add("required_platform", platform="openstack", users=True)
 @scenario.configure(context={"cleanup": ["murano.packages"]},
@@ -118,7 +118,7 @@ class PackageLifecycle(utils.MuranoScenario):
             os.remove(package_path)
 
 
-@validation.file_exists(param_name="package", mode=os.F_OK)
+@validation.add("file_exists", param_name="package", mode=os.F_OK)
 @validation.add("required_services", services=[consts.Service.MURANO])
 @validation.add("required_platform", platform="openstack", users=True)
 @scenario.configure(context={"cleanup": ["murano.packages"]},

rally/plugins/openstack/scenarios/vm/utils.py

@@ -124,8 +124,9 @@ class VMScenario(nova_utils.NovaScenario):
                                  "or list type")
             cmd.extend(remote_path)
             if command.get("local_path"):
-                ssh.put_file(command["local_path"], remote_path[-1],
+                ssh.put_file(os.path.expanduser(
-                             mode=self.USER_RWX_OTHERS_RX_ACCESS_MODE)
+                    command["local_path"]), remote_path[-1],
+                    mode=self.USER_RWX_OTHERS_RX_ACCESS_MODE)
 
         if command.get("script_file"):
             stdin = open(os.path.expanduser(command["script_file"]), "rb")

rally/task/validation.py

@@ -115,30 +115,6 @@ def _file_access_ok(filename, mode, param_name, required=True):
     return ValidationResult(True)
 
 
-@validator
-def file_exists(config, clients, deployment, param_name, mode=os.R_OK,
-                required=True):
-    """Validator checks parameter is proper path to file with proper mode.
-
-    Ensure a file exists and can be accessed with the specified mode.
-    Note that path to file will be expanded before access checking.
-
-    :param param_name: Name of parameter to validate
-    :param mode: Access mode to test for. This should be one of:
-        * os.F_OK (file exists)
-        * os.R_OK (file is readable)
-        * os.W_OK (file is writable)
-        * os.X_OK (file is executable)
-
-        If multiple modes are required they can be added, eg:
-            mode=os.R_OK+os.W_OK
-    :param required: Boolean indicating whether this argument is required.
-    """
-
-    return _file_access_ok(config.get("args", {}).get(param_name), mode,
-                           param_name, required)
-
-
 def check_command_dict(command):
     """Check command-specifying dict `command', raise ValueError on error."""
 
@@ -409,3 +385,5 @@ required_param_or_context = deprecated_validator("required_param_or_context",
 volume_type_exists = deprecated_validator("volume_type_exists",
                                           "volume_type_exists",
                                           "0.10.0")
+
+file_exists = deprecated_validator("file_exists", "file_exists", "0.10.0")

tests/unit/plugins/common/test_validators.py

@@ -1,4 +1,3 @@
-# Copyright 2017: Mirantis Inc.
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -15,7 +14,11 @@
 
 import ddt
 import mock
+import os
+import shutil
+import tempfile
 
+import rally
 from rally.common.plugin import plugin
 from rally.common import validation
 from rally.plugins.common import validators
@@ -340,3 +343,38 @@ class RequiredParamOrContextValidatorTestCase(test.TestCase):
             self.assertEqual(err_msg, result.msg)
         else:
             self.assertIsNone(result)
+
+
+class FileExistsValidatorTestCase(test.TestCase):
+    rally_jobs_path = os.path.join(
+        os.path.dirname(rally.__file__), "..", "rally-jobs")
+
+    def setUp(self):
+        super(FileExistsValidatorTestCase, self).setUp()
+        self.validator = validators.FileExistsValidator(param_name="p",
+                                                        required=False)
+        self.credentials = dict(openstack={"admin": mock.MagicMock(),
+                                           "users": [mock.MagicMock()], })
+        self.tmp_dir = tempfile.mkdtemp()
+        os.makedirs(os.path.join(self.tmp_dir, ".rally"))
+        shutil.copytree(os.path.join(self.rally_jobs_path, "extra"),
+                        os.path.join(self.tmp_dir, ".rally", "extra"))
+
+        self.original_home = os.environ["HOME"]
+        os.environ["HOME"] = self.tmp_dir
+
+        def return_home():
+            os.environ["HOME"] = self.original_home
+        self.addCleanup(shutil.rmtree, self.tmp_dir)
+
+        self.addCleanup(return_home)
+
+    @mock.patch("rally.plugins.common.validators."
+                "ValidatorUtils._file_access_ok")
+    def test_file_exists(self, mock__file_access_ok):
+        mock__file_access_ok.return_value = "foobar"
+        result = self.validator.validate({"args": {"p": "test_file"}},
+                                         self.credentials, None, None)
+        self.assertEqual("foobar", result)
+        mock__file_access_ok.assert_called_once_with(
+            "test_file", os.R_OK, "p", False)

tests/unit/rally_jobs/test_jobs.py

@@ -13,6 +13,8 @@
 #    under the License.
 
 import os
+import shutil
+import tempfile
 import traceback
 
 import mock
@@ -30,6 +32,22 @@ class RallyJobsTestCase(test.TestCase):
     rally_jobs_path = os.path.join(
         os.path.dirname(rally.__file__), "..", "rally-jobs")
 
+    def setUp(self):
+        super(RallyJobsTestCase, self).setUp()
+        self.tmp_dir = tempfile.mkdtemp()
+        os.makedirs(os.path.join(self.tmp_dir, ".rally"))
+        shutil.copytree(os.path.join(self.rally_jobs_path, "extra"),
+                        os.path.join(self.tmp_dir, ".rally", "extra"))
+
+        self.original_home = os.environ["HOME"]
+        os.environ["HOME"] = self.tmp_dir
+
+        def return_home():
+            os.environ["HOME"] = self.original_home
+        self.addCleanup(shutil.rmtree, self.tmp_dir)
+
+        self.addCleanup(return_home)
+
     def test_schema_is_valid(self):
         discover.load_plugins(os.path.join(self.rally_jobs_path, "plugins"))
 

tests/unit/task/test_validation.py

@@ -192,17 +192,6 @@ class ValidatorsTestCase(test.TestCase):
             "foobar", os.R_OK, "p", False)
         self.assertFalse(result.is_valid, result.msg)
 
-    @mock.patch(MODULE + "_file_access_ok")
-    def test_file_exists(self, mock__file_access_ok):
-        mock__file_access_ok.return_value = "foobar"
-        validator = self._unwrap_validator(validation.file_exists,
-                                           param_name="p",
-                                           required=False)
-        result = validator({"args": {"p": "test_file"}}, None, None)
-        self.assertEqual("foobar", result)
-        mock__file_access_ok.assert_called_once_with(
-            "test_file", os.R_OK, "p", False)
-
     @ddt.data({"raises_message": "Command must be a dictionary"},
               {"command": "foo",
                "raises_message": "Command must be a dictionary"},