Negative tests: Add result check for resources

This add an result check if a invalid resource is given. Introducing a
new dict that can be enhanced later for other cases.

Change-Id: Ief3e51fafb9437a8e6b9a71aad79459716bac3d4
Partially-implements: bp negative-tests

etc/schemas/compute/flavors/flavor_details.json

@@ -2,5 +2,7 @@
     "name": "get-flavor-details",
     "http-method": "GET",
     "url": "flavors/%s",
-    "resources": ["flavor"]
+    "resources": [
+        {"name": "flavor", "expected_result": 404}
+    ]
 }

etc/schemas/compute/servers/get_console_output.json

@@ -2,7 +2,9 @@
     "name": "get-console-output",
     "http-method": "POST",
     "url": "servers/%s/action",
-    "resources": ["server"],
+    "resources": [
+        {"name":"server", "expected_result": 404}
+    ],
     "json-schema": {
         "type": "object",
         "properties": {

tempest/common/generate_json.py

@@ -203,36 +203,62 @@ def gen_inv_prop_obj(schema):
     return invalids
 
 
-type_map_valid = {"string": generate_valid_string,
-                  "integer": generate_valid_integer,
-                  "object": generate_valid_object}
+type_map_valid = {
+    "string": generate_valid_string,
+    "integer": generate_valid_integer,
+    "object": generate_valid_object
+}
 
-type_map_invalid = {"string": [gen_int,
-                               gen_none,
-                               gen_str_min_length,
-                               gen_str_max_length],
-                    "integer": [gen_string,
-                                gen_none,
-                                gen_int_min,
-                                gen_int_max],
-                    "object": [gen_obj_remove_attr,
-                               gen_obj_add_attr,
-                               gen_inv_prop_obj]}
+type_map_invalid = {
+    "string": [
+        gen_int,
+        gen_none,
+        gen_str_min_length,
+        gen_str_max_length],
+    "integer": [
+        gen_string,
+        gen_none,
+        gen_int_min,
+        gen_int_max],
+    "object": [
+        gen_obj_remove_attr,
+        gen_obj_add_attr,
+        gen_inv_prop_obj]
+}
 
-schema = {"type": "object",
-          "properties":
-          {"name": {"type": "string"},
-           "http-method": {"enum": ["GET", "PUT", "HEAD",
-                                    "POST", "PATCH", "DELETE", 'COPY']},
-           "url": {"type": "string"},
-           "json-schema": jsonschema._utils.load_schema("draft4"),
-           "resources": {"type": "array", "items": {"type": "string"}},
-           "results": {"type": "object",
-                       "properties": {}}
-           },
-          "required": ["name", "http-method", "url"],
-          "additionalProperties": False,
-          }
+schema = {
+    "type": "object",
+    "properties": {
+        "name": {"type": "string"},
+        "http-method": {
+            "enum": ["GET", "PUT", "HEAD",
+                     "POST", "PATCH", "DELETE", 'COPY']
+        },
+        "url": {"type": "string"},
+        "json-schema": jsonschema._utils.load_schema("draft4"),
+        "resources": {
+            "type": "array",
+            "items": {
+                "oneOf": [
+                    {"type": "string"},
+                    {
+                        "type": "object",
+                        "properties": {
+                            "name": {"type": "string"},
+                            "expected_result": {"type": "integer"}
+                        }
+                    }
+                ]
+            }
+        },
+        "results": {
+            "type": "object",
+            "properties": {}
+        }
+    },
+    "required": ["name", "http-method", "url"],
+    "additionalProperties": False,
+}
 
 
 def validate_negative_test_schema(nts):

tempest/test.py

@@ -405,11 +405,16 @@ class NegativeAutoTest(BaseTestCase):
         schema = description.get("json-schema", None)
         resources = description.get("resources", [])
         scenario_list = []
+        expected_result = None
         for resource in resources:
+            if isinstance(resource, dict):
+                expected_result = resource['expected_result']
+                resource = resource['name']
             LOG.debug("Add resource to test %s" % resource)
             scn_name = "inv_res_%s" % (resource)
             scenario_list.append((scn_name, {"resource": (resource,
-                                                          str(uuid.uuid4()))
+                                                          str(uuid.uuid4())),
+                                             "expected_result": expected_result
                                              }))
         if schema is not None:
             for invalid in generate_json.generate_invalid(schema):
@@ -460,16 +465,12 @@ class NegativeAutoTest(BaseTestCase):
             if schema:
                 valid = generate_json.generate_valid(schema)
             new_url, body = self._http_arguments(valid, url, method)
-            resp, resp_body = self.client.send_request(method, new_url,
-                                                       resources, body=body)
-            self._check_negative_response(resp.status, resp_body)
-            return
-
-        if hasattr(self, "schema"):
+        elif hasattr(self, "schema"):
             new_url, body = self._http_arguments(self.schema, url, method)
-            resp, resp_body = self.client.send_request(method, new_url,
-                                                       resources, body=body)
-            self._check_negative_response(resp.status, resp_body)
+
+        resp, resp_body = self.client.send_request(method, new_url,
+                                                   resources, body=body)
+        self._check_negative_response(resp.status, resp_body)
 
     def _http_arguments(self, json_dict, url, method):
         LOG.debug("dict: %s url: %s method: %s" % (json_dict, url, method))
@@ -510,6 +511,8 @@ class NegativeAutoTest(BaseTestCase):
         :param name: The name of the kind of resource such as "flavor", "role",
             etc.
         """
+        if isinstance(name, dict):
+            name = name['name']
         if hasattr(self, "resource") and self.resource[0] == name:
             LOG.debug("Return invalid resource (%s) value: %s" %
                       (self.resource[0], self.resource[1]))