Negative tests: Add result check for resources

This add an result check if a invalid resource is given. Introducing a
new dict that can be enhanced later for other cases.

Change-Id: Ief3e51fafb9437a8e6b9a71aad79459716bac3d4
Partially-implements: bp negative-tests

etc/schemas/compute/flavors/flavor_details.json

@@ -2,5 +2,7 @@
     "name": "get-flavor-details",
     "http-method": "GET",
     "url": "flavors/%s",
-    "resources": ["flavor"]
+    "resources": [
+        {"name": "flavor", "expected_result": 404}
+    ]
 }

etc/schemas/compute/servers/get_console_output.json

@@ -2,7 +2,9 @@
     "name": "get-console-output",
     "http-method": "POST",
     "url": "servers/%s/action",
-    "resources": ["server"],
+    "resources": [
+        {"name":"server", "expected_result": 404}
+    ],
     "json-schema": {
         "type": "object",
         "properties": {

tempest/common/generate_json.py

@@ -203,36 +203,62 @@ def gen_inv_prop_obj(schema):
     return invalids
 
 
-type_map_valid = {"string": generate_valid_string,
+type_map_valid = {
+    "string": generate_valid_string,
     "integer": generate_valid_integer,
-                  "object": generate_valid_object}
+    "object": generate_valid_object
+}
 
-type_map_invalid = {"string": [gen_int,
+type_map_invalid = {
+    "string": [
+        gen_int,
         gen_none,
         gen_str_min_length,
         gen_str_max_length],
-                    "integer": [gen_string,
+    "integer": [
+        gen_string,
         gen_none,
         gen_int_min,
         gen_int_max],
-                    "object": [gen_obj_remove_attr,
+    "object": [
+        gen_obj_remove_attr,
         gen_obj_add_attr,
-                               gen_inv_prop_obj]}
+        gen_inv_prop_obj]
+}
 
-schema = {"type": "object",
-          "properties":
-          {"name": {"type": "string"},
-           "http-method": {"enum": ["GET", "PUT", "HEAD",
-                                    "POST", "PATCH", "DELETE", 'COPY']},
+schema = {
+    "type": "object",
+    "properties": {
+        "name": {"type": "string"},
+        "http-method": {
+            "enum": ["GET", "PUT", "HEAD",
+                     "POST", "PATCH", "DELETE", 'COPY']
+        },
         "url": {"type": "string"},
         "json-schema": jsonschema._utils.load_schema("draft4"),
-           "resources": {"type": "array", "items": {"type": "string"}},
-           "results": {"type": "object",
-                       "properties": {}}
+        "resources": {
+            "type": "array",
+            "items": {
+                "oneOf": [
+                    {"type": "string"},
+                    {
+                        "type": "object",
+                        "properties": {
+                            "name": {"type": "string"},
+                            "expected_result": {"type": "integer"}
+                        }
+                    }
+                ]
+            }
+        },
+        "results": {
+            "type": "object",
+            "properties": {}
+        }
     },
     "required": ["name", "http-method", "url"],
     "additionalProperties": False,
-          }
+}
 
 
 def validate_negative_test_schema(nts):

tempest/test.py

@@ -405,11 +405,16 @@ class NegativeAutoTest(BaseTestCase):
         schema = description.get("json-schema", None)
         resources = description.get("resources", [])
         scenario_list = []
+        expected_result = None
         for resource in resources:
+            if isinstance(resource, dict):
+                expected_result = resource['expected_result']
+                resource = resource['name']
             LOG.debug("Add resource to test %s" % resource)
             scn_name = "inv_res_%s" % (resource)
             scenario_list.append((scn_name, {"resource": (resource,
-                                                          str(uuid.uuid4()))
+                                                          str(uuid.uuid4())),
+                                             "expected_result": expected_result
                                              }))
         if schema is not None:
             for invalid in generate_json.generate_invalid(schema):
@@ -460,13 +465,9 @@ class NegativeAutoTest(BaseTestCase):
             if schema:
                 valid = generate_json.generate_valid(schema)
             new_url, body = self._http_arguments(valid, url, method)
-            resp, resp_body = self.client.send_request(method, new_url,
-                                                       resources, body=body)
-            self._check_negative_response(resp.status, resp_body)
-            return
-
-        if hasattr(self, "schema"):
+        elif hasattr(self, "schema"):
             new_url, body = self._http_arguments(self.schema, url, method)
+
         resp, resp_body = self.client.send_request(method, new_url,
                                                    resources, body=body)
         self._check_negative_response(resp.status, resp_body)
@@ -510,6 +511,8 @@ class NegativeAutoTest(BaseTestCase):
         :param name: The name of the kind of resource such as "flavor", "role",
             etc.
         """
+        if isinstance(name, dict):
+            name = name['name']
         if hasattr(self, "resource") and self.resource[0] == name:
             LOG.debug("Return invalid resource (%s) value: %s" %
                       (self.resource[0], self.resource[1]))