From fa2b1c809f8f3909bae35ce9d70622bddc200764 Mon Sep 17 00:00:00 2001 From: Enol Fernandez Date: Tue, 14 Apr 2015 10:17:15 +0200 Subject: [PATCH] Improved parsers and validators Everything is now under the same module (wsgi). Request object includes methods for parsing a validating the user input. Fixed the compute method which was still using the old body for creating the VM. Added tests for parser. --- ooi/api/__init__.py | 69 ------------------------------- ooi/api/base.py | 9 ++++ ooi/api/compute.py | 32 +++++++------- ooi/tests/test_wsgi.py | 25 ++++++++++- ooi/wsgi/__init__.py | 26 +++++++----- ooi/wsgi/parsers.py | 94 ++++++++++++++++++++++++++++++++++-------- 6 files changed, 140 insertions(+), 115 deletions(-) diff --git a/ooi/api/__init__.py b/ooi/api/__init__.py index 9888593..e69de29 100644 --- a/ooi/api/__init__.py +++ b/ooi/api/__init__.py @@ -1,69 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2015 Spanish National Research Council -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -import copy - -from ooi import exception -from ooi.occi import helpers - - -def compare_schemes(expected_type, actual): - actual_scheme, actual_term = helpers.decompose_type(actual) - if expected_type.scheme != actual_scheme: - return False - try: - if expected_type.term != actual_term: - return False - except AttributeError: - # ignore the fact the type does not have a term - pass - return True - - -def validate(schema): - def accepts(f): - # TODO(enolfc): proper testing and attribute checking. - def _validate(obj, req, body, *args, **kwargs): - parsed_obj = req.parse() - if "kind" in schema: - try: - if schema["kind"].type_id != parsed_obj["kind"]: - raise exception.OCCISchemaMismatch( - expected=schema["kind"].type_id, - found=parsed_obj["kind"]) - except KeyError: - raise exception.OCCIMissingType( - type_id=schema["kind"].type_id) - unmatched = copy.copy(parsed_obj["mixins"]) - for m in schema.get("mixins", []): - for um in unmatched: - if compare_schemes(m, um): - unmatched[um] -= 1 - break - else: - raise exception.OCCIMissingType(type_id=m.scheme) - for m in schema.get("optional_mixins", []): - for um in unmatched: - if compare_schemes(m, um): - unmatched[um] -= 1 - unexpected = [m for m in unmatched if unmatched[m]] - if unexpected: - raise exception.OCCISchemaMismatch(expected="", - found=unexpected) - return f(obj, parsed_obj, req, body, *args, **kwargs) - return _validate - return accepts diff --git a/ooi/api/base.py b/ooi/api/base.py index f6a15d2..6cdf432 100644 --- a/ooi/api/base.py +++ b/ooi/api/base.py @@ -74,6 +74,15 @@ class Controller(object): else: raise exception_from_response(response) + @staticmethod + def validate(schema): + def accepts(f): + def _validate(obj, req, body, *args, **kwargs): + parsed_obj = req.validate(schema) + return f(obj, parsed_obj, req, body, *args, **kwargs) + return _validate + return accepts + def exception_from_response(response): """Convert an OpenStack V2 Fault into a webob exception. diff --git a/ooi/api/compute.py b/ooi/api/compute.py index a599e9f..f2a8efd 100644 --- a/ooi/api/compute.py +++ b/ooi/api/compute.py @@ -16,7 +16,6 @@ import json -import ooi.api import ooi.api.base from ooi.occi.core import collection from ooi.occi.infrastructure import compute @@ -67,16 +66,17 @@ class Controller(ooi.api.base.Controller): return collection.Collection(resources=occi_compute_resources) - @ooi.api.validate({"kind": compute.ComputeResource.kind, - "mixins": [ - templates.OpenStackOSTemplate, - templates.OpenStackResourceTemplate, - ], - "optional_mixins": [ - contextualization.user_data, - contextualization.public_key, - ] - }) + @ooi.api.base.Controller.validate( + {"kind": compute.ComputeResource.kind, + "mixins": [ + templates.OpenStackOSTemplate, + templates.OpenStackResourceTemplate, + ], + "optional_mixins": [ + contextualization.user_data, + contextualization.public_key, + ] + }) def create(self, obj, req, body): tenant_id = req.environ["keystone.token_auth"].user.project_id name = obj.get("occi.core.title", "OCCI VM") @@ -89,15 +89,13 @@ class Controller(ooi.api.base.Controller): }} if contextualization.user_data.scheme in obj["schemes"]: req_body["user_data"] = obj.get("org.openstack.compute.user_data") + # TODO(enolfc): add here the correct metadata info + # if contextualization.public_key.scheme in obj["schemes"]: + # req_body["metadata"] = XXX req = self._get_req(req, path="/%s/servers" % tenant_id, content_type="application/json", - body=json.dumps({ - "server": { - "name": name, - "imageRef": image, - "flavorRef": flavor, - }})) + body=json.dumps(req_body)) response = req.get_response(self.app) # We only get one server server = self.get_from_response(response, "server", {}) diff --git a/ooi/tests/test_wsgi.py b/ooi/tests/test_wsgi.py index bb1ef85..2642fa2 100644 --- a/ooi/tests/test_wsgi.py +++ b/ooi/tests/test_wsgi.py @@ -113,12 +113,33 @@ class TestMiddleware(base.TestCase): result = req.get_response(self.app) self.assertEqual(406, result.status_code) - def test_bad_content_type(self): + def test_bad_content_type_post(self): + req = webob.Request.blank("/foos", + method="POST", + content_type="foo/bazonk") + result = req.get_response(self.app) + self.assertEqual(406, result.status_code) + + def test_bad_content_type_put(self): + req = webob.Request.blank("/foos", + method="PUT", + content_type="foo/bazonk") + result = req.get_response(self.app) + self.assertEqual(404, result.status_code) + + def test_bad_content_type_get(self): req = webob.Request.blank("/foos", method="GET", content_type="foo/bazonk") result = req.get_response(self.app) - self.assertEqual(406, result.status_code) + self.assertEqual(200, result.status_code) + + def test_bad_content_type_delete(self): + req = webob.Request.blank("/foos", + method="DELETE", + content_type="foo/bazonk") + result = req.get_response(self.app) + self.assertEqual(404, result.status_code) class TestOCCIMiddleware(base.TestCase): diff --git a/ooi/wsgi/__init__.py b/ooi/wsgi/__init__.py index bdc9835..cfa57d9 100644 --- a/ooi/wsgi/__init__.py +++ b/ooi/wsgi/__init__.py @@ -31,13 +31,17 @@ LOG = logging.getLogger(__name__) class Request(webob.Request): + def should_have_body(self): + return self.method in ("POST", "PUT") + def get_content_type(self): """Determine content type of the request body.""" if not self.content_type: return None - # FIXME: we should change this, since the content type does not depend - # on the serializers, but on the parsers + if not self.should_have_body(): + return None + if self.content_type not in parsers.get_supported_content_types(): LOG.debug("Unrecognized Content-Type provided in request") raise exception.InvalidContentType(content_type=self.content_type) @@ -53,9 +57,15 @@ class Request(webob.Request): raise exception.InvalidAccept(content_type=content_type) return content_type - def parse(self): - parser = parsers.HeaderParser() - return parser.parse(self.headers, self.body) + def get_parser(self): + mtype = parsers.get_media_map().get(self.get_content_type, + "header") + return parsers.get_default_parsers()[mtype] + + def validate(self, schema): + parser = self.get_parser()(self.headers, self.body) + parser.validate(schema) + return parser.parsed_obj class OCCIMiddleware(object): @@ -173,10 +183,6 @@ class Resource(object): return args - @staticmethod - def _should_have_body(request): - return request.method in ("POST", "PUT") - def __call__(self, request, args): """Control the method dispatch.""" action_args = self.get_action_args(args) @@ -201,7 +207,7 @@ class Resource(object): return Fault(webob.exc.HTTPBadRequest(explanation=msg)) contents = {} - if self._should_have_body(request): + if request.should_have_body(): # allow empty body with PUT and POST if request.content_length == 0: contents = {'body': None} diff --git a/ooi/wsgi/parsers.py b/ooi/wsgi/parsers.py index 71ad703..498b273 100644 --- a/ooi/wsgi/parsers.py +++ b/ooi/wsgi/parsers.py @@ -15,17 +15,86 @@ # under the License. import collections +import copy import shlex from ooi import exception +from ooi.occi import helpers _MEDIA_TYPE_MAP = collections.OrderedDict([ - # ('text/plain', 'text'), + ('text/plain', 'text'), ('text/occi', 'header') ]) +class BaseParser(object): + def __init__(self, headers, body): + self.headers = headers + self.body = body + + def parse(self): + raise NotImplemented + + def _validate_kind(self, kind): + try: + if kind.type_id != self.parsed_obj["kind"]: + raise exception.OCCISchemaMismatch( + expected=kind.type_id, found=self.parsed_obj["kind"]) + except KeyError: + raise exception.OCCIMissingType( + type_id=kind.type_id) + + def _compare_schemes(self, expected_type, actual): + actual_scheme, actual_term = helpers.decompose_type(actual) + if expected_type.scheme != actual_scheme: + return False + try: + if expected_type.term != actual_term: + return False + except AttributeError: + # ignore the fact the type does not have a term + pass + return True + + def _validate_mandatory_mixins(self, mixins, unmatched): + for m in mixins: + for um in unmatched: + if self._compare_schemes(m, um): + unmatched[um] -= 1 + break + else: + raise exception.OCCIMissingType(type_id=m.scheme) + return unmatched + + def _validate_optional_mixins(self, mixins, unmatched): + for m in mixins: + for um in unmatched: + if self._compare_schemes(m, um): + unmatched[um] -= 1 + break + return unmatched + + def validate(self, schema): + self.parsed_obj = self.parse() + if "kind" in schema: + self._validate_kind(schema["kind"]) + unmatched = copy.copy(self.parsed_obj["mixins"]) + unmatched = self._validate_mandatory_mixins( + schema.get("mixins", []), unmatched) + unmatched = self._validate_optional_mixins( + schema.get("optional_mixins", []), unmatched) + unexpected = [m for m in unmatched if unmatched[m]] + if unexpected: + raise exception.OCCISchemaMismatch(expected="", + found=unexpected) + return True + + +class TextParser(BaseParser): + pass + + def _lexize(s, separator, ignore_whitespace=False): lex = shlex.shlex(instream=s, posix=True) lex.commenters = "" @@ -37,22 +106,13 @@ def _lexize(s, separator, ignore_whitespace=False): return list(lex) -class BaseParser(object): - def validate(self): - return False - - -class TextParser(BaseParser): - pass - - class HeaderParser(BaseParser): - def parse_categories(self, headers, body): + def parse_categories(self): kind = None mixins = collections.Counter() schemes = collections.defaultdict(list) try: - categories = headers["Category"] + categories = self.headers["Category"] except KeyError: raise exception.OCCIInvalidSchema("No categories") for ctg in _lexize(categories, separator=",", ignore_whitespace=True): @@ -74,10 +134,10 @@ class HeaderParser(BaseParser): "schemes": schemes, } - def parse_attributes(self, headers, body): + def parse_attributes(self): attrs = {} try: - header_attrs = headers["X-OCCI-Attribute"] + header_attrs = self.headers["X-OCCI-Attribute"] for attr in _lexize(header_attrs, separator=",", ignore_whitespace=True): n, v = attr.split('=', 1) @@ -86,9 +146,9 @@ class HeaderParser(BaseParser): pass return attrs - def parse(self, headers, body): - obj = self.parse_categories(headers, body) - obj['attributes'] = self.parse_attributes(headers, body) + def parse(self): + obj = self.parse_categories() + obj['attributes'] = self.parse_attributes() return obj