deb-murano/murano/common/wsgi.py

# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Utility methods for working with WSGI servers."""

import datetime
import errno
import re
import socket
import sys
import time
from xml.dom import minidom
from xml.parsers import expat

import eventlet
import eventlet.wsgi
import jsonschema
from oslo_config import cfg
from oslo_log import log as logging
from oslo_serialization import jsonutils
from oslo_service import service
from oslo_service import sslutils
import routes
import routes.middleware
import six
import webob.dec
import webob.exc

from murano.api.v1 import validation_schemas
from murano.common import config
from murano.common import exceptions
from murano.common.i18n import _, _LE, _LW
from murano.common import xmlutils

eventlet.patcher.monkey_patch(all=False, socket=True)

wsgi_opts = [
    cfg.IntOpt('backlog',
               default=4096,
               help="Number of backlog requests to configure the socket with"),
    cfg.IntOpt('tcp_keepidle',
               default=600,
               help="Sets the value of TCP_KEEPIDLE in seconds for each "
                    "server socket. Not supported on OS X."),
    cfg.IntOpt('max_header_line',
               default=16384,
               help="Maximum line size of message headers to be accepted. "
                    "max_header_line may need to be increased when using "
                    "large tokens (typically those generated by the "
                    "Keystone v3 API with big service catalogs)."),
]

CONF = cfg.CONF
CONF.register_opts(wsgi_opts)

LOG = logging.getLogger(__name__)


class Service(service.Service):
    """Provides a Service API for wsgi servers.

    This gives us the ability to launch wsgi servers with the
    Launcher classes in oslo_service.service.py.
    """

    def __init__(self, application, port,
                 host='0.0.0.0', backlog=4096, threads=1000):
        self.application = application
        self._port = port
        self._host = host
        self._backlog = backlog if backlog else CONF.backlog
        self._logger = logging.getLogger('eventlet.wsgi')
        self.greenthread = None
        config.set_middleware_defaults()
        super(Service, self).__init__(threads)

    def _get_socket(self, host, port, backlog):
        # TODO(dims): eventlet's green dns/socket module does not actually
        # support IPv6 in getaddrinfo(). We need to get around this in the
        # future or monitor upstream for a fix
        info = socket.getaddrinfo(host,
                                  port,
                                  socket.AF_UNSPEC,
                                  socket.SOCK_STREAM)[0]
        family = info[0]
        bind_addr = info[-1]

        sock = None
        retry_until = time.time() + 30
        while not sock and time.time() < retry_until:
            try:
                sock = eventlet.listen(bind_addr,
                                       backlog=backlog,
                                       family=family)
                if sslutils.is_enabled(CONF):
                    sock = sslutils.wrap(CONF, sock)

            except socket.error as err:
                if err.args[0] != errno.EADDRINUSE:
                    raise
                eventlet.sleep(0.1)
        if not sock:
            raise RuntimeError(_("Could not bind to %(host)s:%(port)s "
                                 "after trying for 30 seconds: Address"
                                 " already in use.") %
                               {'host': host, 'port': port})
        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        # sockets can hang around forever without keepalive
        sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)

        # This option isn't available in the OS X version of eventlet
        if hasattr(socket, 'TCP_KEEPIDLE'):
            sock.setsockopt(socket.IPPROTO_TCP,
                            socket.TCP_KEEPIDLE,
                            CONF.tcp_keepidle)

        return sock

    def start(self):
        """Start serving this service using the provided server instance.

        :returns: None

        """
        super(Service, self).start()
        self._socket = self._get_socket(self._host, self._port, self._backlog)
        self.greenthread = eventlet.spawn(
            self._run, self.application, self._socket)

    @property
    def backlog(self):
        return self._backlog

    @property
    def host(self):
        return self._socket.getsockname()[0] if self._socket else self._host

    @property
    def port(self):
        return self._socket.getsockname()[1] if self._socket else self._port

    def stop(self):
        """Stop serving this API.

        :returns: None

        """
        super(Service, self).stop()
        if self.greenthread is not None:
            self.greenthread.kill()

    def reset(self):
        super(Service, self).reset()
        logging.setup(cfg.CONF, 'murano')

    def _run(self, application, socket):
        """Start a WSGI server in a new green thread."""
        eventlet.wsgi.MAX_HEADER_LINE = CONF.max_header_line
        eventlet.wsgi.server(socket,
                             application,
                             custom_pool=self.tg.pool,
                             log=self._logger)


class Middleware(object):
    """Base WSGI middleware wrapper.

    These classes require an application to be
    initialized that will be called next.  By default the middleware will
    simply call its wrapped app, or you can override __call__ to customize its
    behavior.
    """

    def __init__(self, application):
        self.application = application

    def process_request(self, req):
        """Called on each request.

        If this returns None, the next application down the stack will be
        executed. If it returns a response then that response will be returned
        and execution will stop here.
        """
        return None

    def process_response(self, response):
        """Do whatever you'd like to the response."""
        return response

    @webob.dec.wsgify
    def __call__(self, req):
        response = self.process_request(req)
        if response:
            return response
        response = req.get_response(self.application)
        return self.process_response(response)


class Debug(Middleware):
    """Helper class to get info about request and response

    Helper class that can be inserted into any WSGI application chain
    to get information about the request and response.
    """

    @webob.dec.wsgify
    def __call__(self, req):
        print(("*" * 40) + " REQUEST ENVIRON")
        for key, value in req.environ.items():
            print(key, "=", value)
        print("")
        resp = req.get_response(self.application)

        print(("*" * 40) + " RESPONSE HEADERS")
        for (key, value) in six.iteritems(resp.headers):
            print(key, "=", value)
        print("")

        resp.app_iter = self.print_generator(resp.app_iter)

        return resp

    @staticmethod
    def print_generator(app_iter):
        """Prints the contents of a wrapper string iterator

        Iterator that prints the contents of a wrapper string iterator
        when iterated.
        """
        print(("*" * 40) + " BODY")
        for part in app_iter:
            sys.stdout.write(part)
            sys.stdout.flush()
            yield part
        print("")


class Router(object):
    """WSGI middleware that maps incoming requests to WSGI apps."""

    def __init__(self, mapper):
        """Create a router for the given routes.Mapper.

        Each route in `mapper` must specify a 'controller', which is a
        WSGI app to call.  You'll probably want to specify an 'action' as
        well and have your controller be a wsgi.Controller, who will route
        the request to the action method.

        Examples:
          mapper = routes.Mapper()
          sc = ServerController()

          # Explicit mapping of one route to a controller+action
          mapper.connect(None, "/svrlist", controller=sc, action="list")

          # Actions are all implicitly defined
          mapper.resource("server", "servers", controller=sc)

          # Pointing to an arbitrary WSGI app.  You can specify the
          # {path_info:.*} parameter so the target app can be handed just that
          # section of the URL.
          mapper.connect(None, "/v1.0/{path_info:.*}", controller=BlogApp())
        """
        self.map = mapper
        self._router = routes.middleware.RoutesMiddleware(self._dispatch,
                                                          self.map)

    @webob.dec.wsgify
    def __call__(self, req):
        """Route the incoming request to a controller

           Route the incoming request to a controller based on self.map.
           If no match, return a 404.
        """
        return self._router

    @staticmethod
    @webob.dec.wsgify
    def _dispatch(req):
        """Looks for routed WSGI app's response

           Called by self._router after matching the incoming request to
           a route and putting the information into req.environ.
           Either returns 404 or the routed WSGI app's response.
        """
        match = req.environ['wsgiorg.routing_args'][1]
        if not match:
            return webob.exc.HTTPNotFound()
        app = match['controller']
        return app


class Request(webob.Request):
    """Add some OpenStack API-specific logic to the base webob.Request."""

    default_request_content_types = ('application/json',
                                     'application/xml',
                                     'application/murano-packages-json-patch',
                                     'application/env-model-json-patch',
                                     'multipart/form-data')
    default_accept_types = ('application/json',
                            'application/xml',
                            'application/octet-stream')

    def best_match_content_type(self, action, supported_content_types=None,
                                specific_content_types=None):
        """Determine the requested response content-type.

        Based on the query extension then the Accept header.
        Raise NotAcceptableContentType exception if we don't find a preference

        """
        supported_content_types = (supported_content_types or
                                   self.default_accept_types)

        parts = self.path.rsplit('.', 1)
        if len(parts) > 1:
            ctype = 'application/{0}'.format(parts[1])
            if ctype in supported_content_types:
                return ctype

        if specific_content_types and action in specific_content_types:
            bm = self.accept.best_match(specific_content_types[action])
        else:
            bm = self.accept.best_match(supported_content_types)

        if not bm:
            raise exceptions.NotAcceptableContentType(content_type=self.accept)
        return bm

    def get_content_type(self, allowed_content_types=None):
        """Determine content type of the request body.

        Does not do any body introspection, only checks header

        """
        if "Content-Type" not in self.headers:
            return None

        content_type = self.content_type
        allowed_content_types = (allowed_content_types or
                                 self.default_request_content_types)

        if content_type not in allowed_content_types:
            raise exceptions.UnsupportedContentType(content_type=content_type)
        return content_type


class ResourceExceptionHandler(object):
    """Context manager to handle Resource exceptions.

    Used when processing exceptions generated by API implementation
    methods. Converts most exceptions to webob exceptions, with the
    appropriate logging.
    """

    def __enter__(self):
        return None

    def __exit__(self, ex_type, ex_value, ex_traceback):
        if not ex_value:
            return True

        # TODO(lin.a.yang): current only handle TypeError here, we should
        # process other kind of internal exceptions generated by API and
        # convert to webob exceptions.
        if isinstance(ex_value, TypeError):
            exc_info = (ex_type, ex_value, ex_traceback)
            LOG.error(_LE("Exception handling resource: {0}").format(ex_value),
                      exc_info=exc_info)
            raise webob.exc.HTTPBadRequest()

        # We didn't handle this kind of exception
        return False


class Resource(object):
    """WSGI app that handles (de)serialization and controller dispatch.

    Reads routing information supplied by RoutesMiddleware and calls
    the requested action method upon its deserializer, controller,
    and serializer. Those three objects may implement any of the basic
    controller action methods (create, update, show, index, delete)
    along with any that may be specified in the api router. A 'default'
    method may also be implemented to be used in place of any
    non-implemented actions. Deserializer methods must accept a request
    argument and return a dictionary. Controller methods must accept a
    request argument. Additionally, they must also accept keyword
    arguments that represent the keys returned by the Deserializer. They
    may raise a webob.exc exception or return a dict, which will be
    serialized by requested content type.
    """

    def __init__(self, controller, deserializer=None, serializer=None):
        """Resource init.

        :param controller: object that implement methods created by routes lib
        :param deserializer: object that supports webob request deserialization
                             through controller-like actions
        :param serializer: object that supports webob response serialization
                           through controller-like actions
        """
        self.controller = controller
        self.serializer = serializer or ResponseSerializer()
        self.deserializer = deserializer or RequestDeserializer()

    @webob.dec.wsgify(RequestClass=Request)
    def __call__(self, request):
        """WSGI method that controls (de)serialization and method dispatch."""

        LOG.debug("{method} {url}\nHEADERS: {headers}".format(
                  method=request.method,
                  url=request.url,
                  headers=self._format_request_headers(request.headers)))

        try:
            action, action_args, accept = self.deserialize_request(request)
        except exceptions.UnsupportedContentType:
            msg = _("Unsupported Content-Type")
            return webob.exc.HTTPUnsupportedMediaType(detail=msg)
        except exceptions.NotAcceptableContentType:
            msg = _("Acceptable response can not be provided")
            return webob.exc.HTTPNotAcceptable(detail=msg)
        except exceptions.MalformedRequestBody:
            msg = _("Malformed request body")
            return webob.exc.HTTPBadRequest(explanation=msg)

        with ResourceExceptionHandler():
            action_result = self.execute_action(action, request, **action_args)

        try:
            return self.serialize_response(action, action_result, accept)
        # return unserializable result (typically a webob exc)
        except Exception:
            return action_result

    def deserialize_request(self, request):
        return self.deserializer.deserialize(request)

    def serialize_response(self, action, action_result, accept):
        return self.serializer.serialize(action_result, accept, action)

    def execute_action(self, action, request, **action_args):
        return self.dispatch(self.controller, action, request, **action_args)

    def dispatch(self, obj, action, *args, **kwargs):
        """Find action-specific method on self and call it."""
        try:
            method = getattr(obj, action)
        except AttributeError:
            method = getattr(obj, 'default')
        return method(*args, **kwargs)

    def get_action_args(self, request_environment):
        """Parse dictionary created by routes library."""
        try:
            args = request_environment['wsgiorg.routing_args'][1].copy()
        except Exception:
            return {}

        try:
            del args['controller']
        except KeyError:
            pass

        try:
            del args['format']
        except KeyError:
            pass

        return args

    def _format_request_headers(self, headers):
        """Format the request headers to be logged.

        To keep log more clear, only show the X-* header include murano own
        header and several useful headers added by keystone auth middleware,
        and skip other X-* headers.
        """
        string_parts = []

        # Only show following X-* header
        useful_headers = ("X-Configuration-Session",
                          "X-Roles",
                          "X-User-Id",
                          "X-Tenant-Id")

        for header, value in six.iteritems(headers):
            if header.startswith("X-") and header not in useful_headers:
                continue
            string_parts.append("{0}: {1}".format(header, value))

        return ', '.join(string_parts)


class ActionDispatcher(object):
    """Maps method name to local methods through action name."""

    def dispatch(self, *args, **kwargs):
        """Find and call local method."""
        action = kwargs.pop('action', 'default')
        action_method = getattr(self, str(action), self.default)
        return action_method(*args, **kwargs)

    def default(self, data):
        raise NotImplementedError()


class DictSerializer(ActionDispatcher):
    """Default request body serialization."""

    def serialize(self, data, action='default'):
        return self.dispatch(data, action=action)

    def default(self, data):
        return ""


class JSONDictSerializer(DictSerializer):
    """Default JSON request body serialization."""

    def default(self, data, result=None):
        def sanitizer(obj):
            if isinstance(obj, datetime.datetime):
                _dtime = obj - datetime.timedelta(microseconds=obj.microsecond)
                return _dtime.isoformat()
            return six.text_type(obj)
        if result:
            data.body = jsonutils.dump_as_bytes(result)
        return jsonutils.dump_as_bytes(data, default=sanitizer)


class XMLDictSerializer(DictSerializer):
    def __init__(self, metadata=None, xmlns=None):
        """Default XML request body serialization.

        :param metadata: information needed to deserialize xml into
                         a dictionary.
        :param xmlns: XML namespace to include with serialized xml
        """
        super(XMLDictSerializer, self).__init__()
        self.metadata = metadata or {}
        self.xmlns = xmlns

    def default(self, data, result=None):
        # We expect data to contain a single key which is the XML root.
        root_key = list(data.keys())[0]
        doc = minidom.Document()
        node = self._to_xml_node(doc, self.metadata, root_key, data[root_key])

        return self.to_xml_string(node)

    def to_xml_string(self, node, has_atom=False):
        self._add_xmlns(node, has_atom)
        return node.toprettyxml(indent='    ', encoding='UTF-8')

    # NOTE (ameade): the has_atom should be removed after all the
    # xml serializers and view builders have been updated to the current
    # spec that required all responses include the xmlns:atom, the has_atom
    # flag is to prevent current tests from breaking
    def _add_xmlns(self, node, has_atom=False):
        if self.xmlns is not None:
            node.setAttribute('xmlns', self.xmlns)
        if has_atom:
            node.setAttribute('xmlns:atom', "http://www.w3.org/2005/Atom")

    def _to_xml_node(self, doc, metadata, nodename, data):
        """Recursive method to convert data members to XML nodes."""
        result = doc.createElement(nodename)

        # Set the xml namespace if one is specified
        # TODO(justinsb): We could also use prefixes on the keys
        xmlns = metadata.get('xmlns', None)
        if xmlns:
            result.setAttribute('xmlns', xmlns)

        # TODO(bcwaldon): accomplish this without a type-check
        if type(data) is list:
            collections = metadata.get('list_collections', {})
            if nodename in collections:
                metadata = collections[nodename]
                for item in data:
                    node = doc.createElement(metadata['item_name'])
                    node.setAttribute(metadata['item_key'], str(item))
                    result.appendChild(node)
                return result
            singular = metadata.get('plurals', {}).get(nodename, None)
            if singular is None:
                if nodename.endswith('s'):
                    singular = nodename[:-1]
                else:
                    singular = 'item'
            for item in data:
                node = self._to_xml_node(doc, metadata, singular, item)
                result.appendChild(node)
        # TODO(bcwaldon): accomplish this without a type-check
        elif type(data) is dict:
            collections = metadata.get('dict_collections', {})
            if nodename in collections:
                metadata = collections[nodename]
                for k, v in data.items():
                    node = doc.createElement(metadata['item_name'])
                    node.setAttribute(metadata['item_key'], str(k))
                    text = doc.createTextNode(str(v))
                    node.appendChild(text)
                    result.appendChild(node)
                return result
            attrs = metadata.get('attributes', {}).get(nodename, {})
            for k, v in data.items():
                if k in attrs:
                    result.setAttribute(k, str(v))
                else:
                    node = self._to_xml_node(doc, metadata, k, v)
                    result.appendChild(node)
        else:
            # Type is atom
            node = doc.createTextNode(str(data))
            result.appendChild(node)
        return result

    def _create_link_nodes(self, xml_doc, links):
        link_nodes = []
        for link in links:
            link_node = xml_doc.createElement('atom:link')
            link_node.setAttribute('rel', link['rel'])
            link_node.setAttribute('href', link['href'])
            if 'type' in link:
                link_node.setAttribute('type', link['type'])
            link_nodes.append(link_node)
        return link_nodes


class BlankSerializer(DictSerializer):
    """Return raw data."""
    def default(self, data):
        return data


class ResponseHeadersSerializer(ActionDispatcher):
    """Default response headers serialization."""

    def serialize(self, response, data, action):
        self.dispatch(response, data, action=action)

    def default(self, response, data):
        response.status_int = 200


class ResponseSerializer(object):
    """Encode the necessary pieces into a response object."""

    def __init__(self, body_serializers=None, headers_serializer=None):
        self.body_serializers = {
            'application/xml': XMLDictSerializer(),
            'application/json': JSONDictSerializer(),
            'text/plain': BlankSerializer(),
            'application/octet-stream': BlankSerializer()
        }
        self.body_serializers.update(body_serializers or {})

        self.headers_serializer = (headers_serializer or
                                   ResponseHeadersSerializer())

    def serialize(self, response_data, content_type, action='default'):
        """Serialize a dict into a string and wrap in a wsgi.Request object.

        :param response_data: dict produced by the Controller
        :param content_type: expected mimetype of serialized response body

        """
        response = webob.Response()
        self.serialize_headers(response, response_data, action)
        self.serialize_body(response, response_data, content_type, action)
        return response

    def serialize_headers(self, response, data, action):
        self.headers_serializer.serialize(response, data, action)

    def serialize_body(self, response, data, content_type, action):
        response.headers['Content-Type'] = content_type
        if data is not None:
            serializer = self.get_body_serializer(content_type)
            response.body = serializer.serialize(data, action)

    def get_body_serializer(self, content_type):
        try:
            return self.body_serializers[content_type]
        except (KeyError, TypeError):
            raise exceptions.UnsupportedContentType(content_type=content_type)


class ServiceBrokerResponseSerializer(ResponseSerializer):
    def __init__(self):
        super(ServiceBrokerResponseSerializer, self).__init__()

    def serialize(self, response_data, content_type, action='default'):
        if isinstance(response_data, webob.Response):
            response = response_data
            self.serialize_body(response, response.data, content_type, action)
        else:
            response = super(ServiceBrokerResponseSerializer, self).serialize(
                response_data, content_type, action='default')
        return response


class RequestHeadersDeserializer(ActionDispatcher):
    """Default request headers deserializer."""

    def deserialize(self, request, action):
        return self.dispatch(request, action=action)

    def default(self, request):
        return {}


class RequestDeserializer(object):
    """Break up a Request object into more useful pieces."""

    def __init__(self, body_deserializers=None, headers_deserializer=None,
                 supported_content_types=None, specific_content_types=None):

        self.supported_content_types = supported_content_types
        self.specific_content_types = specific_content_types

        self.body_deserializers = {
            'application/xml': XMLDeserializer(),
            'application/json': JSONDeserializer(),
            'application/murano-packages-json-patch':
                MuranoPackageJSONPatchDeserializer(),
            'application/env-model-json-patch':
                EnvModelJSONPatchDeserializer(),
            'multipart/form-data': FormDataDeserializer()
        }
        self.body_deserializers.update(body_deserializers or {})

        self.headers_deserializer = (headers_deserializer or
                                     RequestHeadersDeserializer())

    def deserialize(self, request):
        """Extract necessary pieces of the request.

        :param request: Request object
        :returns: tuple of (expected controller action name, dictionary of
                  keyword arguments to pass to the controller, the expected
                  content type of the response)

        """
        action_args = self.get_action_args(request.environ)
        action = action_args.pop('action', None)

        action_args.update(self.deserialize_headers(request, action))
        action_args.update(self.deserialize_body(request, action))

        accept = self.get_expected_content_type(request, action)

        return (action, action_args, accept)

    def deserialize_headers(self, request, action):
        return self.headers_deserializer.deserialize(request, action)

    def deserialize_body(self, request, action):
        if not len(request.body) > 0:
            LOG.debug("Empty body provided in request")
            return {}

        try:
            content_type = request.get_content_type()
        except exceptions.UnsupportedContentType as e:
            LOG.error(_LE("Unrecognized Content-Type provided in request: "
                          "{error}").format(error=str(e)))
            raise

        if content_type is None:
            LOG.debug("No Content-Type provided in request")
            return {}

        try:
            deserializer = self.get_body_deserializer(content_type)
        except exceptions.UnsupportedContentType:
            LOG.debug("Unable to deserialize body as provided Content-Type")
            raise

        return deserializer.deserialize(request, action)

    def get_body_deserializer(self, content_type):
        try:
            return self.body_deserializers[content_type]
        except (KeyError, TypeError):
            raise exceptions.UnsupportedContentType(content_type=content_type)

    def get_expected_content_type(self, request, action):
        return request.best_match_content_type(action,
                                               self.supported_content_types,
                                               self.specific_content_types)

    def get_action_args(self, request_environment):
        """Parse dictionary created by routes library."""
        try:
            args = request_environment['wsgiorg.routing_args'][1].copy()
        except Exception:
            return {}

        try:
            del args['controller']
        except KeyError:
            pass

        try:
            del args['format']
        except KeyError:
            pass

        return args


class TextDeserializer(ActionDispatcher):
    """Default request body deserialization."""

    def deserialize(self, request, action='default'):
        return self.dispatch(request, action=action)

    def default(self, request):
        return {}


class JSONDeserializer(TextDeserializer):
    def _from_json(self, datastring):
        try:
            return jsonutils.loads(datastring)
        except ValueError:
            msg = _("cannot understand JSON")
            raise exceptions.MalformedRequestBody(reason=msg)

    def default(self, request):
        datastring = request.body
        return {'body': self._from_json(datastring)}


class JSONPatchDeserializer(TextDeserializer):
    allowed_operations = {}
    schema = None
    allow_unknown_path = False

    def _from_json_patch(self, datastring):
        try:
            operations = jsonutils.loads(datastring)
        except ValueError:
            msg = _("cannot understand JSON")
            raise exceptions.MalformedRequestBody(reason=msg)

        if not isinstance(operations, list):
            msg = _('JSON-patch must be a list.')
            raise webob.exc.HTTPBadRequest(explanation=msg)

        changes = []
        for raw_change in operations:
            if not isinstance(raw_change, dict):
                msg = _('Operations must be JSON objects.')
                raise webob.exc.HTTPBadRequest(explanation=msg)

            (op, path) = self._parse_json_schema_change(raw_change)

            self._validate_path(path)
            change = {'op': op, 'path': path}

            change['value'] = self._get_change_value(raw_change, op)
            self._validate_change(change)

            changes.append(change)
        return changes

    def _get_change_value(self, raw_change, op):
        if 'value' not in raw_change:
            msg = _('Operation "%s" requires a member named "value".')
            raise webob.exc.HTTPBadRequest(explanation=msg % op)
        return raw_change['value']

    def _get_change_operation(self, raw_change):
        try:
            return raw_change['op']
        except KeyError:
            msg = _("Unable to find '%s' in JSON Schema change") % 'op'
            raise webob.exc.HTTPBadRequest(explanation=msg)

    def _get_change_path(self, raw_change):
        try:
            return raw_change['path']
        except KeyError:
            msg = _("Unable to find '%s' in JSON Schema change") % 'path'
            raise webob.exc.HTTPBadRequest(explanation=msg)

    def _validate_change(self, change):
        self._validate_allowed_methods(change, self.allow_unknown_path)

        if self.schema:
            self._validate_schema(change)

    def _validate_allowed_methods(self, change, allow_unknown_path=False):
        full_path = '/'.join(change['path'])
        change_op = change['op']
        allowed_methods = self.allowed_operations.get(full_path)

        if allowed_methods is None:
            if allow_unknown_path:
                allowed_methods = ['add', 'replace', 'remove']
            else:
                msg = _("Attribute '{0}' is invalid").format(full_path)
                raise webob.exc.HTTPForbidden(explanation=six.text_type(msg))

        if change_op not in allowed_methods:
            ops = ', '.join(allowed_methods) if allowed_methods\
                else 'no operations'
            msg = _("Method '{method}' is not allowed for a path with name "
                    "'{name}'. Allowed operations are: "
                    "{ops}").format(method=change_op, name=full_path, ops=ops)

            raise webob.exc.HTTPForbidden(explanation=six.text_type(msg))

    def _validate_schema(self, change):
        property_to_update = change['value']
        can_validate = True
        schema = self.schema
        for p in change['path']:
            if schema['type'] == 'array':
                try:
                    schema = schema['items']
                except KeyError:
                    can_validate = False
            elif schema['type'] == 'object':
                try:
                    schema = schema['properties'][p]
                except KeyError:
                    can_validate = False
        if can_validate:
            try:
                jsonschema.validate(property_to_update, schema)
            except jsonschema.ValidationError as e:
                LOG.error(_LE("Schema validation error occurred: %s"), e)
                raise webob.exc.HTTPBadRequest(explanation=e.message)

    def _decode_json_pointer(self, pointer):
        """Parse a json pointer.

            Json Pointers are defined in
            http://tools.ietf.org/html/draft-pbryan-zyp-json-pointer .
            The pointers use '/' for separation between object attributes, such
            that '/A/B' would evaluate to C in {"A": {"B": "C"}}. A '/'
            character
            in an attribute name is encoded as "~1" and a '~' character is
            encoded
            as "~0".
            """
        self._validate_json_pointer(pointer)
        ret = []
        for part in pointer.lstrip('/').split('/'):
            ret.append(part.replace('~1', '/').replace('~0', '~').strip())
        return ret

    def _validate_json_pointer(self, pointer):
        """Validate a json pointer.

        Only limited form of json pointers is accepted.
        """
        if not pointer.startswith('/'):
            msg = _('Pointer `%s` does not start with "/".') % pointer
            raise webob.exc.HTTPBadRequest(explanation=msg)
        if re.search('/\s*?/', pointer[1:]):
            msg = _('Pointer `%s` contains adjacent "/".') % pointer
            raise webob.exc.HTTPBadRequest(explanation=msg)
        if len(pointer) > 1 and pointer.endswith('/'):
            msg = _('Pointer `%s` ends with "/".') % pointer
            raise webob.exc.HTTPBadRequest(explanation=msg)
        if pointer[1:].strip() == '/':
            msg = _('Pointer `%s` does not contain a valid token.') % pointer
            raise webob.exc.HTTPBadRequest(explanation=msg)
        if re.search('~[^01]', pointer) or pointer.endswith('~'):
            msg = _('Pointer `%s` contains "~", which is not part of'
                    ' a recognized escape sequence.') % pointer
            raise webob.exc.HTTPBadRequest(explanation=msg)

    def _parse_json_schema_change(self, raw_change):
        op = self._get_change_operation(raw_change)
        path = self._get_change_path(raw_change)

        path_list = self._decode_json_pointer(path)
        return op, path_list

    def _validate_path(self, path):
        pass

    def default(self, request):
        return {'body': self._from_json_patch(request.body)}


class MuranoPackageJSONPatchDeserializer(JSONPatchDeserializer):
    allowed_operations = {"categories": ["add", "replace", "remove"],
                          "tags": ["add", "replace", "remove"],
                          "is_public": ["replace"],
                          "enabled": ["replace"],
                          "description": ["replace"],
                          "name": ["replace"]}
    allow_unknown_path = False
    schema = validation_schemas.PKG_UPDATE_SCHEMA

    def _validate_path(self, path):
        if len(path) > 1:
            msg = _('Nested paths are not allowed')
            raise webob.exc.HTTPBadRequest(explanation=msg)


class EnvModelJSONPatchDeserializer(JSONPatchDeserializer):
    allowed_operations = {"": [],
                          "defaultNetworks": ["replace"],
                          "defaultNetworks/environment": ["replace"],
                          "defaultNetworks/environment/?/id": [],
                          "defaultNetworks/flat": ["replace"],
                          "name": ["replace"],
                          "region": ["replace"],
                          "?/type": ["replace"],
                          "?/id": []
                          }
    allow_unknown_path = True
    schema = validation_schemas.ENV_SCHEMA


class XMLDeserializer(TextDeserializer):
    def __init__(self, metadata=None):
        """XMLDeserializer.

        :param metadata: information needed to deserialize xml into
                         a dictionary.
        """
        super(XMLDeserializer, self).__init__()
        self.metadata = metadata or {}

    def _from_xml(self, request):
        datastring = request.body
        plurals = set(self.metadata.get('plurals', {}))

        try:
            node = xmlutils.safe_minidom_parse_string(datastring).childNodes[0]
            return {node.nodeName: self._from_xml_node(node, plurals)}
        except expat.ExpatError:
            msg = _("cannot understand XML")
            raise exceptions.MalformedRequestBody(reason=msg)

    def _from_xml_node(self, node, listnames):
        """Convert a minidom node to a simple Python type.

        :param listnames: list of XML node names whose subnodes should
                          be considered list items.

        """

        if len(node.childNodes) == 1 and node.childNodes[0].nodeType == 3:
            return node.childNodes[0].nodeValue
        elif node.nodeName in listnames:
            return [self._from_xml_node(n, listnames) for n in node.childNodes]
        else:
            result = dict()
            for attr in node.attributes.keys():
                result[attr] = node.attributes[attr].nodeValue
            for child in node.childNodes:
                if child.nodeType != node.TEXT_NODE:
                    result[child.nodeName] = self._from_xml_node(child,
                                                                 listnames)
            return result

    def find_first_child_named(self, parent, name):
        """Find first child which has the given name of a node."""
        for node in parent.childNodes:
            if node.nodeName == name:
                return node
        return None

    def find_children_named(self, parent, name):
        """Return all children of a node with the given name."""
        for node in parent.childNodes:
            if node.nodeName == name:
                yield node

    def extract_text(self, node):
        """Get the text field contained by the given node."""
        if len(node.childNodes) == 1:
            child = node.childNodes[0]
            if child.nodeType == child.TEXT_NODE:
                return child.nodeValue
        return ""

    def default(self, datastring):
        return {'body': self._from_xml(datastring)}


class FormDataDeserializer(TextDeserializer):
    def _from_json(self, datastring):
        value = datastring
        try:
            LOG.debug("Trying to deserialize '{data}' to json".format(
                data=datastring))
            value = jsonutils.loads(datastring)
        except ValueError:
            LOG.warning(_LW("Unable to deserialize to json, using raw text"))
        return value

    def default(self, request):
        form_data_parts = request.POST
        for key, value in six.iteritems(form_data_parts):
            if isinstance(value, six.string_types):
                form_data_parts[key] = self._from_json(value)
        return {'body': form_data_parts}