Implement upload call to the repository API

* Add multiform/part-data to the allowed content type
* Get json and file from request

Partially-implements blueprint murano-repository-api-v2
Implements blueprint publish-app-to-catalog
Change-Id: I2ed8887a235739f6316695eb13879cc494f2f042

muranoapi/api/v1/__init__.py

@@ -25,6 +25,14 @@ SEARCH_MAPPING = {'fqn': 'fully_qualified_name',
                   'name': 'name',
                   'created': 'created'
                   }
+PKG_PARAMS_MAP = {'display_name': 'name',
+                  'full_name': 'fully_qualified_name',
+                  'raw_ui': 'ui_definition',
+                  'logo': 'logo',
+                  'package_type': 'type',
+                  'description': 'description',
+                  'author': 'author',
+                  'classes': 'class_definition'}
 
 
 def get_draft(environment_id=None, session_id=None):

muranoapi/api/v1/catalog.py

@@ -13,16 +13,23 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import cgi
+import jsonschema
+import tempfile
+
 from oslo.config import cfg
+from sqlalchemy import exc as sql_exc
 from webob import exc
 
 import muranoapi.api.v1
+from muranoapi.api.v1 import schemas
 from muranoapi.db.catalog import api as db_api
 from muranoapi.openstack.common import exception
 from muranoapi.openstack.common.gettextutils import _  # noqa
 from muranoapi.openstack.common import log as logging
 from muranoapi.openstack.common import wsgi
-
+from muranoapi.packages import application_package as app_pkg
+from muranoapi.packages import exceptions as pkg_exc
 
 LOG = logging.getLogger(__name__)
 CONF = cfg.CONF
@@ -30,6 +37,7 @@ CONF = cfg.CONF
 SUPPORTED_PARAMS = muranoapi.api.v1.SUPPORTED_PARAMS
 LIST_PARAMS = muranoapi.api.v1.LIST_PARAMS
 ORDER_VALUES = muranoapi.api.v1.ORDER_VALUES
+PKG_PARAMS_MAP = muranoapi.api.v1.PKG_PARAMS_MAP
 
 
 def _check_content_type(req, content_type):
@@ -61,10 +69,34 @@ def _get_filters(query_params):
                 LOG.warning(_("Value of 'order_by' parameter is not valid. "
                               "Allowed values are: {0}. Skipping it.").format(
                             ", ".join(ORDER_VALUES)))
-
     return filters
 
 
+def _validate_body(body):
+    if len(body.keys()) != 2:
+        msg = "multipart/form-data request should contain " \
+              "two parts: json and tar.gz archive"
+        LOG.error(msg)
+        raise exc.HTTPBadRequest(msg)
+    file_obj = None
+    package_meta = None
+    for part in body.values():
+        if isinstance(part, cgi.FieldStorage):
+            file_obj = part
+            # dict if json deserialized successfully
+        if isinstance(part, dict):
+            package_meta = part
+    if file_obj is None:
+        msg = _("There is no file package with application description")
+        LOG.error(msg)
+        raise exc.HTTPBadRequest(msg)
+    if package_meta is None:
+        msg = _("There is no json with meta information about package")
+        LOG.error(msg)
+        raise exc.HTTPBadRequest(msg)
+    return file_obj, package_meta
+
+
 class Controller(object):
     """
         WSGI controller for application catalog resource in Murano v1 API
@@ -100,6 +132,46 @@ class Controller(object):
         packages = db_api.package_search(filters, req.context)
         return {"packages": [package.to_dict() for package in packages]}
 
+    def upload(self, req, body=None):
+        """
+        Upload new file archive for the new package
+        together with package metadata
+        """
+        _check_content_type(req, 'multipart/form-data')
+        file_obj, package_meta = _validate_body(body)
+        try:
+            jsonschema.validate(package_meta, schemas.PKG_UPLOAD_SCHEMA)
+        except jsonschema.ValidationError as e:
+            LOG.exception(e)
+            raise exc.HTTPBadRequest(explanation=e.message)
+
+        with tempfile.NamedTemporaryFile() as tempf:
+            content = file_obj.file.read()
+            if not content:
+                msg = _("Uploading file can't be empty")
+                raise exc.HTTPBadRequest(msg)
+            tempf.write(content)
+            package_meta['archive'] = content
+            try:
+                pkg_to_upload = app_pkg.load_from_file(tempf.name,
+                                                       target_dir=None,
+                                                       drop_dir=True)
+            except pkg_exc.PackageLoadError as e:
+                LOG.exception(e)
+                raise exc.HTTPBadRequest(e.message)
+
+        # extend dictionary for update db
+        for k, v in PKG_PARAMS_MAP.iteritems():
+            if hasattr(pkg_to_upload, k):
+                package_meta[v] = getattr(pkg_to_upload, k)
+        try:
+            package = db_api.package_upload(package_meta, req.context.tenant)
+        except sql_exc.SQLAlchemyError:
+            msg = _('Unable to save package in database')
+            LOG.exception(msg)
+            raise exc.HTTPServerError(msg)
+        return package.to_dict()
+
 
 def create_resource():
     return wsgi.Resource(Controller())

muranoapi/api/v1/router.py

@@ -133,4 +133,8 @@ class API(wsgi.Router):
                        controller=catalog_resource,
                        action='search',
                        conditions={'method': ['GET']})
+        mapper.connect('/catalog/packages',
+                       controller=catalog_resource,
+                       action='upload',
+                       conditions={'method': ['POST']})
         super(API, self).__init__(mapper)

muranoapi/api/v1/schemas.py

@@ -24,6 +24,32 @@ ENV_SCHEMA = {
     "required": ["id", "name"]
 }
 
+PKG_UPLOAD_SCHEMA = {
+    "$schema": "http://json-schema.org/draft-04/schema#",
+
+    "type": "object",
+    "properties": {
+        "tags": {
+            "type": "array",
+            "minItems": 1,
+            "items": {"type": "string"},
+            "uniqueItems": True
+        },
+        "categories": {
+            "type": "array",
+            "minItems": 1,
+            "items": {"type": "string"},
+            "uniqueItems": True
+        },
+        "description": {"type": "string"},
+        "name": {"type": "string"},
+        "is_public": {"type": "boolean"},
+        "enabled": {"type": "boolean"}
+    },
+    "required": ["categories"],
+    "additionalProperties": False
+}
+
 PKG_UPDATE_SCHEMA = {
     "$schema": "http://json-schema.org/draft-04/schema#",
 

muranoapi/db/catalog/api.py

@@ -26,11 +26,6 @@ SEARCH_MAPPING = muranoapi.api.v1.SEARCH_MAPPING
 LOG = logging.getLogger(__name__)
 
 
-def get_category_by_name(name):
-    session = db_session.get_session()
-    return session.query(models.Category).filter_by(name=name).first()
-
-
 def category_get_names():
     session = db_session.get_session()
     categories = []
@@ -40,11 +35,6 @@ def category_get_names():
     return categories
 
 
-def get_tag_by_name(name):
-    session = db_session.get_session()
-    return session.query(models.Tag).filter_by(name=name).first()
-
-
 def _package_get(package_id, session):
     package = session.query(models.Package).get(package_id)
     if not package:
@@ -83,15 +73,18 @@ def package_get(package_id, context):
     return package
 
 
-def _get_categories(category_names):
+def _get_categories(category_names, session=None):
     """
     Return existing category objects or raise an exception
     :param category_names: name of categories to associate with package, list
     :returns: list of Category objects to associate with package, list
     """
+    if session is None:
+        session = db_session.get_session()
     categories = []
     for ctg_name in category_names:
-        ctg_obj = get_category_by_name(ctg_name)
+        ctg_obj = session.query(models.Category).filter_by(
+            name=ctg_name).first()
         if not ctg_obj:
             # it's not allowed to specify non-existent categories
             raise exc.HTTPBadRequest(
@@ -100,24 +93,39 @@ def _get_categories(category_names):
     return categories
 
 
-def _get_tags(tag_names):
+def _get_tags(tag_names, session=None):
     """
     Return existing tags object or create new ones
     :param tag_names: name of tags to associate with package, list
     :returns: list of Tag objects to associate with package, list
     """
+    if session is None:
+        session = db_session.get_session()
     tags = []
-    if tag_names:
+    for tag_name in tag_names:
-        for tag_name in tag_names:
+        tag_obj = session.query(models.Tag).filter_by(name=tag_name).first()
-            tag_obj = get_tag_by_name(tag_name)
+        if tag_obj:
-            if tag_obj:
+            tags.append(tag_obj)
-                tags.append(tag_obj)
+        else:
-            else:
+            tag_record = models.Tag(name=tag_name)
-                tag_record = models.Tag(name=tag_name)
+            tags.append(tag_record)
-                tags.append(tag_record)
     return tags
 
 
+def _get_class_definitions(class_names, session):
+    if session is None:
+        session = db_session.get_session()
+    classes = []
+    for name in class_names:
+        class_obj = session.query(models.Class).filter_by(name=name).first()
+        if class_obj:
+            classes.append(class_obj)
+        else:
+            class_record = models.Class(name=name)
+            classes.append(class_record)
+    return classes
+
+
 def _do_replace(package, change):
     path = change['path'][0]
     value = change['value']
@@ -199,13 +207,12 @@ def package_update(pkg_id, changes, context):
                          'replace': _do_replace,
                          'remove': _do_remove}
     session = db_session.get_session()
-    pkg = _package_get(pkg_id, session)
-    _authorize_package(pkg, context)
-
-    for change in changes:
-        pkg = operation_methods[change['op']](pkg, change)
-
     with session.begin():
+        pkg = _package_get(pkg_id, session)
+        _authorize_package(pkg, context)
+
+        for change in changes:
+            pkg = operation_methods[change['op']](pkg, change)
         session.add(pkg)
     return pkg
 
@@ -309,3 +316,28 @@ def package_search(filters, context):
         query = query.limit(limit)
 
     return query.all()
+
+
+def package_upload(values, tenant_id):
+    """
+    Upload a package with new application
+    :param values: parameters describing the new package
+    :returns: detailed information about new package, dict
+    """
+    session = db_session.get_session()
+    package = models.Package()
+
+    composite_attr_to_func = {'categories': _get_categories,
+                              'tags': _get_tags,
+                              'class_definition': _get_class_definitions}
+    with session.begin():
+        for attr, func in composite_attr_to_func.iteritems():
+            if values.get(attr):
+                result = func(values[attr], session)
+                setattr(package, attr, result)
+                del values[attr]
+
+        package.update(values)
+        package.owner_id = tenant_id
+        package.save(session)
+    return package

muranoapi/db/models.py

@@ -263,12 +263,12 @@ class Package(BASE, ModelBase):
                             'archive',
                             'logo',
                             'ui_definition']
-        nested_objects = ['categories', 'tags']
+        nested_objects = ['categories', 'tags', 'class_definition']
         for key in not_serializable:
             if key in d.keys():
                 del d[key]
         for key in nested_objects:
-            d[key] = [a.name for a in d.get(key)]
+            d[key] = [a.name for a in d.get(key, [])]
         return d
 
 

muranoapi/openstack/common/wsgi.py

@@ -296,7 +296,8 @@ class Request(webob.Request):
 
     default_request_content_types = ('application/json',
                                      'application/xml',
-                                     'application/murano-packages-json-patch')
+                                     'application/murano-packages-json-patch',
+                                     'multipart/form-data')
     default_accept_types = ('application/json', 'application/xml')
     default_accept_type = 'application/json'
 
@@ -632,7 +633,8 @@ class RequestDeserializer(object):
         self.body_deserializers = {
             'application/xml': XMLDeserializer(),
             'application/json': JSONDeserializer(),
-            'application/murano-packages-json-patch': JSONPatchDeserializer()
+            'application/murano-packages-json-patch': JSONPatchDeserializer(),
+            'multipart/form-data': FormDataDeserializer()
         }
         self.body_deserializers.update(body_deserializers or {})
 
@@ -682,7 +684,7 @@ class RequestDeserializer(object):
             LOG.debug(_("Unable to deserialize body as provided Content-Type"))
             raise
 
-        return deserializer.deserialize(request.body, action)
+        return deserializer.deserialize(request, action)
 
     def get_body_deserializer(self, content_type):
         try:
@@ -716,10 +718,10 @@ class RequestDeserializer(object):
 class TextDeserializer(ActionDispatcher):
     """Default request body deserialization"""
 
-    def deserialize(self, datastring, action='default'):
+    def deserialize(self, request, action='default'):
-        return self.dispatch(datastring, action=action)
+        return self.dispatch(request, action=action)
 
-    def default(self, datastring):
+    def default(self, request):
         return {}
 
 
@@ -731,7 +733,8 @@ class JSONDeserializer(TextDeserializer):
             msg = _("cannot understand JSON")
             raise exception.MalformedRequestBody(reason=msg)
 
-    def default(self, datastring):
+    def default(self, request):
+        datastring = request.body
         return {'body': self._from_json(datastring)}
 
 
@@ -831,7 +834,6 @@ class JSONPatchDeserializer(TextDeserializer):
             ret.append(part.replace('~1', '/').replace('~0', '~').strip())
         return ret
 
-
     def _validate_json_pointer(self, pointer):
         """Validate a json pointer.
 
@@ -866,8 +868,8 @@ class JSONPatchDeserializer(TextDeserializer):
             msg = _('Nested paths are not allowed')
             raise webob.exc.HTTPBadRequest(explanation=msg)
 
-    def default(self, datastring):
+    def default(self, request):
-        return {'body': self._from_json_patch(datastring)}
+        return {'body': self._from_json_patch(request.body)}
 
 
 class XMLDeserializer(TextDeserializer):
@@ -879,7 +881,8 @@ class XMLDeserializer(TextDeserializer):
         super(XMLDeserializer, self).__init__()
         self.metadata = metadata or {}
 
-    def _from_xml(self, datastring):
+    def _from_xml(self, request):
+        datastring = request.body
         plurals = set(self.metadata.get('plurals', {}))
 
         try:
@@ -934,3 +937,22 @@ class XMLDeserializer(TextDeserializer):
 
     def default(self, datastring):
         return {'body': self._from_xml(datastring)}
+
+
+class FormDataDeserializer(TextDeserializer):
+    def _from_json(self, datastring):
+        value = datastring
+        try:
+            LOG.debug(_("Trying deserialize '{0}' to json".format(datastring)))
+            value = jsonutils.loads(datastring)
+        except ValueError:
+            LOG.debug(_("Unable deserialize to json, using raw text"))
+        return value
+
+    def default(self, request):
+        form_data_parts = request.POST
+        result = []
+        for key, value in form_data_parts.iteritems():
+            if isinstance(value, basestring):
+                form_data_parts[key] = self._from_json(value)
+        return {'body': form_data_parts}

muranoapi/packages/application_package.py

@@ -206,6 +206,9 @@ def load_from_file(archive_path, target_dir=None, drop_dir=False):
             raise e.PackageLoadError('Target directory is not empty')
 
     try:
+        if not tarfile.is_tarfile(archive_path):
+            raise e.PackageFormatError("Uploading file should be a"
+                                       " 'tar.gz' archive")
         package = tarfile.open(archive_path)
         package.extractall(path=target_dir)
         return load_from_dir(target_dir, preload=True)