From 70f4bdb7ccea4c5edd388943043100837f3cfe84 Mon Sep 17 00:00:00 2001 From: Ekaterina Fedorova Date: Mon, 31 Mar 2014 20:00:55 +0400 Subject: [PATCH] Implement upload call to the repository API * Add multiform/part-data to the allowed content type * Get json and file from request Partially-implements blueprint murano-repository-api-v2 Implements blueprint publish-app-to-catalog Change-Id: I2ed8887a235739f6316695eb13879cc494f2f042 --- muranoapi/api/v1/__init__.py | 8 +++ muranoapi/api/v1/catalog.py | 76 +++++++++++++++++++- muranoapi/api/v1/router.py | 4 ++ muranoapi/api/v1/schemas.py | 26 +++++++ muranoapi/db/catalog/api.py | 86 ++++++++++++++++------- muranoapi/db/models.py | 4 +- muranoapi/openstack/common/wsgi.py | 44 +++++++++--- muranoapi/packages/application_package.py | 3 + 8 files changed, 209 insertions(+), 42 deletions(-) diff --git a/muranoapi/api/v1/__init__.py b/muranoapi/api/v1/__init__.py index 0a985c22..0d484279 100644 --- a/muranoapi/api/v1/__init__.py +++ b/muranoapi/api/v1/__init__.py @@ -25,6 +25,14 @@ SEARCH_MAPPING = {'fqn': 'fully_qualified_name', 'name': 'name', 'created': 'created' } +PKG_PARAMS_MAP = {'display_name': 'name', + 'full_name': 'fully_qualified_name', + 'raw_ui': 'ui_definition', + 'logo': 'logo', + 'package_type': 'type', + 'description': 'description', + 'author': 'author', + 'classes': 'class_definition'} def get_draft(environment_id=None, session_id=None): diff --git a/muranoapi/api/v1/catalog.py b/muranoapi/api/v1/catalog.py index 50f6f516..cb1e2c30 100644 --- a/muranoapi/api/v1/catalog.py +++ b/muranoapi/api/v1/catalog.py @@ -13,16 +13,23 @@ # License for the specific language governing permissions and limitations # under the License. +import cgi +import jsonschema +import tempfile + from oslo.config import cfg +from sqlalchemy import exc as sql_exc from webob import exc import muranoapi.api.v1 +from muranoapi.api.v1 import schemas from muranoapi.db.catalog import api as db_api from muranoapi.openstack.common import exception from muranoapi.openstack.common.gettextutils import _ # noqa from muranoapi.openstack.common import log as logging from muranoapi.openstack.common import wsgi - +from muranoapi.packages import application_package as app_pkg +from muranoapi.packages import exceptions as pkg_exc LOG = logging.getLogger(__name__) CONF = cfg.CONF @@ -30,6 +37,7 @@ CONF = cfg.CONF SUPPORTED_PARAMS = muranoapi.api.v1.SUPPORTED_PARAMS LIST_PARAMS = muranoapi.api.v1.LIST_PARAMS ORDER_VALUES = muranoapi.api.v1.ORDER_VALUES +PKG_PARAMS_MAP = muranoapi.api.v1.PKG_PARAMS_MAP def _check_content_type(req, content_type): @@ -61,10 +69,34 @@ def _get_filters(query_params): LOG.warning(_("Value of 'order_by' parameter is not valid. " "Allowed values are: {0}. Skipping it.").format( ", ".join(ORDER_VALUES))) - return filters +def _validate_body(body): + if len(body.keys()) != 2: + msg = "multipart/form-data request should contain " \ + "two parts: json and tar.gz archive" + LOG.error(msg) + raise exc.HTTPBadRequest(msg) + file_obj = None + package_meta = None + for part in body.values(): + if isinstance(part, cgi.FieldStorage): + file_obj = part + # dict if json deserialized successfully + if isinstance(part, dict): + package_meta = part + if file_obj is None: + msg = _("There is no file package with application description") + LOG.error(msg) + raise exc.HTTPBadRequest(msg) + if package_meta is None: + msg = _("There is no json with meta information about package") + LOG.error(msg) + raise exc.HTTPBadRequest(msg) + return file_obj, package_meta + + class Controller(object): """ WSGI controller for application catalog resource in Murano v1 API @@ -100,6 +132,46 @@ class Controller(object): packages = db_api.package_search(filters, req.context) return {"packages": [package.to_dict() for package in packages]} + def upload(self, req, body=None): + """ + Upload new file archive for the new package + together with package metadata + """ + _check_content_type(req, 'multipart/form-data') + file_obj, package_meta = _validate_body(body) + try: + jsonschema.validate(package_meta, schemas.PKG_UPLOAD_SCHEMA) + except jsonschema.ValidationError as e: + LOG.exception(e) + raise exc.HTTPBadRequest(explanation=e.message) + + with tempfile.NamedTemporaryFile() as tempf: + content = file_obj.file.read() + if not content: + msg = _("Uploading file can't be empty") + raise exc.HTTPBadRequest(msg) + tempf.write(content) + package_meta['archive'] = content + try: + pkg_to_upload = app_pkg.load_from_file(tempf.name, + target_dir=None, + drop_dir=True) + except pkg_exc.PackageLoadError as e: + LOG.exception(e) + raise exc.HTTPBadRequest(e.message) + + # extend dictionary for update db + for k, v in PKG_PARAMS_MAP.iteritems(): + if hasattr(pkg_to_upload, k): + package_meta[v] = getattr(pkg_to_upload, k) + try: + package = db_api.package_upload(package_meta, req.context.tenant) + except sql_exc.SQLAlchemyError: + msg = _('Unable to save package in database') + LOG.exception(msg) + raise exc.HTTPServerError(msg) + return package.to_dict() + def create_resource(): return wsgi.Resource(Controller()) diff --git a/muranoapi/api/v1/router.py b/muranoapi/api/v1/router.py index c0f0e7cd..db2b5241 100644 --- a/muranoapi/api/v1/router.py +++ b/muranoapi/api/v1/router.py @@ -133,4 +133,8 @@ class API(wsgi.Router): controller=catalog_resource, action='search', conditions={'method': ['GET']}) + mapper.connect('/catalog/packages', + controller=catalog_resource, + action='upload', + conditions={'method': ['POST']}) super(API, self).__init__(mapper) diff --git a/muranoapi/api/v1/schemas.py b/muranoapi/api/v1/schemas.py index e39fc7c8..50c9b2f3 100644 --- a/muranoapi/api/v1/schemas.py +++ b/muranoapi/api/v1/schemas.py @@ -24,6 +24,32 @@ ENV_SCHEMA = { "required": ["id", "name"] } +PKG_UPLOAD_SCHEMA = { + "$schema": "http://json-schema.org/draft-04/schema#", + + "type": "object", + "properties": { + "tags": { + "type": "array", + "minItems": 1, + "items": {"type": "string"}, + "uniqueItems": True + }, + "categories": { + "type": "array", + "minItems": 1, + "items": {"type": "string"}, + "uniqueItems": True + }, + "description": {"type": "string"}, + "name": {"type": "string"}, + "is_public": {"type": "boolean"}, + "enabled": {"type": "boolean"} + }, + "required": ["categories"], + "additionalProperties": False +} + PKG_UPDATE_SCHEMA = { "$schema": "http://json-schema.org/draft-04/schema#", diff --git a/muranoapi/db/catalog/api.py b/muranoapi/db/catalog/api.py index 4c89e31f..45a6c6c2 100644 --- a/muranoapi/db/catalog/api.py +++ b/muranoapi/db/catalog/api.py @@ -26,11 +26,6 @@ SEARCH_MAPPING = muranoapi.api.v1.SEARCH_MAPPING LOG = logging.getLogger(__name__) -def get_category_by_name(name): - session = db_session.get_session() - return session.query(models.Category).filter_by(name=name).first() - - def category_get_names(): session = db_session.get_session() categories = [] @@ -40,11 +35,6 @@ def category_get_names(): return categories -def get_tag_by_name(name): - session = db_session.get_session() - return session.query(models.Tag).filter_by(name=name).first() - - def _package_get(package_id, session): package = session.query(models.Package).get(package_id) if not package: @@ -83,15 +73,18 @@ def package_get(package_id, context): return package -def _get_categories(category_names): +def _get_categories(category_names, session=None): """ Return existing category objects or raise an exception :param category_names: name of categories to associate with package, list :returns: list of Category objects to associate with package, list """ + if session is None: + session = db_session.get_session() categories = [] for ctg_name in category_names: - ctg_obj = get_category_by_name(ctg_name) + ctg_obj = session.query(models.Category).filter_by( + name=ctg_name).first() if not ctg_obj: # it's not allowed to specify non-existent categories raise exc.HTTPBadRequest( @@ -100,24 +93,39 @@ def _get_categories(category_names): return categories -def _get_tags(tag_names): +def _get_tags(tag_names, session=None): """ Return existing tags object or create new ones :param tag_names: name of tags to associate with package, list :returns: list of Tag objects to associate with package, list """ + if session is None: + session = db_session.get_session() tags = [] - if tag_names: - for tag_name in tag_names: - tag_obj = get_tag_by_name(tag_name) - if tag_obj: - tags.append(tag_obj) - else: - tag_record = models.Tag(name=tag_name) - tags.append(tag_record) + for tag_name in tag_names: + tag_obj = session.query(models.Tag).filter_by(name=tag_name).first() + if tag_obj: + tags.append(tag_obj) + else: + tag_record = models.Tag(name=tag_name) + tags.append(tag_record) return tags +def _get_class_definitions(class_names, session): + if session is None: + session = db_session.get_session() + classes = [] + for name in class_names: + class_obj = session.query(models.Class).filter_by(name=name).first() + if class_obj: + classes.append(class_obj) + else: + class_record = models.Class(name=name) + classes.append(class_record) + return classes + + def _do_replace(package, change): path = change['path'][0] value = change['value'] @@ -199,13 +207,12 @@ def package_update(pkg_id, changes, context): 'replace': _do_replace, 'remove': _do_remove} session = db_session.get_session() - pkg = _package_get(pkg_id, session) - _authorize_package(pkg, context) - - for change in changes: - pkg = operation_methods[change['op']](pkg, change) - with session.begin(): + pkg = _package_get(pkg_id, session) + _authorize_package(pkg, context) + + for change in changes: + pkg = operation_methods[change['op']](pkg, change) session.add(pkg) return pkg @@ -309,3 +316,28 @@ def package_search(filters, context): query = query.limit(limit) return query.all() + + +def package_upload(values, tenant_id): + """ + Upload a package with new application + :param values: parameters describing the new package + :returns: detailed information about new package, dict + """ + session = db_session.get_session() + package = models.Package() + + composite_attr_to_func = {'categories': _get_categories, + 'tags': _get_tags, + 'class_definition': _get_class_definitions} + with session.begin(): + for attr, func in composite_attr_to_func.iteritems(): + if values.get(attr): + result = func(values[attr], session) + setattr(package, attr, result) + del values[attr] + + package.update(values) + package.owner_id = tenant_id + package.save(session) + return package diff --git a/muranoapi/db/models.py b/muranoapi/db/models.py index 0dc9beb7..29917085 100644 --- a/muranoapi/db/models.py +++ b/muranoapi/db/models.py @@ -263,12 +263,12 @@ class Package(BASE, ModelBase): 'archive', 'logo', 'ui_definition'] - nested_objects = ['categories', 'tags'] + nested_objects = ['categories', 'tags', 'class_definition'] for key in not_serializable: if key in d.keys(): del d[key] for key in nested_objects: - d[key] = [a.name for a in d.get(key)] + d[key] = [a.name for a in d.get(key, [])] return d diff --git a/muranoapi/openstack/common/wsgi.py b/muranoapi/openstack/common/wsgi.py index 931f81a7..39db4e97 100644 --- a/muranoapi/openstack/common/wsgi.py +++ b/muranoapi/openstack/common/wsgi.py @@ -296,7 +296,8 @@ class Request(webob.Request): default_request_content_types = ('application/json', 'application/xml', - 'application/murano-packages-json-patch') + 'application/murano-packages-json-patch', + 'multipart/form-data') default_accept_types = ('application/json', 'application/xml') default_accept_type = 'application/json' @@ -632,7 +633,8 @@ class RequestDeserializer(object): self.body_deserializers = { 'application/xml': XMLDeserializer(), 'application/json': JSONDeserializer(), - 'application/murano-packages-json-patch': JSONPatchDeserializer() + 'application/murano-packages-json-patch': JSONPatchDeserializer(), + 'multipart/form-data': FormDataDeserializer() } self.body_deserializers.update(body_deserializers or {}) @@ -682,7 +684,7 @@ class RequestDeserializer(object): LOG.debug(_("Unable to deserialize body as provided Content-Type")) raise - return deserializer.deserialize(request.body, action) + return deserializer.deserialize(request, action) def get_body_deserializer(self, content_type): try: @@ -716,10 +718,10 @@ class RequestDeserializer(object): class TextDeserializer(ActionDispatcher): """Default request body deserialization""" - def deserialize(self, datastring, action='default'): - return self.dispatch(datastring, action=action) + def deserialize(self, request, action='default'): + return self.dispatch(request, action=action) - def default(self, datastring): + def default(self, request): return {} @@ -731,7 +733,8 @@ class JSONDeserializer(TextDeserializer): msg = _("cannot understand JSON") raise exception.MalformedRequestBody(reason=msg) - def default(self, datastring): + def default(self, request): + datastring = request.body return {'body': self._from_json(datastring)} @@ -831,7 +834,6 @@ class JSONPatchDeserializer(TextDeserializer): ret.append(part.replace('~1', '/').replace('~0', '~').strip()) return ret - def _validate_json_pointer(self, pointer): """Validate a json pointer. @@ -866,8 +868,8 @@ class JSONPatchDeserializer(TextDeserializer): msg = _('Nested paths are not allowed') raise webob.exc.HTTPBadRequest(explanation=msg) - def default(self, datastring): - return {'body': self._from_json_patch(datastring)} + def default(self, request): + return {'body': self._from_json_patch(request.body)} class XMLDeserializer(TextDeserializer): @@ -879,7 +881,8 @@ class XMLDeserializer(TextDeserializer): super(XMLDeserializer, self).__init__() self.metadata = metadata or {} - def _from_xml(self, datastring): + def _from_xml(self, request): + datastring = request.body plurals = set(self.metadata.get('plurals', {})) try: @@ -934,3 +937,22 @@ class XMLDeserializer(TextDeserializer): def default(self, datastring): return {'body': self._from_xml(datastring)} + + +class FormDataDeserializer(TextDeserializer): + def _from_json(self, datastring): + value = datastring + try: + LOG.debug(_("Trying deserialize '{0}' to json".format(datastring))) + value = jsonutils.loads(datastring) + except ValueError: + LOG.debug(_("Unable deserialize to json, using raw text")) + return value + + def default(self, request): + form_data_parts = request.POST + result = [] + for key, value in form_data_parts.iteritems(): + if isinstance(value, basestring): + form_data_parts[key] = self._from_json(value) + return {'body': form_data_parts} diff --git a/muranoapi/packages/application_package.py b/muranoapi/packages/application_package.py index f2ce95a1..c1e97467 100644 --- a/muranoapi/packages/application_package.py +++ b/muranoapi/packages/application_package.py @@ -206,6 +206,9 @@ def load_from_file(archive_path, target_dir=None, drop_dir=False): raise e.PackageLoadError('Target directory is not empty') try: + if not tarfile.is_tarfile(archive_path): + raise e.PackageFormatError("Uploading file should be a" + " 'tar.gz' archive") package = tarfile.open(archive_path) package.extractall(path=target_dir) return load_from_dir(target_dir, preload=True)