openstack/deb-murano
Code Issues Proposed changes

Implement upload call to the repository API

Browse Source 
* Add multiform/part-data to the allowed content type
* Get json and file from request

Partially-implements blueprint murano-repository-api-v2
Implements blueprint publish-app-to-catalog
Change-Id: I2ed8887a235739f6316695eb13879cc494f2f042
This commit is contained in:
Ekaterina Fedorova 2014-03-31 20:00:55 +04:00
parent 39c2726540
commit 70f4bdb7cc
8 changed files with 209 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
muranoapi/api/v1/__init__.py
View File

@ -25,6 +25,14 @@ SEARCH_MAPPING = {'fqn': 'fully_qualified_name',
'name': 'name',
'created': 'created'
}
PKG_PARAMS_MAP = {'display_name': 'name',
'full_name': 'fully_qualified_name',
'raw_ui': 'ui_definition',
'logo': 'logo',
'package_type': 'type',
'description': 'description',
'author': 'author',
'classes': 'class_definition'}
def get_draft(environment_id=None, session_id=None):

76
muranoapi/api/v1/catalog.py
View File

@ -13,16 +13,23 @@
# License for the specific language governing permissions and limitations
# under the License.
import cgi
import jsonschema
import tempfile
from oslo.config import cfg
from sqlalchemy import exc as sql_exc
from webob import exc
import muranoapi.api.v1
from muranoapi.api.v1 import schemas
from muranoapi.db.catalog import api as db_api
from muranoapi.openstack.common import exception
from muranoapi.openstack.common.gettextutils import _ # noqa
from muranoapi.openstack.common import log as logging
from muranoapi.openstack.common import wsgi
from muranoapi.packages import application_package as app_pkg
from muranoapi.packages import exceptions as pkg_exc
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
@ -30,6 +37,7 @@ CONF = cfg.CONF
SUPPORTED_PARAMS = muranoapi.api.v1.SUPPORTED_PARAMS
LIST_PARAMS = muranoapi.api.v1.LIST_PARAMS
ORDER_VALUES = muranoapi.api.v1.ORDER_VALUES
PKG_PARAMS_MAP = muranoapi.api.v1.PKG_PARAMS_MAP
def _check_content_type(req, content_type):
@ -61,10 +69,34 @@ def _get_filters(query_params):
LOG.warning(_("Value of 'order_by' parameter is not valid. "
"Allowed values are: {0}. Skipping it.").format(
", ".join(ORDER_VALUES)))
return filters
def _validate_body(body):
if len(body.keys()) != 2:
msg = "multipart/form-data request should contain " \
"two parts: json and tar.gz archive"
LOG.error(msg)
raise exc.HTTPBadRequest(msg)
file_obj = None
package_meta = None
for part in body.values():
if isinstance(part, cgi.FieldStorage):
file_obj = part
# dict if json deserialized successfully
if isinstance(part, dict):
package_meta = part
if file_obj is None:
msg = _("There is no file package with application description")
LOG.error(msg)
raise exc.HTTPBadRequest(msg)
if package_meta is None:
msg = _("There is no json with meta information about package")
LOG.error(msg)
raise exc.HTTPBadRequest(msg)
return file_obj, package_meta
class Controller(object):
"""
WSGI controller for application catalog resource in Murano v1 API
@ -100,6 +132,46 @@ class Controller(object):
packages = db_api.package_search(filters, req.context)
return {"packages": [package.to_dict() for package in packages]}
def upload(self, req, body=None):
"""
Upload new file archive for the new package
together with package metadata
"""
_check_content_type(req, 'multipart/form-data')
file_obj, package_meta = _validate_body(body)
try:
jsonschema.validate(package_meta, schemas.PKG_UPLOAD_SCHEMA)
except jsonschema.ValidationError as e:
LOG.exception(e)
raise exc.HTTPBadRequest(explanation=e.message)
with tempfile.NamedTemporaryFile() as tempf:
content = file_obj.file.read()
if not content:
msg = _("Uploading file can't be empty")
raise exc.HTTPBadRequest(msg)
tempf.write(content)
package_meta['archive'] = content
try:
pkg_to_upload = app_pkg.load_from_file(tempf.name,
target_dir=None,
drop_dir=True)
except pkg_exc.PackageLoadError as e:
LOG.exception(e)
raise exc.HTTPBadRequest(e.message)
# extend dictionary for update db
for k, v in PKG_PARAMS_MAP.iteritems():
if hasattr(pkg_to_upload, k):
package_meta[v] = getattr(pkg_to_upload, k)
try:
package = db_api.package_upload(package_meta, req.context.tenant)
except sql_exc.SQLAlchemyError:
msg = _('Unable to save package in database')
LOG.exception(msg)
raise exc.HTTPServerError(msg)
return package.to_dict()
def create_resource():
return wsgi.Resource(Controller())

4
muranoapi/api/v1/router.py
View File

@ -133,4 +133,8 @@ class API(wsgi.Router):
controller=catalog_resource,
action='search',
conditions={'method': ['GET']})
mapper.connect('/catalog/packages',
controller=catalog_resource,
action='upload',
conditions={'method': ['POST']})
super(API, self).__init__(mapper)

26
muranoapi/api/v1/schemas.py
View File

@ -24,6 +24,32 @@ ENV_SCHEMA = {
"required": ["id", "name"]
}
PKG_UPLOAD_SCHEMA = {
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": {
"tags": {
"type": "array",
"minItems": 1,
"items": {"type": "string"},
"uniqueItems": True
},
"categories": {
"type": "array",
"minItems": 1,
"items": {"type": "string"},
"uniqueItems": True
},
"description": {"type": "string"},
"name": {"type": "string"},
"is_public": {"type": "boolean"},
"enabled": {"type": "boolean"}
},
"required": ["categories"],
"additionalProperties": False
}
PKG_UPDATE_SCHEMA = {
"$schema": "http://json-schema.org/draft-04/schema#",

86
muranoapi/db/catalog/api.py
View File

@ -26,11 +26,6 @@ SEARCH_MAPPING = muranoapi.api.v1.SEARCH_MAPPING
LOG = logging.getLogger(__name__)
def get_category_by_name(name):
session = db_session.get_session()
return session.query(models.Category).filter_by(name=name).first()
def category_get_names():
session = db_session.get_session()
categories = []
@ -40,11 +35,6 @@ def category_get_names():
return categories
def get_tag_by_name(name):
session = db_session.get_session()
return session.query(models.Tag).filter_by(name=name).first()
def _package_get(package_id, session):
package = session.query(models.Package).get(package_id)
if not package:
@ -83,15 +73,18 @@ def package_get(package_id, context):
return package
def _get_categories(category_names):
def _get_categories(category_names, session=None):
"""
Return existing category objects or raise an exception
:param category_names: name of categories to associate with package, list
:returns: list of Category objects to associate with package, list
"""
if session is None:
session = db_session.get_session()
categories = []
for ctg_name in category_names:
ctg_obj = get_category_by_name(ctg_name)
ctg_obj = session.query(models.Category).filter_by(
name=ctg_name).first()
if not ctg_obj:
# it's not allowed to specify non-existent categories
raise exc.HTTPBadRequest(
@ -100,24 +93,39 @@ def _get_categories(category_names):
return categories
def _get_tags(tag_names):
def _get_tags(tag_names, session=None):
"""
Return existing tags object or create new ones
:param tag_names: name of tags to associate with package, list
:returns: list of Tag objects to associate with package, list
"""
if session is None:
session = db_session.get_session()
tags = []
if tag_names:
for tag_name in tag_names:
tag_obj = get_tag_by_name(tag_name)
if tag_obj:
tags.append(tag_obj)
else:
tag_record = models.Tag(name=tag_name)
tags.append(tag_record)
for tag_name in tag_names:
tag_obj = session.query(models.Tag).filter_by(name=tag_name).first()
if tag_obj:
tags.append(tag_obj)
else:
tag_record = models.Tag(name=tag_name)
tags.append(tag_record)
return tags
def _get_class_definitions(class_names, session):
if session is None:
session = db_session.get_session()
classes = []
for name in class_names:
class_obj = session.query(models.Class).filter_by(name=name).first()
if class_obj:
classes.append(class_obj)
else:
class_record = models.Class(name=name)
classes.append(class_record)
return classes
def _do_replace(package, change):
path = change['path'][0]
value = change['value']
@ -199,13 +207,12 @@ def package_update(pkg_id, changes, context):
'replace': _do_replace,
'remove': _do_remove}
session = db_session.get_session()
pkg = _package_get(pkg_id, session)
_authorize_package(pkg, context)
for change in changes:
pkg = operation_methods[change['op']](pkg, change)
with session.begin():
pkg = _package_get(pkg_id, session)
_authorize_package(pkg, context)
for change in changes:
pkg = operation_methods[change['op']](pkg, change)
session.add(pkg)
return pkg
@ -309,3 +316,28 @@ def package_search(filters, context):
query = query.limit(limit)
return query.all()
def package_upload(values, tenant_id):
"""
Upload a package with new application
:param values: parameters describing the new package
:returns: detailed information about new package, dict
"""
session = db_session.get_session()
package = models.Package()
composite_attr_to_func = {'categories': _get_categories,
'tags': _get_tags,
'class_definition': _get_class_definitions}
with session.begin():
for attr, func in composite_attr_to_func.iteritems():
if values.get(attr):
result = func(values[attr], session)
setattr(package, attr, result)
del values[attr]
package.update(values)
package.owner_id = tenant_id
package.save(session)
return package

4
muranoapi/db/models.py
View File

@ -263,12 +263,12 @@ class Package(BASE, ModelBase):
'archive',
'logo',
'ui_definition']
nested_objects = ['categories', 'tags']
nested_objects = ['categories', 'tags', 'class_definition']
for key in not_serializable:
if key in d.keys():
del d[key]
for key in nested_objects:
d[key] = [a.name for a in d.get(key)]
d[key] = [a.name for a in d.get(key, [])]
return d

44
muranoapi/openstack/common/wsgi.py
View File

@ -296,7 +296,8 @@ class Request(webob.Request):
default_request_content_types = ('application/json',
'application/xml',
'application/murano-packages-json-patch')
'application/murano-packages-json-patch',
'multipart/form-data')
default_accept_types = ('application/json', 'application/xml')
default_accept_type = 'application/json'
@ -632,7 +633,8 @@ class RequestDeserializer(object):
self.body_deserializers = {
'application/xml': XMLDeserializer(),
'application/json': JSONDeserializer(),
'application/murano-packages-json-patch': JSONPatchDeserializer()
'application/murano-packages-json-patch': JSONPatchDeserializer(),
'multipart/form-data': FormDataDeserializer()
}
self.body_deserializers.update(body_deserializers or {})
@ -682,7 +684,7 @@ class RequestDeserializer(object):
LOG.debug(_("Unable to deserialize body as provided Content-Type"))
raise
return deserializer.deserialize(request.body, action)
return deserializer.deserialize(request, action)
def get_body_deserializer(self, content_type):
try:
@ -716,10 +718,10 @@ class RequestDeserializer(object):
class TextDeserializer(ActionDispatcher):
"""Default request body deserialization"""
def deserialize(self, datastring, action='default'):
return self.dispatch(datastring, action=action)
def deserialize(self, request, action='default'):
return self.dispatch(request, action=action)
def default(self, datastring):
def default(self, request):
return {}
@ -731,7 +733,8 @@ class JSONDeserializer(TextDeserializer):
msg = _("cannot understand JSON")
raise exception.MalformedRequestBody(reason=msg)
def default(self, datastring):
def default(self, request):
datastring = request.body
return {'body': self._from_json(datastring)}
@ -831,7 +834,6 @@ class JSONPatchDeserializer(TextDeserializer):
ret.append(part.replace('~1', '/').replace('~0', '~').strip())
return ret
def _validate_json_pointer(self, pointer):
"""Validate a json pointer.
@ -866,8 +868,8 @@ class JSONPatchDeserializer(TextDeserializer):
msg = _('Nested paths are not allowed')
raise webob.exc.HTTPBadRequest(explanation=msg)
def default(self, datastring):
return {'body': self._from_json_patch(datastring)}
def default(self, request):
return {'body': self._from_json_patch(request.body)}
class XMLDeserializer(TextDeserializer):
@ -879,7 +881,8 @@ class XMLDeserializer(TextDeserializer):
super(XMLDeserializer, self).__init__()
self.metadata = metadata or {}
def _from_xml(self, datastring):
def _from_xml(self, request):
datastring = request.body
plurals = set(self.metadata.get('plurals', {}))
try:
@ -934,3 +937,22 @@ class XMLDeserializer(TextDeserializer):
def default(self, datastring):
return {'body': self._from_xml(datastring)}
class FormDataDeserializer(TextDeserializer):
def _from_json(self, datastring):
value = datastring
try:
LOG.debug(_("Trying deserialize '{0}' to json".format(datastring)))
value = jsonutils.loads(datastring)
except ValueError:
LOG.debug(_("Unable deserialize to json, using raw text"))
return value
def default(self, request):
form_data_parts = request.POST
result = []
for key, value in form_data_parts.iteritems():
if isinstance(value, basestring):
form_data_parts[key] = self._from_json(value)
return {'body': form_data_parts}

3
muranoapi/packages/application_package.py
View File

@ -206,6 +206,9 @@ def load_from_file(archive_path, target_dir=None, drop_dir=False):
raise e.PackageLoadError('Target directory is not empty')
try:
if not tarfile.is_tarfile(archive_path):
raise e.PackageFormatError("Uploading file should be a"
" 'tar.gz' archive")
package = tarfile.open(archive_path)
package.extractall(path=target_dir)
return load_from_dir(target_dir, preload=True)