From b79b2791b8c3e1ac2a65ce7e76c703aec0b723d9 Mon Sep 17 00:00:00 2001 From: Lingxian Kong Date: Tue, 16 May 2017 15:24:46 +1200 Subject: [PATCH] Add swift object support for function creation With this patch, users can create function using swift object: http POST http://127.0.0.1:7070/v1/functions name=swift_function \ runtime_id=xxx \ code='{"source": "swift", "swift": {"container": "container_name", "object": "object_name"}}' Implements: blueprint support-swift-object-as-code --- README.rst | 13 +++-- qinling/api/controllers/v1/function.py | 65 +++++++++++++++++----- qinling/orchestrator/kubernetes/manager.py | 8 ++- qinling/storage/file_system.py | 6 ++ qinling/utils/common.py | 19 +++++++ qinling/utils/openstack/__init__.py | 0 qinling/utils/openstack/keystone.py | 43 ++++++++++++++ qinling/utils/openstack/swift.py | 53 ++++++++++++++++++ requirements.txt | 2 + runtimes/python2/server.py | 16 ++++-- 10 files changed, 200 insertions(+), 25 deletions(-) create mode 100644 qinling/utils/openstack/__init__.py create mode 100644 qinling/utils/openstack/keystone.py create mode 100644 qinling/utils/openstack/swift.py diff --git a/README.rst b/README.rst index d7e41977..0cd31e5a 100644 --- a/README.rst +++ b/README.rst @@ -100,11 +100,14 @@ python-qinlingclient is still under development.** Perform following commands on your local host, the process will create runtime/function/execution in Qinling. -#. First, build a docker image that is used to create runtime in Qinling and - upload to docker hub. Only ``Python 2`` runtime is supported for now, but it - is very easy to add another program language support. Run the commands in - ``qinling`` repo directory, replace ``DOCKER_USER`` with your docker hub - username: +#. (Optional) Prepare a docker image including development environment for a + specific programming language. For your convenience, I already build one + (``lingxiankong/python-runtime``) in my docker hub account that you could + directly use to create runtime in Qinling. Only ``Python 2`` runtime is + supported for now, but it is very easy to add another program language + support. If you indeed want to build a new image, run the following commands + in ``qinling`` repo directory, replace ``DOCKER_USER`` with your own docker + hub username: .. code-block:: console diff --git a/qinling/api/controllers/v1/function.py b/qinling/api/controllers/v1/function.py index f74606d6..052e5587 100644 --- a/qinling/api/controllers/v1/function.py +++ b/qinling/api/controllers/v1/function.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import collections import json import os @@ -29,16 +30,19 @@ from qinling import context from qinling.db import api as db_api from qinling import exceptions as exc from qinling.storage import base as storage_base +from qinling.utils.openstack import swift as swift_util from qinling.utils import rest_utils LOG = logging.getLogger(__name__) +CONF = cfg.CONF POST_REQUIRED = set(['name', 'runtime_id', 'code']) +CODE_SOURCE = set(['package', 'swift', 'image']) class FunctionsController(rest.RestController): def __init__(self, *args, **kwargs): - self.storage_provider = storage_base.load_storage_provider(cfg.CONF) + self.storage_provider = storage_base.load_storage_provider(CONF) super(FunctionsController, self).__init__(*args, **kwargs) @@ -57,15 +61,20 @@ class FunctionsController(rest.RestController): pecan.override_template('json') return resources.Function.from_dict(func_db.to_dict()).to_dict() else: - f = self.storage_provider.retrieve( - ctx.projectid, - id, - ) + source = func_db.code['source'] - pecan.response.app_iter = FileIter(f) + if source == 'package': + f = self.storage_provider.retrieve(ctx.projectid, id) + elif source == 'swift': + container = func_db.code['swift']['container'] + obj = func_db.code['swift']['object'] + f = swift_util.download_object(container, obj) + + pecan.response.app_iter = (f if isinstance(f, collections.Iterable) + else FileIter(f)) pecan.response.headers['Content-Type'] = 'application/zip' pecan.response.headers['Content-Disposition'] = ( - 'attachment; filename="%s"' % os.path.basename(f.name) + 'attachment; filename="%s"' % os.path.basename(func_db.name) ) @rest_utils.wrap_pecan_controller_exception @@ -92,19 +101,40 @@ class FunctionsController(rest.RestController): 'entry': kwargs.get('entry', 'main'), } - if values['code'].get('package', False): + source = values['code'].get('source') + if not source or source not in CODE_SOURCE: + raise exc.InputException( + 'Invalid code source specified, available sources: %s' % + CODE_SOURCE + ) + + store = False + if values['code']['source'] == 'package': + store = True data = kwargs['package'].file.read() + elif values['code']['source'] == 'swift': + # Auth needs to be enabled because qinling needs to check swift + # object using user's credential. + if not CONF.pecan.auth_enable: + raise exc.InputException('Swift object not supported.') + + container = values['code']['swift'].get('container') + object = values['code']['swift'].get('object') + + if not swift_util.check_object(container, object): + raise exc.InputException('Object does not exist in Swift.') ctx = context.get_ctx() with db_api.transaction(): func_db = db_api.create_function(values) - self.storage_provider.store( - ctx.projectid, - func_db.id, - data - ) + if store: + self.storage_provider.store( + ctx.projectid, + func_db.id, + data + ) pecan.response.status = 201 return resources.Function.from_dict(func_db.to_dict()).to_dict() @@ -126,6 +156,11 @@ class FunctionsController(rest.RestController): LOG.info("Delete function [id=%s]", id) with db_api.transaction(): - db_api.delete_function(id) + func_db = db_api.get_function(id) + source = func_db.code['source'] - self.storage_provider.delete(context.get_ctx().projectid, id) + if source == 'package': + self.storage_provider.delete(context.get_ctx().projectid, id) + + # This will also delete function service mapping as well. + db_api.delete_function(id) diff --git a/qinling/orchestrator/kubernetes/manager.py b/qinling/orchestrator/kubernetes/manager.py index 27ffbe90..8b5d0325 100644 --- a/qinling/orchestrator/kubernetes/manager.py +++ b/qinling/orchestrator/kubernetes/manager.py @@ -21,6 +21,7 @@ from oslo_log import log as logging import requests import yaml +from qinling import context from qinling import exceptions as exc from qinling.orchestrator import base from qinling.utils import common @@ -223,7 +224,12 @@ class KubernetesManager(base.OrchestratorBase): (self.conf.kubernetes.qinling_service_address, self.conf.api.port, function_id) ) - data = {'download_url': download_url, 'function_id': function_id} + + data = { + 'download_url': download_url, + 'function_id': function_id, + 'token': context.get_ctx().auth_token + } LOG.debug( 'Send request to pod %s, request_url: %s, data: %s', diff --git a/qinling/storage/file_system.py b/qinling/storage/file_system.py index f962754c..e5a68a49 100644 --- a/qinling/storage/file_system.py +++ b/qinling/storage/file_system.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import zipfile from oslo_config import cfg from oslo_log import log as logging @@ -43,6 +44,11 @@ class FileSystemStorage(base.PackageStorage): with open(func_zip, 'wb') as fd: fd.write(data) + if not zipfile.is_zipfile(func_zip): + fileutils.delete_if_exists(func_zip) + + raise exc.InputException("Package is not a valid ZIP package.") + def retrieve(self, project_id, function): LOG.info( 'Get package data, function: %s, project: %s', function, project_id diff --git a/qinling/utils/common.py b/qinling/utils/common.py index 43f9835e..be8a7e12 100644 --- a/qinling/utils/common.py +++ b/qinling/utils/common.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import functools +import warnings from oslo_utils import uuidutils @@ -23,3 +25,20 @@ def convert_dict_to_string(d): def generate_unicode_uuid(): return uuidutils.generate_uuid() + + +def disable_ssl_warnings(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="A true SSLContext object is not available" + ) + warnings.filterwarnings( + "ignore", + message="Unverified HTTPS request is being made" + ) + return func(*args, **kwargs) + + return wrapper diff --git a/qinling/utils/openstack/__init__.py b/qinling/utils/openstack/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/qinling/utils/openstack/keystone.py b/qinling/utils/openstack/keystone.py new file mode 100644 index 00000000..6e07275a --- /dev/null +++ b/qinling/utils/openstack/keystone.py @@ -0,0 +1,43 @@ +# Copyright 2017 Catalyst IT Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from keystoneauth1.identity import generic +from keystoneauth1 import session +from oslo_config import cfg +import swiftclient + +from qinling import context + +CONF = cfg.CONF + +KS_SESSION = None + + +def _get_user_keystone_session(): + ctx = context.get_ctx() + + auth = generic.Token( + auth_url=CONF.keystone_authtoken.auth_url, + token=ctx.auth_token, + ) + + return session.Session(auth=auth, verify=False) + + +def get_swiftclient(): + session = _get_user_keystone_session() + + conn = swiftclient.Connection(session=session) + + return conn diff --git a/qinling/utils/openstack/swift.py b/qinling/utils/openstack/swift.py new file mode 100644 index 00000000..872d8f54 --- /dev/null +++ b/qinling/utils/openstack/swift.py @@ -0,0 +1,53 @@ +# Copyright 2017 Catalyst IT Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_log import log as logging +from swiftclient.exceptions import ClientException + +from qinling.utils import common +from qinling.utils.openstack import keystone + +LOG = logging.getLogger(__name__) + + +@common.disable_ssl_warnings +def check_object(container, object): + """Check if object exists in Swift. + + :param container: Container name. + :param object: Object name. + :return: True if object exists, otherwise return False. + """ + swift_conn = keystone.get_swiftclient() + + try: + swift_conn.head_object(container, object) + return True + except ClientException: + LOG.error( + 'The object %s in container %s was not found', object, container + ) + return False + + +@common.disable_ssl_warnings +def download_object(container, object): + swift_conn = keystone.get_swiftclient() + + # Specify 'resp_chunk_size' here to return a file reader. + _, obj_reader = swift_conn.get_object( + container, object, resp_chunk_size=65536 + ) + + return obj_reader diff --git a/requirements.txt b/requirements.txt index e0079061..898ab3d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ pbr>=2.0 # Apache-2.0 Babel!=2.4.0,>=2.3.4 # BSD eventlet!=0.18.3,>=0.18.2 # MIT +keystoneauth1>=2.20.0 # Apache-2.0 keystonemiddleware>=4.12.0 # Apache-2.0 oslo.concurrency>=3.8.0 # Apache-2.0 oslo.config>=3.22.0 # Apache-2.0 @@ -24,3 +25,4 @@ stevedore>=1.20.0 # Apache-2.0 WSME>=0.8 # MIT kubernetes>=1.0.0b1 # Apache-2.0 PyYAML>=3.10.0 # MIT +python-swiftclient>=3.2.0 # Apache-2.0 diff --git a/runtimes/python2/server.py b/runtimes/python2/server.py index 6b43e309..b077cef7 100644 --- a/runtimes/python2/server.py +++ b/runtimes/python2/server.py @@ -31,18 +31,26 @@ file_name = '' @app.route('/download', methods=['POST']) def download(): - service_url = request.form['download_url'] + download_url = request.form['download_url'] function_id = request.form['function_id'] + token = request.form['token'] + + headers = {} + if token: + headers = {'X-Auth-Token': token} global file_name file_name = '%s.zip' % function_id - app.logger.info('Request received, service_url:%s' % service_url) + app.logger.info( + 'Request received, download_url:%s, headers: %s' % + (download_url, headers) + ) - r = requests.get(service_url, stream=True) + r = requests.get(download_url, headers=headers, stream=True) with open(file_name, 'wb') as fd: - for chunk in r.iter_content(chunk_size=128): + for chunk in r.iter_content(chunk_size=65535): fd.write(chunk) if not zipfile.is_zipfile(file_name):