Add swift object support for function creation

With this patch, users can create function using swift object:

http POST http://127.0.0.1:7070/v1/functions name=swift_function \
  runtime_id=xxx \
  code='{"source": "swift", "swift": {"container": "container_name", "object": "object_name"}}'

Implements: blueprint support-swift-object-as-code

README.rst

@@ -100,11 +100,14 @@ python-qinlingclient is still under development.**
 Perform following commands on your local host, the process will create
 runtime/function/execution in Qinling.
 
-#. First, build a docker image that is used to create runtime in Qinling and
-   upload to docker hub. Only ``Python 2`` runtime is supported for now, but it
-   is very easy to add another program language support. Run the commands in
-   ``qinling`` repo directory, replace ``DOCKER_USER`` with your docker hub
-   username:
+#. (Optional) Prepare a docker image including development environment for a
+   specific programming language. For your convenience, I already build one
+   (``lingxiankong/python-runtime``) in my docker hub account that you could
+   directly use to create runtime in Qinling. Only ``Python 2`` runtime is
+   supported for now, but it is very easy to add another program language
+   support. If you indeed want to build a new image, run the following commands
+   in ``qinling`` repo directory, replace ``DOCKER_USER`` with your own docker
+   hub username:
 
    .. code-block:: console
 

qinling/api/controllers/v1/function.py

@@ -12,6 +12,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 
+import collections
 import json
 import os
 
@@ -29,16 +30,19 @@ from qinling import context
 from qinling.db import api as db_api
 from qinling import exceptions as exc
 from qinling.storage import base as storage_base
+from qinling.utils.openstack import swift as swift_util
 from qinling.utils import rest_utils
 
 LOG = logging.getLogger(__name__)
+CONF = cfg.CONF
 
 POST_REQUIRED = set(['name', 'runtime_id', 'code'])
+CODE_SOURCE = set(['package', 'swift', 'image'])
 
 
 class FunctionsController(rest.RestController):
     def __init__(self, *args, **kwargs):
-        self.storage_provider = storage_base.load_storage_provider(cfg.CONF)
+        self.storage_provider = storage_base.load_storage_provider(CONF)
 
         super(FunctionsController, self).__init__(*args, **kwargs)
 
@@ -57,15 +61,20 @@ class FunctionsController(rest.RestController):
             pecan.override_template('json')
             return resources.Function.from_dict(func_db.to_dict()).to_dict()
         else:
-            f = self.storage_provider.retrieve(
-                ctx.projectid,
-                id,
-            )
+            source = func_db.code['source']
+
+            if source == 'package':
+                f = self.storage_provider.retrieve(ctx.projectid, id)
+            elif source == 'swift':
+                container = func_db.code['swift']['container']
+                obj = func_db.code['swift']['object']
+                f = swift_util.download_object(container, obj)
 
-            pecan.response.app_iter = FileIter(f)
+            pecan.response.app_iter = (f if isinstance(f, collections.Iterable)
+                                       else FileIter(f))
             pecan.response.headers['Content-Type'] = 'application/zip'
             pecan.response.headers['Content-Disposition'] = (
-                'attachment; filename="%s"' % os.path.basename(f.name)
+                'attachment; filename="%s"' % os.path.basename(func_db.name)
             )
 
     @rest_utils.wrap_pecan_controller_exception
@@ -92,19 +101,40 @@ class FunctionsController(rest.RestController):
             'entry': kwargs.get('entry', 'main'),
         }
 
-        if values['code'].get('package', False):
+        source = values['code'].get('source')
+        if not source or source not in CODE_SOURCE:
+            raise exc.InputException(
+                'Invalid code source specified, available sources: %s' %
+                CODE_SOURCE
+            )
+
+        store = False
+        if values['code']['source'] == 'package':
+            store = True
             data = kwargs['package'].file.read()
+        elif values['code']['source'] == 'swift':
+            # Auth needs to be enabled because qinling needs to check swift
+            # object using user's credential.
+            if not CONF.pecan.auth_enable:
+                raise exc.InputException('Swift object not supported.')
+
+            container = values['code']['swift'].get('container')
+            object = values['code']['swift'].get('object')
+
+            if not swift_util.check_object(container, object):
+                raise exc.InputException('Object does not exist in Swift.')
 
         ctx = context.get_ctx()
 
         with db_api.transaction():
             func_db = db_api.create_function(values)
 
-            self.storage_provider.store(
-                ctx.projectid,
-                func_db.id,
-                data
-            )
+            if store:
+                self.storage_provider.store(
+                    ctx.projectid,
+                    func_db.id,
+                    data
+                )
 
         pecan.response.status = 201
         return resources.Function.from_dict(func_db.to_dict()).to_dict()
@@ -126,6 +156,11 @@ class FunctionsController(rest.RestController):
         LOG.info("Delete function [id=%s]", id)
 
         with db_api.transaction():
-            db_api.delete_function(id)
+            func_db = db_api.get_function(id)
+            source = func_db.code['source']
 
-            self.storage_provider.delete(context.get_ctx().projectid, id)
+            if source == 'package':
+                self.storage_provider.delete(context.get_ctx().projectid, id)
+
+            # This will also delete function service mapping as well.
+            db_api.delete_function(id)

qinling/orchestrator/kubernetes/manager.py

@@ -21,6 +21,7 @@ from oslo_log import log as logging
 import requests
 import yaml
 
+from qinling import context
 from qinling import exceptions as exc
 from qinling.orchestrator import base
 from qinling.utils import common
@@ -223,7 +224,12 @@ class KubernetesManager(base.OrchestratorBase):
             (self.conf.kubernetes.qinling_service_address,
              self.conf.api.port, function_id)
         )
-        data = {'download_url': download_url, 'function_id': function_id}
+
+        data = {
+            'download_url': download_url,
+            'function_id': function_id,
+            'token': context.get_ctx().auth_token
+        }
 
         LOG.debug(
             'Send request to pod %s, request_url: %s, data: %s',

qinling/storage/file_system.py

@@ -13,6 +13,7 @@
 #    limitations under the License.
 
 import os
+import zipfile
 
 from oslo_config import cfg
 from oslo_log import log as logging
@@ -43,6 +44,11 @@ class FileSystemStorage(base.PackageStorage):
         with open(func_zip, 'wb') as fd:
             fd.write(data)
 
+        if not zipfile.is_zipfile(func_zip):
+            fileutils.delete_if_exists(func_zip)
+
+            raise exc.InputException("Package is not a valid ZIP package.")
+
     def retrieve(self, project_id, function):
         LOG.info(
             'Get package data, function: %s, project: %s', function, project_id

qinling/utils/common.py

@@ -11,6 +11,8 @@
 #    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
+import functools
+import warnings
 
 from oslo_utils import uuidutils
 
@@ -23,3 +25,20 @@ def convert_dict_to_string(d):
 
 def generate_unicode_uuid():
     return uuidutils.generate_uuid()
+
+
+def disable_ssl_warnings(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        with warnings.catch_warnings():
+            warnings.filterwarnings(
+                "ignore",
+                message="A true SSLContext object is not available"
+            )
+            warnings.filterwarnings(
+                "ignore",
+                message="Unverified HTTPS request is being made"
+            )
+            return func(*args, **kwargs)
+
+    return wrapper

qinling/utils/openstack/keystone.py

@@ -0,0 +1,43 @@
+# Copyright 2017 Catalyst IT Limited
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+from keystoneauth1.identity import generic
+from keystoneauth1 import session
+from oslo_config import cfg
+import swiftclient
+
+from qinling import context
+
+CONF = cfg.CONF
+
+KS_SESSION = None
+
+
+def _get_user_keystone_session():
+    ctx = context.get_ctx()
+
+    auth = generic.Token(
+        auth_url=CONF.keystone_authtoken.auth_url,
+        token=ctx.auth_token,
+    )
+
+    return session.Session(auth=auth, verify=False)
+
+
+def get_swiftclient():
+    session = _get_user_keystone_session()
+
+    conn = swiftclient.Connection(session=session)
+
+    return conn

qinling/utils/openstack/swift.py

@@ -0,0 +1,53 @@
+# Copyright 2017 Catalyst IT Limited
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+from oslo_log import log as logging
+from swiftclient.exceptions import ClientException
+
+from qinling.utils import common
+from qinling.utils.openstack import keystone
+
+LOG = logging.getLogger(__name__)
+
+
+@common.disable_ssl_warnings
+def check_object(container, object):
+    """Check if object exists in Swift.
+
+    :param container: Container name.
+    :param object: Object name.
+    :return: True if object exists, otherwise return False.
+    """
+    swift_conn = keystone.get_swiftclient()
+
+    try:
+        swift_conn.head_object(container, object)
+        return True
+    except ClientException:
+        LOG.error(
+            'The object %s in container %s was not found', object, container
+        )
+        return False
+
+
+@common.disable_ssl_warnings
+def download_object(container, object):
+    swift_conn = keystone.get_swiftclient()
+
+    # Specify 'resp_chunk_size' here to return a file reader.
+    _, obj_reader = swift_conn.get_object(
+        container, object, resp_chunk_size=65536
+    )
+
+    return obj_reader

requirements.txt

@@ -5,6 +5,7 @@
 pbr>=2.0 # Apache-2.0
 Babel!=2.4.0,>=2.3.4 # BSD
 eventlet!=0.18.3,>=0.18.2 # MIT
+keystoneauth1>=2.20.0  # Apache-2.0
 keystonemiddleware>=4.12.0 # Apache-2.0
 oslo.concurrency>=3.8.0 # Apache-2.0
 oslo.config>=3.22.0 # Apache-2.0
@@ -24,3 +25,4 @@ stevedore>=1.20.0 # Apache-2.0
 WSME>=0.8 # MIT
 kubernetes>=1.0.0b1  # Apache-2.0
 PyYAML>=3.10.0 # MIT
+python-swiftclient>=3.2.0  # Apache-2.0

runtimes/python2/server.py

@@ -31,18 +31,26 @@ file_name = ''
 
 @app.route('/download', methods=['POST'])
 def download():
-    service_url = request.form['download_url']
+    download_url = request.form['download_url']
     function_id = request.form['function_id']
+    token = request.form['token']
+
+    headers = {}
+    if token:
+        headers = {'X-Auth-Token': token}
 
     global file_name
     file_name = '%s.zip' % function_id
 
-    app.logger.info('Request received, service_url:%s' % service_url)
+    app.logger.info(
+        'Request received, download_url:%s, headers: %s' %
+        (download_url, headers)
+    )
 
-    r = requests.get(service_url, stream=True)
+    r = requests.get(download_url, headers=headers, stream=True)
 
     with open(file_name, 'wb') as fd:
-        for chunk in r.iter_content(chunk_size=128):
+        for chunk in r.iter_content(chunk_size=65535):
             fd.write(chunk)
 
     if not zipfile.is_zipfile(file_name):