Add swift object support for function creation

With this patch, users can create function using swift object:

http POST http://127.0.0.1:7070/v1/functions name=swift_function \
  runtime_id=xxx \
  code='{"source": "swift", "swift": {"container": "container_name", "object": "object_name"}}'

Implements: blueprint support-swift-object-as-code
This commit is contained in:
Lingxian Kong 2017-05-16 15:24:46 +12:00
parent 6c71504654
commit b79b2791b8
10 changed files with 200 additions and 25 deletions

View File

@ -100,11 +100,14 @@ python-qinlingclient is still under development.**
Perform following commands on your local host, the process will create Perform following commands on your local host, the process will create
runtime/function/execution in Qinling. runtime/function/execution in Qinling.
#. First, build a docker image that is used to create runtime in Qinling and #. (Optional) Prepare a docker image including development environment for a
upload to docker hub. Only ``Python 2`` runtime is supported for now, but it specific programming language. For your convenience, I already build one
is very easy to add another program language support. Run the commands in (``lingxiankong/python-runtime``) in my docker hub account that you could
``qinling`` repo directory, replace ``DOCKER_USER`` with your docker hub directly use to create runtime in Qinling. Only ``Python 2`` runtime is
username: supported for now, but it is very easy to add another program language
support. If you indeed want to build a new image, run the following commands
in ``qinling`` repo directory, replace ``DOCKER_USER`` with your own docker
hub username:
.. code-block:: console .. code-block:: console

View File

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import collections
import json import json
import os import os
@ -29,16 +30,19 @@ from qinling import context
from qinling.db import api as db_api from qinling.db import api as db_api
from qinling import exceptions as exc from qinling import exceptions as exc
from qinling.storage import base as storage_base from qinling.storage import base as storage_base
from qinling.utils.openstack import swift as swift_util
from qinling.utils import rest_utils from qinling.utils import rest_utils
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
CONF = cfg.CONF
POST_REQUIRED = set(['name', 'runtime_id', 'code']) POST_REQUIRED = set(['name', 'runtime_id', 'code'])
CODE_SOURCE = set(['package', 'swift', 'image'])
class FunctionsController(rest.RestController): class FunctionsController(rest.RestController):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self.storage_provider = storage_base.load_storage_provider(cfg.CONF) self.storage_provider = storage_base.load_storage_provider(CONF)
super(FunctionsController, self).__init__(*args, **kwargs) super(FunctionsController, self).__init__(*args, **kwargs)
@ -57,15 +61,20 @@ class FunctionsController(rest.RestController):
pecan.override_template('json') pecan.override_template('json')
return resources.Function.from_dict(func_db.to_dict()).to_dict() return resources.Function.from_dict(func_db.to_dict()).to_dict()
else: else:
f = self.storage_provider.retrieve( source = func_db.code['source']
ctx.projectid,
id,
)
pecan.response.app_iter = FileIter(f) if source == 'package':
f = self.storage_provider.retrieve(ctx.projectid, id)
elif source == 'swift':
container = func_db.code['swift']['container']
obj = func_db.code['swift']['object']
f = swift_util.download_object(container, obj)
pecan.response.app_iter = (f if isinstance(f, collections.Iterable)
else FileIter(f))
pecan.response.headers['Content-Type'] = 'application/zip' pecan.response.headers['Content-Type'] = 'application/zip'
pecan.response.headers['Content-Disposition'] = ( pecan.response.headers['Content-Disposition'] = (
'attachment; filename="%s"' % os.path.basename(f.name) 'attachment; filename="%s"' % os.path.basename(func_db.name)
) )
@rest_utils.wrap_pecan_controller_exception @rest_utils.wrap_pecan_controller_exception
@ -92,19 +101,40 @@ class FunctionsController(rest.RestController):
'entry': kwargs.get('entry', 'main'), 'entry': kwargs.get('entry', 'main'),
} }
if values['code'].get('package', False): source = values['code'].get('source')
if not source or source not in CODE_SOURCE:
raise exc.InputException(
'Invalid code source specified, available sources: %s' %
CODE_SOURCE
)
store = False
if values['code']['source'] == 'package':
store = True
data = kwargs['package'].file.read() data = kwargs['package'].file.read()
elif values['code']['source'] == 'swift':
# Auth needs to be enabled because qinling needs to check swift
# object using user's credential.
if not CONF.pecan.auth_enable:
raise exc.InputException('Swift object not supported.')
container = values['code']['swift'].get('container')
object = values['code']['swift'].get('object')
if not swift_util.check_object(container, object):
raise exc.InputException('Object does not exist in Swift.')
ctx = context.get_ctx() ctx = context.get_ctx()
with db_api.transaction(): with db_api.transaction():
func_db = db_api.create_function(values) func_db = db_api.create_function(values)
self.storage_provider.store( if store:
ctx.projectid, self.storage_provider.store(
func_db.id, ctx.projectid,
data func_db.id,
) data
)
pecan.response.status = 201 pecan.response.status = 201
return resources.Function.from_dict(func_db.to_dict()).to_dict() return resources.Function.from_dict(func_db.to_dict()).to_dict()
@ -126,6 +156,11 @@ class FunctionsController(rest.RestController):
LOG.info("Delete function [id=%s]", id) LOG.info("Delete function [id=%s]", id)
with db_api.transaction(): with db_api.transaction():
db_api.delete_function(id) func_db = db_api.get_function(id)
source = func_db.code['source']
self.storage_provider.delete(context.get_ctx().projectid, id) if source == 'package':
self.storage_provider.delete(context.get_ctx().projectid, id)
# This will also delete function service mapping as well.
db_api.delete_function(id)

View File

@ -21,6 +21,7 @@ from oslo_log import log as logging
import requests import requests
import yaml import yaml
from qinling import context
from qinling import exceptions as exc from qinling import exceptions as exc
from qinling.orchestrator import base from qinling.orchestrator import base
from qinling.utils import common from qinling.utils import common
@ -223,7 +224,12 @@ class KubernetesManager(base.OrchestratorBase):
(self.conf.kubernetes.qinling_service_address, (self.conf.kubernetes.qinling_service_address,
self.conf.api.port, function_id) self.conf.api.port, function_id)
) )
data = {'download_url': download_url, 'function_id': function_id}
data = {
'download_url': download_url,
'function_id': function_id,
'token': context.get_ctx().auth_token
}
LOG.debug( LOG.debug(
'Send request to pod %s, request_url: %s, data: %s', 'Send request to pod %s, request_url: %s, data: %s',

View File

@ -13,6 +13,7 @@
# limitations under the License. # limitations under the License.
import os import os
import zipfile
from oslo_config import cfg from oslo_config import cfg
from oslo_log import log as logging from oslo_log import log as logging
@ -43,6 +44,11 @@ class FileSystemStorage(base.PackageStorage):
with open(func_zip, 'wb') as fd: with open(func_zip, 'wb') as fd:
fd.write(data) fd.write(data)
if not zipfile.is_zipfile(func_zip):
fileutils.delete_if_exists(func_zip)
raise exc.InputException("Package is not a valid ZIP package.")
def retrieve(self, project_id, function): def retrieve(self, project_id, function):
LOG.info( LOG.info(
'Get package data, function: %s, project: %s', function, project_id 'Get package data, function: %s, project: %s', function, project_id

View File

@ -11,6 +11,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import functools
import warnings
from oslo_utils import uuidutils from oslo_utils import uuidutils
@ -23,3 +25,20 @@ def convert_dict_to_string(d):
def generate_unicode_uuid(): def generate_unicode_uuid():
return uuidutils.generate_uuid() return uuidutils.generate_uuid()
def disable_ssl_warnings(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
with warnings.catch_warnings():
warnings.filterwarnings(
"ignore",
message="A true SSLContext object is not available"
)
warnings.filterwarnings(
"ignore",
message="Unverified HTTPS request is being made"
)
return func(*args, **kwargs)
return wrapper

View File

View File

@ -0,0 +1,43 @@
# Copyright 2017 Catalyst IT Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from keystoneauth1.identity import generic
from keystoneauth1 import session
from oslo_config import cfg
import swiftclient
from qinling import context
CONF = cfg.CONF
KS_SESSION = None
def _get_user_keystone_session():
ctx = context.get_ctx()
auth = generic.Token(
auth_url=CONF.keystone_authtoken.auth_url,
token=ctx.auth_token,
)
return session.Session(auth=auth, verify=False)
def get_swiftclient():
session = _get_user_keystone_session()
conn = swiftclient.Connection(session=session)
return conn

View File

@ -0,0 +1,53 @@
# Copyright 2017 Catalyst IT Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_log import log as logging
from swiftclient.exceptions import ClientException
from qinling.utils import common
from qinling.utils.openstack import keystone
LOG = logging.getLogger(__name__)
@common.disable_ssl_warnings
def check_object(container, object):
"""Check if object exists in Swift.
:param container: Container name.
:param object: Object name.
:return: True if object exists, otherwise return False.
"""
swift_conn = keystone.get_swiftclient()
try:
swift_conn.head_object(container, object)
return True
except ClientException:
LOG.error(
'The object %s in container %s was not found', object, container
)
return False
@common.disable_ssl_warnings
def download_object(container, object):
swift_conn = keystone.get_swiftclient()
# Specify 'resp_chunk_size' here to return a file reader.
_, obj_reader = swift_conn.get_object(
container, object, resp_chunk_size=65536
)
return obj_reader

View File

@ -5,6 +5,7 @@
pbr>=2.0 # Apache-2.0 pbr>=2.0 # Apache-2.0
Babel!=2.4.0,>=2.3.4 # BSD Babel!=2.4.0,>=2.3.4 # BSD
eventlet!=0.18.3,>=0.18.2 # MIT eventlet!=0.18.3,>=0.18.2 # MIT
keystoneauth1>=2.20.0 # Apache-2.0
keystonemiddleware>=4.12.0 # Apache-2.0 keystonemiddleware>=4.12.0 # Apache-2.0
oslo.concurrency>=3.8.0 # Apache-2.0 oslo.concurrency>=3.8.0 # Apache-2.0
oslo.config>=3.22.0 # Apache-2.0 oslo.config>=3.22.0 # Apache-2.0
@ -24,3 +25,4 @@ stevedore>=1.20.0 # Apache-2.0
WSME>=0.8 # MIT WSME>=0.8 # MIT
kubernetes>=1.0.0b1 # Apache-2.0 kubernetes>=1.0.0b1 # Apache-2.0
PyYAML>=3.10.0 # MIT PyYAML>=3.10.0 # MIT
python-swiftclient>=3.2.0 # Apache-2.0

View File

@ -31,18 +31,26 @@ file_name = ''
@app.route('/download', methods=['POST']) @app.route('/download', methods=['POST'])
def download(): def download():
service_url = request.form['download_url'] download_url = request.form['download_url']
function_id = request.form['function_id'] function_id = request.form['function_id']
token = request.form['token']
headers = {}
if token:
headers = {'X-Auth-Token': token}
global file_name global file_name
file_name = '%s.zip' % function_id file_name = '%s.zip' % function_id
app.logger.info('Request received, service_url:%s' % service_url) app.logger.info(
'Request received, download_url:%s, headers: %s' %
(download_url, headers)
)
r = requests.get(service_url, stream=True) r = requests.get(download_url, headers=headers, stream=True)
with open(file_name, 'wb') as fd: with open(file_name, 'wb') as fd:
for chunk in r.iter_content(chunk_size=128): for chunk in r.iter_content(chunk_size=65535):
fd.write(chunk) fd.write(chunk)
if not zipfile.is_zipfile(file_name): if not zipfile.is_zipfile(file_name):