Browse Source

Add swift object support for function creation

With this patch, users can create function using swift object:

http POST http://127.0.0.1:7070/v1/functions name=swift_function \
  runtime_id=xxx \
  code='{"source": "swift", "swift": {"container": "container_name", "object": "object_name"}}'

Implements: blueprint support-swift-object-as-code
changes/17/465917/1
Lingxian Kong 4 years ago
parent
commit
b79b2791b8
10 changed files with 200 additions and 25 deletions
  1. +8
    -5
      README.rst
  2. +50
    -15
      qinling/api/controllers/v1/function.py
  3. +7
    -1
      qinling/orchestrator/kubernetes/manager.py
  4. +6
    -0
      qinling/storage/file_system.py
  5. +19
    -0
      qinling/utils/common.py
  6. +0
    -0
      qinling/utils/openstack/__init__.py
  7. +43
    -0
      qinling/utils/openstack/keystone.py
  8. +53
    -0
      qinling/utils/openstack/swift.py
  9. +2
    -0
      requirements.txt
  10. +12
    -4
      runtimes/python2/server.py

+ 8
- 5
README.rst View File

@ -100,11 +100,14 @@ python-qinlingclient is still under development.**
Perform following commands on your local host, the process will create
runtime/function/execution in Qinling.
#. First, build a docker image that is used to create runtime in Qinling and
upload to docker hub. Only ``Python 2`` runtime is supported for now, but it
is very easy to add another program language support. Run the commands in
``qinling`` repo directory, replace ``DOCKER_USER`` with your docker hub
username:
#. (Optional) Prepare a docker image including development environment for a
specific programming language. For your convenience, I already build one
(``lingxiankong/python-runtime``) in my docker hub account that you could
directly use to create runtime in Qinling. Only ``Python 2`` runtime is
supported for now, but it is very easy to add another program language
support. If you indeed want to build a new image, run the following commands
in ``qinling`` repo directory, replace ``DOCKER_USER`` with your own docker
hub username:
.. code-block:: console


+ 50
- 15
qinling/api/controllers/v1/function.py View File

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import collections
import json
import os
@ -29,16 +30,19 @@ from qinling import context
from qinling.db import api as db_api
from qinling import exceptions as exc
from qinling.storage import base as storage_base
from qinling.utils.openstack import swift as swift_util
from qinling.utils import rest_utils
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
POST_REQUIRED = set(['name', 'runtime_id', 'code'])
CODE_SOURCE = set(['package', 'swift', 'image'])
class FunctionsController(rest.RestController):
def __init__(self, *args, **kwargs):
self.storage_provider = storage_base.load_storage_provider(cfg.CONF)
self.storage_provider = storage_base.load_storage_provider(CONF)
super(FunctionsController, self).__init__(*args, **kwargs)
@ -57,15 +61,20 @@ class FunctionsController(rest.RestController):
pecan.override_template('json')
return resources.Function.from_dict(func_db.to_dict()).to_dict()
else:
f = self.storage_provider.retrieve(
ctx.projectid,
id,
)
source = func_db.code['source']
if source == 'package':
f = self.storage_provider.retrieve(ctx.projectid, id)
elif source == 'swift':
container = func_db.code['swift']['container']
obj = func_db.code['swift']['object']
f = swift_util.download_object(container, obj)
pecan.response.app_iter = FileIter(f)
pecan.response.app_iter = (f if isinstance(f, collections.Iterable)
else FileIter(f))
pecan.response.headers['Content-Type'] = 'application/zip'
pecan.response.headers['Content-Disposition'] = (
'attachment; filename="%s"' % os.path.basename(f.name)
'attachment; filename="%s"' % os.path.basename(func_db.name)
)
@rest_utils.wrap_pecan_controller_exception
@ -92,19 +101,40 @@ class FunctionsController(rest.RestController):
'entry': kwargs.get('entry', 'main'),
}
if values['code'].get('package', False):
source = values['code'].get('source')
if not source or source not in CODE_SOURCE:
raise exc.InputException(
'Invalid code source specified, available sources: %s' %
CODE_SOURCE
)
store = False
if values['code']['source'] == 'package':
store = True
data = kwargs['package'].file.read()
elif values['code']['source'] == 'swift':
# Auth needs to be enabled because qinling needs to check swift
# object using user's credential.
if not CONF.pecan.auth_enable:
raise exc.InputException('Swift object not supported.')
container = values['code']['swift'].get('container')
object = values['code']['swift'].get('object')
if not swift_util.check_object(container, object):
raise exc.InputException('Object does not exist in Swift.')
ctx = context.get_ctx()
with db_api.transaction():
func_db = db_api.create_function(values)
self.storage_provider.store(
ctx.projectid,
func_db.id,
data
)
if store:
self.storage_provider.store(
ctx.projectid,
func_db.id,
data
)
pecan.response.status = 201
return resources.Function.from_dict(func_db.to_dict()).to_dict()
@ -126,6 +156,11 @@ class FunctionsController(rest.RestController):
LOG.info("Delete function [id=%s]", id)
with db_api.transaction():
db_api.delete_function(id)
func_db = db_api.get_function(id)
source = func_db.code['source']
self.storage_provider.delete(context.get_ctx().projectid, id)
if source == 'package':
self.storage_provider.delete(context.get_ctx().projectid, id)
# This will also delete function service mapping as well.
db_api.delete_function(id)

+ 7
- 1
qinling/orchestrator/kubernetes/manager.py View File

@ -21,6 +21,7 @@ from oslo_log import log as logging
import requests
import yaml
from qinling import context
from qinling import exceptions as exc
from qinling.orchestrator import base
from qinling.utils import common
@ -223,7 +224,12 @@ class KubernetesManager(base.OrchestratorBase):
(self.conf.kubernetes.qinling_service_address,
self.conf.api.port, function_id)
)
data = {'download_url': download_url, 'function_id': function_id}
data = {
'download_url': download_url,
'function_id': function_id,
'token': context.get_ctx().auth_token
}
LOG.debug(
'Send request to pod %s, request_url: %s, data: %s',


+ 6
- 0
qinling/storage/file_system.py View File

@ -13,6 +13,7 @@
# limitations under the License.
import os
import zipfile
from oslo_config import cfg
from oslo_log import log as logging
@ -43,6 +44,11 @@ class FileSystemStorage(base.PackageStorage):
with open(func_zip, 'wb') as fd:
fd.write(data)
if not zipfile.is_zipfile(func_zip):
fileutils.delete_if_exists(func_zip)
raise exc.InputException("Package is not a valid ZIP package.")
def retrieve(self, project_id, function):
LOG.info(
'Get package data, function: %s, project: %s', function, project_id


+ 19
- 0
qinling/utils/common.py View File

@ -11,6 +11,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import functools
import warnings
from oslo_utils import uuidutils
@ -23,3 +25,20 @@ def convert_dict_to_string(d):
def generate_unicode_uuid():
return uuidutils.generate_uuid()
def disable_ssl_warnings(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
with warnings.catch_warnings():
warnings.filterwarnings(
"ignore",
message="A true SSLContext object is not available"
)
warnings.filterwarnings(
"ignore",
message="Unverified HTTPS request is being made"
)
return func(*args, **kwargs)
return wrapper

+ 0
- 0
qinling/utils/openstack/__init__.py View File


+ 43
- 0
qinling/utils/openstack/keystone.py View File

@ -0,0 +1,43 @@
# Copyright 2017 Catalyst IT Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from keystoneauth1.identity import generic
from keystoneauth1 import session
from oslo_config import cfg
import swiftclient
from qinling import context
CONF = cfg.CONF
KS_SESSION = None
def _get_user_keystone_session():
ctx = context.get_ctx()
auth = generic.Token(
auth_url=CONF.keystone_authtoken.auth_url,
token=ctx.auth_token,
)
return session.Session(auth=auth, verify=False)
def get_swiftclient():
session = _get_user_keystone_session()
conn = swiftclient.Connection(session=session)
return conn

+ 53
- 0
qinling/utils/openstack/swift.py View File

@ -0,0 +1,53 @@
# Copyright 2017 Catalyst IT Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_log import log as logging
from swiftclient.exceptions import ClientException
from qinling.utils import common
from qinling.utils.openstack import keystone
LOG = logging.getLogger(__name__)
@common.disable_ssl_warnings
def check_object(container, object):
"""Check if object exists in Swift.
:param container: Container name.
:param object: Object name.
:return: True if object exists, otherwise return False.
"""
swift_conn = keystone.get_swiftclient()
try:
swift_conn.head_object(container, object)
return True
except ClientException:
LOG.error(
'The object %s in container %s was not found', object, container
)
return False
@common.disable_ssl_warnings
def download_object(container, object):
swift_conn = keystone.get_swiftclient()
# Specify 'resp_chunk_size' here to return a file reader.
_, obj_reader = swift_conn.get_object(
container, object, resp_chunk_size=65536
)
return obj_reader

+ 2
- 0
requirements.txt View File

@ -5,6 +5,7 @@
pbr>=2.0 # Apache-2.0
Babel!=2.4.0,>=2.3.4 # BSD
eventlet!=0.18.3,>=0.18.2 # MIT
keystoneauth1>=2.20.0 # Apache-2.0
keystonemiddleware>=4.12.0 # Apache-2.0
oslo.concurrency>=3.8.0 # Apache-2.0
oslo.config>=3.22.0 # Apache-2.0
@ -24,3 +25,4 @@ stevedore>=1.20.0 # Apache-2.0
WSME>=0.8 # MIT
kubernetes>=1.0.0b1 # Apache-2.0
PyYAML>=3.10.0 # MIT
python-swiftclient>=3.2.0 # Apache-2.0

+ 12
- 4
runtimes/python2/server.py View File

@ -31,18 +31,26 @@ file_name = ''
@app.route('/download', methods=['POST'])
def download():
service_url = request.form['download_url']
download_url = request.form['download_url']
function_id = request.form['function_id']
token = request.form['token']
headers = {}
if token:
headers = {'X-Auth-Token': token}
global file_name
file_name = '%s.zip' % function_id
app.logger.info('Request received, service_url:%s' % service_url)
app.logger.info(
'Request received, download_url:%s, headers: %s' %
(download_url, headers)
)
r = requests.get(service_url, stream=True)
r = requests.get(download_url, headers=headers, stream=True)
with open(file_name, 'wb') as fd:
for chunk in r.iter_content(chunk_size=128):
for chunk in r.iter_content(chunk_size=65535):
fd.write(chunk)
if not zipfile.is_zipfile(file_name):


Loading…
Cancel
Save