Add swift object support for function creation
With this patch, users can create function using swift object: http POST http://127.0.0.1:7070/v1/functions name=swift_function \ runtime_id=xxx \ code='{"source": "swift", "swift": {"container": "container_name", "object": "object_name"}}' Implements: blueprint support-swift-object-as-code
This commit is contained in:
parent
6c71504654
commit
b79b2791b8
13
README.rst
13
README.rst
@ -100,11 +100,14 @@ python-qinlingclient is still under development.**
|
|||||||
Perform following commands on your local host, the process will create
|
Perform following commands on your local host, the process will create
|
||||||
runtime/function/execution in Qinling.
|
runtime/function/execution in Qinling.
|
||||||
|
|
||||||
#. First, build a docker image that is used to create runtime in Qinling and
|
#. (Optional) Prepare a docker image including development environment for a
|
||||||
upload to docker hub. Only ``Python 2`` runtime is supported for now, but it
|
specific programming language. For your convenience, I already build one
|
||||||
is very easy to add another program language support. Run the commands in
|
(``lingxiankong/python-runtime``) in my docker hub account that you could
|
||||||
``qinling`` repo directory, replace ``DOCKER_USER`` with your docker hub
|
directly use to create runtime in Qinling. Only ``Python 2`` runtime is
|
||||||
username:
|
supported for now, but it is very easy to add another program language
|
||||||
|
support. If you indeed want to build a new image, run the following commands
|
||||||
|
in ``qinling`` repo directory, replace ``DOCKER_USER`` with your own docker
|
||||||
|
hub username:
|
||||||
|
|
||||||
.. code-block:: console
|
.. code-block:: console
|
||||||
|
|
||||||
|
@ -12,6 +12,7 @@
|
|||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
import collections
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
|
||||||
@ -29,16 +30,19 @@ from qinling import context
|
|||||||
from qinling.db import api as db_api
|
from qinling.db import api as db_api
|
||||||
from qinling import exceptions as exc
|
from qinling import exceptions as exc
|
||||||
from qinling.storage import base as storage_base
|
from qinling.storage import base as storage_base
|
||||||
|
from qinling.utils.openstack import swift as swift_util
|
||||||
from qinling.utils import rest_utils
|
from qinling.utils import rest_utils
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
LOG = logging.getLogger(__name__)
|
||||||
|
CONF = cfg.CONF
|
||||||
|
|
||||||
POST_REQUIRED = set(['name', 'runtime_id', 'code'])
|
POST_REQUIRED = set(['name', 'runtime_id', 'code'])
|
||||||
|
CODE_SOURCE = set(['package', 'swift', 'image'])
|
||||||
|
|
||||||
|
|
||||||
class FunctionsController(rest.RestController):
|
class FunctionsController(rest.RestController):
|
||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
self.storage_provider = storage_base.load_storage_provider(cfg.CONF)
|
self.storage_provider = storage_base.load_storage_provider(CONF)
|
||||||
|
|
||||||
super(FunctionsController, self).__init__(*args, **kwargs)
|
super(FunctionsController, self).__init__(*args, **kwargs)
|
||||||
|
|
||||||
@ -57,15 +61,20 @@ class FunctionsController(rest.RestController):
|
|||||||
pecan.override_template('json')
|
pecan.override_template('json')
|
||||||
return resources.Function.from_dict(func_db.to_dict()).to_dict()
|
return resources.Function.from_dict(func_db.to_dict()).to_dict()
|
||||||
else:
|
else:
|
||||||
f = self.storage_provider.retrieve(
|
source = func_db.code['source']
|
||||||
ctx.projectid,
|
|
||||||
id,
|
|
||||||
)
|
|
||||||
|
|
||||||
pecan.response.app_iter = FileIter(f)
|
if source == 'package':
|
||||||
|
f = self.storage_provider.retrieve(ctx.projectid, id)
|
||||||
|
elif source == 'swift':
|
||||||
|
container = func_db.code['swift']['container']
|
||||||
|
obj = func_db.code['swift']['object']
|
||||||
|
f = swift_util.download_object(container, obj)
|
||||||
|
|
||||||
|
pecan.response.app_iter = (f if isinstance(f, collections.Iterable)
|
||||||
|
else FileIter(f))
|
||||||
pecan.response.headers['Content-Type'] = 'application/zip'
|
pecan.response.headers['Content-Type'] = 'application/zip'
|
||||||
pecan.response.headers['Content-Disposition'] = (
|
pecan.response.headers['Content-Disposition'] = (
|
||||||
'attachment; filename="%s"' % os.path.basename(f.name)
|
'attachment; filename="%s"' % os.path.basename(func_db.name)
|
||||||
)
|
)
|
||||||
|
|
||||||
@rest_utils.wrap_pecan_controller_exception
|
@rest_utils.wrap_pecan_controller_exception
|
||||||
@ -92,19 +101,40 @@ class FunctionsController(rest.RestController):
|
|||||||
'entry': kwargs.get('entry', 'main'),
|
'entry': kwargs.get('entry', 'main'),
|
||||||
}
|
}
|
||||||
|
|
||||||
if values['code'].get('package', False):
|
source = values['code'].get('source')
|
||||||
|
if not source or source not in CODE_SOURCE:
|
||||||
|
raise exc.InputException(
|
||||||
|
'Invalid code source specified, available sources: %s' %
|
||||||
|
CODE_SOURCE
|
||||||
|
)
|
||||||
|
|
||||||
|
store = False
|
||||||
|
if values['code']['source'] == 'package':
|
||||||
|
store = True
|
||||||
data = kwargs['package'].file.read()
|
data = kwargs['package'].file.read()
|
||||||
|
elif values['code']['source'] == 'swift':
|
||||||
|
# Auth needs to be enabled because qinling needs to check swift
|
||||||
|
# object using user's credential.
|
||||||
|
if not CONF.pecan.auth_enable:
|
||||||
|
raise exc.InputException('Swift object not supported.')
|
||||||
|
|
||||||
|
container = values['code']['swift'].get('container')
|
||||||
|
object = values['code']['swift'].get('object')
|
||||||
|
|
||||||
|
if not swift_util.check_object(container, object):
|
||||||
|
raise exc.InputException('Object does not exist in Swift.')
|
||||||
|
|
||||||
ctx = context.get_ctx()
|
ctx = context.get_ctx()
|
||||||
|
|
||||||
with db_api.transaction():
|
with db_api.transaction():
|
||||||
func_db = db_api.create_function(values)
|
func_db = db_api.create_function(values)
|
||||||
|
|
||||||
self.storage_provider.store(
|
if store:
|
||||||
ctx.projectid,
|
self.storage_provider.store(
|
||||||
func_db.id,
|
ctx.projectid,
|
||||||
data
|
func_db.id,
|
||||||
)
|
data
|
||||||
|
)
|
||||||
|
|
||||||
pecan.response.status = 201
|
pecan.response.status = 201
|
||||||
return resources.Function.from_dict(func_db.to_dict()).to_dict()
|
return resources.Function.from_dict(func_db.to_dict()).to_dict()
|
||||||
@ -126,6 +156,11 @@ class FunctionsController(rest.RestController):
|
|||||||
LOG.info("Delete function [id=%s]", id)
|
LOG.info("Delete function [id=%s]", id)
|
||||||
|
|
||||||
with db_api.transaction():
|
with db_api.transaction():
|
||||||
db_api.delete_function(id)
|
func_db = db_api.get_function(id)
|
||||||
|
source = func_db.code['source']
|
||||||
|
|
||||||
self.storage_provider.delete(context.get_ctx().projectid, id)
|
if source == 'package':
|
||||||
|
self.storage_provider.delete(context.get_ctx().projectid, id)
|
||||||
|
|
||||||
|
# This will also delete function service mapping as well.
|
||||||
|
db_api.delete_function(id)
|
||||||
|
@ -21,6 +21,7 @@ from oslo_log import log as logging
|
|||||||
import requests
|
import requests
|
||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
|
from qinling import context
|
||||||
from qinling import exceptions as exc
|
from qinling import exceptions as exc
|
||||||
from qinling.orchestrator import base
|
from qinling.orchestrator import base
|
||||||
from qinling.utils import common
|
from qinling.utils import common
|
||||||
@ -223,7 +224,12 @@ class KubernetesManager(base.OrchestratorBase):
|
|||||||
(self.conf.kubernetes.qinling_service_address,
|
(self.conf.kubernetes.qinling_service_address,
|
||||||
self.conf.api.port, function_id)
|
self.conf.api.port, function_id)
|
||||||
)
|
)
|
||||||
data = {'download_url': download_url, 'function_id': function_id}
|
|
||||||
|
data = {
|
||||||
|
'download_url': download_url,
|
||||||
|
'function_id': function_id,
|
||||||
|
'token': context.get_ctx().auth_token
|
||||||
|
}
|
||||||
|
|
||||||
LOG.debug(
|
LOG.debug(
|
||||||
'Send request to pod %s, request_url: %s, data: %s',
|
'Send request to pod %s, request_url: %s, data: %s',
|
||||||
|
@ -13,6 +13,7 @@
|
|||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import zipfile
|
||||||
|
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
from oslo_log import log as logging
|
from oslo_log import log as logging
|
||||||
@ -43,6 +44,11 @@ class FileSystemStorage(base.PackageStorage):
|
|||||||
with open(func_zip, 'wb') as fd:
|
with open(func_zip, 'wb') as fd:
|
||||||
fd.write(data)
|
fd.write(data)
|
||||||
|
|
||||||
|
if not zipfile.is_zipfile(func_zip):
|
||||||
|
fileutils.delete_if_exists(func_zip)
|
||||||
|
|
||||||
|
raise exc.InputException("Package is not a valid ZIP package.")
|
||||||
|
|
||||||
def retrieve(self, project_id, function):
|
def retrieve(self, project_id, function):
|
||||||
LOG.info(
|
LOG.info(
|
||||||
'Get package data, function: %s, project: %s', function, project_id
|
'Get package data, function: %s, project: %s', function, project_id
|
||||||
|
@ -11,6 +11,8 @@
|
|||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
import functools
|
||||||
|
import warnings
|
||||||
|
|
||||||
from oslo_utils import uuidutils
|
from oslo_utils import uuidutils
|
||||||
|
|
||||||
@ -23,3 +25,20 @@ def convert_dict_to_string(d):
|
|||||||
|
|
||||||
def generate_unicode_uuid():
|
def generate_unicode_uuid():
|
||||||
return uuidutils.generate_uuid()
|
return uuidutils.generate_uuid()
|
||||||
|
|
||||||
|
|
||||||
|
def disable_ssl_warnings(func):
|
||||||
|
@functools.wraps(func)
|
||||||
|
def wrapper(*args, **kwargs):
|
||||||
|
with warnings.catch_warnings():
|
||||||
|
warnings.filterwarnings(
|
||||||
|
"ignore",
|
||||||
|
message="A true SSLContext object is not available"
|
||||||
|
)
|
||||||
|
warnings.filterwarnings(
|
||||||
|
"ignore",
|
||||||
|
message="Unverified HTTPS request is being made"
|
||||||
|
)
|
||||||
|
return func(*args, **kwargs)
|
||||||
|
|
||||||
|
return wrapper
|
||||||
|
0
qinling/utils/openstack/__init__.py
Normal file
0
qinling/utils/openstack/__init__.py
Normal file
43
qinling/utils/openstack/keystone.py
Normal file
43
qinling/utils/openstack/keystone.py
Normal file
@ -0,0 +1,43 @@
|
|||||||
|
# Copyright 2017 Catalyst IT Limited
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
from keystoneauth1.identity import generic
|
||||||
|
from keystoneauth1 import session
|
||||||
|
from oslo_config import cfg
|
||||||
|
import swiftclient
|
||||||
|
|
||||||
|
from qinling import context
|
||||||
|
|
||||||
|
CONF = cfg.CONF
|
||||||
|
|
||||||
|
KS_SESSION = None
|
||||||
|
|
||||||
|
|
||||||
|
def _get_user_keystone_session():
|
||||||
|
ctx = context.get_ctx()
|
||||||
|
|
||||||
|
auth = generic.Token(
|
||||||
|
auth_url=CONF.keystone_authtoken.auth_url,
|
||||||
|
token=ctx.auth_token,
|
||||||
|
)
|
||||||
|
|
||||||
|
return session.Session(auth=auth, verify=False)
|
||||||
|
|
||||||
|
|
||||||
|
def get_swiftclient():
|
||||||
|
session = _get_user_keystone_session()
|
||||||
|
|
||||||
|
conn = swiftclient.Connection(session=session)
|
||||||
|
|
||||||
|
return conn
|
53
qinling/utils/openstack/swift.py
Normal file
53
qinling/utils/openstack/swift.py
Normal file
@ -0,0 +1,53 @@
|
|||||||
|
# Copyright 2017 Catalyst IT Limited
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
from oslo_log import log as logging
|
||||||
|
from swiftclient.exceptions import ClientException
|
||||||
|
|
||||||
|
from qinling.utils import common
|
||||||
|
from qinling.utils.openstack import keystone
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@common.disable_ssl_warnings
|
||||||
|
def check_object(container, object):
|
||||||
|
"""Check if object exists in Swift.
|
||||||
|
|
||||||
|
:param container: Container name.
|
||||||
|
:param object: Object name.
|
||||||
|
:return: True if object exists, otherwise return False.
|
||||||
|
"""
|
||||||
|
swift_conn = keystone.get_swiftclient()
|
||||||
|
|
||||||
|
try:
|
||||||
|
swift_conn.head_object(container, object)
|
||||||
|
return True
|
||||||
|
except ClientException:
|
||||||
|
LOG.error(
|
||||||
|
'The object %s in container %s was not found', object, container
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
@common.disable_ssl_warnings
|
||||||
|
def download_object(container, object):
|
||||||
|
swift_conn = keystone.get_swiftclient()
|
||||||
|
|
||||||
|
# Specify 'resp_chunk_size' here to return a file reader.
|
||||||
|
_, obj_reader = swift_conn.get_object(
|
||||||
|
container, object, resp_chunk_size=65536
|
||||||
|
)
|
||||||
|
|
||||||
|
return obj_reader
|
@ -5,6 +5,7 @@
|
|||||||
pbr>=2.0 # Apache-2.0
|
pbr>=2.0 # Apache-2.0
|
||||||
Babel!=2.4.0,>=2.3.4 # BSD
|
Babel!=2.4.0,>=2.3.4 # BSD
|
||||||
eventlet!=0.18.3,>=0.18.2 # MIT
|
eventlet!=0.18.3,>=0.18.2 # MIT
|
||||||
|
keystoneauth1>=2.20.0 # Apache-2.0
|
||||||
keystonemiddleware>=4.12.0 # Apache-2.0
|
keystonemiddleware>=4.12.0 # Apache-2.0
|
||||||
oslo.concurrency>=3.8.0 # Apache-2.0
|
oslo.concurrency>=3.8.0 # Apache-2.0
|
||||||
oslo.config>=3.22.0 # Apache-2.0
|
oslo.config>=3.22.0 # Apache-2.0
|
||||||
@ -24,3 +25,4 @@ stevedore>=1.20.0 # Apache-2.0
|
|||||||
WSME>=0.8 # MIT
|
WSME>=0.8 # MIT
|
||||||
kubernetes>=1.0.0b1 # Apache-2.0
|
kubernetes>=1.0.0b1 # Apache-2.0
|
||||||
PyYAML>=3.10.0 # MIT
|
PyYAML>=3.10.0 # MIT
|
||||||
|
python-swiftclient>=3.2.0 # Apache-2.0
|
||||||
|
@ -31,18 +31,26 @@ file_name = ''
|
|||||||
|
|
||||||
@app.route('/download', methods=['POST'])
|
@app.route('/download', methods=['POST'])
|
||||||
def download():
|
def download():
|
||||||
service_url = request.form['download_url']
|
download_url = request.form['download_url']
|
||||||
function_id = request.form['function_id']
|
function_id = request.form['function_id']
|
||||||
|
token = request.form['token']
|
||||||
|
|
||||||
|
headers = {}
|
||||||
|
if token:
|
||||||
|
headers = {'X-Auth-Token': token}
|
||||||
|
|
||||||
global file_name
|
global file_name
|
||||||
file_name = '%s.zip' % function_id
|
file_name = '%s.zip' % function_id
|
||||||
|
|
||||||
app.logger.info('Request received, service_url:%s' % service_url)
|
app.logger.info(
|
||||||
|
'Request received, download_url:%s, headers: %s' %
|
||||||
|
(download_url, headers)
|
||||||
|
)
|
||||||
|
|
||||||
r = requests.get(service_url, stream=True)
|
r = requests.get(download_url, headers=headers, stream=True)
|
||||||
|
|
||||||
with open(file_name, 'wb') as fd:
|
with open(file_name, 'wb') as fd:
|
||||||
for chunk in r.iter_content(chunk_size=128):
|
for chunk in r.iter_content(chunk_size=65535):
|
||||||
fd.write(chunk)
|
fd.write(chunk)
|
||||||
|
|
||||||
if not zipfile.is_zipfile(file_name):
|
if not zipfile.is_zipfile(file_name):
|
||||||
|
Loading…
Reference in New Issue
Block a user