Browse Source

Add session support for function

Also add the documentation.

Partially implements: blueprint qinling-support-user-session
Change-Id: Ic11b3b9f6ffe65bb08cfd880b81d41a40590919b
changes/77/505977/6
Lingxian Kong 4 years ago
parent
commit
d2dd717a2f
  1. 2
      doc/source/contributor/index.rst
  2. 33
      doc/source/function_developer/index.rst
  3. 51
      doc/source/function_developer/openstack_integration.rst
  4. 18
      doc/source/index.rst
  5. 33
      qinling/config.py
  6. 2
      qinling/db/sqlalchemy/migration/alembic_migrations/versions/001_pike.py
  7. 9
      qinling/engine/default_engine.py
  8. 3
      qinling/engine/service.py
  9. 27
      qinling/orchestrator/kubernetes/manager.py
  10. 14
      qinling/tests/unit/api/base.py
  11. 14
      qinling/tests/unit/base.py
  12. 26
      qinling/tests/unit/config.py
  13. 2
      qinling_tempest_plugin/functions/python_test.py
  14. 36
      runtimes/python2/server.py

2
doc/source/contributor/index.rst

@ -22,8 +22,6 @@ In this section you will find information helpful for contributing to qinling
project.
Programming HowTos and Tutorials
--------------------------------
.. toctree::
:maxdepth: 2

33
doc/source/function_developer/index.rst

@ -0,0 +1,33 @@
..
Copyright 2010-2011 United States Government as represented by the
Administrator of the National Aeronautics and Space Administration.
All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
Function Programming Guide
==========================
In this section you will find information helpful for developing functions.
.. toctree::
:maxdepth: 2
openstack_integration
Indices and tables
------------------
* :ref:`genindex`
* :ref:`search`

51
doc/source/function_developer/openstack_integration.rst

@ -0,0 +1,51 @@
..
Copyright 2017 Catalyst IT Ltd
All Rights Reserved.
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
Interact with other OpenStack services
======================================
It's very easy to interact with other OpenStack services in your function.
Let's take Python programming language and integration with Swift service for
an example.
At the time you create a function, you specify an entry, which is a function
in your code module that Qinling can invoke when the service executes your
code. Use the following general syntax structure when creating a function in
Python which will interact with Swift service in OpenStack.
.. code-block:: python
import swiftclient
def main(region_name, container, object, context=None, **kwargs):
conn = swiftclient.Connection(
session=context['os_session'],
os_options={'region_name': region_name},
)
obj_info = conn.head_object(container, object)
return obj_info
In the above code, note the following:
- Qinling supports most of OpenStack service clients, so you don't need to
install ``python-swiftclient`` in your code package.
- There is a parameter named ``context``, this is a parameter provided by
Qinling that is usually of the Python dict type. You can easily get a valid
Keystone session object from it. As a result, you don't need to pass any
sensitive data to Qinling in order to interact with OpenStack services.
.. note::
Please avoid using ``context`` as your own parameter in the code.

18
doc/source/index.rst

@ -27,24 +27,32 @@ Lambda). Qinling supports different container orchestration platforms
* IRC channel on Freenode: #openstack-qinling
Table of Contents
=================
Overview
--------
.. toctree::
:maxdepth: 1
quick_start
Contributor/Developer Docs
==========================
Contributor/Developer Guide
---------------------------
.. toctree::
:maxdepth: 1
contributor/index
Function Programming Guide
--------------------------
.. toctree::
:maxdepth: 1
function_developer/index
Indices and tables
==================
------------------
* :ref:`genindex`
* :ref:`search`

33
qinling/config.py

@ -11,17 +11,15 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Configuration options registration and useful routines.
"""
import itertools
from keystoneauth1 import loading
from keystonemiddleware import auth_token
from oslo_config import cfg
from oslo_log import log
from qinling import version
CONF = cfg.CONF
launch_opt = cfg.ListOpt(
'server',
default=['all'],
@ -137,25 +135,24 @@ kubernetes_opts = [
),
]
CONF = cfg.CONF
CLI_OPTS = [launch_opt]
CONF.register_cli_opts(CLI_OPTS)
default_group_opts = itertools.chain(CLI_OPTS, [])
def list_opts():
return [
keystone_middleware_opts = auth_token.list_opts()
keystone_loading_opts = [(
'keystone_authtoken', loading.get_auth_plugin_conf_options('password')
)]
qinling_opts = [
(API_GROUP, api_opts),
(PECAN_GROUP, pecan_opts),
(ENGINE_GROUP, engine_opts),
(STORAGE_GROUP, storage_opts),
(KUBERNETES_GROUP, kubernetes_opts),
(None, default_group_opts)
(None, [launch_opt])
]
return keystone_middleware_opts + keystone_loading_opts + qinling_opts
for group, options in list_opts():
CONF.register_opts(list(options), group)
_DEFAULT_LOG_LEVELS = [
'eventlet.wsgi.server=WARN',
@ -176,6 +173,12 @@ def parse_args(args=None, usage=None, default_config_files=None):
log.set_defaults(default_log_levels=default_log_levels)
log.register_options(CONF)
CLI_OPTS = [launch_opt]
CONF.register_cli_opts(CLI_OPTS)
for group, options in list_opts():
CONF.register_opts(list(options), group)
CONF(
args=args,
project='qinling',

2
qinling/db/sqlalchemy/migration/alembic_migrations/versions/001_pike.py

@ -134,7 +134,7 @@ def upgrade():
sa.Column('function_input', st.JsonLongDictType(), nullable=True),
sa.Column('status', sa.String(length=32), nullable=False),
sa.Column('name', sa.String(length=255), nullable=True),
sa.Column('pattern', sa.String(length=32), nullable=False),
sa.Column('pattern', sa.String(length=32), nullable=True),
sa.Column('first_execution_time', sa.DateTime(), nullable=True),
sa.Column('next_execution_time', sa.DateTime(), nullable=False),
sa.Column('count', sa.Integer(), nullable=True),

9
qinling/engine/default_engine.py

@ -16,6 +16,7 @@ from oslo_config import cfg
from oslo_log import log as logging
import requests
from qinling import context
from qinling.db import api as db_api
from qinling import status
from qinling.utils import common
@ -103,6 +104,14 @@ class DefaultEngine(object):
)
data = {'input': input, 'execution_id': execution_id}
if CONF.pecan.auth_enable:
data.update(
{
'token': context.get_ctx().auth_token,
'trust_id': context.get_ctx().trust_id
}
)
r = self.session.post(func_url, json=data)
res = r.json()

3
qinling/engine/service.py

@ -55,7 +55,8 @@ class EngineService(service.Service):
executor='eventlet',
access_policy=access_policy,
serializer=rpc.ContextSerializer(
messaging.serializer.JsonPayloadSerializer())
messaging.serializer.JsonPayloadSerializer()
)
)
LOG.info('Starting engine...')

27
qinling/orchestrator/kubernetes/manager.py

@ -298,12 +298,19 @@ class KubernetesManager(base.OrchestratorBase):
'download_url': download_url,
'function_id': actual_function,
'entry': entry,
'token': context.get_ctx().auth_token,
}
if self.conf.pecan.auth_enable:
data.update(
{
'token': context.get_ctx().auth_token,
'auth_url': self.conf.keystone_authtoken.auth_uri,
'username': self.conf.keystone_authtoken.username,
'password': self.conf.keystone_authtoken.password,
}
)
LOG.debug(
'Send request to pod %s, request_url: %s, data: %s',
name, request_url, data
LOG.info(
'Send request to pod %s, request_url: %s', name, request_url
)
exception = None
@ -390,7 +397,17 @@ class KubernetesManager(base.OrchestratorBase):
identifier=None, service_url=None):
if service_url:
func_url = '%s/execute' % service_url
data = {'input': input, 'execution_id': execution_id}
data = {
'input': input,
'execution_id': execution_id,
}
if self.conf.pecan.auth_enable:
data.update(
{
'token': context.get_ctx().auth_token,
'trust_id': context.get_ctx().trust_id
}
)
LOG.info('Invoke function %s, url: %s', function_id, func_url)

14
qinling/tests/unit/api/base.py

@ -23,8 +23,7 @@ from webtest import app as webtest_app
from qinling.tests.unit import base
# Disable authentication by default for API tests.
cfg.CONF.set_default('auth_enable', False, group='pecan')
CONF = cfg.CONF
class APITest(base.DbTestCase):
@ -36,7 +35,13 @@ class APITest(base.DbTestCase):
self.override_config('file_system_dir', package_dir, 'storage')
self.addCleanup(shutil.rmtree, package_dir, True)
pecan_opts = cfg.CONF.pecan
# Disable authentication by default for API tests.
CONF.set_default('auth_enable', False, group='pecan')
self.addCleanup(
CONF.set_default, 'auth_enable', False, group='pecan'
)
pecan_opts = CONF.pecan
self.app = pecan.testing.load_test_app({
'app': {
'root': pecan_opts.root,
@ -47,9 +52,6 @@ class APITest(base.DbTestCase):
})
self.addCleanup(pecan.set_config, {}, overwrite=True)
self.addCleanup(
cfg.CONF.set_default, 'auth_enable', False, group='pecan'
)
self.patch_ctx = mock.patch('qinling.context.Context.from_environ')
self.mock_ctx = self.patch_ctx.start()

14
qinling/tests/unit/base.py

@ -19,13 +19,12 @@ import random
from oslo_config import cfg
from oslotest import base
from qinling import config
from qinling import context as auth_context
from qinling.db import api as db_api
from qinling.db.sqlalchemy import sqlite_lock
from qinling import status
from qinling.tests.unit import config as test_config
test_config.parse_args()
DEFAULT_PROJECT_ID = 'default'
OPT_PROJECT_ID = '55-66-77-88'
@ -132,6 +131,17 @@ class DbTestCase(BaseTest):
cfg.CONF.set_default('max_overflow', -1, group='database')
cfg.CONF.set_default('max_pool_size', 1000, group='database')
qinling_opts = [
(config.API_GROUP, config.api_opts),
(config.PECAN_GROUP, config.pecan_opts),
(config.ENGINE_GROUP, config.engine_opts),
(config.STORAGE_GROUP, config.storage_opts),
(config.KUBERNETES_GROUP, config.kubernetes_opts),
(None, [config.launch_opt])
]
for group, options in qinling_opts:
cfg.CONF.register_opts(list(options), group)
db_api.setup_db()
@classmethod

26
qinling/tests/unit/config.py

@ -1,26 +0,0 @@
# Copyright 2017 Catalyst IT Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from qinling import config
def parse_args():
# Look for .mistral.conf in the project directory by default.
project_dir = '%s/../..' % os.path.dirname(__file__)
config_file = '%s/.qinling.conf' % os.path.realpath(project_dir)
config_files = [config_file] if os.path.isfile(config_file) else None
config.parse_args(args=[], default_config_files=config_files)

2
qinling_tempest_plugin/functions/python_test.py

@ -13,5 +13,5 @@
# limitations under the License.
def main(name='World'):
def main(name='World', **kwargs):
print('Hello, %s' % name)

36
runtimes/python2/server.py

@ -27,21 +27,34 @@ from flask import abort
from flask import Flask
from flask import request
from flask import Response
from keystoneauth1.identity import generic
from keystoneauth1 import session
import requests
app = Flask(__name__)
zip_file = ''
function_module = 'main'
function_method = 'main'
auth_url = None
username = None
password = None
@app.route('/download', methods=['POST'])
def download():
params = request.get_json() or {}
download_url = params.get('download_url')
function_id = params.get('function_id')
entry = params.get('entry')
global auth_url
global username
global password
token = params.get('token')
auth_url = params.get('auth_url')
username = params.get('username')
password = params.get('password')
headers = {}
if token:
@ -101,10 +114,32 @@ def execute():
global zip_file
global function_module
global function_method
global auth_url
global username
global password
params = request.get_json() or {}
input = params.get('input') or {}
execution_id = params['execution_id']
token = params.get('token')
trust_id = params.get('trust_id')
# Provide an openstack session to user's function
os_session = None
if auth_url:
if not trust_id:
auth = generic.Token(auth_url=auth_url, token=token)
else:
auth = generic.Password(
username=username,
password=password,
auth_url=auth_url,
trust_id=trust_id,
user_domain_name='Default'
)
os_session = session.Session(auth=auth, verify=False)
input.update({'context': {'os_session': os_session}})
manager = Manager()
return_dict = manager.dict()
@ -124,7 +159,6 @@ def execute():
with open('%s.out' % execution_id) as f:
logs = f.read()
os.remove('%s.out' % execution_id)
return Response(

Loading…
Cancel
Save