diff --git a/doc/source/contributor/index.rst b/doc/source/contributor/index.rst index d45d21ad..cd8c3547 100644 --- a/doc/source/contributor/index.rst +++ b/doc/source/contributor/index.rst @@ -22,8 +22,6 @@ In this section you will find information helpful for contributing to qinling project. -Programming HowTos and Tutorials --------------------------------- .. toctree:: :maxdepth: 2 diff --git a/doc/source/function_developer/index.rst b/doc/source/function_developer/index.rst new file mode 100644 index 00000000..fa099b6e --- /dev/null +++ b/doc/source/function_developer/index.rst @@ -0,0 +1,33 @@ +.. + Copyright 2010-2011 United States Government as represented by the + Administrator of the National Aeronautics and Space Administration. + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Function Programming Guide +========================== + +In this section you will find information helpful for developing functions. + + +.. toctree:: + :maxdepth: 2 + + openstack_integration + +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`search` diff --git a/doc/source/function_developer/openstack_integration.rst b/doc/source/function_developer/openstack_integration.rst new file mode 100644 index 00000000..ad8b87f1 --- /dev/null +++ b/doc/source/function_developer/openstack_integration.rst @@ -0,0 +1,51 @@ +.. + Copyright 2017 Catalyst IT Ltd + All Rights Reserved. + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Interact with other OpenStack services +====================================== + +It's very easy to interact with other OpenStack services in your function. +Let's take Python programming language and integration with Swift service for +an example. + +At the time you create a function, you specify an entry, which is a function +in your code module that Qinling can invoke when the service executes your +code. Use the following general syntax structure when creating a function in +Python which will interact with Swift service in OpenStack. + +.. code-block:: python + + import swiftclient + + def main(region_name, container, object, context=None, **kwargs): + conn = swiftclient.Connection( + session=context['os_session'], + os_options={'region_name': region_name}, + ) + + obj_info = conn.head_object(container, object) + return obj_info + +In the above code, note the following: + +- Qinling supports most of OpenStack service clients, so you don't need to + install ``python-swiftclient`` in your code package. +- There is a parameter named ``context``, this is a parameter provided by + Qinling that is usually of the Python dict type. You can easily get a valid + Keystone session object from it. As a result, you don't need to pass any + sensitive data to Qinling in order to interact with OpenStack services. + +.. note:: + + Please avoid using ``context`` as your own parameter in the code. diff --git a/doc/source/index.rst b/doc/source/index.rst index 3eb5e48d..15e9f72f 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -27,24 +27,32 @@ Lambda). Qinling supports different container orchestration platforms * IRC channel on Freenode: #openstack-qinling -Table of Contents -================= +Overview +-------- .. toctree:: :maxdepth: 1 quick_start -Contributor/Developer Docs -========================== +Contributor/Developer Guide +--------------------------- .. toctree:: :maxdepth: 1 contributor/index +Function Programming Guide +-------------------------- + +.. toctree:: + :maxdepth: 1 + + function_developer/index + Indices and tables -================== +------------------ * :ref:`genindex` * :ref:`search` diff --git a/qinling/config.py b/qinling/config.py index b068f12b..3731a710 100644 --- a/qinling/config.py +++ b/qinling/config.py @@ -11,17 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -""" -Configuration options registration and useful routines. -""" -import itertools - +from keystoneauth1 import loading +from keystonemiddleware import auth_token from oslo_config import cfg from oslo_log import log from qinling import version +CONF = cfg.CONF + launch_opt = cfg.ListOpt( 'server', default=['all'], @@ -137,25 +135,24 @@ kubernetes_opts = [ ), ] -CONF = cfg.CONF -CLI_OPTS = [launch_opt] -CONF.register_cli_opts(CLI_OPTS) -default_group_opts = itertools.chain(CLI_OPTS, []) - def list_opts(): - return [ + keystone_middleware_opts = auth_token.list_opts() + keystone_loading_opts = [( + 'keystone_authtoken', loading.get_auth_plugin_conf_options('password') + )] + + qinling_opts = [ (API_GROUP, api_opts), (PECAN_GROUP, pecan_opts), (ENGINE_GROUP, engine_opts), (STORAGE_GROUP, storage_opts), (KUBERNETES_GROUP, kubernetes_opts), - (None, default_group_opts) + (None, [launch_opt]) ] + return keystone_middleware_opts + keystone_loading_opts + qinling_opts -for group, options in list_opts(): - CONF.register_opts(list(options), group) _DEFAULT_LOG_LEVELS = [ 'eventlet.wsgi.server=WARN', @@ -176,6 +173,12 @@ def parse_args(args=None, usage=None, default_config_files=None): log.set_defaults(default_log_levels=default_log_levels) log.register_options(CONF) + CLI_OPTS = [launch_opt] + CONF.register_cli_opts(CLI_OPTS) + + for group, options in list_opts(): + CONF.register_opts(list(options), group) + CONF( args=args, project='qinling', diff --git a/qinling/db/sqlalchemy/migration/alembic_migrations/versions/001_pike.py b/qinling/db/sqlalchemy/migration/alembic_migrations/versions/001_pike.py index 74858a75..ee666126 100644 --- a/qinling/db/sqlalchemy/migration/alembic_migrations/versions/001_pike.py +++ b/qinling/db/sqlalchemy/migration/alembic_migrations/versions/001_pike.py @@ -134,7 +134,7 @@ def upgrade(): sa.Column('function_input', st.JsonLongDictType(), nullable=True), sa.Column('status', sa.String(length=32), nullable=False), sa.Column('name', sa.String(length=255), nullable=True), - sa.Column('pattern', sa.String(length=32), nullable=False), + sa.Column('pattern', sa.String(length=32), nullable=True), sa.Column('first_execution_time', sa.DateTime(), nullable=True), sa.Column('next_execution_time', sa.DateTime(), nullable=False), sa.Column('count', sa.Integer(), nullable=True), diff --git a/qinling/engine/default_engine.py b/qinling/engine/default_engine.py index f3602e24..493d0066 100644 --- a/qinling/engine/default_engine.py +++ b/qinling/engine/default_engine.py @@ -16,6 +16,7 @@ from oslo_config import cfg from oslo_log import log as logging import requests +from qinling import context from qinling.db import api as db_api from qinling import status from qinling.utils import common @@ -103,6 +104,14 @@ class DefaultEngine(object): ) data = {'input': input, 'execution_id': execution_id} + if CONF.pecan.auth_enable: + data.update( + { + 'token': context.get_ctx().auth_token, + 'trust_id': context.get_ctx().trust_id + } + ) + r = self.session.post(func_url, json=data) res = r.json() diff --git a/qinling/engine/service.py b/qinling/engine/service.py index f3b4217a..07df1c42 100644 --- a/qinling/engine/service.py +++ b/qinling/engine/service.py @@ -55,7 +55,8 @@ class EngineService(service.Service): executor='eventlet', access_policy=access_policy, serializer=rpc.ContextSerializer( - messaging.serializer.JsonPayloadSerializer()) + messaging.serializer.JsonPayloadSerializer() + ) ) LOG.info('Starting engine...') diff --git a/qinling/orchestrator/kubernetes/manager.py b/qinling/orchestrator/kubernetes/manager.py index 7e496de6..44cc0b4c 100644 --- a/qinling/orchestrator/kubernetes/manager.py +++ b/qinling/orchestrator/kubernetes/manager.py @@ -298,12 +298,19 @@ class KubernetesManager(base.OrchestratorBase): 'download_url': download_url, 'function_id': actual_function, 'entry': entry, - 'token': context.get_ctx().auth_token, } + if self.conf.pecan.auth_enable: + data.update( + { + 'token': context.get_ctx().auth_token, + 'auth_url': self.conf.keystone_authtoken.auth_uri, + 'username': self.conf.keystone_authtoken.username, + 'password': self.conf.keystone_authtoken.password, + } + ) - LOG.debug( - 'Send request to pod %s, request_url: %s, data: %s', - name, request_url, data + LOG.info( + 'Send request to pod %s, request_url: %s', name, request_url ) exception = None @@ -390,7 +397,17 @@ class KubernetesManager(base.OrchestratorBase): identifier=None, service_url=None): if service_url: func_url = '%s/execute' % service_url - data = {'input': input, 'execution_id': execution_id} + data = { + 'input': input, + 'execution_id': execution_id, + } + if self.conf.pecan.auth_enable: + data.update( + { + 'token': context.get_ctx().auth_token, + 'trust_id': context.get_ctx().trust_id + } + ) LOG.info('Invoke function %s, url: %s', function_id, func_url) diff --git a/qinling/tests/unit/api/base.py b/qinling/tests/unit/api/base.py index d0aa327a..1b18fc51 100644 --- a/qinling/tests/unit/api/base.py +++ b/qinling/tests/unit/api/base.py @@ -23,8 +23,7 @@ from webtest import app as webtest_app from qinling.tests.unit import base -# Disable authentication by default for API tests. -cfg.CONF.set_default('auth_enable', False, group='pecan') +CONF = cfg.CONF class APITest(base.DbTestCase): @@ -36,7 +35,13 @@ class APITest(base.DbTestCase): self.override_config('file_system_dir', package_dir, 'storage') self.addCleanup(shutil.rmtree, package_dir, True) - pecan_opts = cfg.CONF.pecan + # Disable authentication by default for API tests. + CONF.set_default('auth_enable', False, group='pecan') + self.addCleanup( + CONF.set_default, 'auth_enable', False, group='pecan' + ) + + pecan_opts = CONF.pecan self.app = pecan.testing.load_test_app({ 'app': { 'root': pecan_opts.root, @@ -47,9 +52,6 @@ class APITest(base.DbTestCase): }) self.addCleanup(pecan.set_config, {}, overwrite=True) - self.addCleanup( - cfg.CONF.set_default, 'auth_enable', False, group='pecan' - ) self.patch_ctx = mock.patch('qinling.context.Context.from_environ') self.mock_ctx = self.patch_ctx.start() diff --git a/qinling/tests/unit/base.py b/qinling/tests/unit/base.py index c7103e33..b9c8cafa 100644 --- a/qinling/tests/unit/base.py +++ b/qinling/tests/unit/base.py @@ -19,13 +19,12 @@ import random from oslo_config import cfg from oslotest import base +from qinling import config from qinling import context as auth_context from qinling.db import api as db_api from qinling.db.sqlalchemy import sqlite_lock from qinling import status -from qinling.tests.unit import config as test_config -test_config.parse_args() DEFAULT_PROJECT_ID = 'default' OPT_PROJECT_ID = '55-66-77-88' @@ -132,6 +131,17 @@ class DbTestCase(BaseTest): cfg.CONF.set_default('max_overflow', -1, group='database') cfg.CONF.set_default('max_pool_size', 1000, group='database') + qinling_opts = [ + (config.API_GROUP, config.api_opts), + (config.PECAN_GROUP, config.pecan_opts), + (config.ENGINE_GROUP, config.engine_opts), + (config.STORAGE_GROUP, config.storage_opts), + (config.KUBERNETES_GROUP, config.kubernetes_opts), + (None, [config.launch_opt]) + ] + for group, options in qinling_opts: + cfg.CONF.register_opts(list(options), group) + db_api.setup_db() @classmethod diff --git a/qinling/tests/unit/config.py b/qinling/tests/unit/config.py deleted file mode 100644 index f368f686..00000000 --- a/qinling/tests/unit/config.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2017 Catalyst IT Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -from qinling import config - - -def parse_args(): - # Look for .mistral.conf in the project directory by default. - project_dir = '%s/../..' % os.path.dirname(__file__) - config_file = '%s/.qinling.conf' % os.path.realpath(project_dir) - config_files = [config_file] if os.path.isfile(config_file) else None - - config.parse_args(args=[], default_config_files=config_files) diff --git a/qinling_tempest_plugin/functions/python_test.py b/qinling_tempest_plugin/functions/python_test.py index 1d5921b1..cea1410b 100644 --- a/qinling_tempest_plugin/functions/python_test.py +++ b/qinling_tempest_plugin/functions/python_test.py @@ -13,5 +13,5 @@ # limitations under the License. -def main(name='World'): +def main(name='World', **kwargs): print('Hello, %s' % name) diff --git a/runtimes/python2/server.py b/runtimes/python2/server.py index 61e0f625..42027900 100644 --- a/runtimes/python2/server.py +++ b/runtimes/python2/server.py @@ -27,21 +27,34 @@ from flask import abort from flask import Flask from flask import request from flask import Response +from keystoneauth1.identity import generic +from keystoneauth1 import session import requests app = Flask(__name__) zip_file = '' function_module = 'main' function_method = 'main' +auth_url = None +username = None +password = None @app.route('/download', methods=['POST']) def download(): params = request.get_json() or {} + download_url = params.get('download_url') function_id = params.get('function_id') entry = params.get('entry') + + global auth_url + global username + global password token = params.get('token') + auth_url = params.get('auth_url') + username = params.get('username') + password = params.get('password') headers = {} if token: @@ -101,10 +114,32 @@ def execute(): global zip_file global function_module global function_method + global auth_url + global username + global password params = request.get_json() or {} input = params.get('input') or {} execution_id = params['execution_id'] + token = params.get('token') + trust_id = params.get('trust_id') + + # Provide an openstack session to user's function + os_session = None + if auth_url: + if not trust_id: + auth = generic.Token(auth_url=auth_url, token=token) + else: + auth = generic.Password( + username=username, + password=password, + auth_url=auth_url, + trust_id=trust_id, + user_domain_name='Default' + ) + os_session = session.Session(auth=auth, verify=False) + + input.update({'context': {'os_session': os_session}}) manager = Manager() return_dict = manager.dict() @@ -124,7 +159,6 @@ def execute(): with open('%s.out' % execution_id) as f: logs = f.read() - os.remove('%s.out' % execution_id) return Response(