Add session support for function

Also add the documentation.

Partially implements: blueprint qinling-support-user-session
Change-Id: Ic11b3b9f6ffe65bb08cfd880b81d41a40590919b

doc/source/contributor/index.rst

@@ -22,8 +22,6 @@ In this section you will find information helpful for contributing to qinling
 project.
 
 
-Programming HowTos and Tutorials
---------------------------------
 .. toctree::
     :maxdepth: 2
 

doc/source/function_developer/index.rst

@@ -0,0 +1,33 @@
+..
+      Copyright 2010-2011 United States Government as represented by the
+      Administrator of the National Aeronautics and Space Administration.
+      All Rights Reserved.
+
+      Licensed under the Apache License, Version 2.0 (the "License"); you may
+      not use this file except in compliance with the License. You may obtain
+      a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+      License for the specific language governing permissions and limitations
+      under the License.
+
+Function Programming Guide
+==========================
+
+In this section you will find information helpful for developing functions.
+
+
+.. toctree::
+    :maxdepth: 2
+
+    openstack_integration
+
+Indices and tables
+------------------
+
+* :ref:`genindex`
+* :ref:`search`

doc/source/function_developer/openstack_integration.rst

@@ -0,0 +1,51 @@
+..
+      Copyright 2017 Catalyst IT Ltd
+      All Rights Reserved.
+      not use this file except in compliance with the License. You may obtain
+      a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+      License for the specific language governing permissions and limitations
+      under the License.
+
+Interact with other OpenStack services
+======================================
+
+It's very easy to interact with other OpenStack services in your function.
+Let's take Python programming language and integration with Swift service for
+an example.
+
+At the time you create a function, you specify an entry, which is a function
+in your code module that Qinling can invoke when the service executes your
+code. Use the following general syntax structure when creating a function in
+Python which will interact with Swift service in OpenStack.
+
+.. code-block:: python
+
+    import swiftclient
+
+    def main(region_name, container, object, context=None, **kwargs):
+        conn = swiftclient.Connection(
+            session=context['os_session'],
+            os_options={'region_name': region_name},
+        )
+
+        obj_info = conn.head_object(container, object)
+        return obj_info
+
+In the above code, note the following:
+
+- Qinling supports most of OpenStack service clients, so you don't need to
+  install ``python-swiftclient`` in your code package.
+- There is a parameter named ``context``, this is a parameter provided by
+  Qinling that is usually of the Python dict type. You can easily get a valid
+  Keystone session object from it. As a result, you don't need to pass any
+  sensitive data to Qinling in order to interact with OpenStack services.
+
+.. note::
+
+    Please avoid using ``context`` as your own parameter in the code.

doc/source/index.rst

@@ -27,24 +27,32 @@ Lambda). Qinling supports different container orchestration platforms
 * IRC channel on Freenode: #openstack-qinling
 
 
-Table of Contents
-=================
+Overview
+--------
 
 .. toctree::
    :maxdepth: 1
 
    quick_start
 
-Contributor/Developer Docs
-==========================
+Contributor/Developer Guide
+---------------------------
 
 .. toctree::
    :maxdepth: 1
 
    contributor/index
 
+Function Programming Guide
+--------------------------
+
+.. toctree::
+   :maxdepth: 1
+
+   function_developer/index
+
 Indices and tables
-==================
+------------------
 
 * :ref:`genindex`
 * :ref:`search`

qinling/config.py

@@ -11,17 +11,15 @@
 #    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
-
-"""
-Configuration options registration and useful routines.
-"""
-import itertools
-
+from keystoneauth1 import loading
+from keystonemiddleware import auth_token
 from oslo_config import cfg
 from oslo_log import log
 
 from qinling import version
 
+CONF = cfg.CONF
+
 launch_opt = cfg.ListOpt(
     'server',
     default=['all'],
@@ -137,25 +135,24 @@ kubernetes_opts = [
     ),
 ]
 
-CONF = cfg.CONF
-CLI_OPTS = [launch_opt]
-CONF.register_cli_opts(CLI_OPTS)
-default_group_opts = itertools.chain(CLI_OPTS, [])
-
 
 def list_opts():
-    return [
+    keystone_middleware_opts = auth_token.list_opts()
+    keystone_loading_opts = [(
+        'keystone_authtoken', loading.get_auth_plugin_conf_options('password')
+    )]
+
+    qinling_opts = [
         (API_GROUP, api_opts),
         (PECAN_GROUP, pecan_opts),
         (ENGINE_GROUP, engine_opts),
         (STORAGE_GROUP, storage_opts),
         (KUBERNETES_GROUP, kubernetes_opts),
-        (None, default_group_opts)
+        (None, [launch_opt])
     ]
 
+    return keystone_middleware_opts + keystone_loading_opts + qinling_opts
 
-for group, options in list_opts():
-    CONF.register_opts(list(options), group)
 
 _DEFAULT_LOG_LEVELS = [
     'eventlet.wsgi.server=WARN',
@@ -176,6 +173,12 @@ def parse_args(args=None, usage=None, default_config_files=None):
     log.set_defaults(default_log_levels=default_log_levels)
     log.register_options(CONF)
 
+    CLI_OPTS = [launch_opt]
+    CONF.register_cli_opts(CLI_OPTS)
+
+    for group, options in list_opts():
+        CONF.register_opts(list(options), group)
+
     CONF(
         args=args,
         project='qinling',

qinling/db/sqlalchemy/migration/alembic_migrations/versions/001_pike.py

@@ -134,7 +134,7 @@ def upgrade():
         sa.Column('function_input', st.JsonLongDictType(), nullable=True),
         sa.Column('status', sa.String(length=32), nullable=False),
         sa.Column('name', sa.String(length=255), nullable=True),
-        sa.Column('pattern', sa.String(length=32), nullable=False),
+        sa.Column('pattern', sa.String(length=32), nullable=True),
         sa.Column('first_execution_time', sa.DateTime(), nullable=True),
         sa.Column('next_execution_time', sa.DateTime(), nullable=False),
         sa.Column('count', sa.Integer(), nullable=True),

qinling/engine/default_engine.py

@@ -16,6 +16,7 @@ from oslo_config import cfg
 from oslo_log import log as logging
 import requests
 
+from qinling import context
 from qinling.db import api as db_api
 from qinling import status
 from qinling.utils import common
@@ -103,6 +104,14 @@ class DefaultEngine(object):
                 )
 
                 data = {'input': input, 'execution_id': execution_id}
+                if CONF.pecan.auth_enable:
+                    data.update(
+                        {
+                            'token': context.get_ctx().auth_token,
+                            'trust_id': context.get_ctx().trust_id
+                        }
+                    )
+
                 r = self.session.post(func_url, json=data)
                 res = r.json()
 

qinling/engine/service.py

@@ -55,7 +55,8 @@ class EngineService(service.Service):
             executor='eventlet',
             access_policy=access_policy,
             serializer=rpc.ContextSerializer(
-                messaging.serializer.JsonPayloadSerializer())
+                messaging.serializer.JsonPayloadSerializer()
+            )
         )
 
         LOG.info('Starting engine...')

qinling/orchestrator/kubernetes/manager.py

@@ -298,12 +298,19 @@ class KubernetesManager(base.OrchestratorBase):
             'download_url': download_url,
             'function_id': actual_function,
             'entry': entry,
-            'token': context.get_ctx().auth_token,
         }
+        if self.conf.pecan.auth_enable:
+            data.update(
+                {
+                    'token': context.get_ctx().auth_token,
+                    'auth_url': self.conf.keystone_authtoken.auth_uri,
+                    'username': self.conf.keystone_authtoken.username,
+                    'password': self.conf.keystone_authtoken.password,
+                }
+            )
 
-        LOG.debug(
-            'Send request to pod %s, request_url: %s, data: %s',
-            name, request_url, data
+        LOG.info(
+            'Send request to pod %s, request_url: %s', name, request_url
         )
 
         exception = None
@@ -390,7 +397,17 @@ class KubernetesManager(base.OrchestratorBase):
                       identifier=None, service_url=None):
         if service_url:
             func_url = '%s/execute' % service_url
-            data = {'input': input, 'execution_id': execution_id}
+            data = {
+                'input': input,
+                'execution_id': execution_id,
+            }
+            if self.conf.pecan.auth_enable:
+                data.update(
+                    {
+                        'token': context.get_ctx().auth_token,
+                        'trust_id': context.get_ctx().trust_id
+                    }
+                )
 
             LOG.info('Invoke function %s, url: %s', function_id, func_url)
 

qinling/tests/unit/api/base.py

@@ -23,8 +23,7 @@ from webtest import app as webtest_app
 
 from qinling.tests.unit import base
 
-# Disable authentication by default for API tests.
-cfg.CONF.set_default('auth_enable', False, group='pecan')
+CONF = cfg.CONF
 
 
 class APITest(base.DbTestCase):
@@ -36,7 +35,13 @@ class APITest(base.DbTestCase):
         self.override_config('file_system_dir', package_dir, 'storage')
         self.addCleanup(shutil.rmtree, package_dir, True)
 
-        pecan_opts = cfg.CONF.pecan
+        # Disable authentication by default for API tests.
+        CONF.set_default('auth_enable', False, group='pecan')
+        self.addCleanup(
+            CONF.set_default, 'auth_enable', False, group='pecan'
+        )
+
+        pecan_opts = CONF.pecan
         self.app = pecan.testing.load_test_app({
             'app': {
                 'root': pecan_opts.root,
@@ -47,9 +52,6 @@ class APITest(base.DbTestCase):
         })
 
         self.addCleanup(pecan.set_config, {}, overwrite=True)
-        self.addCleanup(
-            cfg.CONF.set_default, 'auth_enable', False, group='pecan'
-        )
 
         self.patch_ctx = mock.patch('qinling.context.Context.from_environ')
         self.mock_ctx = self.patch_ctx.start()

qinling/tests/unit/base.py

@@ -19,13 +19,12 @@ import random
 from oslo_config import cfg
 from oslotest import base
 
+from qinling import config
 from qinling import context as auth_context
 from qinling.db import api as db_api
 from qinling.db.sqlalchemy import sqlite_lock
 from qinling import status
-from qinling.tests.unit import config as test_config
 
-test_config.parse_args()
 DEFAULT_PROJECT_ID = 'default'
 OPT_PROJECT_ID = '55-66-77-88'
 
@@ -132,6 +131,17 @@ class DbTestCase(BaseTest):
         cfg.CONF.set_default('max_overflow', -1, group='database')
         cfg.CONF.set_default('max_pool_size', 1000, group='database')
 
+        qinling_opts = [
+            (config.API_GROUP, config.api_opts),
+            (config.PECAN_GROUP, config.pecan_opts),
+            (config.ENGINE_GROUP, config.engine_opts),
+            (config.STORAGE_GROUP, config.storage_opts),
+            (config.KUBERNETES_GROUP, config.kubernetes_opts),
+            (None, [config.launch_opt])
+        ]
+        for group, options in qinling_opts:
+            cfg.CONF.register_opts(list(options), group)
+
         db_api.setup_db()
 
     @classmethod

qinling/tests/unit/config.py

@@ -1,26 +0,0 @@
-# Copyright 2017 Catalyst IT Limited
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-
-import os
-
-from qinling import config
-
-
-def parse_args():
-    # Look for .mistral.conf in the project directory by default.
-    project_dir = '%s/../..' % os.path.dirname(__file__)
-    config_file = '%s/.qinling.conf' % os.path.realpath(project_dir)
-    config_files = [config_file] if os.path.isfile(config_file) else None
-
-    config.parse_args(args=[], default_config_files=config_files)

qinling_tempest_plugin/functions/python_test.py

@@ -13,5 +13,5 @@
 #    limitations under the License.
 
 
-def main(name='World'):
+def main(name='World', **kwargs):
     print('Hello, %s' % name)

runtimes/python2/server.py

@@ -27,21 +27,34 @@ from flask import abort
 from flask import Flask
 from flask import request
 from flask import Response
+from keystoneauth1.identity import generic
+from keystoneauth1 import session
 import requests
 
 app = Flask(__name__)
 zip_file = ''
 function_module = 'main'
 function_method = 'main'
+auth_url = None
+username = None
+password = None
 
 
 @app.route('/download', methods=['POST'])
 def download():
     params = request.get_json() or {}
+
     download_url = params.get('download_url')
     function_id = params.get('function_id')
     entry = params.get('entry')
+
+    global auth_url
+    global username
+    global password
     token = params.get('token')
+    auth_url = params.get('auth_url')
+    username = params.get('username')
+    password = params.get('password')
 
     headers = {}
     if token:
@@ -101,10 +114,32 @@ def execute():
     global zip_file
     global function_module
     global function_method
+    global auth_url
+    global username
+    global password
 
     params = request.get_json() or {}
     input = params.get('input') or {}
     execution_id = params['execution_id']
+    token = params.get('token')
+    trust_id = params.get('trust_id')
+
+    # Provide an openstack session to user's function
+    os_session = None
+    if auth_url:
+        if not trust_id:
+            auth = generic.Token(auth_url=auth_url, token=token)
+        else:
+            auth = generic.Password(
+                username=username,
+                password=password,
+                auth_url=auth_url,
+                trust_id=trust_id,
+                user_domain_name='Default'
+            )
+        os_session = session.Session(auth=auth, verify=False)
+
+    input.update({'context': {'os_session': os_session}})
 
     manager = Manager()
     return_dict = manager.dict()
@@ -124,7 +159,6 @@ def execute():
 
     with open('%s.out' % execution_id) as f:
         logs = f.read()
-
     os.remove('%s.out' % execution_id)
 
     return Response(