Support policy check for api access

- Support oslo.policy in api layer
- Add tempest test for basic qinling operation

Change-Id: Icf12ee4df54652e81013e951b91163c085d961b7

devstack/plugin.sh

@@ -64,11 +64,15 @@ function configure_qinling {
     mkdir_chown_stack "$QINLING_FUNCTION_STORAGE_DIR"
     rm -f "$QINLING_FUNCTION_STORAGE_DIR"/*
 
+    cp $QINLING_DIR/etc/policy.json.sample $QINLING_POLICY_FILE
+
     # Generate Qinling configuration file and configure common parameters.
     oslo-config-generator --config-file $QINLING_DIR/tools/config/config-generator.qinling.conf --output-file $QINLING_CONF_FILE
 
+    iniset $QINLING_CONF_FILE oslo_policy policy_file $QINLING_POLICY_FILE
     iniset $QINLING_CONF_FILE DEFAULT debug $QINLING_DEBUG
     iniset $QINLING_CONF_FILE DEFAULT server all
+    iniset $QINLING_CONF_FILE DEFAULT logging_default_format_string $QINLING_LOG_FORMAT
     iniset $QINLING_CONF_FILE storage file_system_dir $QINLING_FUNCTION_STORAGE_DIR
     iniset $QINLING_CONF_FILE kubernetes qinling_service_address $DEFAULT_HOST_IP
 

devstack/settings

@@ -19,6 +19,8 @@ QINLING_SERVICE_PROTOCOL=${QINLING_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
 QINLING_DEBUG=${QINLING_DEBUG:-True}
 QINLING_CONF_DIR=${QINLING_CONF_DIR:-/etc/qinling}
 QINLING_CONF_FILE=${QINLING_CONF_DIR}/qinling.conf
+QINLING_POLICY_FILE=${QINLING_CONF_DIR}/policy.json
 QINLING_AUTH_CACHE_DIR=${QINLING_AUTH_CACHE_DIR:-/var/cache/qinling}
 QINLING_FUNCTION_STORAGE_DIR=${QINLING_FUNCTION_STORAGE_DIR:-/opt/qinling/funtion/packages}
 QINLING_PYTHON_RUNTIME_IMAGE=${QINLING_PYTHON_RUNTIME_IMAGE:-openstackqinling/python-runtime}
+QINLING_LOG_FORMAT=${QINLING_LOG_FORMAT:-"%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s %(resource)s"}

etc/policy.json.sample

@@ -0,0 +1,10 @@
+{
+    "context_is_admin": "role:admin or is_admin:1",
+    "owner" : "project_id:%(project_id)s",
+    "admin_or_owner": "rule:context_is_admin or rule:owner",
+    "default": "rule:admin_or_owner",
+
+    "runtime:create": "rule:context_is_admin",
+    "runtime:update": "rule:context_is_admin",
+    "runtime:delete": "rule:context_is_admin",
+}

qinling/api/access_control.py

@@ -56,10 +56,14 @@ def enforce(action, context, target=None, do_raise=True,
     :return: returns True if authorized and False if not authorized and
              do_raise is False.
     """
+    if not cfg.CONF.pecan.auth_enable:
+        return
+
+    ctx_dict = context.to_policy_values()
 
     target_obj = {
-        'project_id': context.project_id,
+        'project_id': ctx_dict['project_id'],
-        'user_id': context.user_id,
+        'user_id': ctx_dict['user_id'],
     }
 
     target_obj.update(target or {})
@@ -68,7 +72,7 @@ def enforce(action, context, target=None, do_raise=True,
     return _ENFORCER.enforce(
         action,
         target_obj,
-        context.to_dict(),
+        ctx_dict,
         do_raise=do_raise,
         exc=exc
     )

qinling/api/controllers/v1/function.py

@@ -37,7 +37,7 @@ from qinling.utils import rest_utils
 LOG = logging.getLogger(__name__)
 CONF = cfg.CONF
 
-POST_REQUIRED = set(['name', 'code'])
+POST_REQUIRED = set(['code'])
 CODE_SOURCE = set(['package', 'swift', 'image'])
 UPDATE_ALLOWED = set(['name', 'description', 'entry'])
 
@@ -106,7 +106,7 @@ class FunctionsController(rest.RestController):
             )
 
         values = {
-            'name': kwargs['name'],
+            'name': kwargs.get('name'),
             'description': kwargs.get('description'),
             'runtime_id': kwargs.get('runtime_id'),
             'code': json.loads(kwargs['code']),

qinling/api/controllers/v1/resources.py

@@ -218,6 +218,7 @@ class Runtime(Resource):
     name = wtypes.text
     image = wtypes.text
     description = wtypes.text
+    is_public = wsme.wsattr(bool, default=True)
     status = wsme.wsattr(wtypes.text, readonly=True)
     project_id = wsme.wsattr(wtypes.text, readonly=True)
     created_at = wsme.wsattr(wtypes.text, readonly=True)
@@ -230,6 +231,7 @@ class Runtime(Resource):
             name='python2.7',
             image='lingxiankong/python',
             status='available',
+            is_public=True,
             project_id='default',
             description='Python 2.7 environment.',
             created_at='1970-01-01T00:00:00.000000',

qinling/api/controllers/v1/runtime.py

@@ -16,8 +16,10 @@ from oslo_log import log as logging
 from pecan import rest
 import wsmeext.pecan as wsme_pecan
 
+from qinling.api import access_control as acl
 from qinling.api.controllers.v1 import resources
 from qinling.api.controllers.v1 import types
+from qinling import context
 from qinling.db import api as db_api
 from qinling import exceptions as exc
 from qinling import rpc
@@ -63,6 +65,8 @@ class RuntimesController(rest.RestController):
         status_code=201
     )
     def post(self, runtime):
+        acl.enforce('runtime:create', context.get_ctx())
+
         params = runtime.to_dict()
 
         if not POST_REQUIRED.issubset(set(params.keys())):
@@ -82,13 +86,15 @@ class RuntimesController(rest.RestController):
     @rest_utils.wrap_wsme_controller_exception
     @wsme_pecan.wsexpose(None, types.uuid, status_code=204)
     def delete(self, id):
+        acl.enforce('runtime:delete', context.get_ctx())
+
         LOG.info("Delete resource.", resource={'type': self.type, 'id': id})
 
         with db_api.transaction():
             runtime_db = db_api.get_runtime(id)
 
             # Runtime can not be deleted if still associate with functions.
-            funcs = db_api.get_functions(runtime_id={'eq': id})
+            funcs = db_api.get_functions(insecure=True, runtime_id={'eq': id})
             if len(funcs):
                 raise exc.NotAllowedException(
                     'Runtime %s is still in use.' % id
@@ -111,6 +117,8 @@ class RuntimesController(rest.RestController):
         Currently, we only support update name, description, image. When
         updating image, send message to engine for asynchronous handling.
         """
+        acl.enforce('runtime:update', context.get_ctx())
+
         values = {}
         for key in UPDATE_ALLOWED:
             if runtime.to_dict().get(key) is not None:

qinling/db/sqlalchemy/api.py

@@ -109,7 +109,15 @@ def _secure_query(model, *columns):
     if not issubclass(model, model_base.QinlingSecureModelBase):
         return query
 
-    query = query.filter(model.project_id == context.get_ctx().projectid)
+    if model == models.Runtime:
+        query_criterion = sa.or_(
+            model.project_id == context.get_ctx().projectid,
+            model.is_public
+        )
+    else:
+        query_criterion = model.project_id == context.get_ctx().projectid
+
+    query = query.filter(query_criterion)
 
     return query
 
@@ -274,7 +282,13 @@ def create_runtime(values, session=None):
 
 @db_base.session_aware()
 def get_runtime(id, session=None):
-    runtime = _get_db_object_by_id(models.Runtime, id)
+    model = models.Runtime
+    filters = sa.and_(
+        model.id == id,
+        sa.or_(model.project_id == context.get_ctx().projectid,
+               model.is_public),
+    )
+    runtime = db_base.model_query(model).filter(filters).first()
 
     if not runtime:
         raise exc.DBEntityNotFoundError("Runtime not found [id=%s]" % id)

qinling/db/sqlalchemy/migration/alembic_migrations/versions/001_pike.py

@@ -54,8 +54,8 @@ def upgrade():
         sa.Column('description', sa.String(length=255), nullable=True),
         sa.Column('image', sa.String(length=255), nullable=False),
         sa.Column('status', sa.String(length=32), nullable=False),
+        sa.Column('is_public', sa.BOOLEAN, nullable=False),
         sa.PrimaryKeyConstraint('id'),
-        sa.UniqueConstraint('image', 'project_id'),
         info={"check_ifexists": True}
     )
 
@@ -74,7 +74,6 @@ def upgrade():
         sa.Column('entry', sa.String(length=80), nullable=False),
         sa.Column('count', sa.Integer, nullable=False),
         sa.PrimaryKeyConstraint('id'),
-        sa.UniqueConstraint('name', 'project_id'),
         sa.ForeignKeyConstraint(['runtime_id'], [u'runtimes.id']),
         info={"check_ifexists": True}
     )

qinling/db/sqlalchemy/models.py

@@ -23,24 +23,17 @@ from qinling.utils import common
 class Runtime(model_base.QinlingSecureModelBase):
     __tablename__ = 'runtimes'
 
-    __table_args__ = (
-        sa.UniqueConstraint('image', 'project_id'),
-    )
-
     name = sa.Column(sa.String(255))
     description = sa.Column(sa.String(255))
     image = sa.Column(sa.String(255), nullable=False)
     status = sa.Column(sa.String(32), nullable=False)
+    is_public = sa.Column(sa.BOOLEAN, default=True)
 
 
 class Function(model_base.QinlingSecureModelBase):
     __tablename__ = 'functions'
 
-    __table_args__ = (
+    name = sa.Column(sa.String(255), nullable=True)
-        sa.UniqueConstraint('name', 'project_id'),
-    )
-
-    name = sa.Column(sa.String(255), nullable=False)
     description = sa.Column(sa.String(255))
     runtime_id = sa.Column(
         sa.String(36), sa.ForeignKey(Runtime.id), nullable=True

qinling/tests/unit/api/controllers/v1/test_runtime.py

@@ -37,7 +37,8 @@ class TestRuntimeController(base.APITest):
             "image": self.db_runtime.image,
             "name": self.db_runtime.name,
             "project_id": test_base.DEFAULT_PROJECT_ID,
-            "status": status.AVAILABLE
+            "status": status.AVAILABLE,
+            "is_public": True,
         }
 
         self.assertEqual(200, resp.status_int)
@@ -51,7 +52,8 @@ class TestRuntimeController(base.APITest):
             "image": self.db_runtime.image,
             "name": self.db_runtime.name,
             "project_id": test_base.DEFAULT_PROJECT_ID,
-            "status": status.AVAILABLE
+            "status": status.AVAILABLE,
+            "is_public": True,
         }
 
         self.assertEqual(200, resp.status_int)

qinling_tempest_plugin/functions/python_test.py

@@ -0,0 +1,17 @@
+# Copyright 2017 Catalyst IT Ltd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+
+def main(name='World'):
+    print('Hello, %s' % name)

qinling_tempest_plugin/services/base.py

@@ -21,9 +21,6 @@ class QinlingClientBase(rest_client.RestClient):
     def __init__(self, auth_provider, **kwargs):
         super(QinlingClientBase, self).__init__(auth_provider, **kwargs)
 
-        self.runtimes = []
-        self.functions = []
-
     def get_list_objs(self, obj):
         resp, body = self.get('/v1/%s' % obj)
 

qinling_tempest_plugin/services/qinling_client.py

@@ -14,19 +14,73 @@
 
 import json
 
+import requests
+
 from qinling_tempest_plugin.services import base as client_base
 
 
 class QinlingClient(client_base.QinlingClientBase):
     """Tempest REST client for Qinling."""
 
+    def delete_resource(self, res, id):
+        resp, _ = self.delete_obj(res, id)
+
+        return resp
+
+    def get_resource(self, res, id):
+        resp, body = self.get_obj(res, id)
+
+        return resp, body
+
+    def get_resources(self, res):
+        resp, body = self.get_list_objs(res)
+
+        return resp, body
+
     def create_runtime(self, image, name=None):
-        body = {"image": image}
+        req_body = {"image": image}
 
         if name:
-            body.update({'name': name})
+            req_body.update({'name': name})
 
-        resp, body = self.post('runtimes', json.dumps(body))
+        resp, body = self.post_json('runtimes', req_body)
-        self.runtimes.append(json.loads(body)['id'])
 
-        return resp, json.loads(body)
+        return resp, body
+
+    def create_function(self, code, runtime_id, name='', package_data=None,
+                        entry=None):
+        """Create function.
+
+        Tempest rest client doesn't support multipart upload, so use requests
+        lib instead.
+        """
+        headers = {'X-Auth-Token': self.auth_provider.get_token()}
+        req_body = {
+            'name': name,
+            'runtime_id': runtime_id,
+            'code': json.dumps(code)
+        }
+        if entry:
+            req_body['entry'] = entry
+
+        req_kwargs = {
+            'headers': headers,
+            'data': req_body
+        }
+        if package_data:
+            req_kwargs.update({'files': {'package': package_data}})
+
+        url_path = '%s/v1/functions' % (self.base_url)
+        resp = requests.post(url_path, **req_kwargs)
+
+        return resp, json.loads(resp.text)
+
+    def create_execution(self, function_id, input=None, sync=True):
+        req_body = {'function_id': function_id, 'sync': sync, 'input': input}
+        resp, body = self.post_json('executions', req_body)
+
+        return resp, body
+
+    def get_execution_log(self, execution_id):
+        return self.get('/v1/executions/%s/log' % execution_id,
+                        headers={'Accept': 'text/plain'})

qinling_tempest_plugin/tests/api/test_runtimes.py

@@ -13,7 +13,6 @@
 #    limitations under the License.
 from tempest.lib.common.utils import data_utils
 from tempest.lib import decorators
-import tenacity
 
 from qinling_tempest_plugin.tests import base
 
@@ -21,33 +20,21 @@ from qinling_tempest_plugin.tests import base
 class RuntimesTest(base.BaseQinlingTest):
     name_prefix = 'RuntimesTest'
 
-    @tenacity.retry(
-        wait=tenacity.wait_fixed(2),
-        stop=tenacity.stop_after_delay(10),
-        retry=tenacity.retry_if_exception_type(AssertionError)
-    )
-    def _await_runtime_available(self, id):
-        resp, body = self.qinling_client.get_obj('runtimes', id)
-
-        self.assertEqual(200, resp.status)
-        self.assertEqual('available', body['status'])
-
     @decorators.idempotent_id('fdc2f07f-dd1d-4981-86d3-5bc7908d9a9b')
     def test_create_list_get_delete_runtime(self):
         name = data_utils.rand_name('runtime', prefix=self.name_prefix)
 
-        req_body = {
+        resp, body = self.admin_client.create_runtime(
-            'name': name,
+            'openstackqinling/python-runtime', name
-            'image': 'openstackqinling/python-runtime'
+        )
-        }
-        resp, body = self.qinling_client.post_json('runtimes', req_body)
-        runtime_id = body['id']
 
         self.assertEqual(201, resp.status)
         self.assertEqual(name, body['name'])
 
+        runtime_id = body['id']
+
         # Get runtimes
-        resp, body = self.qinling_client.get_list_objs('runtimes')
+        resp, body = self.client.get_resources('runtimes')
 
         self.assertEqual(200, resp.status)
         self.assertIn(
@@ -56,7 +43,7 @@ class RuntimesTest(base.BaseQinlingTest):
         )
 
         # Wait for runtime to be available
-        self._await_runtime_available(runtime_id)
+        self.await_runtime_available(runtime_id)
 
         # Check k8s resource
         deploy = self.k8s_v1extention.read_namespaced_deployment(
@@ -70,6 +57,6 @@ class RuntimesTest(base.BaseQinlingTest):
         )
 
         # Delete runtime
-        resp, _ = self.qinling_client.delete_obj('runtimes', runtime_id)
+        resp = self.admin_client.delete_resource('runtimes', runtime_id)
 
         self.assertEqual(204, resp.status)

qinling_tempest_plugin/tests/base.py

@@ -11,17 +11,16 @@
 #    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
-
 from kubernetes import client as k8s_client
 from tempest import config
 from tempest import test
+import tenacity
 
 CONF = config.CONF
 
 
 class BaseQinlingTest(test.BaseTestCase):
-    credentials = ('primary',)
+    credentials = ('admin', 'primary', 'alt')
-    force_tenant_isolation = False
 
     @classmethod
     def skip_checks(cls):
@@ -34,19 +33,23 @@ class BaseQinlingTest(test.BaseTestCase):
     def setup_clients(cls):
         super(BaseQinlingTest, cls).setup_clients()
 
-        # os here is tempest.lib.services.clients.ServiceClients object
+        cls.client = cls.os_primary.qinling.QinlingClient()
-        os = getattr(cls, 'os_%s' % cls.credentials[0])
+        cls.alt_client = cls.os_alt.qinling.QinlingClient()
-        cls.qinling_client = os.qinling.QinlingClient()
+        cls.admin_client = cls.os_admin.qinling.QinlingClient()
-
-        if CONF.identity.auth_version == 'v3':
-            project_id = os.auth_provider.auth_data[1]['project']['id']
-        else:
-            project_id = os.auth_provider.auth_data[1]['token']['tenant']['id']
-        cls.tenant_id = project_id
-        cls.user_id = os.auth_provider.auth_data[1]['user']['id']
 
         # Initilize k8s client
         k8s_client.Configuration().host = CONF.qinling.kube_host
         cls.k8s_v1 = k8s_client.CoreV1Api()
         cls.k8s_v1extention = k8s_client.ExtensionsV1beta1Api()
         cls.namespace = 'qinling'
+
+    @tenacity.retry(
+        wait=tenacity.wait_fixed(2),
+        stop=tenacity.stop_after_delay(10),
+        retry=tenacity.retry_if_exception_type(AssertionError)
+    )
+    def await_runtime_available(self, id):
+        resp, body = self.client.get_resource('runtimes', id)
+
+        self.assertEqual(200, resp.status)
+        self.assertEqual('available', body['status'])

qinling_tempest_plugin/tests/scenario/test_basic_ops.py

@@ -0,0 +1,113 @@
+# Copyright 2017 Catalyst IT Ltd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+import os
+import tempfile
+import zipfile
+
+from tempest.lib.common.utils import data_utils
+from tempest.lib import decorators
+
+from qinling_tempest_plugin.tests import base
+
+
+class BasicOpsTest(base.BaseQinlingTest):
+    name_prefix = 'BasicOpsTest'
+
+    def setUp(self):
+        super(BasicOpsTest, self).setUp()
+
+        python_file_path = os.path.abspath(
+            os.path.join(
+                os.path.dirname(__file__),
+                os.pardir,
+                os.pardir,
+                'functions/python_test.py'
+            )
+        )
+
+        base_name, extention = os.path.splitext(python_file_path)
+        self.base_name = os.path.basename(base_name)
+        self.python_zip_file = os.path.join(
+            tempfile.gettempdir(),
+            '%s.zip' % self.base_name
+        )
+
+        zf = zipfile.ZipFile(self.python_zip_file, mode='w')
+        try:
+            # Use default compression mode, may change in future.
+            zf.write(
+                python_file_path,
+                '%s%s' % (self.base_name, extention),
+                compress_type=zipfile.ZIP_STORED
+            )
+        finally:
+            zf.close()
+
+    @decorators.idempotent_id('205fd749-2468-4d9f-9c05-45558d6d8f9e')
+    def test_basic_ops(self):
+        """Basic qinling operations test case, including following steps:
+
+        1. Admin user creates a runtime.
+        2. Normal user creates function.
+        3. Normal user creates execution(invoke function).
+        4. Check result and execution log.
+        """
+        name = data_utils.rand_name('runtime', prefix=self.name_prefix)
+
+        resp, body = self.admin_client.create_runtime(
+            'openstackqinling/python-runtime', name
+        )
+
+        self.assertEqual(201, resp.status)
+        self.assertEqual(name, body['name'])
+
+        # Wait for runtime to be available
+        runtime_id = body['id']
+        self.await_runtime_available(runtime_id)
+        self.addCleanup(self.admin_client.delete_resource, 'runtimes',
+                        runtime_id)
+
+        # Create function
+        function_name = data_utils.rand_name('function',
+                                             prefix=self.name_prefix)
+        with open(self.python_zip_file, 'rb') as package_data:
+            resp, body = self.client.create_function(
+                {"source": "package"},
+                runtime_id,
+                name=function_name,
+                package_data=package_data,
+                entry='%s.main' % self.base_name
+            )
+            function_id = body['id']
+
+        self.assertEqual(201, resp.status_code)
+        self.addCleanup(self.client.delete_resource, 'functions',
+                        function_id)
+
+        # Invoke function
+        resp, body = self.client.create_execution(function_id,
+                                                  input={'name': 'Qinling'})
+
+        self.assertEqual(201, resp.status)
+        self.assertEqual('success', body['status'])
+
+        execution_id = body['id']
+        self.addCleanup(self.client.delete_resource, 'executions',
+                        execution_id)
+
+        # Get execution log
+        resp, body = self.client.get_execution_log(execution_id)
+
+        self.assertEqual(200, resp.status)
+        self.assertIn('Hello, Qinling', body)