diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 89396ffa..cad563c0 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -64,11 +64,15 @@ function configure_qinling { mkdir_chown_stack "$QINLING_FUNCTION_STORAGE_DIR" rm -f "$QINLING_FUNCTION_STORAGE_DIR"/* + cp $QINLING_DIR/etc/policy.json.sample $QINLING_POLICY_FILE + # Generate Qinling configuration file and configure common parameters. oslo-config-generator --config-file $QINLING_DIR/tools/config/config-generator.qinling.conf --output-file $QINLING_CONF_FILE + iniset $QINLING_CONF_FILE oslo_policy policy_file $QINLING_POLICY_FILE iniset $QINLING_CONF_FILE DEFAULT debug $QINLING_DEBUG iniset $QINLING_CONF_FILE DEFAULT server all + iniset $QINLING_CONF_FILE DEFAULT logging_default_format_string $QINLING_LOG_FORMAT iniset $QINLING_CONF_FILE storage file_system_dir $QINLING_FUNCTION_STORAGE_DIR iniset $QINLING_CONF_FILE kubernetes qinling_service_address $DEFAULT_HOST_IP diff --git a/devstack/settings b/devstack/settings index 8a112fc1..58b0384f 100644 --- a/devstack/settings +++ b/devstack/settings @@ -19,6 +19,8 @@ QINLING_SERVICE_PROTOCOL=${QINLING_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL} QINLING_DEBUG=${QINLING_DEBUG:-True} QINLING_CONF_DIR=${QINLING_CONF_DIR:-/etc/qinling} QINLING_CONF_FILE=${QINLING_CONF_DIR}/qinling.conf +QINLING_POLICY_FILE=${QINLING_CONF_DIR}/policy.json QINLING_AUTH_CACHE_DIR=${QINLING_AUTH_CACHE_DIR:-/var/cache/qinling} QINLING_FUNCTION_STORAGE_DIR=${QINLING_FUNCTION_STORAGE_DIR:-/opt/qinling/funtion/packages} QINLING_PYTHON_RUNTIME_IMAGE=${QINLING_PYTHON_RUNTIME_IMAGE:-openstackqinling/python-runtime} +QINLING_LOG_FORMAT=${QINLING_LOG_FORMAT:-"%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s %(resource)s"} diff --git a/etc/policy.json.sample b/etc/policy.json.sample new file mode 100644 index 00000000..9c2deb74 --- /dev/null +++ b/etc/policy.json.sample @@ -0,0 +1,10 @@ +{ + "context_is_admin": "role:admin or is_admin:1", + "owner" : "project_id:%(project_id)s", + "admin_or_owner": "rule:context_is_admin or rule:owner", + "default": "rule:admin_or_owner", + + "runtime:create": "rule:context_is_admin", + "runtime:update": "rule:context_is_admin", + "runtime:delete": "rule:context_is_admin", +} diff --git a/qinling/api/access_control.py b/qinling/api/access_control.py index a6107697..4f15fbf2 100644 --- a/qinling/api/access_control.py +++ b/qinling/api/access_control.py @@ -56,10 +56,14 @@ def enforce(action, context, target=None, do_raise=True, :return: returns True if authorized and False if not authorized and do_raise is False. """ + if not cfg.CONF.pecan.auth_enable: + return + + ctx_dict = context.to_policy_values() target_obj = { - 'project_id': context.project_id, - 'user_id': context.user_id, + 'project_id': ctx_dict['project_id'], + 'user_id': ctx_dict['user_id'], } target_obj.update(target or {}) @@ -68,7 +72,7 @@ def enforce(action, context, target=None, do_raise=True, return _ENFORCER.enforce( action, target_obj, - context.to_dict(), + ctx_dict, do_raise=do_raise, exc=exc ) diff --git a/qinling/api/controllers/v1/function.py b/qinling/api/controllers/v1/function.py index 6449dac5..547b7de1 100644 --- a/qinling/api/controllers/v1/function.py +++ b/qinling/api/controllers/v1/function.py @@ -37,7 +37,7 @@ from qinling.utils import rest_utils LOG = logging.getLogger(__name__) CONF = cfg.CONF -POST_REQUIRED = set(['name', 'code']) +POST_REQUIRED = set(['code']) CODE_SOURCE = set(['package', 'swift', 'image']) UPDATE_ALLOWED = set(['name', 'description', 'entry']) @@ -106,7 +106,7 @@ class FunctionsController(rest.RestController): ) values = { - 'name': kwargs['name'], + 'name': kwargs.get('name'), 'description': kwargs.get('description'), 'runtime_id': kwargs.get('runtime_id'), 'code': json.loads(kwargs['code']), diff --git a/qinling/api/controllers/v1/resources.py b/qinling/api/controllers/v1/resources.py index 1d70c5c8..6c099a5d 100644 --- a/qinling/api/controllers/v1/resources.py +++ b/qinling/api/controllers/v1/resources.py @@ -218,6 +218,7 @@ class Runtime(Resource): name = wtypes.text image = wtypes.text description = wtypes.text + is_public = wsme.wsattr(bool, default=True) status = wsme.wsattr(wtypes.text, readonly=True) project_id = wsme.wsattr(wtypes.text, readonly=True) created_at = wsme.wsattr(wtypes.text, readonly=True) @@ -230,6 +231,7 @@ class Runtime(Resource): name='python2.7', image='lingxiankong/python', status='available', + is_public=True, project_id='default', description='Python 2.7 environment.', created_at='1970-01-01T00:00:00.000000', diff --git a/qinling/api/controllers/v1/runtime.py b/qinling/api/controllers/v1/runtime.py index e3a14510..fc963104 100644 --- a/qinling/api/controllers/v1/runtime.py +++ b/qinling/api/controllers/v1/runtime.py @@ -16,8 +16,10 @@ from oslo_log import log as logging from pecan import rest import wsmeext.pecan as wsme_pecan +from qinling.api import access_control as acl from qinling.api.controllers.v1 import resources from qinling.api.controllers.v1 import types +from qinling import context from qinling.db import api as db_api from qinling import exceptions as exc from qinling import rpc @@ -63,6 +65,8 @@ class RuntimesController(rest.RestController): status_code=201 ) def post(self, runtime): + acl.enforce('runtime:create', context.get_ctx()) + params = runtime.to_dict() if not POST_REQUIRED.issubset(set(params.keys())): @@ -82,13 +86,15 @@ class RuntimesController(rest.RestController): @rest_utils.wrap_wsme_controller_exception @wsme_pecan.wsexpose(None, types.uuid, status_code=204) def delete(self, id): + acl.enforce('runtime:delete', context.get_ctx()) + LOG.info("Delete resource.", resource={'type': self.type, 'id': id}) with db_api.transaction(): runtime_db = db_api.get_runtime(id) # Runtime can not be deleted if still associate with functions. - funcs = db_api.get_functions(runtime_id={'eq': id}) + funcs = db_api.get_functions(insecure=True, runtime_id={'eq': id}) if len(funcs): raise exc.NotAllowedException( 'Runtime %s is still in use.' % id @@ -111,6 +117,8 @@ class RuntimesController(rest.RestController): Currently, we only support update name, description, image. When updating image, send message to engine for asynchronous handling. """ + acl.enforce('runtime:update', context.get_ctx()) + values = {} for key in UPDATE_ALLOWED: if runtime.to_dict().get(key) is not None: diff --git a/qinling/db/sqlalchemy/api.py b/qinling/db/sqlalchemy/api.py index 54ac76e8..0cb0e533 100644 --- a/qinling/db/sqlalchemy/api.py +++ b/qinling/db/sqlalchemy/api.py @@ -109,7 +109,15 @@ def _secure_query(model, *columns): if not issubclass(model, model_base.QinlingSecureModelBase): return query - query = query.filter(model.project_id == context.get_ctx().projectid) + if model == models.Runtime: + query_criterion = sa.or_( + model.project_id == context.get_ctx().projectid, + model.is_public + ) + else: + query_criterion = model.project_id == context.get_ctx().projectid + + query = query.filter(query_criterion) return query @@ -274,7 +282,13 @@ def create_runtime(values, session=None): @db_base.session_aware() def get_runtime(id, session=None): - runtime = _get_db_object_by_id(models.Runtime, id) + model = models.Runtime + filters = sa.and_( + model.id == id, + sa.or_(model.project_id == context.get_ctx().projectid, + model.is_public), + ) + runtime = db_base.model_query(model).filter(filters).first() if not runtime: raise exc.DBEntityNotFoundError("Runtime not found [id=%s]" % id) diff --git a/qinling/db/sqlalchemy/migration/alembic_migrations/versions/001_pike.py b/qinling/db/sqlalchemy/migration/alembic_migrations/versions/001_pike.py index a7c8bcc5..74858a75 100644 --- a/qinling/db/sqlalchemy/migration/alembic_migrations/versions/001_pike.py +++ b/qinling/db/sqlalchemy/migration/alembic_migrations/versions/001_pike.py @@ -54,8 +54,8 @@ def upgrade(): sa.Column('description', sa.String(length=255), nullable=True), sa.Column('image', sa.String(length=255), nullable=False), sa.Column('status', sa.String(length=32), nullable=False), + sa.Column('is_public', sa.BOOLEAN, nullable=False), sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('image', 'project_id'), info={"check_ifexists": True} ) @@ -74,7 +74,6 @@ def upgrade(): sa.Column('entry', sa.String(length=80), nullable=False), sa.Column('count', sa.Integer, nullable=False), sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('name', 'project_id'), sa.ForeignKeyConstraint(['runtime_id'], [u'runtimes.id']), info={"check_ifexists": True} ) diff --git a/qinling/db/sqlalchemy/models.py b/qinling/db/sqlalchemy/models.py index 0a9b59d5..00ce486d 100644 --- a/qinling/db/sqlalchemy/models.py +++ b/qinling/db/sqlalchemy/models.py @@ -23,24 +23,17 @@ from qinling.utils import common class Runtime(model_base.QinlingSecureModelBase): __tablename__ = 'runtimes' - __table_args__ = ( - sa.UniqueConstraint('image', 'project_id'), - ) - name = sa.Column(sa.String(255)) description = sa.Column(sa.String(255)) image = sa.Column(sa.String(255), nullable=False) status = sa.Column(sa.String(32), nullable=False) + is_public = sa.Column(sa.BOOLEAN, default=True) class Function(model_base.QinlingSecureModelBase): __tablename__ = 'functions' - __table_args__ = ( - sa.UniqueConstraint('name', 'project_id'), - ) - - name = sa.Column(sa.String(255), nullable=False) + name = sa.Column(sa.String(255), nullable=True) description = sa.Column(sa.String(255)) runtime_id = sa.Column( sa.String(36), sa.ForeignKey(Runtime.id), nullable=True diff --git a/qinling/tests/unit/api/controllers/v1/test_runtime.py b/qinling/tests/unit/api/controllers/v1/test_runtime.py index ca181fbf..30a95746 100644 --- a/qinling/tests/unit/api/controllers/v1/test_runtime.py +++ b/qinling/tests/unit/api/controllers/v1/test_runtime.py @@ -37,7 +37,8 @@ class TestRuntimeController(base.APITest): "image": self.db_runtime.image, "name": self.db_runtime.name, "project_id": test_base.DEFAULT_PROJECT_ID, - "status": status.AVAILABLE + "status": status.AVAILABLE, + "is_public": True, } self.assertEqual(200, resp.status_int) @@ -51,7 +52,8 @@ class TestRuntimeController(base.APITest): "image": self.db_runtime.image, "name": self.db_runtime.name, "project_id": test_base.DEFAULT_PROJECT_ID, - "status": status.AVAILABLE + "status": status.AVAILABLE, + "is_public": True, } self.assertEqual(200, resp.status_int) diff --git a/qinling_tempest_plugin/functions/python_test.py b/qinling_tempest_plugin/functions/python_test.py new file mode 100644 index 00000000..1d5921b1 --- /dev/null +++ b/qinling_tempest_plugin/functions/python_test.py @@ -0,0 +1,17 @@ +# Copyright 2017 Catalyst IT Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def main(name='World'): + print('Hello, %s' % name) diff --git a/qinling_tempest_plugin/services/base.py b/qinling_tempest_plugin/services/base.py index 2aa95f9c..9c8bfe0b 100644 --- a/qinling_tempest_plugin/services/base.py +++ b/qinling_tempest_plugin/services/base.py @@ -21,9 +21,6 @@ class QinlingClientBase(rest_client.RestClient): def __init__(self, auth_provider, **kwargs): super(QinlingClientBase, self).__init__(auth_provider, **kwargs) - self.runtimes = [] - self.functions = [] - def get_list_objs(self, obj): resp, body = self.get('/v1/%s' % obj) diff --git a/qinling_tempest_plugin/services/qinling_client.py b/qinling_tempest_plugin/services/qinling_client.py index 93514e18..ecde652b 100644 --- a/qinling_tempest_plugin/services/qinling_client.py +++ b/qinling_tempest_plugin/services/qinling_client.py @@ -14,19 +14,73 @@ import json +import requests + from qinling_tempest_plugin.services import base as client_base class QinlingClient(client_base.QinlingClientBase): """Tempest REST client for Qinling.""" + def delete_resource(self, res, id): + resp, _ = self.delete_obj(res, id) + + return resp + + def get_resource(self, res, id): + resp, body = self.get_obj(res, id) + + return resp, body + + def get_resources(self, res): + resp, body = self.get_list_objs(res) + + return resp, body + def create_runtime(self, image, name=None): - body = {"image": image} + req_body = {"image": image} if name: - body.update({'name': name}) + req_body.update({'name': name}) - resp, body = self.post('runtimes', json.dumps(body)) - self.runtimes.append(json.loads(body)['id']) + resp, body = self.post_json('runtimes', req_body) - return resp, json.loads(body) + return resp, body + + def create_function(self, code, runtime_id, name='', package_data=None, + entry=None): + """Create function. + + Tempest rest client doesn't support multipart upload, so use requests + lib instead. + """ + headers = {'X-Auth-Token': self.auth_provider.get_token()} + req_body = { + 'name': name, + 'runtime_id': runtime_id, + 'code': json.dumps(code) + } + if entry: + req_body['entry'] = entry + + req_kwargs = { + 'headers': headers, + 'data': req_body + } + if package_data: + req_kwargs.update({'files': {'package': package_data}}) + + url_path = '%s/v1/functions' % (self.base_url) + resp = requests.post(url_path, **req_kwargs) + + return resp, json.loads(resp.text) + + def create_execution(self, function_id, input=None, sync=True): + req_body = {'function_id': function_id, 'sync': sync, 'input': input} + resp, body = self.post_json('executions', req_body) + + return resp, body + + def get_execution_log(self, execution_id): + return self.get('/v1/executions/%s/log' % execution_id, + headers={'Accept': 'text/plain'}) diff --git a/qinling_tempest_plugin/tests/api/test_runtimes.py b/qinling_tempest_plugin/tests/api/test_runtimes.py index d706914d..2b539987 100644 --- a/qinling_tempest_plugin/tests/api/test_runtimes.py +++ b/qinling_tempest_plugin/tests/api/test_runtimes.py @@ -13,7 +13,6 @@ # limitations under the License. from tempest.lib.common.utils import data_utils from tempest.lib import decorators -import tenacity from qinling_tempest_plugin.tests import base @@ -21,33 +20,21 @@ from qinling_tempest_plugin.tests import base class RuntimesTest(base.BaseQinlingTest): name_prefix = 'RuntimesTest' - @tenacity.retry( - wait=tenacity.wait_fixed(2), - stop=tenacity.stop_after_delay(10), - retry=tenacity.retry_if_exception_type(AssertionError) - ) - def _await_runtime_available(self, id): - resp, body = self.qinling_client.get_obj('runtimes', id) - - self.assertEqual(200, resp.status) - self.assertEqual('available', body['status']) - @decorators.idempotent_id('fdc2f07f-dd1d-4981-86d3-5bc7908d9a9b') def test_create_list_get_delete_runtime(self): name = data_utils.rand_name('runtime', prefix=self.name_prefix) - req_body = { - 'name': name, - 'image': 'openstackqinling/python-runtime' - } - resp, body = self.qinling_client.post_json('runtimes', req_body) - runtime_id = body['id'] + resp, body = self.admin_client.create_runtime( + 'openstackqinling/python-runtime', name + ) self.assertEqual(201, resp.status) self.assertEqual(name, body['name']) + runtime_id = body['id'] + # Get runtimes - resp, body = self.qinling_client.get_list_objs('runtimes') + resp, body = self.client.get_resources('runtimes') self.assertEqual(200, resp.status) self.assertIn( @@ -56,7 +43,7 @@ class RuntimesTest(base.BaseQinlingTest): ) # Wait for runtime to be available - self._await_runtime_available(runtime_id) + self.await_runtime_available(runtime_id) # Check k8s resource deploy = self.k8s_v1extention.read_namespaced_deployment( @@ -70,6 +57,6 @@ class RuntimesTest(base.BaseQinlingTest): ) # Delete runtime - resp, _ = self.qinling_client.delete_obj('runtimes', runtime_id) + resp = self.admin_client.delete_resource('runtimes', runtime_id) self.assertEqual(204, resp.status) diff --git a/qinling_tempest_plugin/tests/base.py b/qinling_tempest_plugin/tests/base.py index 0596de99..d49a4893 100644 --- a/qinling_tempest_plugin/tests/base.py +++ b/qinling_tempest_plugin/tests/base.py @@ -11,17 +11,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - from kubernetes import client as k8s_client from tempest import config from tempest import test +import tenacity CONF = config.CONF class BaseQinlingTest(test.BaseTestCase): - credentials = ('primary',) - force_tenant_isolation = False + credentials = ('admin', 'primary', 'alt') @classmethod def skip_checks(cls): @@ -34,19 +33,23 @@ class BaseQinlingTest(test.BaseTestCase): def setup_clients(cls): super(BaseQinlingTest, cls).setup_clients() - # os here is tempest.lib.services.clients.ServiceClients object - os = getattr(cls, 'os_%s' % cls.credentials[0]) - cls.qinling_client = os.qinling.QinlingClient() - - if CONF.identity.auth_version == 'v3': - project_id = os.auth_provider.auth_data[1]['project']['id'] - else: - project_id = os.auth_provider.auth_data[1]['token']['tenant']['id'] - cls.tenant_id = project_id - cls.user_id = os.auth_provider.auth_data[1]['user']['id'] + cls.client = cls.os_primary.qinling.QinlingClient() + cls.alt_client = cls.os_alt.qinling.QinlingClient() + cls.admin_client = cls.os_admin.qinling.QinlingClient() # Initilize k8s client k8s_client.Configuration().host = CONF.qinling.kube_host cls.k8s_v1 = k8s_client.CoreV1Api() cls.k8s_v1extention = k8s_client.ExtensionsV1beta1Api() cls.namespace = 'qinling' + + @tenacity.retry( + wait=tenacity.wait_fixed(2), + stop=tenacity.stop_after_delay(10), + retry=tenacity.retry_if_exception_type(AssertionError) + ) + def await_runtime_available(self, id): + resp, body = self.client.get_resource('runtimes', id) + + self.assertEqual(200, resp.status) + self.assertEqual('available', body['status']) diff --git a/qinling_tempest_plugin/tests/scenario/test_basic_ops.py b/qinling_tempest_plugin/tests/scenario/test_basic_ops.py new file mode 100644 index 00000000..7f5adede --- /dev/null +++ b/qinling_tempest_plugin/tests/scenario/test_basic_ops.py @@ -0,0 +1,113 @@ +# Copyright 2017 Catalyst IT Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import tempfile +import zipfile + +from tempest.lib.common.utils import data_utils +from tempest.lib import decorators + +from qinling_tempest_plugin.tests import base + + +class BasicOpsTest(base.BaseQinlingTest): + name_prefix = 'BasicOpsTest' + + def setUp(self): + super(BasicOpsTest, self).setUp() + + python_file_path = os.path.abspath( + os.path.join( + os.path.dirname(__file__), + os.pardir, + os.pardir, + 'functions/python_test.py' + ) + ) + + base_name, extention = os.path.splitext(python_file_path) + self.base_name = os.path.basename(base_name) + self.python_zip_file = os.path.join( + tempfile.gettempdir(), + '%s.zip' % self.base_name + ) + + zf = zipfile.ZipFile(self.python_zip_file, mode='w') + try: + # Use default compression mode, may change in future. + zf.write( + python_file_path, + '%s%s' % (self.base_name, extention), + compress_type=zipfile.ZIP_STORED + ) + finally: + zf.close() + + @decorators.idempotent_id('205fd749-2468-4d9f-9c05-45558d6d8f9e') + def test_basic_ops(self): + """Basic qinling operations test case, including following steps: + + 1. Admin user creates a runtime. + 2. Normal user creates function. + 3. Normal user creates execution(invoke function). + 4. Check result and execution log. + """ + name = data_utils.rand_name('runtime', prefix=self.name_prefix) + + resp, body = self.admin_client.create_runtime( + 'openstackqinling/python-runtime', name + ) + + self.assertEqual(201, resp.status) + self.assertEqual(name, body['name']) + + # Wait for runtime to be available + runtime_id = body['id'] + self.await_runtime_available(runtime_id) + self.addCleanup(self.admin_client.delete_resource, 'runtimes', + runtime_id) + + # Create function + function_name = data_utils.rand_name('function', + prefix=self.name_prefix) + with open(self.python_zip_file, 'rb') as package_data: + resp, body = self.client.create_function( + {"source": "package"}, + runtime_id, + name=function_name, + package_data=package_data, + entry='%s.main' % self.base_name + ) + function_id = body['id'] + + self.assertEqual(201, resp.status_code) + self.addCleanup(self.client.delete_resource, 'functions', + function_id) + + # Invoke function + resp, body = self.client.create_execution(function_id, + input={'name': 'Qinling'}) + + self.assertEqual(201, resp.status) + self.assertEqual('success', body['status']) + + execution_id = body['id'] + self.addCleanup(self.client.delete_resource, 'executions', + execution_id) + + # Get execution log + resp, body = self.client.get_execution_log(execution_id) + + self.assertEqual(200, resp.status) + self.assertIn('Hello, Qinling', body)