openstack/qinling
Code Issues Proposed changes

Support policy check for api access

Browse Source 
- Support oslo.policy in api layer
- Add tempest test for basic qinling operation

Change-Id: Icf12ee4df54652e81013e951b91163c085d961b7
This commit is contained in:
Lingxian Kong 2017-09-15 23:21:25 +12:00
parent 851227deea
commit cff88e1b64
17 changed files with 272 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
devstack/plugin.sh
View File

@ -64,11 +64,15 @@ function configure_qinling {
mkdir_chown_stack "$QINLING_FUNCTION_STORAGE_DIR"
rm -f "$QINLING_FUNCTION_STORAGE_DIR"/*
cp $QINLING_DIR/etc/policy.json.sample $QINLING_POLICY_FILE
# Generate Qinling configuration file and configure common parameters.
oslo-config-generator --config-file $QINLING_DIR/tools/config/config-generator.qinling.conf --output-file $QINLING_CONF_FILE
iniset $QINLING_CONF_FILE oslo_policy policy_file $QINLING_POLICY_FILE
iniset $QINLING_CONF_FILE DEFAULT debug $QINLING_DEBUG
iniset $QINLING_CONF_FILE DEFAULT server all
iniset $QINLING_CONF_FILE DEFAULT logging_default_format_string $QINLING_LOG_FORMAT
iniset $QINLING_CONF_FILE storage file_system_dir $QINLING_FUNCTION_STORAGE_DIR
iniset $QINLING_CONF_FILE kubernetes qinling_service_address $DEFAULT_HOST_IP

2
devstack/settings
View File

@ -19,6 +19,8 @@ QINLING_SERVICE_PROTOCOL=${QINLING_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
QINLING_DEBUG=${QINLING_DEBUG:-True}
QINLING_CONF_DIR=${QINLING_CONF_DIR:-/etc/qinling}
QINLING_CONF_FILE=${QINLING_CONF_DIR}/qinling.conf
QINLING_POLICY_FILE=${QINLING_CONF_DIR}/policy.json
QINLING_AUTH_CACHE_DIR=${QINLING_AUTH_CACHE_DIR:-/var/cache/qinling}
QINLING_FUNCTION_STORAGE_DIR=${QINLING_FUNCTION_STORAGE_DIR:-/opt/qinling/funtion/packages}
QINLING_PYTHON_RUNTIME_IMAGE=${QINLING_PYTHON_RUNTIME_IMAGE:-openstackqinling/python-runtime}
QINLING_LOG_FORMAT=${QINLING_LOG_FORMAT:-"%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s %(resource)s"}

10
etc/policy.json.sample Normal file
View File

@ -0,0 +1,10 @@
{
"context_is_admin": "role:admin or is_admin:1",
"owner" : "project_id:%(project_id)s",
"admin_or_owner": "rule:context_is_admin or rule:owner",
"default": "rule:admin_or_owner",
"runtime:create": "rule:context_is_admin",
"runtime:update": "rule:context_is_admin",
"runtime:delete": "rule:context_is_admin",
}

10
qinling/api/access_control.py
View File

@ -56,10 +56,14 @@ def enforce(action, context, target=None, do_raise=True,
:return: returns True if authorized and False if not authorized and
do_raise is False.
"""
if not cfg.CONF.pecan.auth_enable:
return
ctx_dict = context.to_policy_values()
target_obj = {
'project_id': context.project_id,
'user_id': context.user_id,
'project_id': ctx_dict['project_id'],
'user_id': ctx_dict['user_id'],
}
target_obj.update(target or {})
@ -68,7 +72,7 @@ def enforce(action, context, target=None, do_raise=True,
return _ENFORCER.enforce(
action,
target_obj,
context.to_dict(),
ctx_dict,
do_raise=do_raise,
exc=exc
)

4
qinling/api/controllers/v1/function.py
View File

@ -37,7 +37,7 @@ from qinling.utils import rest_utils
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
POST_REQUIRED = set(['name', 'code'])
POST_REQUIRED = set(['code'])
CODE_SOURCE = set(['package', 'swift', 'image'])
UPDATE_ALLOWED = set(['name', 'description', 'entry'])
@ -106,7 +106,7 @@ class FunctionsController(rest.RestController):
)
values = {
'name': kwargs['name'],
'name': kwargs.get('name'),
'description': kwargs.get('description'),
'runtime_id': kwargs.get('runtime_id'),
'code': json.loads(kwargs['code']),

2
qinling/api/controllers/v1/resources.py
View File

@ -218,6 +218,7 @@ class Runtime(Resource):
name = wtypes.text
image = wtypes.text
description = wtypes.text
is_public = wsme.wsattr(bool, default=True)
status = wsme.wsattr(wtypes.text, readonly=True)
project_id = wsme.wsattr(wtypes.text, readonly=True)
created_at = wsme.wsattr(wtypes.text, readonly=True)
@ -230,6 +231,7 @@ class Runtime(Resource):
name='python2.7',
image='lingxiankong/python',
status='available',
is_public=True,
project_id='default',
description='Python 2.7 environment.',
created_at='1970-01-01T00:00:00.000000',

10
qinling/api/controllers/v1/runtime.py
View File

@ -16,8 +16,10 @@ from oslo_log import log as logging
from pecan import rest
import wsmeext.pecan as wsme_pecan
from qinling.api import access_control as acl
from qinling.api.controllers.v1 import resources
from qinling.api.controllers.v1 import types
from qinling import context
from qinling.db import api as db_api
from qinling import exceptions as exc
from qinling import rpc
@ -63,6 +65,8 @@ class RuntimesController(rest.RestController):
status_code=201
)
def post(self, runtime):
acl.enforce('runtime:create', context.get_ctx())
params = runtime.to_dict()
if not POST_REQUIRED.issubset(set(params.keys())):
@ -82,13 +86,15 @@ class RuntimesController(rest.RestController):
@rest_utils.wrap_wsme_controller_exception
@wsme_pecan.wsexpose(None, types.uuid, status_code=204)
def delete(self, id):
acl.enforce('runtime:delete', context.get_ctx())
LOG.info("Delete resource.", resource={'type': self.type, 'id': id})
with db_api.transaction():
runtime_db = db_api.get_runtime(id)
# Runtime can not be deleted if still associate with functions.
funcs = db_api.get_functions(runtime_id={'eq': id})
funcs = db_api.get_functions(insecure=True, runtime_id={'eq': id})
if len(funcs):
raise exc.NotAllowedException(
'Runtime %s is still in use.' % id
@ -111,6 +117,8 @@ class RuntimesController(rest.RestController):
Currently, we only support update name, description, image. When
updating image, send message to engine for asynchronous handling.
"""
acl.enforce('runtime:update', context.get_ctx())
values = {}
for key in UPDATE_ALLOWED:
if runtime.to_dict().get(key) is not None:

18
qinling/db/sqlalchemy/api.py
View File

@ -109,7 +109,15 @@ def _secure_query(model, *columns):
if not issubclass(model, model_base.QinlingSecureModelBase):
return query
query = query.filter(model.project_id == context.get_ctx().projectid)
if model == models.Runtime:
query_criterion = sa.or_(
model.project_id == context.get_ctx().projectid,
model.is_public
)
else:
query_criterion = model.project_id == context.get_ctx().projectid
query = query.filter(query_criterion)
return query
@ -274,7 +282,13 @@ def create_runtime(values, session=None):
@db_base.session_aware()
def get_runtime(id, session=None):
runtime = _get_db_object_by_id(models.Runtime, id)
model = models.Runtime
filters = sa.and_(
model.id == id,
sa.or_(model.project_id == context.get_ctx().projectid,
model.is_public),
)
runtime = db_base.model_query(model).filter(filters).first()
if not runtime:
raise exc.DBEntityNotFoundError("Runtime not found [id=%s]" % id)

3
qinling/db/sqlalchemy/migration/alembic_migrations/versions/001_pike.py
View File

@ -54,8 +54,8 @@ def upgrade():
sa.Column('description', sa.String(length=255), nullable=True),
sa.Column('image', sa.String(length=255), nullable=False),
sa.Column('status', sa.String(length=32), nullable=False),
sa.Column('is_public', sa.BOOLEAN, nullable=False),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('image', 'project_id'),
info={"check_ifexists": True}
)
@ -74,7 +74,6 @@ def upgrade():
sa.Column('entry', sa.String(length=80), nullable=False),
sa.Column('count', sa.Integer, nullable=False),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name', 'project_id'),
sa.ForeignKeyConstraint(['runtime_id'], [u'runtimes.id']),
info={"check_ifexists": True}
)

11
qinling/db/sqlalchemy/models.py
View File

@ -23,24 +23,17 @@ from qinling.utils import common
class Runtime(model_base.QinlingSecureModelBase):
__tablename__ = 'runtimes'
__table_args__ = (
sa.UniqueConstraint('image', 'project_id'),
)
name = sa.Column(sa.String(255))
description = sa.Column(sa.String(255))
image = sa.Column(sa.String(255), nullable=False)
status = sa.Column(sa.String(32), nullable=False)
is_public = sa.Column(sa.BOOLEAN, default=True)
class Function(model_base.QinlingSecureModelBase):
__tablename__ = 'functions'
__table_args__ = (
sa.UniqueConstraint('name', 'project_id'),
)
name = sa.Column(sa.String(255), nullable=False)
name = sa.Column(sa.String(255), nullable=True)
description = sa.Column(sa.String(255))
runtime_id = sa.Column(
sa.String(36), sa.ForeignKey(Runtime.id), nullable=True

6
qinling/tests/unit/api/controllers/v1/test_runtime.py
View File

@ -37,7 +37,8 @@ class TestRuntimeController(base.APITest):
"image": self.db_runtime.image,
"name": self.db_runtime.name,
"project_id": test_base.DEFAULT_PROJECT_ID,
"status": status.AVAILABLE
"status": status.AVAILABLE,
"is_public": True,
}
self.assertEqual(200, resp.status_int)
@ -51,7 +52,8 @@ class TestRuntimeController(base.APITest):
"image": self.db_runtime.image,
"name": self.db_runtime.name,
"project_id": test_base.DEFAULT_PROJECT_ID,
"status": status.AVAILABLE
"status": status.AVAILABLE,
"is_public": True,
}
self.assertEqual(200, resp.status_int)

17
qinling_tempest_plugin/functions/python_test.py Normal file
View File

@ -0,0 +1,17 @@
# Copyright 2017 Catalyst IT Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
def main(name='World'):
print('Hello, %s' % name)

3
qinling_tempest_plugin/services/base.py
View File

@ -21,9 +21,6 @@ class QinlingClientBase(rest_client.RestClient):
def __init__(self, auth_provider, **kwargs):
super(QinlingClientBase, self).__init__(auth_provider, **kwargs)
self.runtimes = []
self.functions = []
def get_list_objs(self, obj):
resp, body = self.get('/v1/%s' % obj)

64
qinling_tempest_plugin/services/qinling_client.py
View File

@ -14,19 +14,73 @@
import json
import requests
from qinling_tempest_plugin.services import base as client_base
class QinlingClient(client_base.QinlingClientBase):
"""Tempest REST client for Qinling."""
def delete_resource(self, res, id):
resp, _ = self.delete_obj(res, id)
return resp
def get_resource(self, res, id):
resp, body = self.get_obj(res, id)
return resp, body
def get_resources(self, res):
resp, body = self.get_list_objs(res)
return resp, body
def create_runtime(self, image, name=None):
body = {"image": image}
req_body = {"image": image}
if name:
body.update({'name': name})
req_body.update({'name': name})
resp, body = self.post('runtimes', json.dumps(body))
self.runtimes.append(json.loads(body)['id'])
resp, body = self.post_json('runtimes', req_body)
return resp, json.loads(body)
return resp, body
def create_function(self, code, runtime_id, name='', package_data=None,
entry=None):
"""Create function.
Tempest rest client doesn't support multipart upload, so use requests
lib instead.
"""
headers = {'X-Auth-Token': self.auth_provider.get_token()}
req_body = {
'name': name,
'runtime_id': runtime_id,
'code': json.dumps(code)
}
if entry:
req_body['entry'] = entry
req_kwargs = {
'headers': headers,
'data': req_body
}
if package_data:
req_kwargs.update({'files': {'package': package_data}})
url_path = '%s/v1/functions' % (self.base_url)
resp = requests.post(url_path, **req_kwargs)
return resp, json.loads(resp.text)
def create_execution(self, function_id, input=None, sync=True):
req_body = {'function_id': function_id, 'sync': sync, 'input': input}
resp, body = self.post_json('executions', req_body)
return resp, body
def get_execution_log(self, execution_id):
return self.get('/v1/executions/%s/log' % execution_id,
headers={'Accept': 'text/plain'})

29
qinling_tempest_plugin/tests/api/test_runtimes.py
View File

@ -13,7 +13,6 @@
# limitations under the License.
from tempest.lib.common.utils import data_utils
from tempest.lib import decorators
import tenacity
from qinling_tempest_plugin.tests import base
@ -21,33 +20,21 @@ from qinling_tempest_plugin.tests import base
class RuntimesTest(base.BaseQinlingTest):
name_prefix = 'RuntimesTest'
@tenacity.retry(
wait=tenacity.wait_fixed(2),
stop=tenacity.stop_after_delay(10),
retry=tenacity.retry_if_exception_type(AssertionError)
)
def _await_runtime_available(self, id):
resp, body = self.qinling_client.get_obj('runtimes', id)
self.assertEqual(200, resp.status)
self.assertEqual('available', body['status'])
@decorators.idempotent_id('fdc2f07f-dd1d-4981-86d3-5bc7908d9a9b')
def test_create_list_get_delete_runtime(self):
name = data_utils.rand_name('runtime', prefix=self.name_prefix)
req_body = {
'name': name,
'image': 'openstackqinling/python-runtime'
}
resp, body = self.qinling_client.post_json('runtimes', req_body)
runtime_id = body['id']
resp, body = self.admin_client.create_runtime(
'openstackqinling/python-runtime', name
)
self.assertEqual(201, resp.status)
self.assertEqual(name, body['name'])
runtime_id = body['id']
# Get runtimes
resp, body = self.qinling_client.get_list_objs('runtimes')
resp, body = self.client.get_resources('runtimes')
self.assertEqual(200, resp.status)
self.assertIn(
@ -56,7 +43,7 @@ class RuntimesTest(base.BaseQinlingTest):
)
# Wait for runtime to be available
self._await_runtime_available(runtime_id)
self.await_runtime_available(runtime_id)
# Check k8s resource
deploy = self.k8s_v1extention.read_namespaced_deployment(
@ -70,6 +57,6 @@ class RuntimesTest(base.BaseQinlingTest):
)
# Delete runtime
resp, _ = self.qinling_client.delete_obj('runtimes', runtime_id)
resp = self.admin_client.delete_resource('runtimes', runtime_id)
self.assertEqual(204, resp.status)

29
qinling_tempest_plugin/tests/base.py
View File

@ -11,17 +11,16 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from kubernetes import client as k8s_client
from tempest import config
from tempest import test
import tenacity
CONF = config.CONF
class BaseQinlingTest(test.BaseTestCase):
credentials = ('primary',)
force_tenant_isolation = False
credentials = ('admin', 'primary', 'alt')
@classmethod
def skip_checks(cls):
@ -34,19 +33,23 @@ class BaseQinlingTest(test.BaseTestCase):
def setup_clients(cls):
super(BaseQinlingTest, cls).setup_clients()
# os here is tempest.lib.services.clients.ServiceClients object
os = getattr(cls, 'os_%s' % cls.credentials[0])
cls.qinling_client = os.qinling.QinlingClient()
if CONF.identity.auth_version == 'v3':
project_id = os.auth_provider.auth_data[1]['project']['id']
else:
project_id = os.auth_provider.auth_data[1]['token']['tenant']['id']
cls.tenant_id = project_id
cls.user_id = os.auth_provider.auth_data[1]['user']['id']
cls.client = cls.os_primary.qinling.QinlingClient()
cls.alt_client = cls.os_alt.qinling.QinlingClient()
cls.admin_client = cls.os_admin.qinling.QinlingClient()
# Initilize k8s client
k8s_client.Configuration().host = CONF.qinling.kube_host
cls.k8s_v1 = k8s_client.CoreV1Api()
cls.k8s_v1extention = k8s_client.ExtensionsV1beta1Api()
cls.namespace = 'qinling'
@tenacity.retry(
wait=tenacity.wait_fixed(2),
stop=tenacity.stop_after_delay(10),
retry=tenacity.retry_if_exception_type(AssertionError)
)
def await_runtime_available(self, id):
resp, body = self.client.get_resource('runtimes', id)
self.assertEqual(200, resp.status)
self.assertEqual('available', body['status'])

113
qinling_tempest_plugin/tests/scenario/test_basic_ops.py Normal file
View File

@ -0,0 +1,113 @@
# Copyright 2017 Catalyst IT Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import tempfile
import zipfile
from tempest.lib.common.utils import data_utils
from tempest.lib import decorators
from qinling_tempest_plugin.tests import base
class BasicOpsTest(base.BaseQinlingTest):
name_prefix = 'BasicOpsTest'
def setUp(self):
super(BasicOpsTest, self).setUp()
python_file_path = os.path.abspath(
os.path.join(
os.path.dirname(__file__),
os.pardir,
os.pardir,
'functions/python_test.py'
)
)
base_name, extention = os.path.splitext(python_file_path)
self.base_name = os.path.basename(base_name)
self.python_zip_file = os.path.join(
tempfile.gettempdir(),
'%s.zip' % self.base_name
)
zf = zipfile.ZipFile(self.python_zip_file, mode='w')
try:
# Use default compression mode, may change in future.
zf.write(
python_file_path,
'%s%s' % (self.base_name, extention),
compress_type=zipfile.ZIP_STORED
)
finally:
zf.close()
@decorators.idempotent_id('205fd749-2468-4d9f-9c05-45558d6d8f9e')
def test_basic_ops(self):
"""Basic qinling operations test case, including following steps:
1. Admin user creates a runtime.
2. Normal user creates function.
3. Normal user creates execution(invoke function).
4. Check result and execution log.
"""
name = data_utils.rand_name('runtime', prefix=self.name_prefix)
resp, body = self.admin_client.create_runtime(
'openstackqinling/python-runtime', name
)
self.assertEqual(201, resp.status)
self.assertEqual(name, body['name'])
# Wait for runtime to be available
runtime_id = body['id']
self.await_runtime_available(runtime_id)
self.addCleanup(self.admin_client.delete_resource, 'runtimes',
runtime_id)
# Create function
function_name = data_utils.rand_name('function',
prefix=self.name_prefix)
with open(self.python_zip_file, 'rb') as package_data:
resp, body = self.client.create_function(
{"source": "package"},
runtime_id,
name=function_name,
package_data=package_data,
entry='%s.main' % self.base_name
)
function_id = body['id']
self.assertEqual(201, resp.status_code)
self.addCleanup(self.client.delete_resource, 'functions',
function_id)
# Invoke function
resp, body = self.client.create_execution(function_id,
input={'name': 'Qinling'})
self.assertEqual(201, resp.status)
self.assertEqual('success', body['status'])
execution_id = body['id']
self.addCleanup(self.client.delete_resource, 'executions',
execution_id)
# Get execution log
resp, body = self.client.get_execution_log(execution_id)
self.assertEqual(200, resp.status)
self.assertIn('Hello, Qinling', body)