openstack
/
qinling
Code Issues Proposed changes

Improve function_alias integration 

When creating webhook using function_alias, the function id
and function version should be updated dynamically. For example,
a function alias A1 is created for function_1 and version 1,
a job is created using A1.

When the user updates A1 with function 1 and version 2, the webhook
should pick up the new version automatically.

For function execution, the execution has to make sure if an alias
is provided then the function id and the function version should be
the same as the one from the alias.

Change-Id: I17320a2a4f55cda8884de928c69ceba366c37f2e
Story: 2006337
Task: 36085
This commit is contained in:
Gaëtan Trellu 2019-08-05 18:49:01 -04:00 committed by Lingxian Kong
parent 32a68d1bfc
commit 9696b34288
12 changed files with 356 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
qinling/api/controllers/v1/resources.py
View File

@ -235,7 +235,7 @@ class Execution(Resource):
id = types.uuid
function_id = wsme.wsattr(types.uuid)
function_version = wsme.wsattr(int, default=0)
function_alias = wtypes.text
function_alias = wsme.wsattr(wtypes.text)
description = wtypes.text
status = wsme.wsattr(wtypes.text, readonly=True)
sync = bool

69
qinling/api/controllers/v1/webhook.py
View File

@ -34,7 +34,8 @@ from qinling.utils import rest_utils
LOG = logging.getLogger(__name__)
UPDATE_ALLOWED = set(['function_id', 'function_version', 'description'])
UPDATE_ALLOWED = set(['function_id', 'function_version', 'description',
'function_alias'])
class WebhooksController(rest.RestController):
@ -108,23 +109,25 @@ class WebhooksController(rest.RestController):
'Either function_alias or function_id must be provided.'
)
# if function_alias provided
function_alias = params.get('function_alias')
function_id = params.get('function_id', "")
version = params.get('function_version', 0)
function_alias = params.get('function_alias', "")
if function_alias:
alias_db = db_api.get_function_alias(function_alias)
function_id = alias_db.function_id
version = alias_db.function_version
params.update({'function_id': function_id,
'function_version': version})
# If function_alias is provided, we don't store either functin id
# or function version.
params.update({'function_id': None,
'function_version': None})
LOG.info("Creating %s, params: %s", self.type, params)
# Even admin user can not expose normal user's function
db_api.get_function(params['function_id'], insecure=False)
version = params.get('function_version', 0)
db_api.get_function(function_id, insecure=False)
if version > 0:
db_api.get_function_version(params['function_id'], version)
db_api.get_function_version(function_id, version)
webhook_d = db_api.create_webhook(params).to_dict()
@ -146,10 +149,6 @@ class WebhooksController(rest.RestController):
body=resources.Webhook
)
def put(self, id, webhook):
"""Update webhook.
Currently, only function_id and function_version are allowed to update.
"""
acl.enforce('webhook:update', context.get_ctx())
values = {}
@ -161,15 +160,32 @@ class WebhooksController(rest.RestController):
# Even admin user can not expose normal user's function
webhook_db = db_api.get_webhook(id, insecure=False)
pre_alias = webhook_db.function_alias
pre_function_id = webhook_db.function_id
pre_version = webhook_db.function_version
new_alias = values.get("function_alias")
new_function_id = values.get("function_id", pre_function_id)
new_version = values.get("function_version", pre_version)
db_api.get_function(new_function_id, insecure=False)
if new_version > 0:
db_api.get_function_version(new_function_id, new_version)
function_id = pre_function_id
version = pre_version
if new_alias and new_alias != pre_alias:
alias_db = db_api.get_function_alias(new_alias)
function_id = alias_db.function_id
version = alias_db.function_version
# If function_alias is provided, we don't store either functin id
# or function version.
values.update({'function_id': None,
'function_version': None})
elif new_function_id != pre_function_id or new_version != pre_version:
function_id = new_function_id
version = new_version
values.update({"function_alias": None})
db_api.get_function(function_id, insecure=False)
if version and version > 0:
db_api.get_function_version(function_id, version)
webhook = db_api.update_webhook(id, values).to_dict()
return resources.Webhook.from_dict(self._add_webhook_url(id, webhook))
@ -181,14 +197,25 @@ class WebhooksController(rest.RestController):
# The webhook url can be accessed without authentication, so
# insecure is used here
webhook_db = db_api.get_webhook(id, insecure=True)
function_db = webhook_db.function
function_alias = webhook_db.function_alias
if function_alias:
alias = db_api.get_function_alias(function_alias,
insecure=True)
function_id = alias.function_id
function_version = alias.function_version
function_db = db_api.get_function(function_id, insecure=True)
else:
function_db = webhook_db.function
function_id = webhook_db.function_id
function_version = webhook_db.function_version
trust_id = function_db.trust_id
project_id = function_db.project_id
version = webhook_db.function_version
LOG.info(
'Invoking function %s(version %s) by webhook %s',
webhook_db.function_id, version, id
function_id, function_version, id
)
# Setup user context
@ -196,8 +223,8 @@ class WebhooksController(rest.RestController):
context.set_ctx(ctx)
params = {
'function_id': webhook_db.function_id,
'function_version': version,
'function_id': function_id,
'function_version': function_version,
'sync': False,
'input': json.dumps(kwargs),
'description': constants.EXECUTION_BY_WEBHOOK % id

31
qinling/db/sqlalchemy/migration/alembic_migrations/versions/008_function_alias_for_execution.py Normal file
View File

@ -0,0 +1,31 @@
# Copyright 2019 - Ormuco Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""add function_alias field for executions table
Revision ID: 008
Revises: 007
"""
revision = '008'
down_revision = '007'
from alembic import op
import sqlalchemy as sa
def upgrade():
op.add_column(
'executions',
sa.Column('function_alias', sa.String(length=255), nullable=True)
)

31
qinling/db/sqlalchemy/migration/alembic_migrations/versions/009_function_alias_for_webhook.py Normal file
View File

@ -0,0 +1,31 @@
# Copyright 2019 - Ormuco Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""add function_alias field for webhooks table
Revision ID: 009
Revises: 008
"""
revision = '009'
down_revision = '008'
from alembic import op
import sqlalchemy as sa
def upgrade():
op.add_column(
'webhooks',
sa.Column('function_alias', sa.String(length=255), nullable=True)
)

34
qinling/db/sqlalchemy/migration/alembic_migrations/versions/010_function_id_for_execution.py Normal file
View File

@ -0,0 +1,34 @@
# Copyright 2019 - Ormuco Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Make function id nullable for executions table
Revision ID: 010
Revises: 009
"""
revision = '010'
down_revision = '009'
from alembic import op
import sqlalchemy as sa
def upgrade():
op.alter_column(
'executions',
'function_id',
existing_type=sa.String(length=36),
nullable=True
)

40
qinling/db/sqlalchemy/migration/alembic_migrations/versions/011_function_id_for_webhook.py Normal file
View File

@ -0,0 +1,40 @@
# Copyright 2019 - Ormuco Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Make function id nullable for webhooks table
Revision ID: 011
Revises: 010
"""
revision = '011'
down_revision = '010'
from alembic import op
import sqlalchemy as sa
def upgrade():
op.alter_column(
'webhooks',
'function_id',
existing_type=sa.String(length=36),
nullable=True
)
op.alter_column(
'webhooks',
'function_version',
existing_type=sa.Integer,
nullable=True
)

12
qinling/db/sqlalchemy/models.py
View File

@ -52,7 +52,8 @@ class Function(model_base.QinlingSecureModelBase):
class Execution(model_base.QinlingSecureModelBase):
__tablename__ = 'executions'
function_id = sa.Column(sa.String(36), nullable=False)
function_alias = sa.Column(sa.String(255), nullable=True)
function_id = sa.Column(sa.String(36), nullable=True)
function_version = sa.Column(sa.Integer, default=0)
status = sa.Column(sa.String(32), nullable=False)
sync = sa.Column(sa.BOOLEAN, default=True)
@ -91,11 +92,13 @@ class Job(model_base.QinlingSecureModelBase):
class Webhook(model_base.QinlingSecureModelBase):
__tablename__ = 'webhooks'
function_alias = sa.Column(sa.String(255), nullable=True)
function_id = sa.Column(
sa.String(36),
sa.ForeignKey(Function.id)
sa.ForeignKey(Function.id),
nullable=True
)
function_version = sa.Column(sa.Integer, default=0)
function_version = sa.Column(sa.Integer, nullable=True)
description = sa.Column(sa.String(255))
@ -169,5 +172,6 @@ Function.versions = relationship(
)
Function.aliases = relationship(
"FunctionAlias",
uselist=True
uselist=True,
backref="function"
)

20
qinling/tests/unit/api/controllers/v1/test_execution.py
View File

@ -28,7 +28,7 @@ class TestExecutionController(base.APITest):
self.func_id = db_func.id
@mock.patch('qinling.rpc.EngineClient.create_execution')
def test_post(self, mock_create_execution):
def test_create_with_function(self, mock_create_execution):
body = {
'function_id': self.func_id,
}
@ -41,7 +41,7 @@ class TestExecutionController(base.APITest):
self.assertEqual(1, resp.json.get('count'))
@mock.patch('qinling.rpc.EngineClient.create_execution')
def test_post_with_version(self, mock_rpc):
def test_create_with_version(self, mock_rpc):
db_api.increase_function_version(self.func_id, 0,
description="version 1")
body = {
@ -59,7 +59,7 @@ class TestExecutionController(base.APITest):
self.assertEqual(1, resp.json.get('count'))
@mock.patch('qinling.rpc.EngineClient.create_execution')
def test_post_with_alias(self, mock_rpc):
def test_create_with_alias(self, mock_rpc):
db_api.increase_function_version(self.func_id, 0,
description="version 1")
name = self.rand_name(name="alias", prefix=self.prefix)
@ -75,6 +75,7 @@ class TestExecutionController(base.APITest):
}
resp = self.app.post_json('/v1/executions', execution_body)
self.assertEqual(201, resp.status_int)
self.assertEqual(name, resp.json.get('function_alias'))
resp = self.app.get('/v1/functions/%s' % self.func_id)
self.assertEqual(0, resp.json.get('count'))
@ -82,7 +83,16 @@ class TestExecutionController(base.APITest):
resp = self.app.get('/v1/functions/%s/versions/1' % self.func_id)
self.assertEqual(1, resp.json.get('count'))
def test_post_without_required_params(self):
def test_create_with_invalid_alias(self):
body = {
'function_alias': 'fake_alias',
}
resp = self.app.post_json('/v1/executions', body, expect_errors=True)
self.assertEqual(404, resp.status_int)
def test_create_without_required_params(self):
resp = self.app.post(
'/v1/executions',
params={},
@ -92,7 +102,7 @@ class TestExecutionController(base.APITest):
self.assertEqual(400, resp.status_int)
@mock.patch('qinling.rpc.EngineClient.create_execution')
def test_post_rpc_error(self, mock_create_execution):
def test_create_rpc_error(self, mock_create_execution):
mock_create_execution.side_effect = exc.QinlingException
body = {
'function_id': self.func_id,

143
qinling/tests/unit/api/controllers/v1/test_webhook.py
View File

@ -11,9 +11,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import mock
from qinling import context
from qinling.db import api as db_api
from qinling.tests.unit.api import base
from qinling.utils import constants
class TestWebhookController(base.APITest):
@ -67,7 +71,7 @@ class TestWebhookController(base.APITest):
resp = self.app.get('/v1/webhooks/%s' % webhook_id, expect_errors=True)
self.assertEqual(404, resp.status_int)
def test_post_with_version(self):
def test_create_with_version(self):
db_api.increase_function_version(self.func_id, 0)
body = {
@ -79,8 +83,9 @@ class TestWebhookController(base.APITest):
self.assertEqual(201, resp.status_int)
self.assertEqual(1, resp.json.get("function_version"))
self.assertIsNone(resp.json.get("function_alias"))
def test_post_with_alias(self):
def test_create_with_alias(self):
db_api.increase_function_version(self.func_id, 0)
name = self.rand_name(name="alias", prefix=self.prefix)
body = {
@ -97,9 +102,21 @@ class TestWebhookController(base.APITest):
resp = self.app.post_json('/v1/webhooks', webhook_body)
self.assertEqual(201, resp.status_int)
self.assertEqual(1, resp.json.get("function_version"))
self.assertEqual(name, resp.json.get('function_alias'))
self.assertIsNone(resp.json.get("function_id"))
self.assertIsNone(resp.json.get("function_version"))
def test_post_without_required_params(self):
def test_create_with_invalid_alias(self):
body = {
'function_alias': 'fake_alias',
'description': 'webhook test'
}
resp = self.app.post_json('/v1/webhooks', body, expect_errors=True)
self.assertEqual(404, resp.status_int)
def test_create_without_required_params(self):
resp = self.app.post(
'/v1/webhooks',
params={},
@ -108,11 +125,11 @@ class TestWebhookController(base.APITest):
self.assertEqual(400, resp.status_int)
def test_put_with_version(self):
def test_update_with_version(self):
db_api.increase_function_version(self.func_id, 0)
webhook = self.create_webhook(self.func_id)
self.assertEqual(0, webhook.function_version)
self.assertIsNone(webhook.function_version)
resp = self.app.put_json(
'/v1/webhooks/%s' % webhook.id,
@ -121,8 +138,9 @@ class TestWebhookController(base.APITest):
self.assertEqual(200, resp.status_int)
self.assertEqual(1, resp.json.get("function_version"))
self.assertIsNone(resp.json.get("function_alias"))
def test_put_without_version(self):
def test_update_only_description(self):
db_api.increase_function_version(self.func_id, 0)
webhook = self.create_webhook(self.func_id, function_version=1)
@ -136,3 +154,114 @@ class TestWebhookController(base.APITest):
self.assertEqual(200, resp.status_int)
self.assertEqual(1, resp.json.get("function_version"))
self.assertEqual('updated description', resp.json.get("description"))
def test_update_function_alias_1(self):
# Create webhook using function alias
db_api.increase_function_version(self.func_id, 0)
name = self.rand_name(name="alias", prefix=self.prefix)
body = {
'function_id': self.func_id,
'function_version': 1,
'name': name
}
db_api.create_function_alias(**body)
webhook = self.create_webhook(function_alias=name)
db_api.increase_function_version(self.func_id, 1)
new_name = self.rand_name(name="alias", prefix=self.prefix)
body = {
'function_id': self.func_id,
'function_version': 2,
'name': new_name
}
db_api.create_function_alias(**body)
# Update webhook with the new alias
resp = self.app.put_json(
'/v1/webhooks/%s' % webhook.id,
{'function_alias': new_name}
)
self.assertEqual(200, resp.status_int)
self.assertEqual(new_name, resp.json.get("function_alias"))
self.assertIsNone(resp.json.get("function_id"))
self.assertIsNone(resp.json.get("function_version"))
def test_update_function_alias_2(self):
# Create webhook using function id
db_api.increase_function_version(self.func_id, 0)
webhook = self.create_webhook(function_id=self.func_id,
function_version=1)
db_api.increase_function_version(self.func_id, 1)
alias_name = self.rand_name(name="alias", prefix=self.prefix)
body = {
'function_id': self.func_id,
'function_version': 2,
'name': alias_name
}
db_api.create_function_alias(**body)
# Update webhook with function alias
resp = self.app.put_json(
'/v1/webhooks/%s' % webhook.id,
{'function_alias': alias_name}
)
self.assertEqual(200, resp.status_int)
self.assertEqual(alias_name, resp.json.get("function_alias"))
self.assertIsNone(resp.json.get("function_id"))
self.assertIsNone(resp.json.get("function_version"))
@mock.patch("qinling.utils.openstack.keystone.create_trust_context")
@mock.patch("qinling.utils.executions.create_execution")
def test_invoke_with_function_id(self, mock_create_execution,
mock_create_context):
exec_mock = mock_create_execution.return_value
exec_mock.id = "fake_id"
webhook = self.create_webhook(function_id=self.func_id)
resp = self.app.post_json('/v1/webhooks/%s/invoke' % webhook.id, {})
context.set_ctx(self.ctx)
self.assertEqual(202, resp.status_int)
params = {
'function_id': self.func_id,
'function_version': None,
'sync': False,
'input': json.dumps({}),
'description': constants.EXECUTION_BY_WEBHOOK % webhook.id
}
mock_create_execution.assert_called_once_with(mock.ANY, params)
@mock.patch("qinling.utils.openstack.keystone.create_trust_context")
@mock.patch("qinling.utils.executions.create_execution")
def test_invoke_with_function_alias(self, mock_create_execution,
mock_create_context):
exec_mock = mock_create_execution.return_value
exec_mock.id = "fake_id"
db_api.increase_function_version(self.func_id, 0)
alias_name = self.rand_name(name="alias", prefix=self.prefix)
body = {
'function_id': self.func_id,
'function_version': 1,
'name': alias_name
}
db_api.create_function_alias(**body)
webhook = self.create_webhook(function_alias=alias_name)
resp = self.app.post_json('/v1/webhooks/%s/invoke' % webhook.id, {})
context.set_ctx(self.ctx)
self.assertEqual(202, resp.status_int)
params = {
'function_id': self.func_id,
'function_version': 1,
'sync': False,
'input': json.dumps({}),
'description': constants.EXECUTION_BY_WEBHOOK % webhook.id
}
mock_create_execution.assert_called_once_with(mock.ANY, params)

11
qinling/tests/unit/base.py
View File

@ -219,11 +219,12 @@ class DbTestCase(BaseTest):
return job
def create_webhook(self, function_id=None, **kwargs):
if not function_id:
def create_webhook(self, function_id=None, function_alias=None, **kwargs):
if not function_id and not function_alias:
function_id = self.create_function().id
webhook_params = {
'function_alias': function_alias,
'function_id': function_id,
# 'auth_enable' is disabled by default
'project_id': DEFAULT_PROJECT_ID,
@ -233,11 +234,13 @@ class DbTestCase(BaseTest):
return webhook
def create_execution(self, function_id=None, **kwargs):
if not function_id:
def create_execution(self, function_id=None, function_alias=None,
**kwargs):
if not function_id and not function_alias:
function_id = self.create_function().id
execution_params = {
'function_alias': function_alias,
'function_id': function_id,
'project_id': DEFAULT_PROJECT_ID,
'status': status.RUNNING,

2
qinling/utils/executions.py
View File

@ -80,7 +80,7 @@ def create_execution(engine_client, params):
function_id = alias_db.function_id
version = alias_db.function_version
params.update({'function_id': function_id,
'version': version})
'function_version': version})
func_db = db_api.get_function(function_id)
runtime_id = func_db.runtime_id

4
releasenotes/notes/add-support-function-alias-for-execution-and-webhook-cf786fc4c9efd0af.yaml Normal file
View File

@ -0,0 +1,4 @@
---
fixes:
- When creating execution and webhook with function alias, they should always pick up the
updated function and its version corresponding to the alias.