Browse Source

Merge "Support to specify workload type for runtime"

changes/12/586012/1
Zuul 3 years ago
committed by Gerrit Code Review
parent
commit
e48089052c
14 changed files with 119 additions and 106 deletions
  1. +1
    -26
      qinling/api/controllers/v1/resources.py
  2. +11
    -8
      qinling/api/controllers/v1/runtime.py
  3. +36
    -0
      qinling/db/sqlalchemy/migration/alembic_migrations/versions/005_add_trusted_for_runtime.py
  4. +1
    -0
      qinling/db/sqlalchemy/models.py
  5. +3
    -4
      qinling/engine/default_engine.py
  6. +2
    -2
      qinling/orchestrator/base.py
  7. +36
    -30
      qinling/orchestrator/kubernetes/manager.py
  8. +2
    -0
      qinling/orchestrator/kubernetes/templates/deployment.j2
  9. +2
    -0
      qinling/orchestrator/kubernetes/templates/pod.j2
  10. +3
    -0
      qinling/tests/unit/api/controllers/v1/test_runtime.py
  11. +2
    -1
      qinling/tests/unit/base.py
  12. +3
    -2
      qinling/tests/unit/engine/test_default_engine.py
  13. +8
    -33
      qinling/tests/unit/orchestrator/kubernetes/test_manager.py
  14. +9
    -0
      releasenotes/notes/workload-type-support-d613cdb7bb90b2a2.yaml

+ 1
- 26
qinling/api/controllers/v1/resources.py View File

@ -241,25 +241,12 @@ class Runtime(Resource):
image = wtypes.text
description = wtypes.text
is_public = wsme.wsattr(bool, default=True)
trusted = bool
status = wsme.wsattr(wtypes.text, readonly=True)
project_id = wsme.wsattr(wtypes.text, readonly=True)
created_at = wsme.wsattr(wtypes.text, readonly=True)
updated_at = wsme.wsattr(wtypes.text, readonly=True)
@classmethod
def sample(cls):
return cls(
id='123e4567-e89b-12d3-a456-426655440000',
name='python2.7',
image='lingxiankong/python',
status='available',
is_public=True,
project_id='default',
description='Python 2.7 environment.',
created_at='1970-01-01T00:00:00.000000',
updated_at='1970-01-01T00:00:00.000000'
)
class Runtimes(ResourceList):
runtimes = [Runtime]
@ -269,18 +256,6 @@ class Runtimes(ResourceList):
super(Runtimes, self).__init__(**kwargs)
@classmethod
def sample(cls):
sample = cls()
sample.runtimes = [Runtime.sample()]
sample.next = (
"http://localhost:7070/v1/environments?"
"sort_keys=id,name&sort_dirs=asc,desc&limit=10&"
"marker=123e4567-e89b-12d3-a456-426655440000"
)
return sample
class RuntimePoolCapacity(Resource):
total = wsme.wsattr(int, readonly=True)


+ 11
- 8
qinling/api/controllers/v1/runtime.py View File

@ -71,6 +71,8 @@ class RuntimesController(rest.RestController):
acl.enforce('runtime:create', context.get_ctx())
params = runtime.to_dict()
if 'trusted' not in params:
params['trusted'] = True
if not POST_REQUIRED.issubset(set(params.keys())):
raise exc.InputException(
@ -117,8 +119,9 @@ class RuntimesController(rest.RestController):
def put(self, id, runtime):
"""Update runtime.
Currently, we only support update name, description, image. When
updating image, send message to engine for asynchronous handling.
Currently, we support update name, description, image. When
updating image, send message to engine for asynchronous
handling.
"""
acl.enforce('runtime:update', context.get_ctx())
@ -130,8 +133,10 @@ class RuntimesController(rest.RestController):
LOG.info('Update resource, params: %s', values,
resource={'type': self.type, 'id': id})
image = values.get('image')
with db_api.transaction():
if 'image' in values:
if image is not None:
pre_runtime = db_api.get_runtime(id)
if pre_runtime.status != status.AVAILABLE:
raise exc.RuntimeNotAvailableException(
@ -139,7 +144,7 @@ class RuntimesController(rest.RestController):
)
pre_image = pre_runtime.image
if pre_image != values['image']:
if pre_image != image:
# Ensure there is no function running in the runtime.
db_funcs = db_api.get_functions(
insecure=True, fields=['id'], runtime_id=id
@ -155,11 +160,9 @@ class RuntimesController(rest.RestController):
values['status'] = status.UPGRADING
self.engine_client.update_runtime(
id,
image=values['image'],
pre_image=pre_image
image=image,
pre_image=pre_image,
)
else:
values.pop('image')
runtime_db = db_api.update_runtime(id, values)


+ 36
- 0
qinling/db/sqlalchemy/migration/alembic_migrations/versions/005_add_trusted_for_runtime.py View File

@ -0,0 +1,36 @@
# Copyright 2018 OpenStack Foundation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""add trusted field for runtimes table
Revision ID: 005
Revises: 004
Create Date: 2018-07-24 12:00:00.888969
"""
# revision identifiers, used by Alembic.
revision = '005'
down_revision = '004'
from alembic import op
import sqlalchemy as sa
def upgrade():
op.add_column(
'runtimes',
sa.Column('trusted', sa.BOOLEAN, nullable=False, default=True,
server_default="1")
)

+ 1
- 0
qinling/db/sqlalchemy/models.py View File

@ -28,6 +28,7 @@ class Runtime(model_base.QinlingSecureModelBase):
image = sa.Column(sa.String(255), nullable=False)
status = sa.Column(sa.String(32), nullable=False)
is_public = sa.Column(sa.BOOLEAN, default=True)
trusted = sa.Column(sa.BOOLEAN, default=True)
class Function(model_base.QinlingSecureModelBase):


+ 3
- 4
qinling/engine/default_engine.py View File

@ -43,7 +43,8 @@ class DefaultEngine(object):
try:
self.orchestrator.create_pool(
runtime_id,
runtime.image
runtime.image,
trusted=runtime.trusted
)
runtime.status = status.AVAILABLE
LOG.info('Runtime %s created.', runtime_id)
@ -69,9 +70,7 @@ class DefaultEngine(object):
runtime_id, image, pre_image
)
ret = self.orchestrator.update_pool(
runtime_id, image=image
)
ret = self.orchestrator.update_pool(runtime_id, image=image)
if ret:
values = {'status': status.AVAILABLE}


+ 2
- 2
qinling/orchestrator/base.py View File

@ -27,7 +27,7 @@ class OrchestratorBase(object):
"""OrchestratorBase interface."""
@abc.abstractmethod
def create_pool(self, name, image, **kwargs):
def create_pool(self, name, image, trusted=True, **kwargs):
raise NotImplementedError
@abc.abstractmethod
@ -35,7 +35,7 @@ class OrchestratorBase(object):
raise NotImplementedError
@abc.abstractmethod
def update_pool(self, name, **kwargs):
def update_pool(self, name, image=None, **kwargs):
raise NotImplementedError
@abc.abstractmethod


+ 36
- 30
qinling/orchestrator/kubernetes/manager.py View File

@ -125,10 +125,8 @@ class KubernetesManager(base.OrchestratorBase):
self.conf.kubernetes.namespace
)
if (
not ret.status.replicas or
ret.status.replicas != ret.status.available_replicas
):
if (not ret.status.replicas or
ret.status.replicas != ret.status.available_replicas):
raise exc.OrchestratorException('Deployment %s not ready.' % name)
def get_pool(self, name):
@ -158,7 +156,7 @@ class KubernetesManager(base.OrchestratorBase):
return {"total": total, "available": available}
def create_pool(self, name, image):
def create_pool(self, name, image, trusted=True):
deployment_body = self.deployment_template.render(
{
"name": name,
@ -166,7 +164,8 @@ class KubernetesManager(base.OrchestratorBase):
"replicas": self.conf.kubernetes.replicas,
"container_name": 'worker',
"image": image,
"sidecar_image": self.conf.engine.sidecar_image
"sidecar_image": self.conf.engine.sidecar_image,
"trusted": str(trusted).lower()
}
)
@ -222,6 +221,21 @@ class KubernetesManager(base.OrchestratorBase):
LOG.info("Pods in deployment %s deleted.", name)
LOG.info("Deployment %s deleted.", name)
@tenacity.retry(
wait=tenacity.wait_fixed(5),
stop=tenacity.stop_after_delay(600),
reraise=True,
retry=tenacity.retry_if_exception_type(exc.OrchestratorException)
)
def _wait_for_upgrade(self, deploy_name):
ret = self.v1extension.read_namespaced_deployment(
deploy_name,
self.conf.kubernetes.namespace
)
if ret.status.unavailable_replicas is not None:
raise exc.OrchestratorException("Deployment %s upgrade not "
"ready." % deploy_name)
def update_pool(self, name, image=None):
"""Deployment rolling-update.
@ -235,7 +249,6 @@ class KubernetesManager(base.OrchestratorBase):
'spec': {
'containers': [
{
# TODO(kong): Make the name configurable.
'name': 'worker',
'image': image
}
@ -248,30 +261,23 @@ class KubernetesManager(base.OrchestratorBase):
name, self.conf.kubernetes.namespace, body
)
unavailable_replicas = 1
# TODO(kong): Make this configurable
retry = 5
while unavailable_replicas != 0 and retry > 0:
time.sleep(5)
retry = retry - 1
deploy = self.v1extension.read_namespaced_deployment_status(
name,
self.conf.kubernetes.namespace
)
unavailable_replicas = deploy.status.unavailable_replicas
try:
time.sleep(10)
self._wait_for_upgrade(name)
except exc.OrchestratorException:
LOG.warn("Timeout when waiting for the deployment %s upgrade, "
"Start to roll back.", name)
# Handle failure of rolling-update.
if unavailable_replicas > 0:
body = {
"name": name,
"rollbackTo": {
"revision": 0
}
}
self.v1extension.create_namespaced_deployment_rollback(
name, self.conf.kubernetes.namespace, body
)
body = {"rollbackTo": {"revision": 0}}
try:
self.v1extension.create_namespaced_deployment_rollback(
name, self.conf.kubernetes.namespace, body
)
except Exception:
# TODO(lxkong): remove the exception catch until kubernetes
# python lib has a new release. Refer to
# https://github.com/kubernetes-client/python/issues/491
pass
return False


+ 2
- 0
qinling/orchestrator/kubernetes/templates/deployment.j2 View File

@ -19,6 +19,8 @@ spec:
{% for key, value in labels.items() %}
{{ key }}: {{ value }}
{% endfor %}
annotations:
io.kubernetes.cri-o.TrustedSandbox: "{{ trusted }}"
spec:
terminationGracePeriodSeconds: 5
automountServiceAccountToken: false


+ 2
- 0
qinling/orchestrator/kubernetes/templates/pod.j2 View File

@ -6,6 +6,8 @@ metadata:
{% for key, value in labels.items() %}
{{ key }}: {{ value }}
{% endfor %}
annotations:
io.kubernetes.cri-o.TrustedSandbox: "false"
spec:
terminationGracePeriodSeconds: 5
automountServiceAccountToken: false


+ 3
- 0
qinling/tests/unit/api/controllers/v1/test_runtime.py View File

@ -71,7 +71,10 @@ class TestRuntimeController(base.APITest):
resp = self.app.post_json('/v1/runtimes', body)
self.assertEqual(201, resp.status_int)
body.update({"trusted": True})
self._assertDictContainsSubset(resp.json, body)
mock_create_time.assert_called_once_with(resp.json['id'])
@mock.patch('qinling.rpc.EngineClient.create_runtime')


+ 2
- 1
qinling/tests/unit/base.py View File

@ -175,7 +175,8 @@ class DbTestCase(BaseTest):
# 'auth_enable' is disabled by default, we create runtime for
# default tenant.
'project_id': DEFAULT_PROJECT_ID,
'status': status.AVAILABLE
'status': status.AVAILABLE,
'trusted': True
}
)


+ 3
- 2
qinling/tests/unit/engine/test_default_engine.py View File

@ -50,7 +50,8 @@ class TestDefaultEngine(base.DbTestCase):
self.default_engine.create_runtime(mock.Mock(), runtime_id)
self.orchestrator.create_pool.assert_called_once_with(
runtime_id, runtime.image)
runtime_id, runtime.image, trusted=True)
runtime = db_api.get_runtime(runtime_id)
self.assertEqual(status.AVAILABLE, runtime.status)
@ -64,7 +65,7 @@ class TestDefaultEngine(base.DbTestCase):
self.default_engine.create_runtime(mock.Mock(), runtime_id)
self.orchestrator.create_pool.assert_called_once_with(
runtime_id, runtime.image)
runtime_id, runtime.image, trusted=True)
runtime = db_api.get_runtime(runtime_id)
self.assertEqual(status.ERROR, runtime.status)


+ 8
- 33
qinling/tests/unit/orchestrator/kubernetes/test_manager.py View File

@ -196,7 +196,8 @@ class TestKubernetesManager(base.DbTestCase):
'replicas': fake_replicas,
'container_name': 'worker',
'image': fake_image,
'sidecar_image': CONF.engine.sidecar_image
'sidecar_image': CONF.engine.sidecar_image,
'trusted': 'true'
}
)
self.k8s_v1_ext.create_namespaced_deployment.assert_called_once_with(
@ -297,8 +298,8 @@ class TestKubernetesManager(base.DbTestCase):
}
}
ret = mock.Mock()
ret.status.unavailable_replicas = 0
self.k8s_v1_ext.read_namespaced_deployment_status.return_value = ret
ret.status.unavailable_replicas = None
self.k8s_v1_ext.read_namespaced_deployment.return_value = ret
update_result = self.manager.update_pool(fake_deployment_name,
image=image)
@ -306,7 +307,7 @@ class TestKubernetesManager(base.DbTestCase):
self.assertTrue(update_result)
self.k8s_v1_ext.patch_namespaced_deployment.assert_called_once_with(
fake_deployment_name, self.fake_namespace, body)
read_status = self.k8s_v1_ext.read_namespaced_deployment_status
read_status = self.k8s_v1_ext.read_namespaced_deployment
read_status.assert_called_once_with(fake_deployment_name,
self.fake_namespace)
@ -316,9 +317,8 @@ class TestKubernetesManager(base.DbTestCase):
ret1 = mock.Mock()
ret1.status.unavailable_replicas = 1
ret2 = mock.Mock()
ret2.status.unavailable_replicas = 0
self.k8s_v1_ext.read_namespaced_deployment_status.side_effect = [
ret1, ret2]
ret2.status.unavailable_replicas = None
self.k8s_v1_ext.read_namespaced_deployment.side_effect = [ret1, ret2]
update_result = self.manager.update_pool(fake_deployment_name,
image=image)
@ -326,34 +326,9 @@ class TestKubernetesManager(base.DbTestCase):
self.assertTrue(update_result)
self.k8s_v1_ext.patch_namespaced_deployment.assert_called_once_with(
fake_deployment_name, self.fake_namespace, mock.ANY)
read_status = self.k8s_v1_ext.read_namespaced_deployment_status
read_status = self.k8s_v1_ext.read_namespaced_deployment
self.assertEqual(2, read_status.call_count)
def test_update_pool_rollback(self):
fake_deployment_name = self.rand_name('deployment', prefix=self.prefix)
image = self.rand_name('image', prefix=self.prefix)
ret = mock.Mock()
ret.status.unavailable_replicas = 1
self.k8s_v1_ext.read_namespaced_deployment_status.return_value = ret
rollback_body = {
"name": fake_deployment_name,
"rollbackTo": {
"revision": 0
}
}
update_result = self.manager.update_pool(fake_deployment_name,
image=image)
self.assertFalse(update_result)
self.k8s_v1_ext.patch_namespaced_deployment.assert_called_once_with(
fake_deployment_name, self.fake_namespace, mock.ANY)
read_status = self.k8s_v1_ext.read_namespaced_deployment_status
self.assertEqual(5, read_status.call_count)
rollback = self.k8s_v1_ext.create_namespaced_deployment_rollback
rollback.assert_called_once_with(
fake_deployment_name, self.fake_namespace, rollback_body)
def test_get_pool(self):
fake_deployment_name = self.rand_name('deployment', prefix=self.prefix)


+ 9
- 0
releasenotes/notes/workload-type-support-d613cdb7bb90b2a2.yaml View File

@ -0,0 +1,9 @@
---
features:
- Support to specify ``trusted`` for runtime creation. In Kubernetes
orchestrator implementation, it's using
``io.kubernetes.cri-o.TrustedSandbox`` annotation in the pod specification
to choose the underlying container runtime. This feature is useful to
leverage the security container technology such as Kata containers or
gVisor. It also gets rid of the security concerns for running image type
function.

Loading…
Cancel
Save