openstack
/
qinling
Code Issues Proposed changes

Support version for execution creation 

Now, all the underlying resources(pod, service) in k8s cluster have
version number in their labels. Different versions of the same function
will have different services exposed in k8s cluster.

Change-Id: Ic0b3045404105175073844b908fa0f6c2ef2ab8a
Story: #2001829
Task: #14350
This commit is contained in:
Lingxian Kong 2018-04-19 16:29:08 +12:00
parent 917132a19c
commit 885ed28234
20 changed files with 580 additions and 346 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
qinling/api/controllers/v1/execution.py
View File

@ -56,6 +56,9 @@ class ExecutionsController(rest.RestController):
status_code=201 status_code=201
) )
def post(self, body): def post(self, body):
ctx = context.get_ctx()
acl.enforce('execution:create', ctx)
params = body.to_dict() params = body.to_dict()
LOG.info("Creating %s. [params=%s]", self.type, params) LOG.info("Creating %s. [params=%s]", self.type, params)

4
qinling/api/controllers/v1/function.py
View File

@ -55,12 +55,12 @@ class FunctionWorkerController(rest.RestController):
acl.enforce('function_worker:get_all', context.get_ctx()) acl.enforce('function_worker:get_all', context.get_ctx())
LOG.info("Get workers for function %s.", function_id) LOG.info("Get workers for function %s.", function_id)
workers = etcd_util.get_workers(function_id, CONF) workers = etcd_util.get_workers(function_id)
workers = [ workers = [
resources.FunctionWorker.from_dict( resources.FunctionWorker.from_dict(
{'function_id': function_id, 'worker_name': w} {'function_id': function_id, 'worker_name': w}
) for w in workers ) for w in workers
] ]
return resources.FunctionWorkers(workers=workers) return resources.FunctionWorkers(workers=workers)

29
qinling/api/controllers/v1/resources.py
View File

@ -278,6 +278,7 @@ class Runtimes(ResourceList):
class Execution(Resource): class Execution(Resource):
id = types.uuid id = types.uuid
function_id = wsme.wsattr(types.uuid, mandatory=True) function_id = wsme.wsattr(types.uuid, mandatory=True)
function_version = wsme.wsattr(int, default=0)
description = wtypes.text description = wtypes.text
status = wsme.wsattr(wtypes.text, readonly=True) status = wsme.wsattr(wtypes.text, readonly=True)
sync = bool sync = bool
@ -303,21 +304,6 @@ class Execution(Resource):
return obj return obj
@classmethod
def sample(cls):
return cls(
id='123e4567-e89b-12d3-a456-426655440000',
function_id='123e4567-e89b-12d3-a456-426655440000',
description='this is the first execution.',
status='success',
sync=True,
input={'data': 'hello, world'},
result={'result': 'hello, world'},
project_id='default',
created_at='1970-01-01T00:00:00.000000',
updated_at='1970-01-01T00:00:00.000000'
)
class Executions(ResourceList): class Executions(ResourceList):
executions = [Execution] executions = [Execution]
@ -327,18 +313,6 @@ class Executions(ResourceList):
super(Executions, self).__init__(**kwargs) super(Executions, self).__init__(**kwargs)
@classmethod
def sample(cls):
sample = cls()
sample.executions = [Execution.sample()]
sample.next = (
"http://localhost:7070/v1/executions?"
"sort_keys=id,name&sort_dirs=asc,desc&limit=10&"
"marker=123e4567-e89b-12d3-a456-426655440000"
)
return sample
class Job(Resource): class Job(Resource):
id = types.uuid id = types.uuid
@ -420,6 +394,7 @@ class FunctionVersion(Resource):
id = types.uuid id = types.uuid
description = wtypes.text description = wtypes.text
version_number = wsme.wsattr(int, readonly=True) version_number = wsme.wsattr(int, readonly=True)
count = wsme.wsattr(int, readonly=True)
project_id = wsme.wsattr(wtypes.text, readonly=True) project_id = wsme.wsattr(wtypes.text, readonly=True)
created_at = wsme.wsattr(wtypes.text, readonly=True) created_at = wsme.wsattr(wtypes.text, readonly=True)
updated_at = wsme.wsattr(wtypes.text, readonly=True) updated_at = wsme.wsattr(wtypes.text, readonly=True)

9
qinling/db/api.py
View File

@ -212,5 +212,14 @@ def get_function_version(function_id, version):
return IMPL.get_function_version(function_id, version) return IMPL.get_function_version(function_id, version)
# This function is only used in unit test.
def update_function_version(function_id, version, **kwargs):
return IMPL.update_function_version(function_id, version, **kwargs)
def delete_function_version(function_id, version): def delete_function_version(function_id, version):
return IMPL.delete_function_version(function_id, version) return IMPL.delete_function_version(function_id, version)
def get_function_versions(**kwargs):
return IMPL.get_function_versions(**kwargs)

14
qinling/db/sqlalchemy/api.py
View File

@ -529,7 +529,21 @@ def get_function_version(function_id, version, session=None):
return version_db return version_db
# This function is only used in unit test.
@db_base.session_aware()
def update_function_version(function_id, version, session=None, **kwargs):
version_db = get_function_version(function_id, version, session=session)
version_db.update(kwargs.copy())
return version_db
@db_base.session_aware() @db_base.session_aware()
def delete_function_version(function_id, version, session=None): def delete_function_version(function_id, version, session=None):
version_db = get_function_version(function_id, version) version_db = get_function_version(function_id, version)
session.delete(version_db) session.delete(version_db)
@db_base.session_aware()
def get_function_versions(session=None, **kwargs):
return _get_collection_sorted_by_time(models.FunctionVersion, **kwargs)

1
qinling/db/sqlalchemy/migration/alembic_migrations/versions/002_add_function_version_support.py
View File

@ -38,6 +38,7 @@ def upgrade():
sa.Column('function_id', sa.String(length=36), nullable=False), sa.Column('function_id', sa.String(length=36), nullable=False),
sa.Column('description', sa.String(length=255), nullable=True), sa.Column('description', sa.String(length=255), nullable=True),
sa.Column('version_number', sa.Integer, nullable=False), sa.Column('version_number', sa.Integer, nullable=False),
sa.Column('count', sa.Integer, nullable=False),
sa.PrimaryKeyConstraint('id'), sa.PrimaryKeyConstraint('id'),
sa.ForeignKeyConstraint(['function_id'], [u'functions.id']), sa.ForeignKeyConstraint(['function_id'], [u'functions.id']),
sa.UniqueConstraint('function_id', 'version_number', 'project_id'), sa.UniqueConstraint('function_id', 'version_number', 'project_id'),

1
qinling/db/sqlalchemy/models.py
View File

@ -119,6 +119,7 @@ class FunctionVersion(model_base.QinlingSecureModelBase):
function = relationship('Function', back_populates="versions") function = relationship('Function', back_populates="versions")
description = sa.Column(sa.String(255), nullable=True) description = sa.Column(sa.String(255), nullable=True)
version_number = sa.Column(sa.Integer, default=0) version_number = sa.Column(sa.Integer, default=0)
count = sa.Column(sa.Integer, default=0)
Runtime.functions = relationship("Function", back_populates="runtime") Runtime.functions = relationship("Function", back_populates="runtime")

97
qinling/engine/default_engine.py
View File

@ -89,33 +89,41 @@ class DefaultEngine(object):
stop=tenacity.stop_after_attempt(30), stop=tenacity.stop_after_attempt(30),
retry=(tenacity.retry_if_result(lambda result: result is False)) retry=(tenacity.retry_if_result(lambda result: result is False))
) )
def function_load_check(self, function_id, runtime_id): def function_load_check(self, function_id, version, runtime_id):
with etcd_util.get_worker_lock() as lock: """Check function load and scale the workers if needed.
:return: None if no need to scale up otherwise return the service url
"""
with etcd_util.get_worker_lock(function_id, version) as lock:
if not lock.is_acquired(): if not lock.is_acquired():
return False return False
workers = etcd_util.get_workers(function_id) workers = etcd_util.get_workers(function_id, version)
running_execs = db_api.get_executions( running_execs = db_api.get_executions(
function_id=function_id, status=status.RUNNING function_id=function_id,
function_version=version,
status=status.RUNNING
) )
concurrency = (len(running_execs) or 1) / (len(workers) or 1) concurrency = (len(running_execs) or 1) / (len(workers) or 1)
if (len(workers) == 0 or if (len(workers) == 0 or
concurrency > CONF.engine.function_concurrency): concurrency > CONF.engine.function_concurrency):
LOG.info( LOG.info(
'Scale up function %s. Current concurrency: %s, execution ' 'Scale up function %s(version %s). Current concurrency: '
'number %s, worker number %s', '%s, execution number %s, worker number %s',
function_id, concurrency, len(running_execs), len(workers) function_id, version, concurrency, len(running_execs),
len(workers)
) )
# NOTE(kong): The increase step could be configurable # NOTE(kong): The increase step could be configurable
return self.scaleup_function(None, function_id, runtime_id, 1) return self.scaleup_function(None, function_id, version,
runtime_id, 1)
def create_execution(self, ctx, execution_id, function_id, runtime_id, def create_execution(self, ctx, execution_id, function_id,
input=None): function_version, runtime_id, input=None):
LOG.info( LOG.info(
'Creating execution. execution_id=%s, function_id=%s, ' 'Creating execution. execution_id=%s, function_id=%s, '
'runtime_id=%s, input=%s', 'function_version=%s, runtime_id=%s, input=%s',
execution_id, function_id, runtime_id, input execution_id, function_id, function_version, runtime_id, input
) )
function = db_api.get_function(function_id) function = db_api.get_function(function_id)
@ -129,22 +137,25 @@ class DefaultEngine(object):
# Auto scale workers if needed # Auto scale workers if needed
if not is_image_source: if not is_image_source:
try: try:
svc_url = self.function_load_check(function_id, runtime_id) svc_url = self.function_load_check(function_id,
function_version,
runtime_id)
except exc.OrchestratorException as e: except exc.OrchestratorException as e:
utils.handle_execution_exception(execution_id, str(e)) utils.handle_execution_exception(execution_id, str(e))
return return
temp_url = etcd_util.get_service_url(function_id) temp_url = etcd_util.get_service_url(function_id, function_version)
svc_url = svc_url or temp_url svc_url = svc_url or temp_url
if svc_url: if svc_url:
func_url = '%s/execute' % svc_url func_url = '%s/execute' % svc_url
LOG.debug( LOG.debug(
'Found service url for function: %s, execution: %s, url: %s', 'Found service url for function: %s(version %s), execution: '
function_id, execution_id, func_url '%s, url: %s',
function_id, function_version, execution_id, func_url
) )
data = utils.get_request_data( data = utils.get_request_data(
CONF, function_id, execution_id, CONF, function_id, function_version, execution_id,
input, function.entry, function.trust_id, input, function.entry, function.trust_id,
self.qinling_endpoint self.qinling_endpoint
) )
@ -152,12 +163,13 @@ class DefaultEngine(object):
self.session, func_url, body=data self.session, func_url, body=data
) )
utils.finish_execution( utils.finish_execution(execution_id, success, res,
execution_id, success, res, is_image_source=is_image_source) is_image_source=is_image_source)
return return
if source == constants.IMAGE_FUNCTION: if source == constants.IMAGE_FUNCTION:
image = function.code['image'] image = function.code['image']
# Be consistent with k8s naming convention
identifier = ('%s-%s' % identifier = ('%s-%s' %
(common.generate_unicode_uuid(dashed=False), (common.generate_unicode_uuid(dashed=False),
function_id) function_id)
@ -167,8 +179,13 @@ class DefaultEngine(object):
labels = {'runtime_id': runtime_id} labels = {'runtime_id': runtime_id}
try: try:
# For image function, it will be executed inside this method; for
# package type function it only sets up underlying resources and
# get a service url. If the service url is already created
# beforehand, nothing happens.
_, svc_url = self.orchestrator.prepare_execution( _, svc_url = self.orchestrator.prepare_execution(
function_id, function_id,
function_version,
image=image, image=image,
identifier=identifier, identifier=identifier,
labels=labels, labels=labels,
@ -178,9 +195,12 @@ class DefaultEngine(object):
utils.handle_execution_exception(execution_id, str(e)) utils.handle_execution_exception(execution_id, str(e))
return return
# For image type function, read the worker log; For package type
# function, invoke and get log
success, res = self.orchestrator.run_execution( success, res = self.orchestrator.run_execution(
execution_id, execution_id,
function_id, function_id,
function_version,
input=input, input=input,
identifier=identifier, identifier=identifier,
service_url=svc_url, service_url=svc_url,
@ -188,34 +208,43 @@ class DefaultEngine(object):
trust_id=function.trust_id trust_id=function.trust_id
) )
utils.finish_execution( utils.finish_execution(execution_id, success, res,
execution_id, success, res, is_image_source=is_image_source) is_image_source=is_image_source)
def delete_function(self, ctx, function_id): def delete_function(self, ctx, function_id, function_version=0):
"""Deletes underlying resources allocated for function.""" """Deletes underlying resources allocated for function."""
LOG.info('Start to delete function %s.', function_id) LOG.info('Start to delete function %s(version %s).', function_id,
function_version)
self.orchestrator.delete_function(function_id) self.orchestrator.delete_function(function_id, function_version)
LOG.info('Deleted function %s.', function_id) LOG.info('Deleted function %s(version %s).', function_id,
function_version)
def scaleup_function(self, ctx, function_id, runtime_id, count=1): def scaleup_function(self, ctx, function_id, function_version, runtime_id,
count=1):
worker_names, service_url = self.orchestrator.scaleup_function( worker_names, service_url = self.orchestrator.scaleup_function(
function_id, function_id,
function_version,
identifier=runtime_id, identifier=runtime_id,
count=count count=count
) )
for name in worker_names: for name in worker_names:
etcd_util.create_worker(function_id, name) etcd_util.create_worker(function_id, name,
version=function_version)
etcd_util.create_service_url(function_id, service_url) etcd_util.create_service_url(function_id, service_url,
version=function_version)
LOG.info('Finished scaling up function %s(version %s).', function_id,
function_version)
LOG.info('Finished scaling up function %s.', function_id)
return service_url return service_url
def scaledown_function(self, ctx, function_id, count=1): def scaledown_function(self, ctx, function_id, function_version=0,
workers = etcd_util.get_workers(function_id) count=1):
workers = etcd_util.get_workers(function_id, function_version)
worker_deleted_num = ( worker_deleted_num = (
count if len(workers) > count else len(workers) - 1 count if len(workers) > count else len(workers) - 1
) )
@ -224,6 +253,8 @@ class DefaultEngine(object):
for worker in workers: for worker in workers:
LOG.debug('Removing worker %s', worker) LOG.debug('Removing worker %s', worker)
self.orchestrator.delete_worker(worker) self.orchestrator.delete_worker(worker)
etcd_util.delete_worker(function_id, worker) etcd_util.delete_worker(function_id, worker,
version=function_version)
LOG.info('Finished scaling down function %s.', function_id) LOG.info('Finished scaling down function %s(version %s).', function_id,
function_version)

24
qinling/engine/utils.py
View File

@ -74,17 +74,29 @@ def url_request(request_session, url, body=None):
return False, {'error': 'Internal service error.'} return False, {'error': 'Internal service error.'}
def get_request_data(conf, function_id, execution_id, input, entry, trust_id, def get_request_data(conf, function_id, version, execution_id, input, entry,
qinling_endpoint): trust_id, qinling_endpoint):
"""Prepare the request body should send to the worker."""
ctx = context.get_ctx() ctx = context.get_ctx()
download_url = (
'%s/%s/functions/%s?download=true' % if version == 0:
(qinling_endpoint.strip('/'), constants.CURRENT_VERSION, function_id) download_url = (
) '%s/%s/functions/%s?download=true' %
(qinling_endpoint.strip('/'), constants.CURRENT_VERSION,
function_id)
)
else:
download_url = (
'%s/%s/functions/%s/versions/%s?download=true' %
(qinling_endpoint.strip('/'), constants.CURRENT_VERSION,
function_id, version)
)
data = { data = {
'execution_id': execution_id, 'execution_id': execution_id,
'input': input, 'input': input,
'function_id': function_id, 'function_id': function_id,
'function_version': version,
'entry': entry, 'entry': entry,
'download_url': download_url, 'download_url': download_url,
'request_id': ctx.request_id, 'request_id': ctx.request_id,

9
qinling/orchestrator/base.py
View File

@ -39,19 +39,20 @@ class OrchestratorBase(object):
raise NotImplementedError raise NotImplementedError
@abc.abstractmethod @abc.abstractmethod
def prepare_execution(self, function_id, **kwargs): def prepare_execution(self, function_id, function_version, **kwargs):
raise NotImplementedError raise NotImplementedError
@abc.abstractmethod @abc.abstractmethod
def run_execution(self, execution_id, function_id, **kwargs): def run_execution(self, execution_id, function_id, function_version,
**kwargs):
raise NotImplementedError raise NotImplementedError
@abc.abstractmethod @abc.abstractmethod
def delete_function(self, function_id, **kwargs): def delete_function(self, function_id, function_version, **kwargs):
raise NotImplementedError raise NotImplementedError
@abc.abstractmethod @abc.abstractmethod
def scaleup_function(self, function_id, **kwargs): def scaleup_function(self, function_id, function_version, **kwargs):
raise NotImplementedError raise NotImplementedError
@abc.abstractmethod @abc.abstractmethod

84
qinling/orchestrator/kubernetes/manager.py
View File

@ -29,6 +29,7 @@ from qinling.orchestrator import base
from qinling.orchestrator.kubernetes import utils as k8s_util from qinling.orchestrator.kubernetes import utils as k8s_util
from qinling.utils import common from qinling.utils import common
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
TEMPLATES_DIR = (os.path.dirname(os.path.realpath(__file__)) + '/templates/') TEMPLATES_DIR = (os.path.dirname(os.path.realpath(__file__)) + '/templates/')
@ -218,17 +219,20 @@ class KubernetesManager(base.OrchestratorBase):
return True return True
def _choose_available_pods(self, labels, count=1, function_id=None): def _choose_available_pods(self, labels, count=1, function_id=None,
function_version=0):
# If there is already a pod for function, reuse it. # If there is already a pod for function, reuse it.
if function_id: if function_id:
ret = self.v1.list_namespaced_pod( ret = self.v1.list_namespaced_pod(
self.conf.kubernetes.namespace, self.conf.kubernetes.namespace,
label_selector='function_id=%s' % function_id label_selector='function_id=%s,function_version=%s' %
(function_id, function_version)
) )
if len(ret.items) >= count: if len(ret.items) >= count:
LOG.debug( LOG.debug(
"Function %s already associates to a pod with at least " "Function %s(version %s) already associates to a pod with "
"%d worker(s). ", function_id, count "at least %d worker(s). ",
function_id, function_version, count
) )
return ret.items[:count] return ret.items[:count]
@ -243,7 +247,8 @@ class KubernetesManager(base.OrchestratorBase):
return ret.items[-count:] return ret.items[-count:]
def _prepare_pod(self, pod, deployment_name, function_id, labels=None): def _prepare_pod(self, pod, deployment_name, function_id, version,
labels=None):
"""Pod preparation. """Pod preparation.
1. Update pod labels. 1. Update pod labels.
@ -253,16 +258,22 @@ class KubernetesManager(base.OrchestratorBase):
labels = labels or {} labels = labels or {}
LOG.info( LOG.info(
'Prepare pod %s in deployment %s for function %s', 'Prepare pod %s in deployment %s for function %s(version %s)',
pod_name, deployment_name, function_id pod_name, deployment_name, function_id, version
) )
# Update pod label. # Update pod label.
pod_labels = self._update_pod_label(pod, {'function_id': function_id}) pod_labels = self._update_pod_label(
pod,
# pod label value should be string
{'function_id': function_id, 'function_version': str(version)}
)
# Create service for the chosen pod. # Create service for the chosen pod.
service_name = "service-%s" % function_id service_name = "service-%s-%s" % (function_id, version)
labels.update({'function_id': function_id}) labels.update(
{'function_id': function_id, 'function_version': str(version)}
)
# TODO(kong): Make the service type configurable. # TODO(kong): Make the service type configurable.
service_body = self.service_template.render( service_body = self.service_template.render(
@ -314,6 +325,7 @@ class KubernetesManager(base.OrchestratorBase):
return pod_name, pod_service_url return pod_name, pod_service_url
def _create_pod(self, image, pod_name, labels, input): def _create_pod(self, image, pod_name, labels, input):
"""Create pod for image type function."""
if not input: if not input:
input_list = [] input_list = []
elif isinstance(input, dict) and input.get('__function_input'): elif isinstance(input, dict) and input.get('__function_input'):
@ -357,15 +369,17 @@ class KubernetesManager(base.OrchestratorBase):
return pod_labels return pod_labels
def prepare_execution(self, function_id, image=None, identifier=None, def prepare_execution(self, function_id, version, image=None,
labels=None, input=None): identifier=None, labels=None, input=None):
"""Prepare service URL for function. """Prepare service URL for function version.
For image function, create a single pod with input, so the function For image function, create a single pod with input, so the function
will be executed. will be executed.
For normal function, choose a pod from the pool and expose a service, For normal function, choose a pod from the pool and expose a service,
return the service URL. return the service URL.
return a tuple includes pod name and the servise url.
""" """
pods = None pods = None
@ -375,7 +389,8 @@ class KubernetesManager(base.OrchestratorBase):
self._create_pod(image, identifier, labels, input) self._create_pod(image, identifier, labels, input)
return identifier, None return identifier, None
else: else:
pods = self._choose_available_pods(labels, function_id=function_id) pods = self._choose_available_pods(labels, function_id=function_id,
function_version=version)
if not pods: if not pods:
LOG.critical('No worker available.') LOG.critical('No worker available.')
@ -383,31 +398,35 @@ class KubernetesManager(base.OrchestratorBase):
try: try:
pod_name, url = self._prepare_pod( pod_name, url = self._prepare_pod(
pods[0], identifier, function_id, labels pods[0], identifier, function_id, version, labels
) )
return pod_name, url return pod_name, url
except Exception: except Exception:
LOG.exception('Pod preparation failed.') LOG.exception('Pod preparation failed.')
self.delete_function(function_id, labels) self.delete_function(function_id, version, labels)
raise exc.OrchestratorException('Execution preparation failed.') raise exc.OrchestratorException('Execution preparation failed.')
def run_execution(self, execution_id, function_id, input=None, def run_execution(self, execution_id, function_id, version, input=None,
identifier=None, service_url=None, entry='main.main', identifier=None, service_url=None, entry='main.main',
trust_id=None): trust_id=None):
"""Run execution and get output.""" """Run execution.
Return a tuple including the result and the output.
"""
if service_url: if service_url:
func_url = '%s/execute' % service_url func_url = '%s/execute' % service_url
data = utils.get_request_data( data = utils.get_request_data(
self.conf, function_id, execution_id, input, entry, trust_id, self.conf, function_id, version, execution_id, input, entry,
self.qinling_endpoint trust_id, self.qinling_endpoint
) )
LOG.debug( LOG.debug(
'Invoke function %s, url: %s, data: %s', 'Invoke function %s(version %s), url: %s, data: %s',
function_id, func_url, data function_id, version, func_url, data
) )
return utils.url_request(self.session, func_url, body=data) return utils.url_request(self.session, func_url, body=data)
else: else:
# Wait for image type function execution to be finished
def _wait_complete(): def _wait_complete():
pod = self.v1.read_namespaced_pod( pod = self.v1.read_namespaced_pod(
identifier, identifier,
@ -437,8 +456,17 @@ class KubernetesManager(base.OrchestratorBase):
) )
return True, output return True, output
def delete_function(self, function_id, labels=None): def delete_function(self, function_id, version, labels=None):
labels = labels or {'function_id': function_id} """Delete related resources for function.
- Delete service
- Delete pods
"""
pre_label = {
'function_id': function_id,
'function_version': str(version)
}
labels = labels or pre_label
selector = common.convert_dict_to_string(labels) selector = common.convert_dict_to_string(labels)
ret = self.v1.list_namespaced_service( ret = self.v1.list_namespaced_service(
@ -456,7 +484,7 @@ class KubernetesManager(base.OrchestratorBase):
label_selector=selector label_selector=selector
) )
def scaleup_function(self, function_id, identifier=None, count=1): def scaleup_function(self, function_id, version, identifier=None, count=1):
pod_names = [] pod_names = []
labels = {'runtime_id': identifier} labels = {'runtime_id': identifier}
pods = self._choose_available_pods(labels, count=count) pods = self._choose_available_pods(labels, count=count)
@ -466,11 +494,13 @@ class KubernetesManager(base.OrchestratorBase):
for pod in pods: for pod in pods:
pod_name, service_url = self._prepare_pod( pod_name, service_url = self._prepare_pod(
pod, identifier, function_id, labels pod, identifier, function_id, version, labels
) )
pod_names.append(pod_name) pod_names.append(pod_name)
LOG.info('Pods scaled up for function %s: %s', function_id, pod_names) LOG.info('Pods scaled up for function %s(version %s): %s', function_id,
version, pod_names)
return pod_names, service_url return pod_names, service_url
def delete_worker(self, pod_name, **kwargs): def delete_worker(self, pod_name, **kwargs):

2
qinling/orchestrator/kubernetes/templates/service.j2
View File

@ -4,7 +4,7 @@ metadata:
name: {{ service_name }} name: {{ service_name }}
labels: labels:
{% for key, value in labels.items() %} {% for key, value in labels.items() %}
{{ key }}: {{ value }} {{ key }}: "{{ value }}"
{% endfor %} {% endfor %}
spec: spec:
type: NodePort type: NodePort

15
qinling/rpc.py
View File

@ -153,7 +153,7 @@ class EngineClient(object):
) )
@wrap_messaging_exception @wrap_messaging_exception
def create_execution(self, execution_id, function_id, runtime_id, def create_execution(self, execution_id, function_id, version, runtime_id,
input=None, is_sync=True): input=None, is_sync=True):
method_client = self._client.prepare(topic=self.topic, server=None) method_client = self._client.prepare(topic=self.topic, server=None)
@ -163,6 +163,7 @@ class EngineClient(object):
'create_execution', 'create_execution',
execution_id=execution_id, execution_id=execution_id,
function_id=function_id, function_id=function_id,
function_version=version,
runtime_id=runtime_id, runtime_id=runtime_id,
input=input input=input
) )
@ -172,33 +173,37 @@ class EngineClient(object):
'create_execution', 'create_execution',
execution_id=execution_id, execution_id=execution_id,
function_id=function_id, function_id=function_id,
function_version=version,
runtime_id=runtime_id, runtime_id=runtime_id,
input=input input=input
) )
@wrap_messaging_exception @wrap_messaging_exception
def delete_function(self, id): def delete_function(self, id, version=0):
return self._client.prepare(topic=self.topic, server=None).cast( return self._client.prepare(topic=self.topic, server=None).cast(
ctx.get_ctx(), ctx.get_ctx(),
'delete_function', 'delete_function',
function_id=id function_id=id,
function_version=version
) )
@wrap_messaging_exception @wrap_messaging_exception
def scaleup_function(self, id, runtime_id, count=1): def scaleup_function(self, id, runtime_id, version=0, count=1):
return self._client.prepare(topic=self.topic, server=None).cast( return self._client.prepare(topic=self.topic, server=None).cast(
ctx.get_ctx(), ctx.get_ctx(),
'scaleup_function', 'scaleup_function',
function_id=id, function_id=id,
runtime_id=runtime_id, runtime_id=runtime_id,
function_version=version,
count=count count=count
) )
@wrap_messaging_exception @wrap_messaging_exception
def scaledown_function(self, id, count=1): def scaledown_function(self, id, version=0, count=1):
return self._client.prepare(topic=self.topic, server=None).cast( return self._client.prepare(topic=self.topic, server=None).cast(
ctx.get_ctx(), ctx.get_ctx(),
'scaledown_function', 'scaledown_function',
function_id=id, function_id=id,
function_version=version,
count=count count=count
) )

33
qinling/services/periodics.py
View File

@ -51,24 +51,43 @@ def handle_function_service_expiration(ctx, engine):
results = db_api.get_functions( results = db_api.get_functions(
sort_keys=['updated_at'], sort_keys=['updated_at'],
insecure=True, insecure=True,
updated_at={'lte': expiry_time} updated_at={'lte': expiry_time},
latest_version=0
) )
if len(results) == 0:
return
for func_db in results: for func_db in results:
if not etcd_util.get_service_url(func_db.id): if not etcd_util.get_service_url(func_db.id, 0):
continue continue
LOG.info( LOG.info(
'Deleting service mapping and workers for function %s', 'Deleting service mapping and workers for function %s(version 0)',
func_db.id func_db.id
) )
# Delete resources related to the function # Delete resources related to the function
engine.delete_function(ctx, func_db.id) engine.delete_function(ctx, func_db.id, 0)
# Delete etcd keys # Delete etcd keys
etcd_util.delete_function(func_db.id) etcd_util.delete_function(func_db.id, 0)
versions = db_api.get_function_versions(
sort_keys=['updated_at'],
insecure=True,
updated_at={'lte': expiry_time},
)
for v in versions:
if not etcd_util.get_service_url(v.function_id, v.version_number):
continue
LOG.info(
'Deleting service mapping and workers for function %s(version %s)',
v.function_id, v.version_number
)
# Delete resources related to the function
engine.delete_function(ctx, v.function_id, v.version_number)
# Delete etcd keys
etcd_util.delete_function(v.function_id, v.version_number)
@periodics.periodic(3) @periodics.periodic(3)

21
qinling/tests/unit/api/controllers/v1/test_execution.py
View File

@ -14,6 +14,7 @@
import mock import mock
from qinling.db import api as db_api
from qinling import exceptions as exc from qinling import exceptions as exc
from qinling import status from qinling import status
from qinling.tests.unit.api import base from qinling.tests.unit.api import base
@ -40,7 +41,25 @@ class TestExecutionController(base.APITest):
self.assertEqual(1, resp.json.get('count')) self.assertEqual(1, resp.json.get('count'))
@mock.patch('qinling.rpc.EngineClient.create_execution') @mock.patch('qinling.rpc.EngineClient.create_execution')
def test_create_rpc_error(self, mock_create_execution): def test_post_with_version(self, mock_rpc):
db_api.increase_function_version(self.func_id, 0,
description="version 1")
body = {
'function_id': self.func_id,
'function_version': 1
}
resp = self.app.post_json('/v1/executions', body)
self.assertEqual(201, resp.status_int)
resp = self.app.get('/v1/functions/%s' % self.func_id)
self.assertEqual(0, resp.json.get('count'))
resp = self.app.get('/v1/functions/%s/versions/1' % self.func_id)
self.assertEqual(1, resp.json.get('count'))
@mock.patch('qinling.rpc.EngineClient.create_execution')
def test_post_rpc_error(self, mock_create_execution):
mock_create_execution.side_effect = exc.QinlingException mock_create_execution.side_effect = exc.QinlingException
body = { body = {
'function_id': self.func_id, 'function_id': self.func_id,

252
qinling/tests/unit/engine/test_default_engine.py
View File

@ -29,10 +29,11 @@ class TestDefaultEngine(base.DbTestCase):
self.orchestrator = mock.Mock() self.orchestrator = mock.Mock()
self.qinling_endpoint = 'http://127.0.0.1:7070' self.qinling_endpoint = 'http://127.0.0.1:7070'
self.default_engine = default_engine.DefaultEngine( self.default_engine = default_engine.DefaultEngine(
self.orchestrator, self.qinling_endpoint) self.orchestrator, self.qinling_endpoint
)
def _create_running_executions(self, function_id, num): def _create_running_executions(self, function_id, num):
for _i in range(num): for _ in range(num):
self.create_execution(function_id=function_id) self.create_execution(function_id=function_id)
def test_create_runtime(self): def test_create_runtime(self):
@ -110,113 +111,91 @@ class TestDefaultEngine(base.DbTestCase):
self.assertEqual(runtime.image, pre_image) self.assertEqual(runtime.image, pre_image)
self.assertEqual(runtime.status, status.AVAILABLE) self.assertEqual(runtime.status, status.AVAILABLE)
@mock.patch('qinling.engine.default_engine.DefaultEngine.scaleup_function')
@mock.patch('qinling.utils.etcd_util.get_workers') @mock.patch('qinling.utils.etcd_util.get_workers')
@mock.patch('qinling.utils.etcd_util.get_worker_lock') @mock.patch('qinling.utils.etcd_util.get_worker_lock')
def test_function_load_check_no_worker_scaleup( def test_function_load_check_no_worker(self, mock_getlock, mock_getworkers,
self, mock_scaleup):
etcd_util_get_worker_lock_mock,
etcd_util_get_workers_mock
):
function_id = common.generate_unicode_uuid() function_id = common.generate_unicode_uuid()
runtime_id = common.generate_unicode_uuid() runtime_id = common.generate_unicode_uuid()
lock = mock.Mock() lock = mock.Mock()
(
etcd_util_get_worker_lock_mock.return_value.__enter__.return_value
) = lock
lock.is_acquired.return_value = True lock.is_acquired.return_value = True
etcd_util_get_workers_mock.return_value = [] # len(workers) = 0 mock_getlock.return_value.__enter__.return_value = lock
self.default_engine.scaleup_function = mock.Mock() mock_getworkers.return_value = []
self.default_engine.function_load_check(function_id, runtime_id) self.default_engine.function_load_check(function_id, 0, runtime_id)
etcd_util_get_workers_mock.assert_called_once_with(function_id) mock_getworkers.assert_called_once_with(function_id, 0)
self.default_engine.scaleup_function.assert_called_once_with( mock_scaleup.assert_called_once_with(None, function_id, 0, runtime_id,
None, function_id, runtime_id, 1) 1)
@mock.patch('qinling.engine.default_engine.DefaultEngine.scaleup_function')
@mock.patch('qinling.utils.etcd_util.get_workers') @mock.patch('qinling.utils.etcd_util.get_workers')
@mock.patch('qinling.utils.etcd_util.get_worker_lock') @mock.patch('qinling.utils.etcd_util.get_worker_lock')
def test_function_load_check_concurrency_scaleup( def test_function_load_check_scaleup(self, mock_getlock, mock_getworkers,
self, mock_scaleup):
etcd_util_get_worker_lock_mock,
etcd_util_get_workers_mock
):
function = self.create_function() function = self.create_function()
function_id = function.id function_id = function.id
runtime_id = function.runtime_id runtime_id = function.runtime_id
lock = mock.Mock() lock = mock.Mock()
(
etcd_util_get_worker_lock_mock.return_value.__enter__.return_value
) = lock
lock.is_acquired.return_value = True lock.is_acquired.return_value = True
mock_getlock.return_value.__enter__.return_value = lock
# The default concurrency is 3, we use 4 running executions against # The default concurrency is 3, we use 4 running executions against
# 1 worker so that there will be a scaling up. # 1 worker so that there will be a scaling up.
etcd_util_get_workers_mock.return_value = range(1) mock_getworkers.return_value = ['worker1']
self._create_running_executions(function_id, 4) self._create_running_executions(function_id, 4)
self.default_engine.scaleup_function = mock.Mock()
self.default_engine.function_load_check(function_id, runtime_id) self.default_engine.function_load_check(function_id, 0, runtime_id)
etcd_util_get_workers_mock.assert_called_once_with(function_id) mock_getworkers.assert_called_once_with(function_id, 0)
self.default_engine.scaleup_function.assert_called_once_with( mock_scaleup.assert_called_once_with(None, function_id, 0, runtime_id,
None, function_id, runtime_id, 1) 1)
@mock.patch('qinling.engine.default_engine.DefaultEngine.scaleup_function')
@mock.patch('qinling.utils.etcd_util.get_workers') @mock.patch('qinling.utils.etcd_util.get_workers')
@mock.patch('qinling.utils.etcd_util.get_worker_lock') @mock.patch('qinling.utils.etcd_util.get_worker_lock')
def test_function_load_check_not_scaleup( def test_function_load_check_not_scaleup(self, mock_getlock,
self, mock_getworkers, mock_scaleup):
etcd_util_get_worker_lock_mock,
etcd_util_get_workers_mock
):
function = self.create_function() function = self.create_function()
function_id = function.id function_id = function.id
runtime_id = function.runtime_id runtime_id = function.runtime_id
lock = mock.Mock() lock = mock.Mock()
(
etcd_util_get_worker_lock_mock.return_value.__enter__.return_value
) = lock
lock.is_acquired.return_value = True lock.is_acquired.return_value = True
mock_getlock.return_value.__enter__.return_value = lock
# The default concurrency is 3, we use 3 running executions against # The default concurrency is 3, we use 3 running executions against
# 1 worker so that there won't be a scaling up. # 1 worker so that there won't be a scaling up.
etcd_util_get_workers_mock.return_value = range(1) mock_getworkers.return_value = ['worker1']
self._create_running_executions(function_id, 3) self._create_running_executions(function_id, 3)
self.default_engine.scaleup_function = mock.Mock()
self.default_engine.function_load_check(function_id, runtime_id) self.default_engine.function_load_check(function_id, 0, runtime_id)
etcd_util_get_workers_mock.assert_called_once_with(function_id) mock_getworkers.assert_called_once_with(function_id, 0)
self.default_engine.scaleup_function.assert_not_called() mock_scaleup.assert_not_called()
@mock.patch('qinling.utils.etcd_util.get_workers') @mock.patch('qinling.utils.etcd_util.get_workers')
@mock.patch('qinling.utils.etcd_util.get_worker_lock') @mock.patch('qinling.utils.etcd_util.get_worker_lock')
def test_function_load_check_lock_wait( def test_function_load_check_lock_wait(self, mock_getlock,
self, mock_getworkers):
etcd_util_get_worker_lock_mock,
etcd_util_get_workers_mock
):
function = self.create_function() function = self.create_function()
function_id = function.id function_id = function.id
runtime_id = function.runtime_id runtime_id = function.runtime_id
lock = mock.Mock() lock = mock.Mock()
( mock_getlock.return_value.__enter__.return_value = lock
etcd_util_get_worker_lock_mock.return_value.__enter__.return_value
) = lock
# Lock is acquired upon the third try. # Lock is acquired upon the third try.
lock.is_acquired.side_effect = [False, False, True] lock.is_acquired.side_effect = [False, False, True]
etcd_util_get_workers_mock.return_value = range(1) mock_getworkers.return_value = ['worker1']
self._create_running_executions(function_id, 3) self._create_running_executions(function_id, 3)
self.default_engine.scaleup_function = mock.Mock()
self.default_engine.function_load_check(function_id, runtime_id) self.default_engine.function_load_check(function_id, 0, runtime_id)
self.assertEqual(3, lock.is_acquired.call_count) self.assertEqual(3, lock.is_acquired.call_count)
etcd_util_get_workers_mock.assert_called_once_with(function_id) mock_getworkers.assert_called_once_with(function_id, 0)
self.default_engine.scaleup_function.assert_not_called()
@mock.patch('qinling.utils.etcd_util.get_service_url') @mock.patch('qinling.utils.etcd_util.get_service_url')
def test_create_execution( def test_create_execution_image_type_function(self, mock_svc_url):
self, """Create 2 executions for an image type function."""
etcd_util_get_service_url_mock
):
function = self.create_function() function = self.create_function()
function_id = function.id function_id = function.id
runtime_id = function.runtime_id runtime_id = function.runtime_id
@ -234,42 +213,47 @@ class TestDefaultEngine(base.DbTestCase):
execution_1_id = execution_1.id execution_1_id = execution_1.id
execution_2 = self.create_execution(function_id=function_id) execution_2 = self.create_execution(function_id=function_id)
execution_2_id = execution_2.id execution_2_id = execution_2.id
self.default_engine.function_load_check = mock.Mock() mock_svc_url.return_value = None
etcd_util_get_service_url_mock.return_value = None
self.orchestrator.prepare_execution.return_value = ( self.orchestrator.prepare_execution.return_value = (
mock.Mock(), None) mock.Mock(), None)
self.orchestrator.run_execution.side_effect = [ self.orchestrator.run_execution.side_effect = [
(True, 'success result'), (True, 'success result'),
(False, 'failed result')] (False, 'failed result')]
# Try create two executions, with different results # Create two executions, with different results
self.default_engine.create_execution( self.default_engine.create_execution(
mock.Mock(), execution_1_id, function_id, runtime_id) mock.Mock(), execution_1_id, function_id, 0, runtime_id
)
self.default_engine.create_execution( self.default_engine.create_execution(
mock.Mock(), execution_2_id, function_id, runtime_id, mock.Mock(), execution_2_id, function_id, 0, runtime_id,
input='input') input='input'
)
self.default_engine.function_load_check.assert_not_called()
get_service_url_calls = [ get_service_url_calls = [
mock.call(function_id), mock.call(function_id)] mock.call(function_id, 0), mock.call(function_id, 0)
etcd_util_get_service_url_mock.assert_has_calls(get_service_url_calls) ]
self.assertEqual(2, etcd_util_get_service_url_mock.call_count) mock_svc_url.assert_has_calls(get_service_url_calls)
prepare_calls = [ prepare_calls = [
mock.call(function_id, mock.call(function_id,
0,
image=function.code['image'], image=function.code['image'],
identifier=mock.ANY, identifier=mock.ANY,
labels=None, labels=None,
input=None), input=None),
mock.call(function_id, mock.call(function_id,
0,
image=function.code['image'], image=function.code['image'],
identifier=mock.ANY, identifier=mock.ANY,
labels=None, labels=None,
input='input')] input='input')
]
self.orchestrator.prepare_execution.assert_has_calls(prepare_calls) self.orchestrator.prepare_execution.assert_has_calls(prepare_calls)
self.assertEqual(2, self.orchestrator.prepare_execution.call_count)
run_calls = [ run_calls = [
mock.call(execution_1_id, mock.call(execution_1_id,
function_id, function_id,
0,
input=None, input=None,
identifier=mock.ANY, identifier=mock.ANY,
service_url=None, service_url=None,
@ -277,15 +261,18 @@ class TestDefaultEngine(base.DbTestCase):
trust_id=function.trust_id), trust_id=function.trust_id),
mock.call(execution_2_id, mock.call(execution_2_id,
function_id, function_id,
0,
input='input', input='input',
identifier=mock.ANY, identifier=mock.ANY,
service_url=None, service_url=None,
entry=function.entry, entry=function.entry,
trust_id=function.trust_id)] trust_id=function.trust_id)
]
self.orchestrator.run_execution.assert_has_calls(run_calls) self.orchestrator.run_execution.assert_has_calls(run_calls)
self.assertEqual(2, self.orchestrator.run_execution.call_count)
execution_1 = db_api.get_execution(execution_1_id) execution_1 = db_api.get_execution(execution_1_id)
execution_2 = db_api.get_execution(execution_2_id) execution_2 = db_api.get_execution(execution_2_id)
self.assertEqual(execution_1.status, status.SUCCESS) self.assertEqual(execution_1.status, status.SUCCESS)
self.assertEqual(execution_1.logs, '') self.assertEqual(execution_1.logs, '')
self.assertEqual(execution_1.result, {'output': 'success result'}) self.assertEqual(execution_1.result, {'output': 'success result'})
@ -298,6 +285,11 @@ class TestDefaultEngine(base.DbTestCase):
self, self,
etcd_util_get_service_url_mock etcd_util_get_service_url_mock
): ):
"""test_create_execution_prepare_execution_exception
Create execution for image type function, prepare_execution method
raises exception.
"""
function = self.create_function() function = self.create_function()
function_id = function.id function_id = function.id
runtime_id = function.runtime_id runtime_id = function.runtime_id
@ -320,7 +312,7 @@ class TestDefaultEngine(base.DbTestCase):
etcd_util_get_service_url_mock.return_value = None etcd_util_get_service_url_mock.return_value = None
self.default_engine.create_execution( self.default_engine.create_execution(
mock.Mock(), execution_id, function_id, runtime_id) mock.Mock(), execution_id, function_id, 0, runtime_id)
execution = db_api.get_execution(execution_id) execution = db_api.get_execution(execution_id)
self.assertEqual(execution.status, status.ERROR) self.assertEqual(execution.status, status.ERROR)
@ -328,9 +320,9 @@ class TestDefaultEngine(base.DbTestCase):
self.assertEqual(execution.result, {}) self.assertEqual(execution.result, {})
@mock.patch('qinling.utils.etcd_util.get_service_url') @mock.patch('qinling.utils.etcd_util.get_service_url')
def test_create_execution_not_image_source( def test_create_execution_package_type_function(
self, self,
etcd_util_get_service_url_mock etcd_util_get_service_url_mock
): ):
function = self.create_function() function = self.create_function()
function_id = function.id function_id = function.id
@ -347,24 +339,26 @@ class TestDefaultEngine(base.DbTestCase):
'output': 'success output'}) 'output': 'success output'})
self.default_engine.create_execution( self.default_engine.create_execution(
mock.Mock(), execution_id, function_id, runtime_id) mock.Mock(), execution_id, function_id, 0, runtime_id)
self.default_engine.function_load_check.assert_called_once_with( self.default_engine.function_load_check.assert_called_once_with(
function_id, runtime_id) function_id, 0, runtime_id)
etcd_util_get_service_url_mock.assert_called_once_with(function_id) etcd_util_get_service_url_mock.assert_called_once_with(function_id, 0)
self.orchestrator.prepare_execution.assert_called_once_with( self.orchestrator.prepare_execution.assert_called_once_with(
function_id, image=None, identifier=runtime_id, function_id, 0, image=None, identifier=runtime_id,
labels={'runtime_id': runtime_id}, input=None) labels={'runtime_id': runtime_id}, input=None)
self.orchestrator.run_execution.assert_called_once_with( self.orchestrator.run_execution.assert_called_once_with(
execution_id, function_id, input=None, identifier=runtime_id, execution_id, function_id, 0, input=None, identifier=runtime_id,
service_url='svc_url', entry=function.entry, service_url='svc_url', entry=function.entry,
trust_id=function.trust_id) trust_id=function.trust_id)
execution = db_api.get_execution(execution_id) execution = db_api.get_execution(execution_id)
self.assertEqual(execution.status, status.SUCCESS) self.assertEqual(execution.status, status.SUCCESS)
self.assertEqual(execution.logs, 'execution log') self.assertEqual(execution.logs, 'execution log')
self.assertEqual(execution.result, {'output': 'success output'}) self.assertEqual(execution.result, {'output': 'success output'})
def test_create_execution_not_image_source_scaleup_exception(self): def test_create_execution_loadcheck_exception(self):
function = self.create_function() function = self.create_function()
function_id = function.id function_id = function.id
runtime_id = function.runtime_id runtime_id = function.runtime_id
@ -377,9 +371,10 @@ class TestDefaultEngine(base.DbTestCase):
) )
self.default_engine.create_execution( self.default_engine.create_execution(
mock.Mock(), execution_id, function_id, runtime_id) mock.Mock(), execution_id, function_id, 0, runtime_id)
execution = db_api.get_execution(execution_id) execution = db_api.get_execution(execution_id)
self.assertEqual(execution.status, status.ERROR) self.assertEqual(execution.status, status.ERROR)
self.assertEqual(execution.logs, '') self.assertEqual(execution.logs, '')
self.assertEqual(execution.result, {}) self.assertEqual(execution.result, {})
@ -388,10 +383,10 @@ class TestDefaultEngine(base.DbTestCase):
@mock.patch('qinling.engine.utils.url_request') @mock.patch('qinling.engine.utils.url_request')
@mock.patch('qinling.utils.etcd_util.get_service_url') @mock.patch('qinling.utils.etcd_util.get_service_url')
def test_create_execution_found_service_url( def test_create_execution_found_service_url(
self, self,
etcd_util_get_service_url_mock, etcd_util_get_service_url_mock,
engine_utils_url_request_mock, engine_utils_url_request_mock,
engine_utils_get_request_data_mock engine_utils_get_request_data_mock
): ):
function = self.create_function() function = self.create_function()
function_id = function.id function_id = function.id
@ -407,18 +402,21 @@ class TestDefaultEngine(base.DbTestCase):
'output': 'failed output'}) 'output': 'failed output'})
self.default_engine.create_execution( self.default_engine.create_execution(
mock.Mock(), execution_id, function_id, runtime_id, input='input') mock.Mock(), execution_id, function_id, 0, runtime_id,
input='input')
self.default_engine.function_load_check.assert_called_once_with( self.default_engine.function_load_check.assert_called_once_with(
function_id, runtime_id) function_id, 0, runtime_id)
etcd_util_get_service_url_mock.assert_called_once_with(function_id) etcd_util_get_service_url_mock.assert_called_once_with(function_id, 0)
engine_utils_get_request_data_mock.assert_called_once_with( engine_utils_get_request_data_mock.assert_called_once_with(
mock.ANY, function_id, execution_id, mock.ANY, function_id, 0, execution_id,
'input', function.entry, function.trust_id, 'input', function.entry, function.trust_id,
self.qinling_endpoint) self.qinling_endpoint)
engine_utils_url_request_mock.assert_called_once_with( engine_utils_url_request_mock.assert_called_once_with(
self.default_engine.session, 'svc_url/execute', body='data') self.default_engine.session, 'svc_url/execute', body='data')
execution = db_api.get_execution(execution_id) execution = db_api.get_execution(execution_id)
self.assertEqual(execution.status, status.FAILED) self.assertEqual(execution.status, status.FAILED)
self.assertEqual(execution.logs, 'execution log') self.assertEqual(execution.logs, 'execution log')
self.assertEqual(execution.result, self.assertEqual(execution.result,
@ -430,35 +428,36 @@ class TestDefaultEngine(base.DbTestCase):
self.default_engine.delete_function(mock.Mock(), function_id) self.default_engine.delete_function(mock.Mock(), function_id)
self.orchestrator.delete_function.assert_called_once_with( self.orchestrator.delete_function.assert_called_once_with(
function_id) function_id, 0
)
@mock.patch('qinling.utils.etcd_util.create_service_url') @mock.patch('qinling.utils.etcd_util.create_service_url')
@mock.patch('qinling.utils.etcd_util.create_worker') @mock.patch('qinling.utils.etcd_util.create_worker')
def test_scaleup_function( def test_scaleup_function(
self, self,
etcd_util_create_worker_mock, etcd_util_create_worker_mock,
etcd_util_create_service_url_mock etcd_util_create_service_url_mock
): ):
function_id = common.generate_unicode_uuid() function_id = common.generate_unicode_uuid()
runtime_id = common.generate_unicode_uuid() runtime_id = common.generate_unicode_uuid()
self.orchestrator.scaleup_function.return_value = (['worker'], 'url') self.orchestrator.scaleup_function.return_value = (['worker'], 'url')
self.default_engine.scaleup_function( self.default_engine.scaleup_function(
mock.Mock(), function_id, runtime_id) mock.Mock(), function_id, 0, runtime_id)
self.orchestrator.scaleup_function.assert_called_once_with( self.orchestrator.scaleup_function.assert_called_once_with(
function_id, identifier=runtime_id, count=1) function_id, 0, identifier=runtime_id, count=1)
etcd_util_create_worker_mock.assert_called_once_with( etcd_util_create_worker_mock.assert_called_once_with(
function_id, 'worker') function_id, 'worker', version=0)
etcd_util_create_service_url_mock.assert_called_once_with( etcd_util_create_service_url_mock.assert_called_once_with(
function_id, 'url') function_id, 'url', version=0)
@mock.patch('qinling.utils.etcd_util.create_service_url') @mock.patch('qinling.utils.etcd_util.create_service_url')
@mock.patch('qinling.utils.etcd_util.create_worker') @mock.patch('qinling.utils.etcd_util.create_worker')
def test_scaleup_function_multiple_workers( def test_scaleup_function_multiple_workers(
self, self,
etcd_util_create_worker_mock, etcd_util_create_worker_mock,
etcd_util_create_service_url_mock etcd_util_create_service_url_mock
): ):
function_id = common.generate_unicode_uuid() function_id = common.generate_unicode_uuid()
runtime_id = common.generate_unicode_uuid() runtime_id = common.generate_unicode_uuid()
@ -466,22 +465,24 @@ class TestDefaultEngine(base.DbTestCase):
['worker0', 'worker1'], 'url') ['worker0', 'worker1'], 'url')
self.default_engine.scaleup_function( self.default_engine.scaleup_function(
mock.Mock(), function_id, runtime_id, count=2) mock.Mock(), function_id, 0, runtime_id, count=2
)
self.orchestrator.scaleup_function.assert_called_once_with( self.orchestrator.scaleup_function.assert_called_once_with(
function_id, identifier=runtime_id, count=2) function_id, 0, identifier=runtime_id, count=2
)
# Two new workers are created. # Two new workers are created.
expected = [mock.call(function_id, 'worker0'), expected = [mock.call(function_id, 'worker0', version=0),
mock.call(function_id, 'worker1')] mock.call(function_id, 'worker1', version=0)]
etcd_util_create_worker_mock.assert_has_calls(expected) etcd_util_create_worker_mock.assert_has_calls(expected)
self.assertEqual(2, etcd_util_create_worker_mock.call_count)
etcd_util_create_service_url_mock.assert_called_once_with( etcd_util_create_service_url_mock.assert_called_once_with(
function_id, 'url') function_id, 'url', version=0
)
@mock.patch('qinling.utils.etcd_util.delete_worker') @mock.patch('qinling.utils.etcd_util.delete_worker')
@mock.patch('qinling.utils.etcd_util.get_workers') @mock.patch('qinling.utils.etcd_util.get_workers')
def test_scaledown_function( def test_scaledown_function(
self, etcd_util_get_workers_mock, etcd_util_delete_workers_mock self, etcd_util_get_workers_mock, etcd_util_delete_workers_mock
): ):
function_id = common.generate_unicode_uuid() function_id = common.generate_unicode_uuid()
etcd_util_get_workers_mock.return_value = [ etcd_util_get_workers_mock.return_value = [
@ -491,33 +492,33 @@ class TestDefaultEngine(base.DbTestCase):
self.default_engine.scaledown_function(mock.Mock(), function_id) self.default_engine.scaledown_function(mock.Mock(), function_id)
etcd_util_get_workers_mock.assert_called_once_with( etcd_util_get_workers_mock.assert_called_once_with(
function_id) function_id, 0)
self.orchestrator.delete_worker.assert_called_once_with('worker_0') self.orchestrator.delete_worker.assert_called_once_with('worker_0')
etcd_util_delete_workers_mock.assert_called_once_with( etcd_util_delete_workers_mock.assert_called_once_with(
function_id, 'worker_0') function_id, 'worker_0', version=0
)
@mock.patch('qinling.utils.etcd_util.delete_worker') @mock.patch('qinling.utils.etcd_util.delete_worker')
@mock.patch('qinling.utils.etcd_util.get_workers') @mock.patch('qinling.utils.etcd_util.get_workers')
def test_scaledown_function_multiple_workers( def test_scaledown_function_multiple_workers(
self, etcd_util_get_workers_mock, etcd_util_delete_workers_mock self, etcd_util_get_workers_mock, etcd_util_delete_workers_mock
): ):
function_id = common.generate_unicode_uuid() function_id = common.generate_unicode_uuid()
etcd_util_get_workers_mock.return_value = [ etcd_util_get_workers_mock.return_value = [
'worker_%d' % i for i in range(4) 'worker_%d' % i for i in range(4)
] ]
self.default_engine.scaledown_function( self.default_engine.scaledown_function(mock.Mock(), function_id,
mock.Mock(), function_id, count=2) count=2)
etcd_util_get_workers_mock.assert_called_once_with( etcd_util_get_workers_mock.assert_called_once_with(function_id, 0)
function_id)
# First two workers will be deleted. # First two workers will be deleted.
expected = [mock.call('worker_0'), mock.call('worker_1')] expected = [mock.call('worker_0'), mock.call('worker_1')]
self.orchestrator.delete_worker.assert_has_calls(expected) self.orchestrator.delete_worker.assert_has_calls(expected)
self.assertEqual(2, self.orchestrator.delete_worker.call_count) self.assertEqual(2, self.orchestrator.delete_worker.call_count)
expected = [ expected = [
mock.call(function_id, 'worker_0'), mock.call(function_id, 'worker_0', version=0),
mock.call(function_id, 'worker_1') mock.call(function_id, 'worker_1', version=0)
] ]
etcd_util_delete_workers_mock.assert_has_calls(expected) etcd_util_delete_workers_mock.assert_has_calls(expected)
self.assertEqual(2, etcd_util_delete_workers_mock.call_count) self.assertEqual(2, etcd_util_delete_workers_mock.call_count)
@ -525,7 +526,7 @@ class TestDefaultEngine(base.DbTestCase):
@mock.patch('qinling.utils.etcd_util.delete_worker') @mock.patch('qinling.utils.etcd_util.delete_worker')
@mock.patch('qinling.utils.etcd_util.get_workers') @mock.patch('qinling.utils.etcd_util.get_workers')
def test_scaledown_function_leaving_one_worker( def test_scaledown_function_leaving_one_worker(
self, etcd_util_get_workers_mock, etcd_util_delete_workers_mock self, etcd_util_get_workers_mock, etcd_util_delete_workers_mock
): ):
function_id = common.generate_unicode_uuid() function_id = common.generate_unicode_uuid()
etcd_util_get_workers_mock.return_value = [ etcd_util_get_workers_mock.return_value = [
@ -535,8 +536,7 @@ class TestDefaultEngine(base.DbTestCase):
self.default_engine.scaledown_function( self.default_engine.scaledown_function(
mock.Mock(), function_id, count=5) # count > len(workers) mock.Mock(), function_id, count=5) # count > len(workers)
etcd_util_get_workers_mock.assert_called_once_with( etcd_util_get_workers_mock.assert_called_once_with(function_id, 0)
function_id)
# Only the first three workers will be deleted # Only the first three workers will be deleted
expected = [ expected = [
mock.call('worker_0'), mock.call('worker_1'), mock.call('worker_2') mock.call('worker_0'), mock.call('worker_1'), mock.call('worker_2')
@ -544,9 +544,9 @@ class TestDefaultEngine(base.DbTestCase):
self.orchestrator.delete_worker.assert_has_calls(expected) self.orchestrator.delete_worker.assert_has_calls(expected)
self.assertEqual(3, self.orchestrator.delete_worker.call_count) self.assertEqual(3, self.orchestrator.delete_worker.call_count)
expected = [ expected = [
mock.call(function_id, 'worker_0'), mock.call(function_id, 'worker_0', version=0),
mock.call(function_id, 'worker_1'), mock.call(function_id, 'worker_1', version=0),
mock.call(function_id, 'worker_2') mock.call(function_id, 'worker_2', version=0)
] ]
etcd_util_delete_workers_mock.assert_has_calls(expected) etcd_util_delete_workers_mock.assert_has_calls(expected)
self.assertEqual(3, etcd_util_delete_workers_mock.call_count) self.assertEqual(3, etcd_util_delete_workers_mock.call_count)

180
qinling/tests/unit/orchestrator/kubernetes/test_manager.py
View File

@ -30,10 +30,10 @@ SERVICE_ADDRESS_EXTERNAL = '1.2.3.4'
SERVICE_ADDRESS_INTERNAL = '127.0.0.1' SERVICE_ADDRESS_INTERNAL = '127.0.0.1'
class TestKubernetesManager(base.BaseTest): class TestKubernetesManager(base.DbTestCase):
def setUp(self): def setUp(self):
super(TestKubernetesManager, self).setUp() super(TestKubernetesManager, self).setUp()
CONF.register_opts(config.kubernetes_opts, config.KUBERNETES_GROUP)
self.conf = CONF self.conf = CONF
self.qinling_endpoint = 'http://127.0.0.1:7070' self.qinling_endpoint = 'http://127.0.0.1:7070'
self.k8s_v1_api = mock.Mock() self.k8s_v1_api = mock.Mock()
@ -48,13 +48,17 @@ class TestKubernetesManager(base.BaseTest):
prefix='TestKubernetesManager') prefix='TestKubernetesManager')
self.override_config('namespace', self.fake_namespace, self.override_config('namespace', self.fake_namespace,
config.KUBERNETES_GROUP) config.KUBERNETES_GROUP)
self.override_config('auth_enable', False, group='pecan')
namespace = mock.Mock() namespace = mock.Mock()
namespace.metadata.name = self.fake_namespace namespace.metadata.name = self.fake_namespace
namespaces = mock.Mock() namespaces = mock.Mock()
namespaces.items = [namespace] namespaces.items = [namespace]
self.k8s_v1_api.list_namespace.return_value = namespaces self.k8s_v1_api.list_namespace.return_value = namespaces
self.manager = k8s_manager.KubernetesManager(
self.conf, self.qinling_endpoint) self.manager = k8s_manager.KubernetesManager(self.conf,
self.qinling_endpoint)
def _create_service(self): def _create_service(self):
port = mock.Mock() port = mock.Mock()
@ -305,7 +309,7 @@ class TestKubernetesManager(base.BaseTest):
rollback.assert_called_once_with( rollback.assert_called_once_with(
fake_deployment_name, self.fake_namespace, rollback_body) fake_deployment_name, self.fake_namespace, rollback_body)
def test_prepare_execution(self): def test_prepare_execution_no_image(self):
pod = mock.Mock() pod = mock.Mock()
pod.metadata.name = self.rand_name('pod', pod.metadata.name = self.rand_name('pod',
prefix='TestKubernetesManager') prefix='TestKubernetesManager')
@ -323,7 +327,7 @@ class TestKubernetesManager(base.BaseTest):
function_id = common.generate_unicode_uuid() function_id = common.generate_unicode_uuid()
pod_names, service_url = self.manager.prepare_execution( pod_names, service_url = self.manager.prepare_execution(
function_id, image=None, identifier=runtime_id, function_id, 0, image=None, identifier=runtime_id,
labels={'runtime_id': runtime_id}) labels={'runtime_id': runtime_id})
self.assertEqual(pod.metadata.name, pod_names) self.assertEqual(pod.metadata.name, pod_names)
@ -334,10 +338,15 @@ class TestKubernetesManager(base.BaseTest):
# in _choose_available_pods # in _choose_available_pods
self.k8s_v1_api.list_namespaced_pod.assert_called_once_with( self.k8s_v1_api.list_namespaced_pod.assert_called_once_with(
self.fake_namespace, self.fake_namespace,
label_selector='function_id=%s' % function_id) label_selector='function_id=%s,function_version=0' % (function_id)
)
# in _prepare_pod -> _update_pod_label # in _prepare_pod -> _update_pod_label
pod_labels = {'pod1_key1': 'pod1_value1', 'function_id': function_id} pod_labels = {
'pod1_key1': 'pod1_value1',
'function_id': function_id,
'function_version': '0'
}
body = {'metadata': {'labels': pod_labels}} body = {'metadata': {'labels': pod_labels}}
self.k8s_v1_api.patch_namespaced_pod.assert_called_once_with( self.k8s_v1_api.patch_namespaced_pod.assert_called_once_with(
pod.metadata.name, self.fake_namespace, body) pod.metadata.name, self.fake_namespace, body)
@ -345,8 +354,9 @@ class TestKubernetesManager(base.BaseTest):
# in _prepare_pod # in _prepare_pod
service_body = self.manager.service_template.render( service_body = self.manager.service_template.render(
{ {
'service_name': 'service-%s' % function_id, 'service_name': 'service-%s-0' % function_id,
'labels': {'function_id': function_id, 'labels': {'function_id': function_id,
'function_version': '0',
'runtime_id': runtime_id}, 'runtime_id': runtime_id},
'selector': pod_labels 'selector': pod_labels
} }
@ -357,13 +367,12 @@ class TestKubernetesManager(base.BaseTest):
def test_prepare_execution_with_image(self): def test_prepare_execution_with_image(self):
function_id = common.generate_unicode_uuid() function_id = common.generate_unicode_uuid()
image = self.rand_name('image', prefix='TestKubernetesManager') image = self.rand_name('image', prefix='TestKubernetesManager')
identifier = ('%s-%s' % ( identifier = ('%s-%s' %
common.generate_unicode_uuid(dashed=False), (common.generate_unicode_uuid(dashed=False), function_id)
function_id)
)[:63] )[:63]
pod_name, url = self.manager.prepare_execution( pod_name, url = self.manager.prepare_execution(
function_id, image=image, identifier=identifier) function_id, 0, image=image, identifier=identifier)
self.assertEqual(identifier, pod_name) self.assertEqual(identifier, pod_name)
self.assertIsNone(url) self.assertIsNone(url)
@ -390,7 +399,7 @@ class TestKubernetesManager(base.BaseTest):
fake_input = {'__function_input': 'input_item1 input_item2'} fake_input = {'__function_input': 'input_item1 input_item2'}
pod_name, url = self.manager.prepare_execution( pod_name, url = self.manager.prepare_execution(
function_id, image=image, identifier=identifier, function_id, 0, image=image, identifier=identifier,
input=fake_input) input=fake_input)
# in _create_pod # in _create_pod
@ -415,7 +424,7 @@ class TestKubernetesManager(base.BaseTest):
fake_input = '["input_item3", "input_item4"]' fake_input = '["input_item3", "input_item4"]'
pod_name, url = self.manager.prepare_execution( pod_name, url = self.manager.prepare_execution(
function_id, image=image, identifier=identifier, function_id, 0, image=image, identifier=identifier,
input=fake_input) input=fake_input)
# in _create_pod # in _create_pod
@ -430,7 +439,7 @@ class TestKubernetesManager(base.BaseTest):
self.k8s_v1_api.create_namespaced_pod.assert_called_once_with( self.k8s_v1_api.create_namespaced_pod.assert_called_once_with(
self.fake_namespace, body=yaml.safe_load(pod_body)) self.fake_namespace, body=yaml.safe_load(pod_body))
def test_prepare_execution_no_worker_available(self): def test_prepare_execution_not_image_no_worker_available(self):
ret_pods = mock.Mock() ret_pods = mock.Mock()
ret_pods.items = [] ret_pods.items = []
self.k8s_v1_api.list_namespaced_pod.return_value = ret_pods self.k8s_v1_api.list_namespaced_pod.return_value = ret_pods
@ -442,14 +451,19 @@ class TestKubernetesManager(base.BaseTest):
exc.OrchestratorException, exc.OrchestratorException,
"^Execution preparation failed\.$", "^Execution preparation failed\.$",
self.manager.prepare_execution, self.manager.prepare_execution,
function_id, image=None, identifier=runtime_id, labels=labels) function_id, 0, image=None, identifier=runtime_id, labels=labels)
# in _choose_available_pods # in _choose_available_pods
list_calls = [ list_calls = [
mock.call(self.fake_namespace, mock.call(
label_selector='function_id=%s' % function_id), self.fake_namespace,
mock.call(self.fake_namespace, label_selector=('function_id=%s,function_version=0' %
label_selector='!function_id,runtime_id=%s' % runtime_id) function_id)
),
mock.call(
self.fake_namespace,
label_selector='!function_id,runtime_id=%s' % runtime_id
)
] ]
self.k8s_v1_api.list_namespaced_pod.assert_has_calls(list_calls) self.k8s_v1_api.list_namespaced_pod.assert_has_calls(list_calls)
self.assertEqual(2, self.k8s_v1_api.list_namespaced_pod.call_count) self.assertEqual(2, self.k8s_v1_api.list_namespaced_pod.call_count)
@ -475,14 +489,14 @@ class TestKubernetesManager(base.BaseTest):
function_id = common.generate_unicode_uuid() function_id = common.generate_unicode_uuid()
pod_names, service_url = self.manager.prepare_execution( pod_names, service_url = self.manager.prepare_execution(
function_id, image=None, identifier=runtime_id, function_id, 0, image=None, identifier=runtime_id,
labels={'runtime_id': runtime_id}) labels={'runtime_id': runtime_id})
# in _prepare_pod # in _prepare_pod
self.k8s_v1_api.read_namespaced_service.assert_called_once_with( self.k8s_v1_api.read_namespaced_service.assert_called_once_with(
'service-%s' % function_id, self.fake_namespace) 'service-%s-0' % function_id, self.fake_namespace)
def test_prepare_execution_pod_preparation_failed(self): def test_prepare_execution_create_service_failed(self):
pod = mock.Mock() pod = mock.Mock()
pod.metadata.name = self.rand_name('pod', pod.metadata.name = self.rand_name('pod',
prefix='TestKubernetesManager') prefix='TestKubernetesManager')
@ -503,12 +517,18 @@ class TestKubernetesManager(base.BaseTest):
exc.OrchestratorException, exc.OrchestratorException,
'^Execution preparation failed\.$', '^Execution preparation failed\.$',
self.manager.prepare_execution, self.manager.prepare_execution,
function_id, image=None, identifier=runtime_id, function_id, 0, image=None, identifier=runtime_id,
labels={'runtime_id': runtime_id}) labels={'runtime_id': runtime_id})
delete_function_mock.assert_called_once_with( delete_function_mock.assert_called_once_with(
function_id, function_id,
{'runtime_id': runtime_id, 'function_id': function_id}) 0,
{
'runtime_id': runtime_id,
'function_id': function_id,
'function_version': '0'
}
)
def test_prepare_execution_service_internal_ip(self): def test_prepare_execution_service_internal_ip(self):
pod = mock.Mock() pod = mock.Mock()
@ -528,7 +548,7 @@ class TestKubernetesManager(base.BaseTest):
function_id = common.generate_unicode_uuid() function_id = common.generate_unicode_uuid()
pod_names, service_url = self.manager.prepare_execution( pod_names, service_url = self.manager.prepare_execution(
function_id, image=None, identifier=runtime_id, function_id, 0, image=None, identifier=runtime_id,
labels={'runtime_id': runtime_id}) labels={'runtime_id': runtime_id})
self.assertEqual(pod.metadata.name, pod_names) self.assertEqual(pod.metadata.name, pod_names)
@ -536,7 +556,7 @@ class TestKubernetesManager(base.BaseTest):
'http://%s:%s' % (SERVICE_ADDRESS_INTERNAL, SERVICE_PORT), 'http://%s:%s' % (SERVICE_ADDRESS_INTERNAL, SERVICE_PORT),
service_url) service_url)
def test_run_execution(self): def test_run_execution_image_type_function(self):
pod = mock.Mock() pod = mock.Mock()
pod.status.phase = 'Succeeded' pod.status.phase = 'Succeeded'
self.k8s_v1_api.read_namespaced_pod.return_value = pod self.k8s_v1_api.read_namespaced_pod.return_value = pod
@ -545,7 +565,8 @@ class TestKubernetesManager(base.BaseTest):
execution_id = common.generate_unicode_uuid() execution_id = common.generate_unicode_uuid()
function_id = common.generate_unicode_uuid() function_id = common.generate_unicode_uuid()
result, output = self.manager.run_execution(execution_id, function_id) result, output = self.manager.run_execution(execution_id, function_id,
0)
self.k8s_v1_api.read_namespaced_pod.assert_called_once_with( self.k8s_v1_api.read_namespaced_pod.assert_called_once_with(
None, self.fake_namespace) None, self.fake_namespace)
@ -554,29 +575,33 @@ class TestKubernetesManager(base.BaseTest):
self.assertTrue(result) self.assertTrue(result)
self.assertEqual(fake_output, output) self.assertEqual(fake_output, output)
@mock.patch('qinling.engine.utils.get_request_data')
@mock.patch('qinling.engine.utils.url_request') @mock.patch('qinling.engine.utils.url_request')
def test_run_execution_with_service_url(self, url_request_mock, def test_run_execution_version_0(self, mock_request):
get_request_data_mock): mock_request.return_value = (True, 'fake output')
fake_output = 'fake output'
url_request_mock.return_value = (True, 'fake output')
fake_data = 'some data'
get_request_data_mock.return_value = fake_data
execution_id = common.generate_unicode_uuid() execution_id = common.generate_unicode_uuid()
function_id = common.generate_unicode_uuid() function_id = common.generate_unicode_uuid()
result, output = self.manager.run_execution( result, output = self.manager.run_execution(
execution_id, function_id, service_url='FAKE_URL') execution_id, function_id, 0, service_url='FAKE_URL'
)
get_request_data_mock.assert_called_once_with( download_url = ('http://127.0.0.1:7070/v1/functions/%s?download=true'
self.conf, function_id, execution_id, None, 'main.main', None, % function_id)
self.qinling_endpoint) data = {
url_request_mock.assert_called_once_with( 'execution_id': execution_id,
self.manager.session, 'FAKE_URL/execute', body=fake_data) 'input': None,
self.assertTrue(result) 'function_id': function_id,
self.assertEqual(fake_output, output) 'function_version': 0,
'entry': 'main.main',
'download_url': download_url,
'request_id': self.ctx.request_id,
}
def test_run_execution_retry(self): mock_request.assert_called_once_with(
self.manager.session, 'FAKE_URL/execute', body=data
)
def test_run_execution_no_service_url_retry(self):
pod1 = mock.Mock() pod1 = mock.Mock()
pod1.status.phase = '' pod1.status.phase = ''
pod2 = mock.Mock() pod2 = mock.Mock()
@ -587,7 +612,8 @@ class TestKubernetesManager(base.BaseTest):
execution_id = common.generate_unicode_uuid() execution_id = common.generate_unicode_uuid()
function_id = common.generate_unicode_uuid() function_id = common.generate_unicode_uuid()
result, output = self.manager.run_execution(execution_id, function_id) result, output = self.manager.run_execution(execution_id, function_id,
0)
self.assertEqual(2, self.k8s_v1_api.read_namespaced_pod.call_count) self.assertEqual(2, self.k8s_v1_api.read_namespaced_pod.call_count)
self.k8s_v1_api.read_namespaced_pod_log.assert_called_once_with( self.k8s_v1_api.read_namespaced_pod_log.assert_called_once_with(
@ -595,12 +621,13 @@ class TestKubernetesManager(base.BaseTest):
self.assertTrue(result) self.assertTrue(result)
self.assertEqual(fake_output, output) self.assertEqual(fake_output, output)
def test_run_execution_failed(self): def test_run_execution_no_service_url_read_pod_exception(self):
self.k8s_v1_api.read_namespaced_pod.side_effect = RuntimeError self.k8s_v1_api.read_namespaced_pod.side_effect = RuntimeError
execution_id = common.generate_unicode_uuid() execution_id = common.generate_unicode_uuid()
function_id = common.generate_unicode_uuid() function_id = common.generate_unicode_uuid()
result, output = self.manager.run_execution(execution_id, function_id) result, output = self.manager.run_execution(execution_id, function_id,
0)
self.k8s_v1_api.read_namespaced_pod.assert_called_once_with( self.k8s_v1_api.read_namespaced_pod.assert_called_once_with(
None, self.fake_namespace) None, self.fake_namespace)
@ -621,10 +648,19 @@ class TestKubernetesManager(base.BaseTest):
self.k8s_v1_api.list_namespaced_service.return_value = services self.k8s_v1_api.list_namespaced_service.return_value = services
function_id = common.generate_unicode_uuid() function_id = common.generate_unicode_uuid()
self.manager.delete_function(function_id) self.manager.delete_function(function_id, 0)
args, kwargs = self.k8s_v1_api.list_namespaced_service.call_args
self.assertIn(self.fake_namespace, args)
self.assertIn(
"function_id=%s" % function_id,
kwargs.get("label_selector")
)
self.assertIn(
"function_version=0",
kwargs.get("label_selector")
)
self.k8s_v1_api.list_namespaced_service.assert_called_once_with(
self.fake_namespace, label_selector='function_id=%s' % function_id)
delete_service_calls = [ delete_service_calls = [
mock.call(svc1_name, self.fake_namespace), mock.call(svc1_name, self.fake_namespace),
mock.call(svc2_name, self.fake_namespace) mock.call(svc2_name, self.fake_namespace)
@ -632,10 +668,20 @@ class TestKubernetesManager(base.BaseTest):
self.k8s_v1_api.delete_namespaced_service.assert_has_calls( self.k8s_v1_api.delete_namespaced_service.assert_has_calls(
delete_service_calls) delete_service_calls)
self.assertEqual( self.assertEqual(
2, self.k8s_v1_api.delete_namespaced_service.call_count) 2, self.k8s_v1_api.delete_namespaced_service.call_count
delete_pod = self.k8s_v1_api.delete_collection_namespaced_pod )
delete_pod.assert_called_once_with(
self.fake_namespace, label_selector='function_id=%s' % function_id) args, kwargs = self.k8s_v1_api.delete_collection_namespaced_pod. \
call_args
self.assertIn(self.fake_namespace, args)
self.assertIn(
"function_id=%s" % function_id,
kwargs.get("label_selector")
)
self.assertIn(
"function_version=0",
kwargs.get("label_selector")
)
def test_delete_function_with_labels(self): def test_delete_function_with_labels(self):
services = mock.Mock() services = mock.Mock()
@ -645,7 +691,7 @@ class TestKubernetesManager(base.BaseTest):
self.k8s_v1_api.list_namespaced_service.return_value = services self.k8s_v1_api.list_namespaced_service.return_value = services
function_id = common.generate_unicode_uuid() function_id = common.generate_unicode_uuid()
self.manager.delete_function(function_id, labels=labels) self.manager.delete_function(function_id, 0, labels=labels)
self.k8s_v1_api.list_namespaced_service.assert_called_once_with( self.k8s_v1_api.list_namespaced_service.assert_called_once_with(
self.fake_namespace, label_selector=selector) self.fake_namespace, label_selector=selector)
@ -672,7 +718,8 @@ class TestKubernetesManager(base.BaseTest):
function_id = common.generate_unicode_uuid() function_id = common.generate_unicode_uuid()
pod_names, service_url = self.manager.scaleup_function( pod_names, service_url = self.manager.scaleup_function(
function_id, identifier=runtime_id) function_id, 0, identifier=runtime_id
)
self.assertEqual([pod.metadata.name], pod_names) self.assertEqual([pod.metadata.name], pod_names)
self.assertEqual( self.assertEqual(
@ -685,7 +732,11 @@ class TestKubernetesManager(base.BaseTest):
label_selector='!function_id,runtime_id=%s' % runtime_id) label_selector='!function_id,runtime_id=%s' % runtime_id)
# in _prepare_pod -> _update_pod_label # in _prepare_pod -> _update_pod_label
pod_labels = {'pod1_key1': 'pod1_value1', 'function_id': function_id} pod_labels = {
'pod1_key1': 'pod1_value1',
'function_id': function_id,
'function_version': '0'
}
body = {'metadata': {'labels': pod_labels}} body = {'metadata': {'labels': pod_labels}}
self.k8s_v1_api.patch_namespaced_pod.assert_called_once_with( self.k8s_v1_api.patch_namespaced_pod.assert_called_once_with(
pod.metadata.name, self.fake_namespace, body) pod.metadata.name, self.fake_namespace, body)
@ -693,8 +744,9 @@ class TestKubernetesManager(base.BaseTest):
# in _prepare_pod # in _prepare_pod
service_body = self.manager.service_template.render( service_body = self.manager.service_template.render(
{ {
'service_name': 'service-%s' % function_id, 'service_name': 'service-%s-0' % function_id,
'labels': {'function_id': function_id, 'labels': {'function_id': function_id,
'function_version': 0,
'runtime_id': runtime_id}, 'runtime_id': runtime_id},
'selector': pod_labels 'selector': pod_labels
} }
@ -713,7 +765,7 @@ class TestKubernetesManager(base.BaseTest):
exc.OrchestratorException, exc.OrchestratorException,
"^Not enough workers available\.$", "^Not enough workers available\.$",
self.manager.scaleup_function, self.manager.scaleup_function,
function_id, identifier=runtime_id, count=2) function_id, 0, identifier=runtime_id, count=2)
def test_scaleup_function_service_already_exists(self): def test_scaleup_function_service_already_exists(self):
pod = mock.Mock() pod = mock.Mock()
@ -736,11 +788,11 @@ class TestKubernetesManager(base.BaseTest):
function_id = common.generate_unicode_uuid() function_id = common.generate_unicode_uuid()
pod_names, service_url = self.manager.scaleup_function( pod_names, service_url = self.manager.scaleup_function(
function_id, identifier=runtime_id) function_id, 0, identifier=runtime_id)
# in _prepare_pod # in _prepare_pod
self.k8s_v1_api.read_namespaced_service.assert_called_once_with( self.k8s_v1_api.read_namespaced_service.assert_called_once_with(
'service-%s' % function_id, self.fake_namespace) 'service-%s-0' % function_id, self.fake_namespace)
def test_scaleup_function_service_create_failed(self): def test_scaleup_function_service_create_failed(self):
pod = mock.Mock() pod = mock.Mock()
@ -759,7 +811,7 @@ class TestKubernetesManager(base.BaseTest):
self.assertRaises( self.assertRaises(
RuntimeError, RuntimeError,
self.manager.scaleup_function, self.manager.scaleup_function,
function_id, identifier=runtime_id) function_id, 0, identifier=runtime_id)
def test_scaleup_function_service_internal_ip(self): def test_scaleup_function_service_internal_ip(self):
pod = mock.Mock() pod = mock.Mock()
@ -779,7 +831,7 @@ class TestKubernetesManager(base.BaseTest):
function_id = common.generate_unicode_uuid() function_id = common.generate_unicode_uuid()
pod_names, service_url = self.manager.scaleup_function( pod_names, service_url = self.manager.scaleup_function(
function_id, identifier=runtime_id) function_id, 0, identifier=runtime_id)
self.assertEqual([pod.metadata.name], pod_names) self.assertEqual([pod.metadata.name], pod_names)
self.assertEqual( self.assertEqual(

44
qinling/tests/unit/services/test_periodics.py
View File

@ -20,7 +20,6 @@ from oslo_config import cfg
from qinling import context from qinling import context
from qinling.db import api as db_api from qinling.db import api as db_api
from qinling.engine import default_engine
from qinling.services import periodics from qinling.services import periodics
from qinling import status from qinling import status
from qinling.tests.unit import base from qinling.tests.unit import base
@ -35,24 +34,50 @@ class TestPeriodics(base.DbTestCase):
@mock.patch('qinling.utils.etcd_util.delete_function') @mock.patch('qinling.utils.etcd_util.delete_function')
@mock.patch('qinling.utils.etcd_util.get_service_url') @mock.patch('qinling.utils.etcd_util.get_service_url')
def test_function_service_expiration_handler(self, mock_etcd_url, def test_handle_function_service_no_function_version(self, mock_etcd_url,
mock_etcd_delete): mock_etcd_delete):
db_func = self.create_function() db_func = self.create_function()
function_id = db_func.id function_id = db_func.id
# Update function to simulate function execution # Update function to simulate function execution
db_api.update_function(function_id, {'count': 1}) db_api.update_function(function_id, {'count': 1})
time.sleep(1.5) time.sleep(1.5)
mock_k8s = mock.Mock()
mock_etcd_url.return_value = 'http://localhost:37718' mock_etcd_url.return_value = 'http://localhost:37718'
self.override_config('function_service_expiration', 1, 'engine') self.override_config('function_service_expiration', 1, 'engine')
engine = default_engine.DefaultEngine(mock_k8s, CONF.qinling_endpoint) mock_engine = mock.Mock()
periodics.handle_function_service_expiration(self.ctx, engine)
self.assertEqual(1, mock_k8s.delete_function.call_count) periodics.handle_function_service_expiration(self.ctx, mock_engine)
args, kwargs = mock_k8s.delete_function.call_args
self.assertEqual(1, mock_engine.delete_function.call_count)
args, _ = mock_engine.delete_function.call_args
self.assertIn(function_id, args) self.assertIn(function_id, args)
mock_etcd_delete.assert_called_once_with(function_id) self.assertIn(0, args)
mock_etcd_delete.assert_called_once_with(function_id, 0)
@mock.patch('qinling.utils.etcd_util.delete_function')
@mock.patch('qinling.utils.etcd_util.get_service_url')
def test_handle_function_service_with_function_versions(self, mock_srv_url,
mock_etcd_delete):
db_func = self.create_function()
function_id = db_func.id
db_api.increase_function_version(function_id, 0,
description="new version")
db_api.update_function_version(function_id, 1, count=1)
time.sleep(1.5)
self.override_config('function_service_expiration', 1, 'engine')
mock_srv_url.return_value = 'http://localhost:37718'
mock_engine = mock.Mock()
periodics.handle_function_service_expiration(self.ctx, mock_engine)
self.assertEqual(1, mock_engine.delete_function.call_count)
args, _ = mock_engine.delete_function.call_args
self.assertIn(function_id, args)
self.assertIn(1, args)
mock_etcd_delete.assert_called_once_with(function_id, 1)
@mock.patch('qinling.utils.jobs.get_next_execution_time') @mock.patch('qinling.utils.jobs.get_next_execution_time')
def test_job_handler(self, mock_get_next): def test_job_handler(self, mock_get_next):
@ -72,6 +97,7 @@ class TestPeriodics(base.DbTestCase):
e_client = mock.Mock() e_client = mock.Mock()
mock_get_next.return_value = now + timedelta(seconds=1) mock_get_next.return_value = now + timedelta(seconds=1)
periodics.handle_job(e_client) periodics.handle_job(e_client)
context.set_ctx(self.ctx) context.set_ctx(self.ctx)

31
qinling/utils/etcd_util.py
View File

@ -29,9 +29,10 @@ def get_client(conf=None):
return CLIENT return CLIENT
def get_worker_lock(): def get_worker_lock(function_id, version=0):
client = get_client() client = get_client()
return client.lock(id='function_worker') lock_id = "function_worker_%s_%s" % (function_id, version)
return client.lock(id=lock_id)
def get_function_version_lock(function_id): def get_function_version_lock(function_id):
@ -40,7 +41,7 @@ def get_function_version_lock(function_id):
return client.lock(id=lock_id) return client.lock(id=lock_id)
def create_worker(function_id, worker): def create_worker(function_id, worker, version=0):
"""Create the worker info in etcd. """Create the worker info in etcd.
The worker parameter is assumed to be unique. The worker parameter is assumed to be unique.
@ -50,34 +51,34 @@ def create_worker(function_id, worker):
# is the name of the pod so it is unique. # is the name of the pod so it is unique.
client = get_client() client = get_client()
client.create( client.create(
'%s/worker_%s' % (function_id, worker), '%s_%s/worker_%s' % (function_id, version, worker),
worker worker
) )
def delete_worker(function_id, worker): def delete_worker(function_id, worker, version=0):
client = get_client() client = get_client()
client.delete('%s/worker_%s' % (function_id, worker)) client.delete('%s_%s/worker_%s' % (function_id, version, worker))
def get_workers(function_id, conf=None): def get_workers(function_id, version=0):
client = get_client(conf) client = get_client()
values = client.get_prefix('%s/worker' % function_id) values = client.get_prefix("%s_%s/worker" % (function_id, version))
workers = [w[0] for w in values] workers = [w[0] for w in values]
return workers return workers
def delete_function(function_id): def delete_function(function_id, version=0):
client = get_client() client = get_client()
client.delete_prefix(function_id) client.delete_prefix("%s_%s" % (function_id, version))
def create_service_url(function_id, url): def create_service_url(function_id, url, version=0):
client = get_client() client = get_client()
client.create('%s/service_url' % function_id, url) client.create('%s_%s/service_url' % (function_id, version), url)
def get_service_url(function_id): def get_service_url(function_id, version=0):
client = get_client() client = get_client()
values = client.get('%s/service_url' % function_id) values = client.get('%s_%s/service_url' % (function_id, version))
return None if not values else values[0] return None if not values else values[0]

73
qinling/utils/executions.py
View File

@ -19,49 +19,86 @@ from qinling.db import api as db_api
from qinling.db.sqlalchemy import models from qinling.db.sqlalchemy import models
from qinling import exceptions as exc from qinling import exceptions as exc
from qinling import status from qinling import status
from qinling.utils import constants
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
def _update_function_db(function_id): def _update_function_db(function_id, pre_count):
with db_api.transaction():
# NOTE(kong): Store function info in cache?
func_db = db_api.get_function(function_id)
runtime_db = func_db.runtime
if runtime_db and runtime_db.status != status.AVAILABLE:
raise exc.RuntimeNotAvailableException(
'Runtime %s is not available.' % func_db.runtime_id
)
# Function update is done using UPDATE ... FROM ... WHERE # Function update is done using UPDATE ... FROM ... WHERE
# non-locking clause. # non-locking clause.
while func_db: while True:
count = func_db.count
modified = db_api.conditional_update( modified = db_api.conditional_update(
models.Function, models.Function,
{ {
'count': count + 1, 'count': pre_count + 1,
}, },
{ {
'id': function_id, 'id': function_id,
'count': count 'count': pre_count
}, },
insecure=True, insecure=True,
) )
if not modified: if not modified:
LOG.warning("Retrying to update function count.") LOG.warning("Retrying to update function count.")
func_db = db_api.get_function(function_id) pre_count += 1
continue continue
else: else:
break break
return func_db.runtime_id
def _update_function_version_db(version_id, pre_count):
# Update is done using UPDATE ... FROM ... WHERE non-locking clause.
while True:
modified = db_api.conditional_update(
models.FunctionVersion,
{
'count': pre_count + 1,
},
{
'id': version_id,
'count': pre_count
},
insecure=True,
)
if not modified:
LOG.warning("Retrying to update function version count.")
pre_count += 1
continue
else:
break
def create_execution(engine_client, params): def create_execution(engine_client, params):
function_id = params['function_id'] function_id = params['function_id']
is_sync = params.get('sync', True) is_sync = params.get('sync', True)
input = params.get('input') input = params.get('input')
version = params.get('function_version', 0)
with db_api.transaction():
func_db = db_api.get_function(function_id)
runtime_id = func_db.runtime_id
runtime_db = func_db.runtime
if runtime_db and runtime_db.status != status.AVAILABLE:
raise exc.RuntimeNotAvailableException(
'Runtime %s is not available.' % func_db.runtime_id
)
if version > 0:
if func_db.code['source'] != constants.PACKAGE_FUNCTION:
raise exc.InputException(
"Can not specify version for %s type function." %
constants.PACKAGE_FUNCTION
)
# update version count
version_db = db_api.get_function_version(function_id, version)
pre_version_count = version_db.count
_update_function_version_db(version_db.id, pre_version_count)
else:
pre_count = func_db.count
_update_function_db(function_id, pre_count)
# input in params should be a string. # input in params should be a string.
if input: if input:
@ -70,14 +107,12 @@ def create_execution(engine_client, params):
except ValueError: except ValueError:
params['input'] = {'__function_input': input} params['input'] = {'__function_input': input}
runtime_id = _update_function_db(function_id)
params.update({'status': status.RUNNING}) params.update({'status': status.RUNNING})
db_model = db_api.create_execution(params) db_model = db_api.create_execution(params)
try: try:
engine_client.create_execution( engine_client.create_execution(
db_model.id, function_id, runtime_id, db_model.id, function_id, version, runtime_id,
input=params.get('input'), is_sync=is_sync input=params.get('input'), is_sync=is_sync
) )
except exc.QinlingException: except exc.QinlingException: